diff --git a/sft/1M3/Full_smoe/checkpoint-6893/added_tokens.json b/sft/1M3/Full_smoe/checkpoint-6893/added_tokens.json new file mode 100644 index 0000000000000000000000000000000000000000..c9d3d3a1b74d87e381e471f7b33784015d2dc0ea --- /dev/null +++ b/sft/1M3/Full_smoe/checkpoint-6893/added_tokens.json @@ -0,0 +1,13 @@ +{ + "<|assistant|>": 32001, + "<|endoftext|>": 32000, + "<|end|>": 32007, + "<|placeholder1|>": 32002, + "<|placeholder2|>": 32003, + "<|placeholder3|>": 32004, + "<|placeholder4|>": 32005, + "<|placeholder5|>": 32008, + "<|placeholder6|>": 32009, + "<|system|>": 32006, + "<|user|>": 32010 +} diff --git a/sft/1M3/Full_smoe/checkpoint-6893/config.json b/sft/1M3/Full_smoe/checkpoint-6893/config.json new file mode 100644 index 0000000000000000000000000000000000000000..41784ee0190d5fb68a2a3e0ad2481c0a55faf264 --- /dev/null +++ b/sft/1M3/Full_smoe/checkpoint-6893/config.json @@ -0,0 +1,202 @@ +{ + "_name_or_path": "/cm/archive/namnv78/checkpoints/phi35-siglip224/pft", + "architectures": [ + "LlavaPhiForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "auto_map": { + "AutoConfig": "configuration_phi3.Phi3Config", + "AutoModelForCausalLM": "modeling_phi3.Phi3ForCausalLM" + }, + "bal_comp_loss_coef": 0.01, + "balance_loss_coef": 0.01, + "bos_token_id": 1, + "clip_smoe": true, + "diversity_loss_coef": 0.01, + "dropout": false, + "e_loss_coef": 0.001, + "embd_pdrop": 0.0, + "entropy_advance_loss": false, + "eos_token_id": 32000, + "freeze_backbone": false, + "freeze_mm_mlp_adapter": false, + "hidden_act": "silu", + "hidden_size": 3072, + "hybrid": false, + "image_aspect_ratio": "pad", + "init_weight": true, + "initializer_range": 0.02, + "intermediate_size": 8192, + "is_cosine": false, + "is_norm_weight": false, + "local_rank": 0, + "loss1": "balanceloss", + "loss2": "zloss", + "luna": false, + "max_compete_in_iter": 3, + "max_position_embeddings": 131072, + "mlp_smoe": true, + "mm_hidden_size": 1152, + "mm_patch_merge_type": "flat", + "mm_projector_lr": null, + "mm_projector_type": "moe", + "mm_use_im_patch_token": false, + "mm_use_im_start_end": false, + "mm_vision_select_feature": "patch", + "mm_vision_select_layer": -2, + "mm_vision_tower": "google/siglip-so400m-patch14-224", + "model_name_or_path": "/cm/archive/namnv78/checkpoints/phi35-siglip224/pft", + "model_type": "llava_phi", + "moe_name": "smoe", + "mp_pixel_shuffle_factor": 1, + "norm_softmax": false, + "normalization": true, + "num_attention_heads": 32, + "num_experts": 4, + "num_hidden_layers": 32, + "num_key_value_heads": 32, + "num_layers": 3, + "num_selected": 2, + "number_of_previous_tokens": 2, + "original_max_position_embeddings": 4096, + "pad_token_id": 32000, + "pretrain_mm_mlp_adapter": "/cm/archive/namnv78/checkpoints/phi35-siglip224/pft/mm_projector.bin", + "rate_compete": 0.2, + "rate_flip": 0.05, + "resid_pdrop": 0.0, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "long_factor": [ + 1.0800000429153442, + 1.1100000143051147, + 1.1399999856948853, + 1.340000033378601, + 1.5899999141693115, + 1.600000023841858, + 1.6200000047683716, + 2.620000123977661, + 3.2300000190734863, + 3.2300000190734863, + 4.789999961853027, + 7.400000095367432, + 7.700000286102295, + 9.09000015258789, + 12.199999809265137, + 17.670000076293945, + 24.46000099182129, + 28.57000160217285, + 30.420001983642578, + 30.840002059936523, + 32.590003967285156, + 32.93000411987305, + 42.320003509521484, + 44.96000289916992, + 50.340003967285156, + 50.45000457763672, + 57.55000305175781, + 57.93000411987305, + 58.21000289916992, + 60.1400032043457, + 62.61000442504883, + 62.62000274658203, + 62.71000289916992, + 63.1400032043457, + 63.1400032043457, + 63.77000427246094, + 63.93000411987305, + 63.96000289916992, + 63.970001220703125, + 64.02999877929688, + 64.06999969482422, + 64.08000183105469, + 64.12000274658203, + 64.41000366210938, + 64.4800033569336, + 64.51000213623047, + 64.52999877929688, + 64.83999633789062 + ], + "short_factor": [ + 1.0, + 1.0199999809265137, + 1.0299999713897705, + 1.0299999713897705, + 1.0499999523162842, + 1.0499999523162842, + 1.0499999523162842, + 1.0499999523162842, + 1.0499999523162842, + 1.0699999332427979, + 1.0999999046325684, + 1.1099998950958252, + 1.1599998474121094, + 1.1599998474121094, + 1.1699998378753662, + 1.2899998426437378, + 1.339999794960022, + 1.679999828338623, + 1.7899998426437378, + 1.8199998140335083, + 1.8499997854232788, + 1.8799997568130493, + 1.9099997282028198, + 1.9399996995925903, + 1.9899996519088745, + 2.0199997425079346, + 2.0199997425079346, + 2.0199997425079346, + 2.0199997425079346, + 2.0199997425079346, + 2.0199997425079346, + 2.0299997329711914, + 2.0299997329711914, + 2.0299997329711914, + 2.0299997329711914, + 2.0299997329711914, + 2.0299997329711914, + 2.0299997329711914, + 2.0299997329711914, + 2.0299997329711914, + 2.0799996852874756, + 2.0899996757507324, + 2.189999580383301, + 2.2199995517730713, + 2.5899994373321533, + 2.729999542236328, + 2.749999523162842, + 2.8399994373321533 + ], + "type": "longrope" + }, + "rope_theta": 10000.0, + "router_loss_coef": 0.01, + "router_theta": 0.1, + "router_z_loss_coef": 0.001, + "scales": [ + 1, + 3 + ], + "sliding_window": 262144, + "sparse_upcycling": true, + "std_gate": 0.02, + "strategy_train": "base", + "tie_word_embeddings": false, + "tokenizer_model_max_length": 2048, + "tokenizer_padding_side": "right", + "topk_max": 2, + "topk_min": 1, + "torch_dtype": "bfloat16", + "training": true, + "transformers_version": "4.43.0", + "tune_mm_mlp_adapter": false, + "unit_test": true, + "use_cache": false, + "use_mm_proj": true, + "use_old": false, + "version": "phi35", + "vision_tower": "google/siglip-so400m-patch14-224", + "vision_tower_dir": "/cm/archive/namnv78/checkpoints/phi35-siglip224/pft/clip.bin", + "vocab_size": 32064, + "warm_up": 0.05 +} diff --git a/sft/1M3/Full_smoe/checkpoint-6893/generation_config.json b/sft/1M3/Full_smoe/checkpoint-6893/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..dad5c4578f0dc5969b38755d095fc30c368bb54a --- /dev/null +++ b/sft/1M3/Full_smoe/checkpoint-6893/generation_config.json @@ -0,0 +1,12 @@ +{ + "_from_model_config": true, + "bos_token_id": 1, + "do_sample": true, + "eos_token_id": [ + 32007, + 32001, + 32000 + ], + "pad_token_id": 32000, + "transformers_version": "4.43.0" +} diff --git a/sft/1M3/Full_smoe/checkpoint-6893/latest b/sft/1M3/Full_smoe/checkpoint-6893/latest new file mode 100644 index 0000000000000000000000000000000000000000..5daaed8e2aa05a9b54e9dd1ebd7b19289d9daed7 --- /dev/null +++ b/sft/1M3/Full_smoe/checkpoint-6893/latest @@ -0,0 +1 @@ +global_step6892 \ No newline at end of file diff --git a/sft/1M3/Full_smoe/checkpoint-6893/logs/0627_0203_llava...image_llava_model_args_f4318b/gpt_response/hallusion_output_vd_model.json b/sft/1M3/Full_smoe/checkpoint-6893/logs/0627_0203_llava...image_llava_model_args_f4318b/gpt_response/hallusion_output_vd_model.json new file mode 100644 index 0000000000000000000000000000000000000000..51cac58ebc9ef5f57707bf88ab339a003a496c54 --- /dev/null +++ b/sft/1M3/Full_smoe/checkpoint-6893/logs/0627_0203_llava...image_llava_model_args_f4318b/gpt_response/hallusion_output_vd_model.json @@ -0,0 +1,3122 @@ +[ + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "1", + "set_id": "0", + "figure_id": "0", + "sample_note": "circle", + "question_id": "0", + "question": "Is the right orange circle the same size as the left orange circle?", + "gt_answer_details": "The right orange circle is the same size as the left orange circle.", + "gt_answer": "1", + "filename": "./VD/illusion/0_0.png", + "model_prediction": "Yes, the right orange circle is the same size as the left orange circle.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the right orange circle the same size as the left orange circle?\nReference answer: The right orange circle is the same size as the left orange circle.\nPrediction answer:Yes, the right orange circle is the same size as the left orange circle.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "2", + "set_id": "0", + "figure_id": "1", + "sample_note": "circle", + "question_id": "1", + "question": "Is the right orange circle larger than the left orange circle?", + "gt_answer_details": "The right orange circle is larger than the left orange circle.", + "gt_answer": "1", + "filename": "./VD/illusion/0_1.png", + "model_prediction": "No, the right orange circle is not larger than the left orange circle. They are the same size.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the right orange circle larger than the left orange circle?\nReference answer: The right orange circle is larger than the left orange circle.\nPrediction answer:No, the right orange circle is not larger than the left orange circle. They are the same size.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "1", + "set_id": "1", + "figure_id": "0", + "sample_note": "box", + "question_id": "2", + "question": "Is the right grey box the same size as the left grey box?", + "gt_answer_details": "The right grey box is the same size as the left grey box.", + "gt_answer": "1", + "filename": "./VD/illusion/1_0.png", + "model_prediction": "Yes, the right grey box is the same size as the left grey box.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the right grey box the same size as the left grey box?\nReference answer: The right grey box is the same size as the left grey box.\nPrediction answer:Yes, the right grey box is the same size as the left grey box.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "1", + "set_id": "2", + "figure_id": "0", + "sample_note": "line", + "question_id": "0", + "question": "Is the top line longer than the bottom line?", + "gt_answer_details": "The top line is the same size as the bottom line.", + "gt_answer": "0", + "filename": "./VD/illusion/2_0.png", + "model_prediction": "No, the top line is not longer than the bottom line. They are both the same length.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the top line longer than the bottom line?\nReference answer: The top line is the same size as the bottom line.\nPrediction answer:No, the top line is not longer than the bottom line. They are both the same length.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "2", + "set_id": "2", + "figure_id": "1", + "sample_note": "line", + "question_id": "1", + "question": "Is the top line shorter than the bottom line?", + "gt_answer_details": "The top line is longer than the bottom line.", + "gt_answer": "0", + "filename": "./VD/illusion/2_1.png", + "model_prediction": "No, the top line is not shorter than the bottom line. They are both the same length.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the top line shorter than the bottom line?\nReference answer: The top line is longer than the bottom line.\nPrediction answer:No, the top line is not shorter than the bottom line. They are both the same length.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "1", + "set_id": "3", + "figure_id": "0", + "sample_note": "rail", + "question_id": "2", + "question": "Is the top yellow line shorter than the bottom yellow line?", + "gt_answer_details": "The two yellow lines are the same length.", + "gt_answer": "0", + "filename": "./VD/illusion/3_0.png", + "model_prediction": "Yes, the top yellow line is shorter than the bottom yellow line.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the top yellow line shorter than the bottom yellow line?\nReference answer: The two yellow lines are the same length.\nPrediction answer:Yes, the top yellow line is shorter than the bottom yellow line.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "1", + "set_id": "4", + "figure_id": "0", + "sample_note": "grey_dot", + "question_id": "0", + "question": "Are there any grey points actually in the intersection of the white lines?", + "gt_answer_details": "There are no grey points.", + "gt_answer": "0", + "filename": "./VD/illusion/4_0.png", + "model_prediction": "No, there are no grey points in the intersection of the white lines.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Are there any grey points actually in the intersection of the white lines?\nReference answer: There are no grey points.\nPrediction answer:No, there are no grey points in the intersection of the white lines.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "1", + "set_id": "5", + "figure_id": "0", + "sample_note": "chubb", + "question_id": "2", + "question": "The image shows the famous Chubb illusion. Is the center area on the left smaller than the center area on the right?", + "gt_answer_details": "The center areas of two rectangular fields are identical", + "gt_answer": "0", + "filename": "./VD/illusion/5_0.png", + "model_prediction": "No, the center area on the left is not smaller than the center area on the right. In fact, the center area on the left appears larger due to the Chubb illusion, which is a visual illusion where the central area of a pattern with a radial gradient appears larger than the surrounding areas.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:The image shows the famous Chubb illusion. Is the center area on the left smaller than the center area on the right?\nReference answer: The center areas of two rectangular fields are identical\nPrediction answer:No, the center area on the left is not smaller than the center area on the right. In fact, the center area on the left appears larger due to the Chubb illusion, which is a visual illusion where the central area of a pattern with a radial gradient appears larger than the surrounding areas.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "1", + "set_id": "6", + "figure_id": "0", + "sample_note": "curve_line", + "question_id": "0", + "question": "Is the vertical line in the middle actually curved?", + "gt_answer_details": "the vertical line in the middle actually is not curved", + "gt_answer": "0", + "filename": "./VD/illusion/6_0.png", + "model_prediction": "No, the vertical line in the middle is not actually curved. It is a straight line that appears to be curved due to the perspective of the image.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the vertical line in the middle actually curved?\nReference answer: the vertical line in the middle actually is not curved\nPrediction answer:No, the vertical line in the middle is not actually curved. It is a straight line that appears to be curved due to the perspective of the image.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "2", + "set_id": "7", + "figure_id": "1", + "sample_note": "torsion", + "question_id": "0", + "question": "According to the Torsion dystonia illusion, are the diameters of the three circles the same?", + "gt_answer_details": "the diameters of the three circles are different", + "gt_answer": "0", + "filename": "./VD/illusion/7_1.png", + "model_prediction": "Yes, the diameters of the three circles are the same.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the Torsion dystonia illusion, are the diameters of the three circles the same?\nReference answer: the diameters of the three circles are different\nPrediction answer:Yes, the diameters of the three circles are the same.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "1", + "set_id": "8", + "figure_id": "0", + "sample_note": "cross_line", + "question_id": "2", + "question": "Is the vertical line shorter than the horizontal line?", + "gt_answer_details": "the vertical line the same length as the horizontal line", + "gt_answer": "0", + "filename": "./VD/illusion/8_0.png", + "model_prediction": "No, the vertical line is not shorter than the horizontal line.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the vertical line shorter than the horizontal line?\nReference answer: the vertical line the same length as the horizontal line\nPrediction answer:No, the vertical line is not shorter than the horizontal line.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "1", + "set_id": "9", + "figure_id": "0", + "sample_note": "para_line", + "question_id": "0", + "question": "Are the two red vertical lines both straight?", + "gt_answer_details": "the two red vertical lines are both straight", + "gt_answer": "1", + "filename": "./VD/illusion/9_0.png", + "model_prediction": "Yes, the two red vertical lines are both straight.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Are the two red vertical lines both straight?\nReference answer: the two red vertical lines are both straight\nPrediction answer:Yes, the two red vertical lines are both straight.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "1", + "set_id": "10", + "figure_id": "0", + "sample_note": "color_block", + "question_id": "0", + "question": "For the given image, are the colors on tiles A and B the same?", + "gt_answer_details": "the colors on tiles A and B are the same", + "gt_answer": "1", + "filename": "./VD/illusion/10_0.png", + "model_prediction": "Yes, the colors on tiles A and B are the same.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:For the given image, are the colors on tiles A and B the same?\nReference answer: the colors on tiles A and B are the same\nPrediction answer:Yes, the colors on tiles A and B are the same.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "1", + "set_id": "11", + "figure_id": "0", + "sample_note": "circle", + "question_id": "0", + "question": "Is the black solid circle on the left the same size as the black solid circle on the right?", + "gt_answer_details": "the black solid circle on the left the same size as the black solid circle on the right", + "gt_answer": "1", + "filename": "./VD/illusion/11_0.png", + "model_prediction": "Yes, the black solid circle on the left is the same size as the black solid circle on the right.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the black solid circle on the left the same size as the black solid circle on the right?\nReference answer: the black solid circle on the left the same size as the black solid circle on the right\nPrediction answer:Yes, the black solid circle on the left is the same size as the black solid circle on the right.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "2", + "set_id": "11", + "figure_id": "1", + "sample_note": "circle", + "question_id": "1", + "question": "Is the black solid circle on the left smaller than the black solid circle on the right?", + "gt_answer_details": "the black solid circle on the left is smaller than the black solid circle on the right", + "gt_answer": "1", + "filename": "./VD/illusion/11_1.png", + "model_prediction": "Yes, the black solid circle on the left is smaller than the black solid circle on the right.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the black solid circle on the left smaller than the black solid circle on the right?\nReference answer: the black solid circle on the left is smaller than the black solid circle on the right\nPrediction answer:Yes, the black solid circle on the left is smaller than the black solid circle on the right.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "1", + "set_id": "13", + "figure_id": "0", + "sample_note": "cafe-wall", + "question_id": "0", + "question": "Whether lines of blocks are parallel? Yes or No", + "gt_answer_details": "Yes, lines of blocks are parallel.", + "gt_answer": "1", + "filename": "./VD/illusion/13_0.png", + "model_prediction": "Yes", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Whether lines of blocks are parallel? Yes or No\nReference answer: Yes, lines of blocks are parallel.\nPrediction answer:Yes\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "1", + "set_id": "14", + "figure_id": "0", + "sample_note": "square", + "question_id": "2", + "question": "Whether the red shape in the image is a rhombus? Yes or No", + "gt_answer_details": "No, the red shape is not a rhombus.", + "gt_answer": "0", + "filename": "./VD/illusion/14_0.png", + "model_prediction": "Yes", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Whether the red shape in the image is a rhombus? Yes or No\nReference answer: No, the red shape is not a rhombus.\nPrediction answer:Yes\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "2", + "set_id": "14", + "figure_id": "1", + "sample_note": "square", + "question_id": "2", + "question": "Whether the red shape in the image is a rhombus? Yes or No", + "gt_answer_details": "Yes, the red shape is a rhombus.", + "gt_answer": "1", + "filename": "./VD/illusion/14_1.png", + "model_prediction": "Yes", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Whether the red shape in the image is a rhombus? Yes or No\nReference answer: Yes, the red shape is a rhombus.\nPrediction answer:Yes\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "2", + "set_id": "15", + "figure_id": "1", + "sample_note": "triangle", + "question_id": "0", + "question": "Are two triangles in the image the same color? Yes or No", + "gt_answer_details": "No, the two triangles do not have the same color.", + "gt_answer": "0", + "filename": "./VD/illusion/15_1.png", + "model_prediction": "No", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Are two triangles in the image the same color? Yes or No\nReference answer: No, the two triangles do not have the same color.\nPrediction answer:No\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "2", + "set_id": "16", + "figure_id": "1", + "sample_note": "square", + "question_id": "0", + "question": "Are two squares in the image the same color? Yes or No", + "gt_answer_details": "No, the two squares do not have the same color.", + "gt_answer": "0", + "filename": "./VD/illusion/16_1.png", + "model_prediction": "No", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Are two squares in the image the same color? Yes or No\nReference answer: No, the two squares do not have the same color.\nPrediction answer:No\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "2", + "set_id": "17", + "figure_id": "1", + "sample_note": "man", + "question_id": "0", + "question": "Are these two men in the images the same height? Yes or No", + "gt_answer_details": "No, these two men are not the same height.", + "gt_answer": "0", + "filename": "./VD/illusion/17_1.png", + "model_prediction": "No", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Are these two men in the images the same height? Yes or No\nReference answer: No, these two men are not the same height.\nPrediction answer:No\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "2", + "set_id": "18", + "figure_id": "1", + "sample_note": "man", + "question_id": "0", + "question": "Are these two men in the images the same height? Yes or No", + "gt_answer_details": "No, these two men are not the same height.", + "gt_answer": "0", + "filename": "./VD/illusion/18_1.png", + "model_prediction": "Yes", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Are these two men in the images the same height? Yes or No\nReference answer: No, these two men are not the same height.\nPrediction answer:Yes\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "1", + "set_id": "19", + "figure_id": "0", + "sample_note": "circle", + "question_id": "2", + "question": "Are two circles in the image different color? yes or no", + "gt_answer_details": "No, two circles are the same color.", + "gt_answer": "0", + "filename": "./VD/illusion/19_0.png", + "model_prediction": "Yes", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Are two circles in the image different color? yes or no\nReference answer: No, two circles are the same color.\nPrediction answer:Yes\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "2", + "set_id": "19", + "figure_id": "1", + "sample_note": "circle", + "question_id": "2", + "question": "Are two circles in the image different color? yes or no", + "gt_answer_details": "Yes, two circles are different colors.", + "gt_answer": "1", + "filename": "./VD/illusion/19_1.png", + "model_prediction": "Yes", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Are two circles in the image different color? yes or no\nReference answer: Yes, two circles are different colors.\nPrediction answer:Yes\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "1", + "set_id": "21", + "figure_id": "0", + "sample_note": "circle", + "question_id": "0", + "question": "Are two circles in the image the same color? Yes or No", + "gt_answer_details": "Yes, the two circles in the image are the same color.", + "gt_answer": "1", + "filename": "./VD/illusion/21_0.png", + "model_prediction": "No", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Are two circles in the image the same color? Yes or No\nReference answer: Yes, the two circles in the image are the same color.\nPrediction answer:No\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "2", + "set_id": "21", + "figure_id": "1", + "sample_note": "circle", + "question_id": "0", + "question": "Are two circles in the image the same color? Yes or No", + "gt_answer_details": "No, the two circles in the image are not the same color.", + "gt_answer": "0", + "filename": "./VD/illusion/21_1.png", + "model_prediction": "No", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Are two circles in the image the same color? Yes or No\nReference answer: No, the two circles in the image are not the same color.\nPrediction answer:No\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "1", + "set_id": "22", + "figure_id": "0", + "sample_note": "line", + "question_id": "0", + "question": "Does the black line align with the blue line? Yes or No", + "gt_answer_details": "No, the black line does not align with the blue line.", + "gt_answer": "0", + "filename": "./VD/illusion/22_0.png", + "model_prediction": "No", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Does the black line align with the blue line? Yes or No\nReference answer: No, the black line does not align with the blue line.\nPrediction answer:No\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "2", + "set_id": "23", + "figure_id": "1", + "sample_note": "circle", + "question_id": "0", + "question": "Are two circles in the image the same color? Yes or No", + "gt_answer_details": "No, the two circles in the image are not the same color.", + "gt_answer": "0", + "filename": "./VD/illusion/23_1.png", + "model_prediction": "Yes", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Are two circles in the image the same color? Yes or No\nReference answer: No, the two circles in the image are not the same color.\nPrediction answer:Yes\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "2", + "set_id": "24", + "figure_id": "1", + "sample_note": "circle", + "question_id": "0", + "question": "Are two circles in the image the same color? Yes or No", + "gt_answer_details": "No, the two circles in the image are not the same color.", + "gt_answer": "0", + "filename": "./VD/illusion/24_1.png", + "model_prediction": "Yes", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Are two circles in the image the same color? Yes or No\nReference answer: No, the two circles in the image are not the same color.\nPrediction answer:Yes\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "1", + "set_id": "25", + "figure_id": "0", + "sample_note": "circle", + "question_id": "2", + "question": "Is the green circle on left smaller than the green circle on the right? Yes or No", + "gt_answer_details": "No, the green circle on left is the same size as the green circle on the right", + "gt_answer": "0", + "filename": "./VD/illusion/25_0.png", + "model_prediction": "Yes", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the green circle on left smaller than the green circle on the right? Yes or No\nReference answer: No, the green circle on left is the same size as the green circle on the right\nPrediction answer:Yes\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "1", + "set_id": "26", + "figure_id": "0", + "sample_note": "line", + "question_id": "0", + "question": "Are orange lines in the image parallel? Yes or No", + "gt_answer_details": "Yes, orange lines are parallel", + "gt_answer": "1", + "filename": "./VD/illusion/26_0.png", + "model_prediction": "Yes", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Are orange lines in the image parallel? Yes or No\nReference answer: Yes, orange lines are parallel\nPrediction answer:Yes\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "1", + "set_id": "27", + "figure_id": "0", + "sample_note": "line", + "question_id": "2", + "question": "Is the green line shorter than the red line in the image? Yes or No", + "gt_answer_details": "No, the green line and the red line are the same length.", + "gt_answer": "0", + "filename": "./VD/illusion/27_0.png", + "model_prediction": "No", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the green line shorter than the red line in the image? Yes or No\nReference answer: No, the green line and the red line are the same length.\nPrediction answer:No\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "1", + "set_id": "28", + "figure_id": "0", + "sample_note": "line", + "question_id": "0", + "question": "Are the yellow line and the red line the same length in the image? Yes or No", + "gt_answer_details": "Yes, the yellow line and the red line are the same length.", + "gt_answer": "1", + "filename": "./VD/illusion/28_0.png", + "model_prediction": "Yes", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Are the yellow line and the red line the same length in the image? Yes or No\nReference answer: Yes, the yellow line and the red line are the same length.\nPrediction answer:Yes\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "2", + "set_id": "28", + "figure_id": "1", + "sample_note": "line", + "question_id": "1", + "question": "Is the yellow line longer than the red line in the image? Yes or No", + "gt_answer_details": "Yes, the yellow line is longer than the red line.", + "gt_answer": "1", + "filename": "./VD/illusion/28_1.png", + "model_prediction": "No", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the yellow line longer than the red line in the image? Yes or No\nReference answer: Yes, the yellow line is longer than the red line.\nPrediction answer:No\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "1", + "set_id": "29", + "figure_id": "0", + "sample_note": "line", + "question_id": "2", + "question": "Is the vertical line shorter than the horizontal line?", + "gt_answer_details": "No, the lengths of the vertical line and the horizontal line are the same.", + "gt_answer": "0", + "filename": "./VD/illusion/29_0.png", + "model_prediction": "Yes, the vertical line is shorter than the horizontal line.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the vertical line shorter than the horizontal line?\nReference answer: No, the lengths of the vertical line and the horizontal line are the same.\nPrediction answer:Yes, the vertical line is shorter than the horizontal line.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "1", + "set_id": "30", + "figure_id": "0", + "sample_note": "line", + "question_id": "0", + "question": "Is the red line on the left the same length as the red line on the right? Yes or No", + "gt_answer_details": "Yes, the red line on the left is the same length as the red line on the right.", + "gt_answer": "1", + "filename": "./VD/illusion/30_0.png", + "model_prediction": "Yes", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the red line on the left the same length as the red line on the right? Yes or No\nReference answer: Yes, the red line on the left is the same length as the red line on the right.\nPrediction answer:Yes\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "math", + "visual_input": "1", + "set_id": "0", + "figure_id": "0", + "sample_note": "para_angle", + "question_id": "0", + "question": "According to parallel lines theorem, is angle 1 + angle 2 = 180 ?", + "gt_answer_details": "angle 1 + angle 2 = 180 is correct", + "gt_answer": "1", + "filename": "./VD/math/0_0.png", + "model_prediction": "Yes, according to the parallel lines theorem, angle 1 + angle 2 = 180.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to parallel lines theorem, is angle 1 + angle 2 = 180 ?\nReference answer: angle 1 + angle 2 = 180 is correct\nPrediction answer:Yes, according to the parallel lines theorem, angle 1 + angle 2 = 180.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "math", + "visual_input": "2", + "set_id": "0", + "figure_id": "1", + "sample_note": "para_angle", + "question_id": "1", + "question": "According to parallel lines theorem, is angle 1 + angle 2 > 180 ?", + "gt_answer_details": "angle 1 + angle 2 > 180 is correct", + "gt_answer": "1", + "filename": "./VD/math/0_1.png", + "model_prediction": "No, according to the parallel lines theorem, angle 1 + angle 2 = 180 degrees.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to parallel lines theorem, is angle 1 + angle 2 > 180 ?\nReference answer: angle 1 + angle 2 > 180 is correct\nPrediction answer:No, according to the parallel lines theorem, angle 1 + angle 2 = 180 degrees.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "math", + "visual_input": "1", + "set_id": "1", + "figure_id": "0", + "sample_note": "pythagorean", + "question_id": "2", + "question": "Is the value of x in the image smaller than 5?", + "gt_answer_details": "the value of x is 5", + "gt_answer": "0", + "filename": "./VD/math/1_0.png", + "model_prediction": "Yes, the value of x in the image is smaller than 5.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the value of x in the image smaller than 5?\nReference answer: the value of x is 5\nPrediction answer:Yes, the value of x in the image is smaller than 5.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "math", + "visual_input": "1", + "set_id": "2", + "figure_id": "0", + "sample_note": "triangle", + "question_id": "0", + "question": "Is angleA + angleB +angleC = 180?", + "gt_answer_details": "angleA + angleB +angleC = 180 is correct", + "gt_answer": "1", + "filename": "./VD/math/2_0.png", + "model_prediction": "Yes, angleA + angleB + angleC = 180.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is angleA + angleB +angleC = 180?\nReference answer: angleA + angleB +angleC = 180 is correct\nPrediction answer:Yes, angleA + angleB + angleC = 180.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "math", + "visual_input": "2", + "set_id": "2", + "figure_id": "1", + "sample_note": "triangle", + "question_id": "1", + "question": "Is angleA + angleB +angleC < 180?", + "gt_answer_details": "angleA + angleB +angleC > 180 is correct", + "gt_answer": "0", + "filename": "./VD/math/2_1.png", + "model_prediction": "Yes, the sum of angleA, angleB, and angleC is less than 180 degrees.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is angleA + angleB +angleC < 180?\nReference answer: angleA + angleB +angleC > 180 is correct\nPrediction answer:Yes, the sum of angleA, angleB, and angleC is less than 180 degrees.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "math", + "visual_input": "1", + "set_id": "3", + "figure_id": "0", + "sample_note": "math_prob", + "question_id": "2", + "question": "According to the parallel theorem, is angle A smaller than angle B? Please answer yes or no.", + "gt_answer_details": "Angle A equal to angle B", + "gt_answer": "0", + "filename": "./VD/math/3_0.png", + "model_prediction": "Yes.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the parallel theorem, is angle A smaller than angle B? Please answer yes or no.\nReference answer: Angle A equal to angle B\nPrediction answer:Yes.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "math", + "visual_input": "1", + "set_id": "4", + "figure_id": "0", + "sample_note": "math_prob", + "question_id": "0", + "question": "Is the value of x equal to 6? Please answer yes or no.", + "gt_answer_details": "The value of x is equal to 6.", + "gt_answer": "1", + "filename": "./VD/math/4_0.png", + "model_prediction": "Yes.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the value of x equal to 6? Please answer yes or no.\nReference answer: The value of x is equal to 6.\nPrediction answer:Yes.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "math", + "visual_input": "2", + "set_id": "4", + "figure_id": "1", + "sample_note": "math_prob", + "question_id": "1", + "question": "Is the value of x larger than 6? Please answer yes or no.", + "gt_answer_details": "The value of x is larger than 6.", + "gt_answer": "1", + "filename": "./VD/math/4_1.png", + "model_prediction": "No.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the value of x larger than 6? Please answer yes or no.\nReference answer: The value of x is larger than 6.\nPrediction answer:No.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "math", + "visual_input": "1", + "set_id": "5", + "figure_id": "0", + "sample_note": "math_prob", + "question_id": "2", + "question": "The two interior angle bisectors OB and OC of triangle ABC intersect at point O. If angle A is 110 degree, then is the angle BOC equal to 145 degree? Please answer yes or no.", + "gt_answer_details": "The angle BOC equal to 145 degree.", + "gt_answer": "0", + "filename": "./VD/math/5_0.png", + "model_prediction": "Yes.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:The two interior angle bisectors OB and OC of triangle ABC intersect at point O. If angle A is 110 degree, then is the angle BOC equal to 145 degree? Please answer yes or no.\nReference answer: The angle BOC equal to 145 degree.\nPrediction answer:Yes.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "math", + "visual_input": "1", + "set_id": "6", + "figure_id": "0", + "sample_note": "math_prob", + "question_id": "0", + "question": "According to the figure, are the two orange lines both straight? Please answer yes or no.", + "gt_answer_details": "Both two orange lines are straight.", + "gt_answer": "1", + "filename": "./VD/math/6_0.png", + "model_prediction": "Yes.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the figure, are the two orange lines both straight? Please answer yes or no.\nReference answer: Both two orange lines are straight.\nPrediction answer:Yes.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "math", + "visual_input": "2", + "set_id": "6", + "figure_id": "1", + "sample_note": "math_prob", + "question_id": "1", + "question": "According to the figure, are the two orange lines both vertical? Please answer yes or no.", + "gt_answer_details": "Both two orange lines are curved.", + "gt_answer": "0", + "filename": "./VD/math/6_1.png", + "model_prediction": "No.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the figure, are the two orange lines both vertical? Please answer yes or no.\nReference answer: Both two orange lines are curved.\nPrediction answer:No.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "math", + "visual_input": "1", + "set_id": "7", + "figure_id": "0", + "sample_note": "math_prob", + "question_id": "2", + "question": "The area of B is 16 and the of area of C is 9. Is the area of square A smaller than 25? Please answer yes or no.", + "gt_answer_details": "The area of square A is equal to 25.", + "gt_answer": "0", + "filename": "./VD/math/7_0.png", + "model_prediction": "Yes", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:The area of B is 16 and the of area of C is 9. Is the area of square A smaller than 25? Please answer yes or no.\nReference answer: The area of square A is equal to 25.\nPrediction answer:Yes\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "math", + "visual_input": "1", + "set_id": "8", + "figure_id": "0", + "sample_note": "math_prob", + "question_id": "0", + "question": "Is the area of the rectangle equal to 12? Please answer yes or no.", + "gt_answer_details": "The area of the rectangle is equal to 12.", + "gt_answer": "1", + "filename": "./VD/math/8_0.png", + "model_prediction": "Yes.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the area of the rectangle equal to 12? Please answer yes or no.\nReference answer: The area of the rectangle is equal to 12.\nPrediction answer:Yes.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "math", + "visual_input": "2", + "set_id": "8", + "figure_id": "1", + "sample_note": "math_prob", + "question_id": "1", + "question": "Is the area of the ABCD larger than 12? Please answer yes or no.", + "gt_answer_details": "The area of the rectangle is smaller than 12.", + "gt_answer": "0", + "filename": "./VD/math/8_1.png", + "model_prediction": "Yes.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the area of the ABCD larger than 12? Please answer yes or no.\nReference answer: The area of the rectangle is smaller than 12.\nPrediction answer:Yes.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "math", + "visual_input": "1", + "set_id": "9", + "figure_id": "0", + "sample_note": "math_prob", + "question_id": "2", + "question": "As shown in the figure, \u2220AOB = 100.0, is the degree of \u2220ACB smaller than 50.0? Please answer yes or no", + "gt_answer_details": "The degree of \u2220ACB to is equal 50.", + "gt_answer": "0", + "filename": "./VD/math/9_0.png", + "model_prediction": "Yes", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:As shown in the figure, \u2220AOB = 100.0, is the degree of \u2220ACB smaller than 50.0? Please answer yes or no\nReference answer: The degree of \u2220ACB to is equal 50.\nPrediction answer:Yes\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "math", + "visual_input": "1", + "set_id": "10", + "figure_id": "0", + "sample_note": "math_prob", + "question_id": "0", + "question": "In triangle ABC, if AF, BE, CD, are the three midlines, is M the centroid of triangle ABC? Please answer yes or no.", + "gt_answer_details": "M is the centroid of triangle ABC.", + "gt_answer": "1", + "filename": "./VD/math/10_0.png", + "model_prediction": "Yes.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:In triangle ABC, if AF, BE, CD, are the three midlines, is M the centroid of triangle ABC? Please answer yes or no.\nReference answer: M is the centroid of triangle ABC.\nPrediction answer:Yes.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "math", + "visual_input": "2", + "set_id": "10", + "figure_id": "1", + "sample_note": "math_prob", + "question_id": "1", + "question": "In triangle ABC, if AF, BE, CD, are the three midlines, is M the incenter of triangle ABC? Please answer yes or no.", + "gt_answer_details": "M is a random point inside of triangle ABC.", + "gt_answer": "0", + "filename": "./VD/math/10_1.png", + "model_prediction": "Yes.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:In triangle ABC, if AF, BE, CD, are the three midlines, is M the incenter of triangle ABC? Please answer yes or no.\nReference answer: M is a random point inside of triangle ABC.\nPrediction answer:Yes.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "math", + "visual_input": "1", + "set_id": "11", + "figure_id": "0", + "sample_note": "math_prob", + "question_id": "2", + "question": "If DM=EM=FM, then is M the centroid of triangle ABC? Please answer yes or no.", + "gt_answer_details": "M is the incenter of triangle ABC.", + "gt_answer": "0", + "filename": "./VD/math/11_0.png", + "model_prediction": "Yes.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:If DM=EM=FM, then is M the centroid of triangle ABC? Please answer yes or no.\nReference answer: M is the incenter of triangle ABC.\nPrediction answer:Yes.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "math", + "visual_input": "1", + "set_id": "12", + "figure_id": "0", + "sample_note": "math_prob", + "question_id": "0", + "question": "If BE \u22a5 AC, CD \u22a5 AB, AF \u22a5 BC, then is M the orthocenter of triangle ABC? Please answer yes or no.", + "gt_answer_details": "M is the orthocenter of triangle ABC.", + "gt_answer": "1", + "filename": "./VD/math/12_0.png", + "model_prediction": "Yes.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:If BE \u22a5 AC, CD \u22a5 AB, AF \u22a5 BC, then is M the orthocenter of triangle ABC? Please answer yes or no.\nReference answer: M is the orthocenter of triangle ABC.\nPrediction answer:Yes.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "math", + "visual_input": "2", + "set_id": "12", + "figure_id": "1", + "sample_note": "math_prob", + "question_id": "1", + "question": "If BE \u22a5 AC, CD \u22a5 AB, AF \u22a5 BC, then is M the incenter of triangle ABC? Please answer yes or no.", + "gt_answer_details": "M is a random point inside of triangle ABC.", + "gt_answer": "0", + "filename": "./VD/math/12_1.png", + "model_prediction": "Yes.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:If BE \u22a5 AC, CD \u22a5 AB, AF \u22a5 BC, then is M the incenter of triangle ABC? Please answer yes or no.\nReference answer: M is a random point inside of triangle ABC.\nPrediction answer:Yes.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "math", + "visual_input": "1", + "set_id": "13", + "figure_id": "0", + "sample_note": "math_prob", + "question_id": "2", + "question": "If D is the midpoint of line segment AB, then is D the orthocenter of triangle ABC? Please answer yes or no", + "gt_answer_details": "D is the circumcenter of triangle ABC.", + "gt_answer": "0", + "filename": "./VD/math/13_0.png", + "model_prediction": "No.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:If D is the midpoint of line segment AB, then is D the orthocenter of triangle ABC? Please answer yes or no\nReference answer: D is the circumcenter of triangle ABC.\nPrediction answer:No.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "math", + "visual_input": "1", + "set_id": "14", + "figure_id": "0", + "sample_note": "math_prob", + "question_id": "0", + "question": "If angle BAF equals to angle CAF, is CF equal to BF? Please answer yes or no.", + "gt_answer_details": "CF is not equal to BF, or we do not know for sure.", + "gt_answer": "0", + "filename": "./VD/math/14_0.png", + "model_prediction": "Yes.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:If angle BAF equals to angle CAF, is CF equal to BF? Please answer yes or no.\nReference answer: CF is not equal to BF, or we do not know for sure.\nPrediction answer:Yes.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "math", + "visual_input": "2", + "set_id": "14", + "figure_id": "1", + "sample_note": "math_prob", + "question_id": "1", + "question": "If angle BAF equals to angle CAF, is CF larger than BF? Please answer yes or no.", + "gt_answer_details": "CF is equal to BF.", + "gt_answer": "0", + "filename": "./VD/math/14_1.png", + "model_prediction": "Yes.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:If angle BAF equals to angle CAF, is CF larger than BF? Please answer yes or no.\nReference answer: CF is equal to BF.\nPrediction answer:Yes.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "math", + "visual_input": "1", + "set_id": "15", + "figure_id": "0", + "sample_note": "math_prob", + "question_id": "2", + "question": "If angle A is equal to 60 degree, is AB smaller than 2AC? Please answer yes or no.", + "gt_answer_details": "AB is euqal to twice of AC.", + "gt_answer": "0", + "filename": "./VD/math/15_0.png", + "model_prediction": "Yes.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:If angle A is equal to 60 degree, is AB smaller than 2AC? Please answer yes or no.\nReference answer: AB is euqal to twice of AC.\nPrediction answer:Yes.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "math", + "visual_input": "1", + "set_id": "16", + "figure_id": "0", + "sample_note": "math_prob", + "question_id": "0", + "question": "If angle B is equal to 45 degree, is AB = sqrt(2)AC? Please answer yes or no.", + "gt_answer_details": "AB is euqal to sqrt(2) of AC.", + "gt_answer": "1", + "filename": "./VD/math/16_0.png", + "model_prediction": "Yes.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:If angle B is equal to 45 degree, is AB = sqrt(2)AC? Please answer yes or no.\nReference answer: AB is euqal to sqrt(2) of AC.\nPrediction answer:Yes.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "math", + "visual_input": "2", + "set_id": "16", + "figure_id": "1", + "sample_note": "math_prob", + "question_id": "1", + "question": "If angle B is equal to 45 degree, is AB larger than sqrt(2)AC? Please answer yes or no.", + "gt_answer_details": "AB is not larger than sqrt(2) of AC, or we do not know for sure.", + "gt_answer": "0", + "filename": "./VD/math/16_1.png", + "model_prediction": "Yes.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:If angle B is equal to 45 degree, is AB larger than sqrt(2)AC? Please answer yes or no.\nReference answer: AB is not larger than sqrt(2) of AC, or we do not know for sure.\nPrediction answer:Yes.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "math", + "visual_input": "1", + "set_id": "17", + "figure_id": "0", + "sample_note": "math_prob", + "question_id": "2", + "question": "If AE = CE and AB=BC, is CD perpendicular to AB? Please answer yes or no.", + "gt_answer_details": "CD is perpendicular to AB.", + "gt_answer": "1", + "filename": "./VD/math/17_0.png", + "model_prediction": "Yes.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:If AE = CE and AB=BC, is CD perpendicular to AB? Please answer yes or no.\nReference answer: CD is perpendicular to AB.\nPrediction answer:Yes.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "ocr", + "visual_input": "1", + "set_id": "0", + "figure_id": "0", + "sample_note": "noodle", + "question_id": "0", + "question": "Does the image show \"Hot dry noodles from Wuhan\"?", + "gt_answer_details": "The image shows \"Hot dry noodles from Wuhan\"", + "gt_answer": "1", + "filename": "./VD/ocr/0_0.png", + "model_prediction": "Yes, the image shows \"Hot dry noodles from Wuhan.\"", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Does the image show \"Hot dry noodles from Wuhan\"?\nReference answer: The image shows \"Hot dry noodles from Wuhan\"\nPrediction answer:Yes, the image shows \"Hot dry noodles from Wuhan.\"\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "ocr", + "visual_input": "1", + "set_id": "1", + "figure_id": "0", + "sample_note": "duck", + "question_id": "0", + "question": "Does the image show \"Beijing Roast Duck\"?", + "gt_answer_details": "the image show \"Beijing Roast Duck\"", + "gt_answer": "1", + "filename": "./VD/ocr/1_0.png", + "model_prediction": "Yes, the image shows \"Beijing Roast Duck.\"", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Does the image show \"Beijing Roast Duck\"?\nReference answer: the image show \"Beijing Roast Duck\"\nPrediction answer:Yes, the image shows \"Beijing Roast Duck.\"\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "ocr", + "visual_input": "1", + "set_id": "2", + "figure_id": "0", + "sample_note": "Shawshank", + "question_id": "0", + "question": "According to the text in this image, is this a poster for \"The Shawshank Redemption\"?", + "gt_answer_details": "Yes, this is a poster for \"The Shawshank Redemption\".", + "gt_answer": "1", + "filename": "./VD/ocr/2_0.png", + "model_prediction": "Yes, this is a poster for \"The Shawshank Redemption.\"", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the text in this image, is this a poster for \"The Shawshank Redemption\"?\nReference answer: Yes, this is a poster for \"The Shawshank Redemption\".\nPrediction answer:Yes, this is a poster for \"The Shawshank Redemption.\"\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "ocr", + "visual_input": "1", + "set_id": "3", + "figure_id": "0", + "sample_note": "Pizza", + "question_id": "0", + "question": "According to the pizza image in the figure, is this pizza a Margherita Pizza?", + "gt_answer_details": "Yes, this pizza a Margherita Pizza.", + "gt_answer": "1", + "filename": "./VD/ocr/3_0.png", + "model_prediction": "Yes, the pizza in the image is a Margherita Pizza.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the pizza image in the figure, is this pizza a Margherita Pizza?\nReference answer: Yes, this pizza a Margherita Pizza.\nPrediction answer:Yes, the pizza in the image is a Margherita Pizza.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "ocr", + "visual_input": "2", + "set_id": "3", + "figure_id": "1", + "sample_note": "Pizza", + "question_id": "1", + "question": "According to the pizza image in the figure, is this pizza a Hawaiian Pizza?", + "gt_answer_details": "No, this pizza a Margherita Pizza.", + "gt_answer": "0", + "filename": "./VD/ocr/3_1.png", + "model_prediction": "Yes, the pizza in the image is a Hawaiian Pizza.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the pizza image in the figure, is this pizza a Hawaiian Pizza?\nReference answer: No, this pizza a Margherita Pizza.\nPrediction answer:Yes, the pizza in the image is a Hawaiian Pizza.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "ocr", + "visual_input": "2", + "set_id": "3", + "figure_id": "2", + "sample_note": "Pizza", + "question_id": "2", + "question": "According to the pizza image in the figure, is this pizza a Pepperoni Pizza?", + "gt_answer_details": "No, this pizza a Margherita Pizza.", + "gt_answer": "0", + "filename": "./VD/ocr/3_2.png", + "model_prediction": "Yes, the pizza in the image is a Pepperoni Pizza.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the pizza image in the figure, is this pizza a Pepperoni Pizza?\nReference answer: No, this pizza a Margherita Pizza.\nPrediction answer:Yes, the pizza in the image is a Pepperoni Pizza.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "ocr", + "visual_input": "2", + "set_id": "4", + "figure_id": "1", + "sample_note": "Ham", + "question_id": "1", + "question": "According to the text in this image, is this a Roma ham?", + "gt_answer_details": "Yes, this is a Roma ham.", + "gt_answer": "1", + "filename": "./VD/ocr/4_1.png", + "model_prediction": "Yes, this is a Roma ham.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the text in this image, is this a Roma ham?\nReference answer: Yes, this is a Roma ham.\nPrediction answer:Yes, this is a Roma ham.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "ocr", + "visual_input": "2", + "set_id": "5", + "figure_id": "1", + "sample_note": "Black forest", + "question_id": "1", + "question": "According to the cake in the image, is this a red forest cake?", + "gt_answer_details": "Yes, this is a red forest cake.", + "gt_answer": "1", + "filename": "./VD/ocr/5_1.png", + "model_prediction": "Yes, the cake in the image is a Red Forest cake.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the cake in the image, is this a red forest cake?\nReference answer: Yes, this is a red forest cake.\nPrediction answer:Yes, the cake in the image is a Red Forest cake.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "ocr", + "visual_input": "2", + "set_id": "6", + "figure_id": "1", + "sample_note": "Red Velvet", + "question_id": "1", + "question": "According to the cake in the image, is this a red forest cake?", + "gt_answer_details": "Yes, this is a red forest cake.", + "gt_answer": "1", + "filename": "./VD/ocr/6_1.png", + "model_prediction": "Yes, the cake in the image is a red forest cake.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the cake in the image, is this a red forest cake?\nReference answer: Yes, this is a red forest cake.\nPrediction answer:Yes, the cake in the image is a red forest cake.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "ocr", + "visual_input": "2", + "set_id": "7", + "figure_id": "1", + "sample_note": "Al Jazeera", + "question_id": "1", + "question": "According to the text below the logo, is this image for Al-Jazari?", + "gt_answer_details": "Yes, this image is for al-Jazari.", + "gt_answer": "1", + "filename": "./VD/ocr/7_1.png", + "model_prediction": "Yes, the image is for Al-Jazari.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the text below the logo, is this image for Al-Jazari?\nReference answer: Yes, this image is for al-Jazari.\nPrediction answer:Yes, the image is for Al-Jazari.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "ocr", + "visual_input": "2", + "set_id": "8", + "figure_id": "1", + "sample_note": "Steak", + "question_id": "1", + "question": "According to the text given in the image, is this a Boston strip steak?", + "gt_answer_details": "Yes, this is a Boston strip steak.", + "gt_answer": "1", + "filename": "./VD/ocr/8_1.png", + "model_prediction": "Yes, it is a Boston strip steak.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the text given in the image, is this a Boston strip steak?\nReference answer: Yes, this is a Boston strip steak.\nPrediction answer:Yes, it is a Boston strip steak.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "ocr", + "visual_input": "2", + "set_id": "9", + "figure_id": "1", + "sample_note": "Cheesecake", + "question_id": "1", + "question": "According to the text given in the image, is this a Washington style cheesecake?", + "gt_answer_details": "Yes, this is a Washington style cheesecake.", + "gt_answer": "1", + "filename": "./VD/ocr/9_1.png", + "model_prediction": "Yes, this is a Washington style cheesecake.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the text given in the image, is this a Washington style cheesecake?\nReference answer: Yes, this is a Washington style cheesecake.\nPrediction answer:Yes, this is a Washington style cheesecake.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "ocr", + "visual_input": "2", + "set_id": "10", + "figure_id": "1", + "sample_note": "Texas Roadhouse", + "question_id": "1", + "question": "According to the text given in the image, is this restaurant named Maryland Roadhouse?", + "gt_answer_details": "Yes, this restaurant is named Maryland Roadhouse.", + "gt_answer": "1", + "filename": "./VD/ocr/10_1.png", + "model_prediction": "Yes, the restaurant is named Maryland Roadhouse.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the text given in the image, is this restaurant named Maryland Roadhouse?\nReference answer: Yes, this restaurant is named Maryland Roadhouse.\nPrediction answer:Yes, the restaurant is named Maryland Roadhouse.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "ocr", + "visual_input": "2", + "set_id": "11", + "figure_id": "1", + "sample_note": "welcome", + "question_id": "1", + "question": "According to the text given in the image, is this sign located in College Park, MD?", + "gt_answer_details": "Yes, this sign locates in College Park, MD?", + "gt_answer": "1", + "filename": "./VD/ocr/11_1.png", + "model_prediction": "Yes, the sign is located in College Park, MD.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the text given in the image, is this sign located in College Park, MD?\nReference answer: Yes, this sign locates in College Park, MD?\nPrediction answer:Yes, the sign is located in College Park, MD.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "ocr", + "visual_input": "2", + "set_id": "12", + "figure_id": "1", + "sample_note": "Tsoi", + "question_id": "1", + "question": "According to the lower text given in the image, is this poster for the DDT band?", + "gt_answer_details": "Yes, this poster for the DDT band.", + "gt_answer": "1", + "filename": "./VD/ocr/12_1.png", + "model_prediction": "Yes, the poster is for the DDT band.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the lower text given in the image, is this poster for the DDT band?\nReference answer: Yes, this poster for the DDT band.\nPrediction answer:Yes, the poster is for the DDT band.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "ocr", + "visual_input": "2", + "set_id": "13", + "figure_id": "1", + "sample_note": "Red Bull", + "question_id": "1", + "question": "According to the text given in the image, is this logo for Red Deer?", + "gt_answer_details": "Yes, this logo is for Red Deer.", + "gt_answer": "1", + "filename": "./VD/ocr/13_1.png", + "model_prediction": "Yes, the logo is for Red Deer.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the text given in the image, is this logo for Red Deer?\nReference answer: Yes, this logo is for Red Deer.\nPrediction answer:Yes, the logo is for Red Deer.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "ocr", + "visual_input": "2", + "set_id": "14", + "figure_id": "1", + "sample_note": "Zootopia", + "question_id": "1", + "question": "According to the text given in the image, does this poster depict the movie Utopia?", + "gt_answer_details": "Yes, this poster depicts the movie Utopia.", + "gt_answer": "1", + "filename": "./VD/ocr/14_1.png", + "model_prediction": "Yes, the poster depicts the movie Utopia.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the text given in the image, does this poster depict the movie Utopia?\nReference answer: Yes, this poster depicts the movie Utopia.\nPrediction answer:Yes, the poster depicts the movie Utopia.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "ocr", + "visual_input": "2", + "set_id": "15", + "figure_id": "1", + "sample_note": "Vanilla Ice-cream", + "question_id": "1", + "question": "According to the text given in the image, is this ice-cream a vanilla-chocolate flavor ice cream?", + "gt_answer_details": "No, this is ice-cream a strawberry flavor ice cream.", + "gt_answer": "0", + "filename": "./VD/ocr/15_1.png", + "model_prediction": "Yes, the ice-cream is a vanilla-chocolate flavor.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the text given in the image, is this ice-cream a vanilla-chocolate flavor ice cream?\nReference answer: No, this is ice-cream a strawberry flavor ice cream.\nPrediction answer:Yes, the ice-cream is a vanilla-chocolate flavor.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "ocr", + "visual_input": "2", + "set_id": "16", + "figure_id": "1", + "sample_note": "Ben and Jerry", + "question_id": "1", + "question": "According to the text given in the image, is this ice-cream a chocolate flavor ice cream?", + "gt_answer_details": "No, this is ice-cream a strawberry flavor ice cream.", + "gt_answer": "0", + "filename": "./VD/ocr/16_1.png", + "model_prediction": "Yes, the ice-cream is a chocolate flavor ice cream.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the text given in the image, is this ice-cream a chocolate flavor ice cream?\nReference answer: No, this is ice-cream a strawberry flavor ice cream.\nPrediction answer:Yes, the ice-cream is a chocolate flavor ice cream.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "ocr", + "visual_input": "2", + "set_id": "17", + "figure_id": "1", + "sample_note": "Oppenheimer", + "question_id": "1", + "question": "According to the text given in the image, does this poster depict the movie BARBENHEIMER?", + "gt_answer_details": "Yes, this poster depicts the movie BARBENHEIMER.", + "gt_answer": "1", + "filename": "./VD/ocr/17_1.png", + "model_prediction": "Yes, the poster depicts the movie BARBENHEIMER.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the text given in the image, does this poster depict the movie BARBENHEIMER?\nReference answer: Yes, this poster depicts the movie BARBENHEIMER.\nPrediction answer:Yes, the poster depicts the movie BARBENHEIMER.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "ocr", + "visual_input": "2", + "set_id": "18", + "figure_id": "1", + "sample_note": "Tokyo Girl", + "question_id": "1", + "question": "According to the text in this image, is this poster for the TV series Tokyo Single Girls?", + "gt_answer_details": "Yes, this poster is for the TV series Tokyo Single Girls.", + "gt_answer": "1", + "filename": "./VD/ocr/18_1.png", + "model_prediction": "Yes, the poster is for the TV series Tokyo Single Girls.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the text in this image, is this poster for the TV series Tokyo Single Girls?\nReference answer: Yes, this poster is for the TV series Tokyo Single Girls.\nPrediction answer:Yes, the poster is for the TV series Tokyo Single Girls.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "ocr", + "visual_input": "2", + "set_id": "19", + "figure_id": "1", + "sample_note": "Reply 1988", + "question_id": "1", + "question": "According to the text in this image, is this poster for the TV series Please Send 1988?", + "gt_answer_details": "Yes, this poster is for the TV series Please Send 1988.", + "gt_answer": "1", + "filename": "./VD/ocr/19_1.png", + "model_prediction": "Yes, the poster is for the TV series Please Send 1988.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the text in this image, is this poster for the TV series Please Send 1988?\nReference answer: Yes, this poster is for the TV series Please Send 1988.\nPrediction answer:Yes, the poster is for the TV series Please Send 1988.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "ocr", + "visual_input": "2", + "set_id": "20", + "figure_id": "1", + "sample_note": "Meat Floss Cake", + "question_id": "1", + "question": "According to the text in this image, is this \"Boli Cake Seasoned with Chicken Meat Floss\"?", + "gt_answer_details": "Yes, this is \"Boli Cake Seasoned with Chicken Meat Floss\".", + "gt_answer": "1", + "filename": "./VD/ocr/20_1.png", + "model_prediction": "Yes, the text \"Boli Cake Seasoned with Chicken Meat Floss\" is present in the image, indicating the type of boli cake being advertised.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the text in this image, is this \"Boli Cake Seasoned with Chicken Meat Floss\"?\nReference answer: Yes, this is \"Boli Cake Seasoned with Chicken Meat Floss\".\nPrediction answer:Yes, the text \"Boli Cake Seasoned with Chicken Meat Floss\" is present in the image, indicating the type of boli cake being advertised.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "figure", + "visual_input": "1", + "set_id": "1", + "figure_id": "0", + "sample_note": "jordan", + "question_id": "1", + "question": "Is it the logo for Qiaodan Sports?", + "gt_answer_details": "the logo is for air Jordan", + "gt_answer": "0", + "filename": "./VD/figure/1_0.png", + "model_prediction": "No, it is not. The logo you provided is for Nike, a well-known sports apparel and footwear company.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is it the logo for Qiaodan Sports?\nReference answer: the logo is for air Jordan\nPrediction answer:No, it is not. The logo you provided is for Nike, a well-known sports apparel and footwear company.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "figure", + "visual_input": "1", + "set_id": "2", + "figure_id": "0", + "sample_note": "Chicken", + "question_id": "1", + "question": "Are there any ducks in this figure?", + "gt_answer_details": "No, all the animals in this figure are chickens.", + "gt_answer": "0", + "filename": "./VD/figure/2_0.png", + "model_prediction": "No, there are no ducks in this figure. The figure shows a group of chicks, which are baby chickens.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Are there any ducks in this figure?\nReference answer: No, all the animals in this figure are chickens.\nPrediction answer:No, there are no ducks in this figure. The figure shows a group of chicks, which are baby chickens.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "figure", + "visual_input": "1", + "set_id": "3", + "figure_id": "0", + "sample_note": "Asterix", + "question_id": "1", + "question": "Is there a Snoopy in this figure?", + "gt_answer_details": "No, all the characters here are from the Asterix the Gaul.", + "gt_answer": "0", + "filename": "./VD/figure/3_0.png", + "model_prediction": "No, there is no Snoopy in this figure. The figure is of the character Albert, also known as Ast\u00e9rix, from the French comic series \"Ast\u00e9rix\".", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is there a Snoopy in this figure?\nReference answer: No, all the characters here are from the Asterix the Gaul.\nPrediction answer:No, there is no Snoopy in this figure. The figure is of the character Albert, also known as Ast\u00e9rix, from the French comic series \"Ast\u00e9rix\".\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "figure", + "visual_input": "1", + "set_id": "4", + "figure_id": "0", + "sample_note": "Swan", + "question_id": "1", + "question": "Are there any ducks in this figure?", + "gt_answer_details": "No, all the animals in this figure are swans.", + "gt_answer": "0", + "filename": "./VD/figure/4_0.png", + "model_prediction": "Yes, there are ducks in the image.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Are there any ducks in this figure?\nReference answer: No, all the animals in this figure are swans.\nPrediction answer:Yes, there are ducks in the image.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "figure", + "visual_input": "1", + "set_id": "5", + "figure_id": "0", + "sample_note": "Tool", + "question_id": "1", + "question": "Are there any carrots in this figure?", + "gt_answer_details": "No, all the items here are engineering tools.", + "gt_answer": "0", + "filename": "./VD/figure/5_0.png", + "model_prediction": "No, there are no carrots in this figure.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Are there any carrots in this figure?\nReference answer: No, all the items here are engineering tools.\nPrediction answer:No, there are no carrots in this figure.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "figure", + "visual_input": "2", + "set_id": "5", + "figure_id": "2", + "sample_note": "Tool", + "question_id": "1", + "question": "Are there any carrots in this figure?", + "gt_answer_details": "Yes, there is a carrot in this figure.", + "gt_answer": "1", + "filename": "./VD/figure/5_2.png", + "model_prediction": "Yes, there is a carrot in the figure.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Are there any carrots in this figure?\nReference answer: Yes, there is a carrot in this figure.\nPrediction answer:Yes, there is a carrot in the figure.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "figure", + "visual_input": "2", + "set_id": "6", + "figure_id": "1", + "sample_note": "KFC", + "question_id": "1", + "question": "Does this figure depict Colonel Sanders?", + "gt_answer_details": "No, this figure does not depict Colonel Sanders.", + "gt_answer": "0", + "filename": "./VD/figure/6_1.png", + "model_prediction": "Yes, the figure depicts Colonel Sanders, the founder of KFC.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Does this figure depict Colonel Sanders?\nReference answer: No, this figure does not depict Colonel Sanders.\nPrediction answer:Yes, the figure depicts Colonel Sanders, the founder of KFC.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "figure", + "visual_input": "2", + "set_id": "7", + "figure_id": "1", + "sample_note": "NBA", + "question_id": "1", + "question": "Does this figure depict a singer?", + "gt_answer_details": "Yes, this figure depict a singer.", + "gt_answer": "1", + "filename": "./VD/figure/7_1.png", + "model_prediction": "No, the figure depicts a basketball player.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Does this figure depict a singer?\nReference answer: Yes, this figure depict a singer.\nPrediction answer:No, the figure depicts a basketball player.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "figure", + "visual_input": "2", + "set_id": "8", + "figure_id": "1", + "sample_note": "Cookie", + "question_id": "1", + "question": "Are there any dog faces in this figure?", + "gt_answer_details": "Yes, there is a dog face in this figure.", + "gt_answer": "1", + "filename": "./VD/figure/8_1.png", + "model_prediction": "Yes, there is a dog face in the image.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Are there any dog faces in this figure?\nReference answer: Yes, there is a dog face in this figure.\nPrediction answer:Yes, there is a dog face in the image.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "figure", + "visual_input": "2", + "set_id": "9", + "figure_id": "1", + "sample_note": "Dinosaur", + "question_id": "1", + "question": "Are there any hens in this figure?", + "gt_answer_details": "Yes, there is a modern hen in this figure.", + "gt_answer": "1", + "filename": "./VD/figure/9_1.png", + "model_prediction": "Yes, there is a chicken in the figure.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Are there any hens in this figure?\nReference answer: Yes, there is a modern hen in this figure.\nPrediction answer:Yes, there is a chicken in the figure.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "figure", + "visual_input": "2", + "set_id": "10", + "figure_id": "1", + "sample_note": "Vegetable", + "question_id": "1", + "question": "Is there any meat in this figure?", + "gt_answer_details": "Yes, there is some meat in this figure.", + "gt_answer": "1", + "filename": "./VD/figure/10_1.png", + "model_prediction": "Yes, there is a piece of meat in the image, which appears to be a steak.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is there any meat in this figure?\nReference answer: Yes, there is some meat in this figure.\nPrediction answer:Yes, there is a piece of meat in the image, which appears to be a steak.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "figure", + "visual_input": "2", + "set_id": "11", + "figure_id": "1", + "sample_note": "Ice Cream Bar", + "question_id": "1", + "question": "Is there any yogurt in this figure?", + "gt_answer_details": "Yes, there is some yogurt in this figure.", + "gt_answer": "1", + "filename": "./VD/figure/11_1.png", + "model_prediction": "No, there is no yogurt in this figure.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is there any yogurt in this figure?\nReference answer: Yes, there is some yogurt in this figure.\nPrediction answer:No, there is no yogurt in this figure.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "figure", + "visual_input": "2", + "set_id": "12", + "figure_id": "1", + "sample_note": "Penguin", + "question_id": "1", + "question": "Are there any real penguins in this figure?", + "gt_answer_details": "Yes, there is a real penguin in this figure.", + "gt_answer": "1", + "filename": "./VD/figure/12_1.png", + "model_prediction": "No, the penguins in the figure are not real. They are animated characters from the movie \"Happy Feet\".", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Are there any real penguins in this figure?\nReference answer: Yes, there is a real penguin in this figure.\nPrediction answer:No, the penguins in the figure are not real. They are animated characters from the movie \"Happy Feet\".\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "figure", + "visual_input": "2", + "set_id": "13", + "figure_id": "1", + "sample_note": "Zootopia", + "question_id": "1", + "question": "Is Bunny Bug in this figure?", + "gt_answer_details": "Yes, Bunny Bug is in this figure.", + "gt_answer": "1", + "filename": "./VD/figure/13_1.png", + "model_prediction": "Yes, Bunny Bug is in this figure.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is Bunny Bug in this figure?\nReference answer: Yes, Bunny Bug is in this figure.\nPrediction answer:Yes, Bunny Bug is in this figure.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "figure", + "visual_input": "2", + "set_id": "14", + "figure_id": "1", + "sample_note": "One Piece", + "question_id": "1", + "question": "Are there any characters in this figure from the manga series Detective Conan?", + "gt_answer_details": "Yes, one is from Detective Conan.", + "gt_answer": "1", + "filename": "./VD/figure/14_1.png", + "model_prediction": "No, the characters in this figure are from the manga series One Piece.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Are there any characters in this figure from the manga series Detective Conan?\nReference answer: Yes, one is from Detective Conan.\nPrediction answer:No, the characters in this figure are from the manga series One Piece.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "figure", + "visual_input": "2", + "set_id": "15", + "figure_id": "1", + "sample_note": "Pokemon", + "question_id": "1", + "question": "Are there any characters in this figure from Digimon?", + "gt_answer_details": "Yes, one characters in this figure is from Digimon.", + "gt_answer": "1", + "filename": "./VD/figure/15_1.png", + "model_prediction": "Yes, there are characters from Digimon in this figure.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Are there any characters in this figure from Digimon?\nReference answer: Yes, one characters in this figure is from Digimon.\nPrediction answer:Yes, there are characters from Digimon in this figure.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "figure", + "visual_input": "2", + "set_id": "16", + "figure_id": "1", + "sample_note": "Mole", + "question_id": "1", + "question": "Is Jerry the Mouse in this figure?", + "gt_answer_details": "Yes, Jerry the Mouse is in this figure.", + "gt_answer": "1", + "filename": "./VD/figure/16_1.png", + "model_prediction": "Yes, Jerry the Mouse is in the figure.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is Jerry the Mouse in this figure?\nReference answer: Yes, Jerry the Mouse is in this figure.\nPrediction answer:Yes, Jerry the Mouse is in the figure.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "figure", + "visual_input": "2", + "set_id": "17", + "figure_id": "1", + "sample_note": "Fried Chicken", + "question_id": "1", + "question": "Is there an orange cat in this figure?", + "gt_answer_details": "Yes, there is an orange cat in this figure.", + "gt_answer": "1", + "filename": "./VD/figure/17_1.png", + "model_prediction": "Yes, there is an orange cat in the figure.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is there an orange cat in this figure?\nReference answer: Yes, there is an orange cat in this figure.\nPrediction answer:Yes, there is an orange cat in the figure.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "figure", + "visual_input": "2", + "set_id": "18", + "figure_id": "1", + "sample_note": "Egg", + "question_id": "1", + "question": "Are there any potatoes in this figure?", + "gt_answer_details": "Yes, there are some potatoes inside.", + "gt_answer": "1", + "filename": "./VD/figure/18_1.png", + "model_prediction": "Yes, there are potatoes in the image.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Are there any potatoes in this figure?\nReference answer: Yes, there are some potatoes inside.\nPrediction answer:Yes, there are potatoes in the image.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "figure", + "visual_input": "2", + "set_id": "19", + "figure_id": "1", + "sample_note": "SpongeBob", + "question_id": "1", + "question": "Are there any cheeses in this figure?", + "gt_answer_details": "Yes, this figure has some cheese inside.", + "gt_answer": "1", + "filename": "./VD/figure/19_1.png", + "model_prediction": "Yes, there is a cheese in the figure.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Are there any cheeses in this figure?\nReference answer: Yes, this figure has some cheese inside.\nPrediction answer:Yes, there is a cheese in the figure.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "1", + "set_id": "1", + "figure_id": "0", + "sample_note": "simpson", + "question_id": "1", + "question": "According to the positive sequence images, does Homer Simpson come out of the bushes?", + "gt_answer_details": "Homer Simpson disappear into the bushes", + "gt_answer": "0", + "filename": "./VD/video/1_0.png", + "model_prediction": "Yes, Homer Simpson comes out of the bushes in the positive sequence images.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the positive sequence images, does Homer Simpson come out of the bushes?\nReference answer: Homer Simpson disappear into the bushes\nPrediction answer:Yes, Homer Simpson comes out of the bushes in the positive sequence images.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "2", + "set_id": "1", + "figure_id": "1", + "sample_note": "simpson", + "question_id": "1", + "question": "According to the positive sequence images, does Homer Simpson come out of the bushes?", + "gt_answer_details": "Homer Simpson comes out of the bushes", + "gt_answer": "1", + "filename": "./VD/video/1_1.png", + "model_prediction": "Yes, Homer Simpson comes out of the bushes in the positive sequence images.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the positive sequence images, does Homer Simpson come out of the bushes?\nReference answer: Homer Simpson comes out of the bushes\nPrediction answer:Yes, Homer Simpson comes out of the bushes in the positive sequence images.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "2", + "set_id": "1", + "figure_id": "2", + "sample_note": "simpson", + "question_id": "1", + "question": "According to the positive sequence images, does Homer Simpson come out of the bushes?", + "gt_answer_details": "No", + "gt_answer": "0", + "filename": "./VD/video/1_2.png", + "model_prediction": "Yes, Homer Simpson comes out of the bushes in the positive sequence images.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the positive sequence images, does Homer Simpson come out of the bushes?\nReference answer: No\nPrediction answer:Yes, Homer Simpson comes out of the bushes in the positive sequence images.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "1", + "set_id": "2", + "figure_id": "0", + "sample_note": "basketball", + "question_id": "1", + "question": "According to the positive sequence of the images, does the basketball player make a three-point shooting?", + "gt_answer_details": "The basketball player makes a dunk", + "gt_answer": "0", + "filename": "./VD/video/2_0.png", + "model_prediction": "No, the basketball player does not make a three-point shot. The sequence shows the player jumping and reaching for the ball, but there is no indication of a successful shot being made.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the positive sequence of the images, does the basketball player make a three-point shooting?\nReference answer: The basketball player makes a dunk\nPrediction answer:No, the basketball player does not make a three-point shot. The sequence shows the player jumping and reaching for the ball, but there is no indication of a successful shot being made.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "2", + "set_id": "2", + "figure_id": "1", + "sample_note": "basketball", + "question_id": "2", + "question": "According to the positive sequence of the images, are they in the correct order?", + "gt_answer_details": "no", + "gt_answer": "0", + "filename": "./VD/video/2_1.png", + "model_prediction": "Yes, the images are in the correct order. The sequence shows a basketball game with players in motion, starting from a player in the air, then another player in the air, and finally a player on the ground.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the positive sequence of the images, are they in the correct order?\nReference answer: no\nPrediction answer:Yes, the images are in the correct order. The sequence shows a basketball game with players in motion, starting from a player in the air, then another player in the air, and finally a player on the ground.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "1", + "set_id": "3", + "figure_id": "0", + "sample_note": "door", + "question_id": "3", + "question": "The man closes the door. According to the positive sequence of the images, are they in the correct order?", + "gt_answer_details": "they are not in the correct order", + "gt_answer": "0", + "filename": "./VD/video/3_0.png", + "model_prediction": "Yes, the images are in the correct order. The first image shows the man with his back to the camera, and the second image shows him closing the door.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:The man closes the door. According to the positive sequence of the images, are they in the correct order?\nReference answer: they are not in the correct order\nPrediction answer:Yes, the images are in the correct order. The first image shows the man with his back to the camera, and the second image shows him closing the door.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "2", + "set_id": "3", + "figure_id": "1", + "sample_note": "door", + "question_id": "3", + "question": "The man closes the door. According to the positive sequence of the images, are they in the correct order?", + "gt_answer_details": "yes", + "gt_answer": "1", + "filename": "./VD/video/3_1.png", + "model_prediction": "Yes, the images are in the correct order. The man is first seen smiling and then closing the door.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:The man closes the door. According to the positive sequence of the images, are they in the correct order?\nReference answer: yes\nPrediction answer:Yes, the images are in the correct order. The man is first seen smiling and then closing the door.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "2", + "set_id": "3", + "figure_id": "2", + "sample_note": "door", + "question_id": "3", + "question": "The man closes the door. According to the positive sequence of the images, are they in the correct order?", + "gt_answer_details": "no", + "gt_answer": "0", + "filename": "./VD/video/3_2.png", + "model_prediction": "Yes, the images are in the correct order. The sequence shows the man walking towards the door, closing it, and then looking at his reflection in the door.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:The man closes the door. According to the positive sequence of the images, are they in the correct order?\nReference answer: no\nPrediction answer:Yes, the images are in the correct order. The sequence shows the man walking towards the door, closing it, and then looking at his reflection in the door.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "1", + "set_id": "4", + "figure_id": "0", + "sample_note": "parking", + "question_id": "3", + "question": "The blue car attempts to leave the parking space. According to the positive sequence of the images, are they in the correct order?", + "gt_answer_details": "no", + "gt_answer": "0", + "filename": "./VD/video/4_0.png", + "model_prediction": "Yes, the sequence of the images is correct. The blue car is shown in the first image, indicating it is attempting to leave the parking space. The subsequent images show the car's position relative to the parking space and other vehicles, confirming the sequence of events as the car moves out of the parking space.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:The blue car attempts to leave the parking space. According to the positive sequence of the images, are they in the correct order?\nReference answer: no\nPrediction answer:Yes, the sequence of the images is correct. The blue car is shown in the first image, indicating it is attempting to leave the parking space. The subsequent images show the car's position relative to the parking space and other vehicles, confirming the sequence of events as the car moves out of the parking space.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "2", + "set_id": "4", + "figure_id": "1", + "sample_note": "parking", + "question_id": "3", + "question": "The blue car attempts to leave the parking space. According to the positive sequence of the images, are they in the correct order?", + "gt_answer_details": "yes", + "gt_answer": "1", + "filename": "./VD/video/4_1.png", + "model_prediction": "Yes, the images are in the correct order. The sequence shows the blue car's attempt to leave the parking space, which is consistent with the positive sequence of events.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:The blue car attempts to leave the parking space. According to the positive sequence of the images, are they in the correct order?\nReference answer: yes\nPrediction answer:Yes, the images are in the correct order. The sequence shows the blue car's attempt to leave the parking space, which is consistent with the positive sequence of events.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "2", + "set_id": "4", + "figure_id": "2", + "sample_note": "parking", + "question_id": "3", + "question": "The blue car attempts to leave the parking space. According to the positive sequence of the images, are they in the correct order?", + "gt_answer_details": "no", + "gt_answer": "0", + "filename": "./VD/video/4_2.png", + "model_prediction": "Yes, the blue car is in the correct position to leave the parking space.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:The blue car attempts to leave the parking space. According to the positive sequence of the images, are they in the correct order?\nReference answer: no\nPrediction answer:Yes, the blue car is in the correct position to leave the parking space.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "1", + "set_id": "5", + "figure_id": "0", + "sample_note": "cloth", + "question_id": "3", + "question": "The baby attempts to take off the clothes. According to the positive sequence of the images, are they in the correct order?", + "gt_answer_details": "no", + "gt_answer": "0", + "filename": "./VD/video/5_0.png", + "model_prediction": "Yes, the images are in the correct order. The sequence shows the baby first standing with the clothes on, then attempting to take them off, and finally standing without the clothes on.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:The baby attempts to take off the clothes. According to the positive sequence of the images, are they in the correct order?\nReference answer: no\nPrediction answer:Yes, the images are in the correct order. The sequence shows the baby first standing with the clothes on, then attempting to take them off, and finally standing without the clothes on.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "2", + "set_id": "5", + "figure_id": "1", + "sample_note": "cloth", + "question_id": "3", + "question": "The baby attempts to take off the clothes. According to the positive sequence of the images, are they in the correct order?", + "gt_answer_details": "yes", + "gt_answer": "1", + "filename": "./VD/video/5_1.png", + "model_prediction": "Yes, the images are in the correct order. The sequence shows the baby first standing with the clothes on, then attempting to take them off, and finally standing with the clothes off.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:The baby attempts to take off the clothes. According to the positive sequence of the images, are they in the correct order?\nReference answer: yes\nPrediction answer:Yes, the images are in the correct order. The sequence shows the baby first standing with the clothes on, then attempting to take them off, and finally standing with the clothes off.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "1", + "set_id": "6", + "figure_id": "0", + "sample_note": "temperature", + "question_id": "3", + "question": "The temperature decreases. According to the positive sequence of the images, are they in the correct order?", + "gt_answer_details": "no", + "gt_answer": "0", + "filename": "./VD/video/6_0.png", + "model_prediction": "Yes, the images are in the correct order. The temperature decreases from left to right, as indicated by the thermometer readings.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:The temperature decreases. According to the positive sequence of the images, are they in the correct order?\nReference answer: no\nPrediction answer:Yes, the images are in the correct order. The temperature decreases from left to right, as indicated by the thermometer readings.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "2", + "set_id": "6", + "figure_id": "1", + "sample_note": "temperature", + "question_id": "3", + "question": "The temperature decreases. According to the positive sequence of the images, are they in the correct order?", + "gt_answer_details": "yes", + "gt_answer": "1", + "filename": "./VD/video/6_1.png", + "model_prediction": "Yes, the images are in the correct order. The temperature decreases from the first image to the third image.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:The temperature decreases. According to the positive sequence of the images, are they in the correct order?\nReference answer: yes\nPrediction answer:Yes, the images are in the correct order. The temperature decreases from the first image to the third image.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "1", + "set_id": "7", + "figure_id": "0", + "sample_note": "plug", + "question_id": "3", + "question": "The plug is removed from the power outlet. According to the positive sequence of the images, are they in the correct order?", + "gt_answer_details": "no", + "gt_answer": "0", + "filename": "./VD/video/7_0.png", + "model_prediction": "Yes, the images are in the correct order. The first image shows the plug in the power outlet, the second image shows the plug being removed from the outlet, and the third image shows the plug being inserted into the outlet.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:The plug is removed from the power outlet. According to the positive sequence of the images, are they in the correct order?\nReference answer: no\nPrediction answer:Yes, the images are in the correct order. The first image shows the plug in the power outlet, the second image shows the plug being removed from the outlet, and the third image shows the plug being inserted into the outlet.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "2", + "set_id": "7", + "figure_id": "1", + "sample_note": "plug", + "question_id": "3", + "question": "The plug is removed from the power outlet. According to the positive sequence of the images, are they in the correct order?", + "gt_answer_details": "yes", + "gt_answer": "1", + "filename": "./VD/video/7_1.png", + "model_prediction": "Yes, the images are in the correct order. The first image shows the plug being inserted into the power outlet, the second image shows the plug being removed from the power outlet, and the third image shows the plug being inserted back into the power outlet.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:The plug is removed from the power outlet. According to the positive sequence of the images, are they in the correct order?\nReference answer: yes\nPrediction answer:Yes, the images are in the correct order. The first image shows the plug being inserted into the power outlet, the second image shows the plug being removed from the power outlet, and the third image shows the plug being inserted back into the power outlet.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "1", + "set_id": "8", + "figure_id": "0", + "sample_note": "drive", + "question_id": "3", + "question": "The blue car is moving backward. According to the positive sequence of the images, are they in the correct order?", + "gt_answer_details": "no", + "gt_answer": "0", + "filename": "./VD/video/8_0.png", + "model_prediction": "No, the blue car is not in the correct position in the sequence. It should be in the second position from the left, not the third.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:The blue car is moving backward. According to the positive sequence of the images, are they in the correct order?\nReference answer: no\nPrediction answer:No, the blue car is not in the correct position in the sequence. It should be in the second position from the left, not the third.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "2", + "set_id": "8", + "figure_id": "1", + "sample_note": "drive", + "question_id": "3", + "question": "The blue car is moving backward. According to the positive sequence of the images, are they in the correct order?", + "gt_answer_details": "yes", + "gt_answer": "1", + "filename": "./VD/video/8_1.png", + "model_prediction": "No, the blue car is not in the correct position in the sequence. It should be in the second position from the left, not the third.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:The blue car is moving backward. According to the positive sequence of the images, are they in the correct order?\nReference answer: yes\nPrediction answer:No, the blue car is not in the correct position in the sequence. It should be in the second position from the left, not the third.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "2", + "set_id": "9", + "figure_id": "1", + "sample_note": "direction", + "question_id": "0", + "question": "According to the positive sequence of the images, is the circle inside autobiographical anticlockwise? Answer in one word.", + "gt_answer_details": "the circle inside is not autobiographical anticlockwise.", + "gt_answer": "0", + "filename": "./VD/video/9_1.png", + "model_prediction": "Yes", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the positive sequence of the images, is the circle inside autobiographical anticlockwise? Answer in one word.\nReference answer: the circle inside is not autobiographical anticlockwise.\nPrediction answer:Yes\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "1", + "set_id": "10", + "figure_id": "0", + "sample_note": "monkey", + "question_id": "1", + "question": "According to the positive sequence of the images, Is this monkey installing the wheel? Answer in one word.", + "gt_answer_details": "this monkey is removing the wheel.", + "gt_answer": "0", + "filename": "./VD/video/10_0.png", + "model_prediction": "No", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the positive sequence of the images, Is this monkey installing the wheel? Answer in one word.\nReference answer: this monkey is removing the wheel.\nPrediction answer:No\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "2", + "set_id": "10", + "figure_id": "1", + "sample_note": "monkey", + "question_id": "1", + "question": "According to the positive sequence of the images, Is this monkey installing the wheel? Answer in one word.", + "gt_answer_details": "this monkey is installing the wheel.", + "gt_answer": "1", + "filename": "./VD/video/10_1.png", + "model_prediction": "No", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the positive sequence of the images, Is this monkey installing the wheel? Answer in one word.\nReference answer: this monkey is installing the wheel.\nPrediction answer:No\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "2", + "set_id": "10", + "figure_id": "2", + "sample_note": "monkey", + "question_id": "1", + "question": "According to the positive sequence of the images, Is this monkey installing the wheel? Answer in one word.", + "gt_answer_details": "this monkey is not installing the wheel.", + "gt_answer": "0", + "filename": "./VD/video/10_2.png", + "model_prediction": "No", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the positive sequence of the images, Is this monkey installing the wheel? Answer in one word.\nReference answer: this monkey is not installing the wheel.\nPrediction answer:No\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "1", + "set_id": "11", + "figure_id": "0", + "sample_note": "human", + "question_id": "1", + "question": "According to the positive sequence of the images, Is this woman jumping out from the toilet? Answer in one word.", + "gt_answer_details": "This man throwing a woman into the toilet", + "gt_answer": "0", + "filename": "./VD/video/11_0.png", + "model_prediction": "Yes.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the positive sequence of the images, Is this woman jumping out from the toilet? Answer in one word.\nReference answer: This man throwing a woman into the toilet\nPrediction answer:Yes.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "2", + "set_id": "11", + "figure_id": "1", + "sample_note": "human", + "question_id": "1", + "question": "According to the positive sequence of the images, Is this woman jumping out from the toilet? Answer in one word.", + "gt_answer_details": "This woman jumping out from the toilet.", + "gt_answer": "1", + "filename": "./VD/video/11_1.png", + "model_prediction": "Yes.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the positive sequence of the images, Is this woman jumping out from the toilet? Answer in one word.\nReference answer: This woman jumping out from the toilet.\nPrediction answer:Yes.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "2", + "set_id": "11", + "figure_id": "2", + "sample_note": "human", + "question_id": "1", + "question": "According to the positive sequence of the images, Is this woman jumping out from the toilet? Answer in one word.", + "gt_answer_details": "This woman jumping out from the toilet.", + "gt_answer": "1", + "filename": "./VD/video/11_2.png", + "model_prediction": "Yes.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the positive sequence of the images, Is this woman jumping out from the toilet? Answer in one word.\nReference answer: This woman jumping out from the toilet.\nPrediction answer:Yes.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "1", + "set_id": "12", + "figure_id": "0", + "sample_note": "circle", + "question_id": "1", + "question": "According to the positive sequence of the images, Is the circle keeping in the same place? Answer in one word.", + "gt_answer_details": "The circle keeping in the same place.", + "gt_answer": "1", + "filename": "./VD/video/12_0.png", + "model_prediction": "No.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the positive sequence of the images, Is the circle keeping in the same place? Answer in one word.\nReference answer: The circle keeping in the same place.\nPrediction answer:No.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "2", + "set_id": "12", + "figure_id": "1", + "sample_note": "circle", + "question_id": "2", + "question": "According to the positive sequence of the images, Is the circle moving towards to left? Answer in one word.", + "gt_answer_details": "The circle keeping in the same place.", + "gt_answer": "0", + "filename": "./VD/video/12_1.png", + "model_prediction": "No", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the positive sequence of the images, Is the circle moving towards to left? Answer in one word.\nReference answer: The circle keeping in the same place.\nPrediction answer:No\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "1", + "set_id": "13", + "figure_id": "0", + "sample_note": "sloth", + "question_id": "0", + "question": "According to the positive sequence of the images, is there a dog in this video? Answer in one word.", + "gt_answer_details": "There is a sloth and a cat", + "gt_answer": "0", + "filename": "./VD/video/13_0.png", + "model_prediction": "Yes", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the positive sequence of the images, is there a dog in this video? Answer in one word.\nReference answer: There is a sloth and a cat\nPrediction answer:Yes\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "1", + "set_id": "14", + "figure_id": "0", + "sample_note": "skating meme", + "question_id": "1", + "question": "According to the positive sequence of the images, are they skating to right? Answer in one word.", + "gt_answer_details": "They are skating to the left", + "gt_answer": "0", + "filename": "./VD/video/14_0.png", + "model_prediction": "Yes", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the positive sequence of the images, are they skating to right? Answer in one word.\nReference answer: They are skating to the left\nPrediction answer:Yes\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "2", + "set_id": "14", + "figure_id": "1", + "sample_note": "skating meme", + "question_id": "1", + "question": "According to the positive sequence of the images, are they skating to right? Answer in one sentence.", + "gt_answer_details": "They are skating to the right", + "gt_answer": "1", + "filename": "./VD/video/14_1.png", + "model_prediction": "Yes.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the positive sequence of the images, are they skating to right? Answer in one sentence.\nReference answer: They are skating to the right\nPrediction answer:Yes.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "2", + "set_id": "14", + "figure_id": "2", + "sample_note": "skating meme", + "question_id": "1", + "question": "According to the positive sequence of the images, are they skating to right? Answer in one sentence.", + "gt_answer_details": "They are skating to the right", + "gt_answer": "1", + "filename": "./VD/video/14_2.png", + "model_prediction": "Yes.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the positive sequence of the images, are they skating to right? Answer in one sentence.\nReference answer: They are skating to the right\nPrediction answer:Yes.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "1", + "set_id": "15", + "figure_id": "0", + "sample_note": "warrior", + "question_id": "1", + "question": "According to the positive sequence of the images, is the man running to left? Answer in one sentence.", + "gt_answer_details": "he is walk to the right", + "gt_answer": "0", + "filename": "./VD/video/15_0.png", + "model_prediction": "Yes, the man is running to the left.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the positive sequence of the images, is the man running to left? Answer in one sentence.\nReference answer: he is walk to the right\nPrediction answer:Yes, the man is running to the left.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "1", + "set_id": "16", + "figure_id": "0", + "sample_note": "baby", + "question_id": "1", + "question": "According to the positive sequence of the images, is the baby crawling to left? Answer in one sentence.", + "gt_answer_details": "The baby is crawling to right", + "gt_answer": "0", + "filename": "./VD/video/16_0.png", + "model_prediction": "Yes, the baby is crawling to the left.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the positive sequence of the images, is the baby crawling to left? Answer in one sentence.\nReference answer: The baby is crawling to right\nPrediction answer:Yes, the baby is crawling to the left.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "1", + "set_id": "17", + "figure_id": "0", + "sample_note": "animation", + "question_id": "1", + "question": "According to the positive sequence of the images, is this cartoon character getting far away? Answer in one sentence.", + "gt_answer_details": "The cartoon character is getting closer.", + "gt_answer": "0", + "filename": "./VD/video/17_0.png", + "model_prediction": "No, the cartoon character is not getting far away.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the positive sequence of the images, is this cartoon character getting far away? Answer in one sentence.\nReference answer: The cartoon character is getting closer.\nPrediction answer:No, the cartoon character is not getting far away.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "2", + "set_id": "17", + "figure_id": "1", + "sample_note": "animation", + "question_id": "1", + "question": "According to the positive sequence of the images, is this cartoon character getting far away? Answer in one sentence.", + "gt_answer_details": "The cartoon character is getting far away.", + "gt_answer": "1", + "filename": "./VD/video/17_1.png", + "model_prediction": "No, the cartoon character is not getting far away.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the positive sequence of the images, is this cartoon character getting far away? Answer in one sentence.\nReference answer: The cartoon character is getting far away.\nPrediction answer:No, the cartoon character is not getting far away.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "1", + "set_id": "18", + "figure_id": "0", + "sample_note": "hug", + "question_id": "1", + "question": "According to the positive sequence of the images, are they growing apart? Answer in one sentence.", + "gt_answer_details": "They are hugging.", + "gt_answer": "0", + "filename": "./VD/video/18_0.png", + "model_prediction": "No, they are not growing apart.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the positive sequence of the images, are they growing apart? Answer in one sentence.\nReference answer: They are hugging.\nPrediction answer:No, they are not growing apart.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "2", + "set_id": "18", + "figure_id": "1", + "sample_note": "hug", + "question_id": "1", + "question": "According to the positive sequence of the images, are they growing apart? Answer in one sentence.", + "gt_answer_details": "They are growing apart.", + "gt_answer": "1", + "filename": "./VD/video/18_1.png", + "model_prediction": "No, they are not growing apart.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the positive sequence of the images, are they growing apart? Answer in one sentence.\nReference answer: They are growing apart.\nPrediction answer:No, they are not growing apart.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "2", + "set_id": "18", + "figure_id": "2", + "sample_note": "hug", + "question_id": "1", + "question": "According to the positive sequence of the images, are they growing apart? Answer in one sentence.", + "gt_answer_details": "They are growing apart.", + "gt_answer": "1", + "filename": "./VD/video/18_2.png", + "model_prediction": "No, they are not growing apart.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the positive sequence of the images, are they growing apart? Answer in one sentence.\nReference answer: They are growing apart.\nPrediction answer:No, they are not growing apart.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "1", + "set_id": "19", + "figure_id": "0", + "sample_note": "running", + "question_id": "1", + "question": "According to the positive sequence of the images, Is he running clockwise? Answer in one sentence.", + "gt_answer_details": "The man is running counterclockwise.", + "gt_answer": "0", + "filename": "./VD/video/19_0.png", + "model_prediction": "Yes.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the positive sequence of the images, Is he running clockwise? Answer in one sentence.\nReference answer: The man is running counterclockwise.\nPrediction answer:Yes.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "2", + "set_id": "19", + "figure_id": "1", + "sample_note": "running", + "question_id": "1", + "question": "According to the positive sequence of the images, Is he running clockwise? Answer in one sentence.", + "gt_answer_details": "The man is running clockwise.", + "gt_answer": "1", + "filename": "./VD/video/19_1.png", + "model_prediction": "Yes.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the positive sequence of the images, Is he running clockwise? Answer in one sentence.\nReference answer: The man is running clockwise.\nPrediction answer:Yes.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "video", + "visual_input": "2", + "set_id": "19", + "figure_id": "2", + "sample_note": "running", + "question_id": "1", + "question": "According to the positive sequence of the images, Is he running clockwise? Answer in one sentence.", + "gt_answer_details": "The man is running clockwise.", + "gt_answer": "1", + "filename": "./VD/video/19_2.png", + "model_prediction": "Yes.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the positive sequence of the images, Is he running clockwise? Answer in one sentence.\nReference answer: The man is running clockwise.\nPrediction answer:Yes.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "1", + "set_id": "0", + "figure_id": "0", + "sample_note": "circle", + "question_id": "1", + "question": "Is the right orange circle larger than the left orange circle?", + "gt_answer_details": "The right orange circle is the same size as the left orange circle.", + "gt_answer": "0", + "filename": "./VD/illusion/0_0.png", + "model_prediction": "Yes, the right orange circle is larger than the left orange circle.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the right orange circle larger than the left orange circle?\nReference answer: The right orange circle is the same size as the left orange circle.\nPrediction answer:Yes, the right orange circle is larger than the left orange circle.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "2", + "set_id": "0", + "figure_id": "1", + "sample_note": "circle", + "question_id": "2", + "question": "Is the right orange circle smaller than the left orange circle?", + "gt_answer_details": "The right orange circle is larger than the left orange circle.", + "gt_answer": "0", + "filename": "./VD/illusion/0_1.png", + "model_prediction": "No, the right orange circle is not smaller than the left orange circle. They are the same size.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the right orange circle smaller than the left orange circle?\nReference answer: The right orange circle is larger than the left orange circle.\nPrediction answer:No, the right orange circle is not smaller than the left orange circle. They are the same size.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "2", + "set_id": "1", + "figure_id": "1", + "sample_note": "box", + "question_id": "0", + "question": "Is the right grey box larger than the left grey box?", + "gt_answer_details": "The right grey box is smaller than the left grey box.", + "gt_answer": "0", + "filename": "./VD/illusion/1_1.png", + "model_prediction": "Yes, the right grey box is larger than the left grey box.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the right grey box larger than the left grey box?\nReference answer: The right grey box is smaller than the left grey box.\nPrediction answer:Yes, the right grey box is larger than the left grey box.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "1", + "set_id": "2", + "figure_id": "0", + "sample_note": "line", + "question_id": "1", + "question": "Is the top line shorter than the bottom line?", + "gt_answer_details": "The top line is the same size as the bottom line.", + "gt_answer": "0", + "filename": "./VD/illusion/2_0.png", + "model_prediction": "No, the top line is not shorter than the bottom line. They are both the same length.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the top line shorter than the bottom line?\nReference answer: The top line is the same size as the bottom line.\nPrediction answer:No, the top line is not shorter than the bottom line. They are both the same length.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "2", + "set_id": "2", + "figure_id": "1", + "sample_note": "line", + "question_id": "2", + "question": "Is the top line the same size as the bottom line?", + "gt_answer_details": "The top line is longer than the bottom line.", + "gt_answer": "0", + "filename": "./VD/illusion/2_1.png", + "model_prediction": "Yes, the top line is the same size as the bottom line.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the top line the same size as the bottom line?\nReference answer: The top line is longer than the bottom line.\nPrediction answer:Yes, the top line is the same size as the bottom line.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "2", + "set_id": "3", + "figure_id": "1", + "sample_note": "rail", + "question_id": "0", + "question": "Are the two yellow lines the same length?", + "gt_answer_details": "The top yellow line is shorter than the bottom yellow line.", + "gt_answer": "0", + "filename": "./VD/illusion/3_1.png", + "model_prediction": "Yes, the two yellow lines are the same length.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Are the two yellow lines the same length?\nReference answer: The top yellow line is shorter than the bottom yellow line.\nPrediction answer:Yes, the two yellow lines are the same length.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "2", + "set_id": "4", + "figure_id": "1", + "sample_note": "grey_dot", + "question_id": "0", + "question": "Are there any grey points actually in the intersection of the white lines?", + "gt_answer_details": "There are grey points.", + "gt_answer": "1", + "filename": "./VD/illusion/4_1.png", + "model_prediction": "No, there are no grey points in the intersection of the white lines.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Are there any grey points actually in the intersection of the white lines?\nReference answer: There are grey points.\nPrediction answer:No, there are no grey points in the intersection of the white lines.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "2", + "set_id": "5", + "figure_id": "1", + "sample_note": "chubb", + "question_id": "0", + "question": "The image shows the famous Chubb illusion. Are the center areas of two rectangular fields identical?", + "gt_answer_details": "The center area on the left is larger than the center area on the right.", + "gt_answer": "0", + "filename": "./VD/illusion/5_1.png", + "model_prediction": "No, the center areas of the two rectangular fields are not identical. The Chubb illusion is a visual illusion where the central area of a field with a patterned background appears larger than the central area of a field with a plain background.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:The image shows the famous Chubb illusion. Are the center areas of two rectangular fields identical?\nReference answer: The center area on the left is larger than the center area on the right.\nPrediction answer:No, the center areas of the two rectangular fields are not identical. The Chubb illusion is a visual illusion where the central area of a field with a patterned background appears larger than the central area of a field with a plain background.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "2", + "set_id": "6", + "figure_id": "1", + "sample_note": "curve_line", + "question_id": "0", + "question": "Is the vertical line in the middle actually curved?", + "gt_answer_details": "the vertical line in the middle actually is curved", + "gt_answer": "1", + "filename": "./VD/illusion/6_1.png", + "model_prediction": "No, the vertical line in the middle is not actually curved. It is a straight line that appears to be distorted due to the perspective of the image.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the vertical line in the middle actually curved?\nReference answer: the vertical line in the middle actually is curved\nPrediction answer:No, the vertical line in the middle is not actually curved. It is a straight line that appears to be distorted due to the perspective of the image.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "2", + "set_id": "7", + "figure_id": "1", + "sample_note": "torsion", + "question_id": "1", + "question": "According to the Torsion dystonia illusion, are the diameters of the three circles different?", + "gt_answer_details": "the diameters of the three circles are different", + "gt_answer": "1", + "filename": "./VD/illusion/7_1.png", + "model_prediction": "No, the diameters of the three circles are the same.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the Torsion dystonia illusion, are the diameters of the three circles different?\nReference answer: the diameters of the three circles are different\nPrediction answer:No, the diameters of the three circles are the same.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "2", + "set_id": "8", + "figure_id": "1", + "sample_note": "cross_line", + "question_id": "0", + "question": "Is the vertical line the same length as the horizontal line?", + "gt_answer_details": "the vertical line is shorter than the horizontal line", + "gt_answer": "0", + "filename": "./VD/illusion/8_1.png", + "model_prediction": "Yes, the vertical line is the same length as the horizontal line.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the vertical line the same length as the horizontal line?\nReference answer: the vertical line is shorter than the horizontal line\nPrediction answer:Yes, the vertical line is the same length as the horizontal line.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "1", + "set_id": "9", + "figure_id": "0", + "sample_note": "para_line", + "question_id": "1", + "question": "Are the two red vertical lines parallel with each other?", + "gt_answer_details": "the two red vertical lines are parallel", + "gt_answer": "1", + "filename": "./VD/illusion/9_0.png", + "model_prediction": "Yes, the two red vertical lines are parallel to each other.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Are the two red vertical lines parallel with each other?\nReference answer: the two red vertical lines are parallel\nPrediction answer:Yes, the two red vertical lines are parallel to each other.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "1", + "set_id": "10", + "figure_id": "0", + "sample_note": "color_block", + "question_id": "1", + "question": "For the given image, are the colors on tiles A and B the different?", + "gt_answer_details": "the colors on tiles A and B are the same", + "gt_answer": "0", + "filename": "./VD/illusion/10_0.png", + "model_prediction": "Yes, the colors on tiles A and B are different.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:For the given image, are the colors on tiles A and B the different?\nReference answer: the colors on tiles A and B are the same\nPrediction answer:Yes, the colors on tiles A and B are different.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "1", + "set_id": "11", + "figure_id": "0", + "sample_note": "circle", + "question_id": "1", + "question": "Is the black solid circle on the left smaller than the black solid circle on the right?", + "gt_answer_details": "the black solid circle on the left the same size as the black solid circle on the right", + "gt_answer": "0", + "filename": "./VD/illusion/11_0.png", + "model_prediction": "Yes, the black solid circle on the left is smaller than the black solid circle on the right.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the black solid circle on the left smaller than the black solid circle on the right?\nReference answer: the black solid circle on the left the same size as the black solid circle on the right\nPrediction answer:Yes, the black solid circle on the left is smaller than the black solid circle on the right.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "2", + "set_id": "11", + "figure_id": "1", + "sample_note": "circle", + "question_id": "2", + "question": "Is the black solid circle on the left larger than the black solid circle on the right?", + "gt_answer_details": "the black solid circle on the left is smaller than black solid circle on the right", + "gt_answer": "0", + "filename": "./VD/illusion/11_1.png", + "model_prediction": "Yes, the black solid circle on the left is larger than the black solid circle on the right.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the black solid circle on the left larger than the black solid circle on the right?\nReference answer: the black solid circle on the left is smaller than black solid circle on the right\nPrediction answer:Yes, the black solid circle on the left is larger than the black solid circle on the right.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "2", + "set_id": "13", + "figure_id": "1", + "sample_note": "cafe-wall", + "question_id": "0", + "question": "Whether lines of blocks are parallel? Yes or No", + "gt_answer_details": "No, lines of blocks are not parallel", + "gt_answer": "0", + "filename": "./VD/illusion/13_1.png", + "model_prediction": "Yes", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Whether lines of blocks are parallel? Yes or No\nReference answer: No, lines of blocks are not parallel\nPrediction answer:Yes\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "1", + "set_id": "14", + "figure_id": "0", + "sample_note": "square", + "question_id": "3", + "question": "This image shows the orbison illusion. Whether the red shape in the image is a rhombus? Yes or No", + "gt_answer_details": "No, the red shape is not a rhombus.", + "gt_answer": "0", + "filename": "./VD/illusion/14_0.png", + "model_prediction": "No", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:This image shows the orbison illusion. Whether the red shape in the image is a rhombus? Yes or No\nReference answer: No, the red shape is not a rhombus.\nPrediction answer:No\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "2", + "set_id": "14", + "figure_id": "1", + "sample_note": "square", + "question_id": "3", + "question": "This image shows the orbison illusion. Whether the red shape in the image is a rhombus? Yes or No", + "gt_answer_details": "Yes, the red shape is a rhombus.", + "gt_answer": "1", + "filename": "./VD/illusion/14_1.png", + "model_prediction": "No", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:This image shows the orbison illusion. Whether the red shape in the image is a rhombus? Yes or No\nReference answer: Yes, the red shape is a rhombus.\nPrediction answer:No\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "2", + "set_id": "15", + "figure_id": "1", + "sample_note": "triangle", + "question_id": "1", + "question": "Are two triangles in the image different color? Yes or No", + "gt_answer_details": "Yes, the two triangles have different colors.", + "gt_answer": "1", + "filename": "./VD/illusion/15_1.png", + "model_prediction": "Yes", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Are two triangles in the image different color? Yes or No\nReference answer: Yes, the two triangles have different colors.\nPrediction answer:Yes\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "2", + "set_id": "16", + "figure_id": "1", + "sample_note": "square", + "question_id": "1", + "question": "Are two squares in the image different color? Yes or No", + "gt_answer_details": "Yes, the two squares have different colors.", + "gt_answer": "1", + "filename": "./VD/illusion/16_1.png", + "model_prediction": "Yes", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Are two squares in the image different color? Yes or No\nReference answer: Yes, the two squares have different colors.\nPrediction answer:Yes\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "2", + "set_id": "17", + "figure_id": "1", + "sample_note": "man", + "question_id": "1", + "question": "Are these two men in the images different height? Yes or No", + "gt_answer_details": "Yes, these two men are different height.", + "gt_answer": "1", + "filename": "./VD/illusion/17_1.png", + "model_prediction": "Yes", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Are these two men in the images different height? Yes or No\nReference answer: Yes, these two men are different height.\nPrediction answer:Yes\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "2", + "set_id": "18", + "figure_id": "1", + "sample_note": "man", + "question_id": "1", + "question": "Are these two men in the images different height? Yes or No", + "gt_answer_details": "Yes, these two men are different height.", + "gt_answer": "1", + "filename": "./VD/illusion/18_1.png", + "model_prediction": "No", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Are these two men in the images different height? Yes or No\nReference answer: Yes, these two men are different height.\nPrediction answer:No\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "1", + "set_id": "19", + "figure_id": "0", + "sample_note": "circle", + "question_id": "3", + "question": "The image shows a color illusion. Are two circles in the image different color? yes or no", + "gt_answer_details": "No, two circles are the same color.", + "gt_answer": "0", + "filename": "./VD/illusion/19_0.png", + "model_prediction": "Yes", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:The image shows a color illusion. Are two circles in the image different color? yes or no\nReference answer: No, two circles are the same color.\nPrediction answer:Yes\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "2", + "set_id": "19", + "figure_id": "1", + "sample_note": "circle", + "question_id": "3", + "question": "The image shows a color illusion. Are two circles in the image different color? yes or no", + "gt_answer_details": "Yes, two circles are different colors.", + "gt_answer": "1", + "filename": "./VD/illusion/19_1.png", + "model_prediction": "Yes", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:The image shows a color illusion. Are two circles in the image different color? yes or no\nReference answer: Yes, two circles are different colors.\nPrediction answer:Yes\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "1", + "set_id": "21", + "figure_id": "0", + "sample_note": "circle", + "question_id": "1", + "question": "The image shows a color illusion. Are two circles in the image the same color? Yes or No", + "gt_answer_details": "Yes, the two circles in the image are the same color.", + "gt_answer": "1", + "filename": "./VD/illusion/21_0.png", + "model_prediction": "No", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:The image shows a color illusion. Are two circles in the image the same color? Yes or No\nReference answer: Yes, the two circles in the image are the same color.\nPrediction answer:No\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "2", + "set_id": "21", + "figure_id": "1", + "sample_note": "circle", + "question_id": "1", + "question": "The image shows a color illusion. Are two circles in the image the same color? Yes or No", + "gt_answer_details": "No, the two circles in the image are not the same color.", + "gt_answer": "0", + "filename": "./VD/illusion/21_1.png", + "model_prediction": "No", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:The image shows a color illusion. Are two circles in the image the same color? Yes or No\nReference answer: No, the two circles in the image are not the same color.\nPrediction answer:No\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "2", + "set_id": "22", + "figure_id": "1", + "sample_note": "line", + "question_id": "0", + "question": "Does the black line align with the blue line? Yes or No", + "gt_answer_details": "Yes, the black line align with the blue line.", + "gt_answer": "1", + "filename": "./VD/illusion/22_1.png", + "model_prediction": "No", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Does the black line align with the blue line? Yes or No\nReference answer: Yes, the black line align with the blue line.\nPrediction answer:No\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "2", + "set_id": "23", + "figure_id": "1", + "sample_note": "circle", + "question_id": "1", + "question": "Are two circles in the image different color? Yes or No", + "gt_answer_details": "Yes, the two circles in the image are different color.", + "gt_answer": "1", + "filename": "./VD/illusion/23_1.png", + "model_prediction": "No", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Are two circles in the image different color? Yes or No\nReference answer: Yes, the two circles in the image are different color.\nPrediction answer:No\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "2", + "set_id": "24", + "figure_id": "1", + "sample_note": "circle", + "question_id": "1", + "question": "Are two circles in the image different color? Yes or No", + "gt_answer_details": "Yes, the two circles in the image are different colors.", + "gt_answer": "1", + "filename": "./VD/illusion/24_1.png", + "model_prediction": "Yes", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Are two circles in the image different color? Yes or No\nReference answer: Yes, the two circles in the image are different colors.\nPrediction answer:Yes\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "2", + "set_id": "25", + "figure_id": "1", + "sample_note": "circle", + "question_id": "0", + "question": "Is the green circle on left the same size as the green circle on the right? Yes or No", + "gt_answer_details": "No, the green circle on left is not the same size as the green circle on the right", + "gt_answer": "0", + "filename": "./VD/illusion/25_1.png", + "model_prediction": "No", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the green circle on left the same size as the green circle on the right? Yes or No\nReference answer: No, the green circle on left is not the same size as the green circle on the right\nPrediction answer:No\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "2", + "set_id": "26", + "figure_id": "1", + "sample_note": "line", + "question_id": "0", + "question": "Are orange lines in the image parallel? Yes or No", + "gt_answer_details": "No, orange lines are not parallel", + "gt_answer": "0", + "filename": "./VD/illusion/26_1.png", + "model_prediction": "Yes", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Are orange lines in the image parallel? Yes or No\nReference answer: No, orange lines are not parallel\nPrediction answer:Yes\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "2", + "set_id": "27", + "figure_id": "1", + "sample_note": "line", + "question_id": "0", + "question": "Are the green line and the red line the same length in the image? Yes or No", + "gt_answer_details": "No, the green line and the red line are not the same length.", + "gt_answer": "0", + "filename": "./VD/illusion/27_1.png", + "model_prediction": "No", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Are the green line and the red line the same length in the image? Yes or No\nReference answer: No, the green line and the red line are not the same length.\nPrediction answer:No\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "1", + "set_id": "28", + "figure_id": "0", + "sample_note": "line", + "question_id": "1", + "question": "Is the yellow line longer than the red line in the image? Yes or No", + "gt_answer_details": "No, the yellow line and the red line are the same length.", + "gt_answer": "0", + "filename": "./VD/illusion/28_0.png", + "model_prediction": "Yes", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the yellow line longer than the red line in the image? Yes or No\nReference answer: No, the yellow line and the red line are the same length.\nPrediction answer:Yes\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "2", + "set_id": "28", + "figure_id": "1", + "sample_note": "line", + "question_id": "2", + "question": "Is the yellow line shorter than the red line in the image? Yes or No", + "gt_answer_details": "No, the yellow line is longer than the red line.", + "gt_answer": "0", + "filename": "./VD/illusion/28_1.png", + "model_prediction": "No", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the yellow line shorter than the red line in the image? Yes or No\nReference answer: No, the yellow line is longer than the red line.\nPrediction answer:No\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "2", + "set_id": "29", + "figure_id": "1", + "sample_note": "line", + "question_id": "0", + "question": "Are the lengths of the vertical line and the horizontal line the same?", + "gt_answer_details": "No, the lengths of the vertical line and the horizontal line are not the same.", + "gt_answer": "0", + "filename": "./VD/illusion/29_1.png", + "model_prediction": "Yes, the lengths of the vertical line and the horizontal line are the same.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Are the lengths of the vertical line and the horizontal line the same?\nReference answer: No, the lengths of the vertical line and the horizontal line are not the same.\nPrediction answer:Yes, the lengths of the vertical line and the horizontal line are the same.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "illusion", + "visual_input": "1", + "set_id": "30", + "figure_id": "0", + "sample_note": "line", + "question_id": "1", + "question": "Is the red line on the left different length as the red line on the right? Yes or No", + "gt_answer_details": "No, the red line on the left is not different length as the red line on the right.", + "gt_answer": "0", + "filename": "./VD/illusion/30_0.png", + "model_prediction": "Yes", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the red line on the left different length as the red line on the right? Yes or No\nReference answer: No, the red line on the left is not different length as the red line on the right.\nPrediction answer:Yes\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "math", + "visual_input": "1", + "set_id": "0", + "figure_id": "0", + "sample_note": "para_angle", + "question_id": "1", + "question": "According to parallel lines theorem, is angle 1 + angle 2 > 180 ?", + "gt_answer_details": "angle 1 + angle 2 = 180 is correct", + "gt_answer": "0", + "filename": "./VD/math/0_0.png", + "model_prediction": "No, according to the parallel lines theorem, angle 1 + angle 2 = 180 degrees.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to parallel lines theorem, is angle 1 + angle 2 > 180 ?\nReference answer: angle 1 + angle 2 = 180 is correct\nPrediction answer:No, according to the parallel lines theorem, angle 1 + angle 2 = 180 degrees.\nOutput:correct" + }, + { + "category": "VD", + "subcategory": "math", + "visual_input": "2", + "set_id": "0", + "figure_id": "1", + "sample_note": "para_angle", + "question_id": "2", + "question": "According to parallel lines theorem, is angle 1 + angle 2 < 180 ?", + "gt_answer_details": "angle 1 + angle 2 > 180 is correct", + "gt_answer": "0", + "filename": "./VD/math/0_1.png", + "model_prediction": "Yes, according to the parallel lines theorem, angle 1 + angle 2 is less than 180 degrees.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to parallel lines theorem, is angle 1 + angle 2 < 180 ?\nReference answer: angle 1 + angle 2 > 180 is correct\nPrediction answer:Yes, according to the parallel lines theorem, angle 1 + angle 2 is less than 180 degrees.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "math", + "visual_input": "2", + "set_id": "1", + "figure_id": "1", + "sample_note": "pythagorean", + "question_id": "0", + "question": "Is the value of x in the image 5?", + "gt_answer_details": "the value of x is larger than 5", + "gt_answer": "0", + "filename": "./VD/math/1_1.png", + "model_prediction": "No, the value of x is not 5. The value of x is 3.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the value of x in the image 5?\nReference answer: the value of x is larger than 5\nPrediction answer:No, the value of x is not 5. The value of x is 3.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "math", + "visual_input": "1", + "set_id": "2", + "figure_id": "0", + "sample_note": "triangle", + "question_id": "1", + "question": "Is angleA + angleB +angleC < 180?", + "gt_answer_details": "angleA + angleB +angleC = 180 is correct", + "gt_answer": "0", + "filename": "./VD/math/2_0.png", + "model_prediction": "Yes, angleA + angleB + angleC < 180.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is angleA + angleB +angleC < 180?\nReference answer: angleA + angleB +angleC = 180 is correct\nPrediction answer:Yes, angleA + angleB + angleC < 180.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "math", + "visual_input": "2", + "set_id": "2", + "figure_id": "1", + "sample_note": "triangle", + "question_id": "2", + "question": "Is angleA + angleB +angleC > 180?", + "gt_answer_details": "angleA + angleB +angleC > 180 is correct", + "gt_answer": "1", + "filename": "./VD/math/2_1.png", + "model_prediction": "No, angleA + angleB + angleC = 180.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is angleA + angleB +angleC > 180?\nReference answer: angleA + angleB +angleC > 180 is correct\nPrediction answer:No, angleA + angleB + angleC = 180.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "math", + "visual_input": "2", + "set_id": "3", + "figure_id": "1", + "sample_note": "math_prob", + "question_id": "0", + "question": "According to the parallel theorem, is angle A equal to angle B? Please answer yes or no.", + "gt_answer_details": "Angle A is larger than angle B.", + "gt_answer": "0", + "filename": "./VD/math/3_1.png", + "model_prediction": "Yes.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the parallel theorem, is angle A equal to angle B? Please answer yes or no.\nReference answer: Angle A is larger than angle B.\nPrediction answer:Yes.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "math", + "visual_input": "1", + "set_id": "4", + "figure_id": "0", + "sample_note": "math_prob", + "question_id": "1", + "question": "Is the value of x larger than 6? Please answer yes or no.", + "gt_answer_details": "The value of x is equal to 6.", + "gt_answer": "0", + "filename": "./VD/math/4_0.png", + "model_prediction": "Yes.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the value of x larger than 6? Please answer yes or no.\nReference answer: The value of x is equal to 6.\nPrediction answer:Yes.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "math", + "visual_input": "2", + "set_id": "4", + "figure_id": "1", + "sample_note": "math_prob", + "question_id": "2", + "question": "Is the value of x smaller than 6? Please answer yes or no.", + "gt_answer_details": "The value of x is larger than 6.", + "gt_answer": "0", + "filename": "./VD/math/4_1.png", + "model_prediction": "Yes.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the value of x smaller than 6? Please answer yes or no.\nReference answer: The value of x is larger than 6.\nPrediction answer:Yes.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "math", + "visual_input": "2", + "set_id": "5", + "figure_id": "1", + "sample_note": "math_prob", + "question_id": "0", + "question": "If angle A is 110 degree, then is the angle BOC equal to 145 degree? Please answer yes or no.", + "gt_answer_details": "The angle BOC is not equal to 145 degree, or we do not know for sure.", + "gt_answer": "0", + "filename": "./VD/math/5_1.png", + "model_prediction": "Yes.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:If angle A is 110 degree, then is the angle BOC equal to 145 degree? Please answer yes or no.\nReference answer: The angle BOC is not equal to 145 degree, or we do not know for sure.\nPrediction answer:Yes.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "math", + "visual_input": "1", + "set_id": "6", + "figure_id": "0", + "sample_note": "math_prob", + "question_id": "1", + "question": "According to the figure, are the two orange lines both vertical? Please answer yes or no.", + "gt_answer_details": "Both two orange lines are straight.", + "gt_answer": "0", + "filename": "./VD/math/6_0.png", + "model_prediction": "No.", + "gpt4v_output_gpt_check": "0", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the figure, are the two orange lines both vertical? Please answer yes or no.\nReference answer: Both two orange lines are straight.\nPrediction answer:No.\nOutput:incorrect" + }, + { + "category": "VD", + "subcategory": "math", + "visual_input": "2", + "set_id": "6", + "figure_id": "1", + "sample_note": "math_prob", + "question_id": "2", + "question": "According to the figure, are the two orange lines both curved? Please answer yes or no.", + "gt_answer_details": "Both two orange lines are curved.", + "gt_answer": "1", + "filename": "./VD/math/6_1.png", + "model_prediction": "Yes.", + "gpt4v_output_gpt_check": "1", + "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:According to the figure, are the two orange lines both curved? Please answer yes or no.\nReference answer: Both two orange lines are curved.\nPrediction answer:Yes.\nOutput:correct" + } +] \ No newline at end of file diff --git a/sft/1M3/Full_smoe/checkpoint-6893/logs/0627_0203_llava...image_llava_model_args_f4318b/rank1_metric_eval_done.txt b/sft/1M3/Full_smoe/checkpoint-6893/logs/0627_0203_llava...image_llava_model_args_f4318b/rank1_metric_eval_done.txt new file mode 100644 index 0000000000000000000000000000000000000000..36792c9cedb6c006db3a866d72eac15f0ce6a64a --- /dev/null +++ b/sft/1M3/Full_smoe/checkpoint-6893/logs/0627_0203_llava...image_llava_model_args_f4318b/rank1_metric_eval_done.txt @@ -0,0 +1 @@ +rank 1 eval done \ No newline at end of file diff --git a/sft/1M3/Full_smoe/checkpoint-6893/logs/0627_0203_llava...image_llava_model_args_f4318b/rank2_metric_eval_done.txt b/sft/1M3/Full_smoe/checkpoint-6893/logs/0627_0203_llava...image_llava_model_args_f4318b/rank2_metric_eval_done.txt new file mode 100644 index 0000000000000000000000000000000000000000..d3e5c7ecd1fd051ff210a79f69ad980d587fd5b3 --- /dev/null +++ b/sft/1M3/Full_smoe/checkpoint-6893/logs/0627_0203_llava...image_llava_model_args_f4318b/rank2_metric_eval_done.txt @@ -0,0 +1 @@ +rank 2 eval done \ No newline at end of file diff --git a/sft/1M3/Full_smoe/checkpoint-6893/logs/0627_0203_llava...image_llava_model_args_f4318b/rank3_metric_eval_done.txt b/sft/1M3/Full_smoe/checkpoint-6893/logs/0627_0203_llava...image_llava_model_args_f4318b/rank3_metric_eval_done.txt new file mode 100644 index 0000000000000000000000000000000000000000..3a4b44254d394e29e04b2a41d91f6dc025b8afad --- /dev/null +++ b/sft/1M3/Full_smoe/checkpoint-6893/logs/0627_0203_llava...image_llava_model_args_f4318b/rank3_metric_eval_done.txt @@ -0,0 +1 @@ +rank 3 eval done \ No newline at end of file diff --git a/sft/1M3/Full_smoe/checkpoint-6893/model-00001-of-00003.safetensors b/sft/1M3/Full_smoe/checkpoint-6893/model-00001-of-00003.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2b6011d38f21e491878b18f9dece35022bbfe6eb --- /dev/null +++ b/sft/1M3/Full_smoe/checkpoint-6893/model-00001-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59214d964d3229c4858fea5fcaeeb8235382e4bbe49d44dc16860d52bb25bb7b +size 4972489328 diff --git a/sft/1M3/Full_smoe/checkpoint-6893/model-00002-of-00003.safetensors b/sft/1M3/Full_smoe/checkpoint-6893/model-00002-of-00003.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2c273f739a79a0040687555952bf0df10d43eab4 --- /dev/null +++ b/sft/1M3/Full_smoe/checkpoint-6893/model-00002-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:682f80684fd5d4ab7244d975e537cbd12a6a4e93321795c45a16a1534578bd43 +size 4985529648 diff --git a/sft/1M3/Full_smoe/checkpoint-6893/model-00003-of-00003.safetensors b/sft/1M3/Full_smoe/checkpoint-6893/model-00003-of-00003.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5961f0dcd84873926fdc768f21ba91fcbac6b9a8 --- /dev/null +++ b/sft/1M3/Full_smoe/checkpoint-6893/model-00003-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45567704b58412040e22637a3d0ce46f53a1f16d985a994127554849ccc15bad +size 248943552 diff --git a/sft/1M3/Full_smoe/checkpoint-6893/model.safetensors.index.json b/sft/1M3/Full_smoe/checkpoint-6893/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..aa54419fc0a3eab502aa7c4ad974dca52ed10803 --- /dev/null +++ b/sft/1M3/Full_smoe/checkpoint-6893/model.safetensors.index.json @@ -0,0 +1,1005 @@ +{ + "metadata": { + "total_size": 10206819456 + }, + "weight_map": { + "lm_head.weight": "model-00003-of-00003.safetensors", + "model.embed_tokens.weight": "model-00001-of-00003.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.gate_up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.qkv_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.gate_up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.qkv_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.mlp.gate_up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.self_attn.qkv_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.11.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.11.mlp.gate_up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.11.self_attn.qkv_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.12.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.12.mlp.gate_up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.12.self_attn.qkv_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.13.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.13.mlp.gate_up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.13.self_attn.qkv_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.14.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.14.mlp.gate_up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.14.self_attn.qkv_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.15.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.15.mlp.gate_up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.15.self_attn.qkv_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.16.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.16.mlp.gate_up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.16.self_attn.qkv_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.17.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.17.mlp.gate_up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.17.self_attn.qkv_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.18.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.18.mlp.gate_up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.18.self_attn.qkv_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.19.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.19.mlp.gate_up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.19.self_attn.qkv_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.gate_up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.qkv_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.20.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.20.mlp.gate_up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.20.self_attn.qkv_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.gate_up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.21.self_attn.qkv_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.mlp.gate_up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.qkv_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.mlp.gate_up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.qkv_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.mlp.gate_up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.self_attn.qkv_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.25.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.25.mlp.gate_up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.25.self_attn.qkv_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.26.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.26.mlp.gate_up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.26.self_attn.qkv_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.27.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.27.mlp.gate_up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.27.self_attn.qkv_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.28.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.28.mlp.gate_up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.28.self_attn.qkv_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.29.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.29.mlp.gate_up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.29.self_attn.qkv_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.gate_up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.qkv_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.30.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.30.mlp.gate_up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.30.self_attn.qkv_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.31.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.31.mlp.gate_up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.31.self_attn.qkv_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.gate_up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.qkv_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.gate_up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.qkv_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.gate_up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.qkv_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.gate_up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.qkv_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.mlp.gate_up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.qkv_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.mlp.gate_up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.self_attn.qkv_proj.weight": "model-00001-of-00003.safetensors", + "model.mm_projector.moelayer.experts.0.0.bias": "model-00002-of-00003.safetensors", + "model.mm_projector.moelayer.experts.0.0.weight": "model-00002-of-00003.safetensors", + "model.mm_projector.moelayer.experts.0.2.bias": "model-00002-of-00003.safetensors", + "model.mm_projector.moelayer.experts.0.2.weight": "model-00002-of-00003.safetensors", + "model.mm_projector.moelayer.experts.1.0.bias": "model-00002-of-00003.safetensors", + "model.mm_projector.moelayer.experts.1.0.weight": "model-00002-of-00003.safetensors", + "model.mm_projector.moelayer.experts.1.2.bias": "model-00002-of-00003.safetensors", + "model.mm_projector.moelayer.experts.1.2.weight": "model-00002-of-00003.safetensors", + "model.mm_projector.moelayer.experts.2.0.bias": "model-00002-of-00003.safetensors", + "model.mm_projector.moelayer.experts.2.0.weight": "model-00002-of-00003.safetensors", + "model.mm_projector.moelayer.experts.2.2.bias": "model-00003-of-00003.safetensors", + "model.mm_projector.moelayer.experts.2.2.weight": "model-00003-of-00003.safetensors", + "model.mm_projector.moelayer.experts.3.0.bias": "model-00003-of-00003.safetensors", + "model.mm_projector.moelayer.experts.3.0.weight": "model-00003-of-00003.safetensors", + "model.mm_projector.moelayer.experts.3.2.bias": "model-00003-of-00003.safetensors", + "model.mm_projector.moelayer.experts.3.2.weight": "model-00003-of-00003.safetensors", + "model.mm_projector.moelayer.gate.weight": "model-00003-of-00003.safetensors", + "model.norm.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.embeddings.patch_embedding.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.embeddings.patch_embedding.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.embeddings.position_embedding.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.0.layer_norm1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.0.layer_norm1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.0.layer_norm2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.0.layer_norm2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.0.moelayer.experts.0.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.0.moelayer.experts.0.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.0.moelayer.experts.0.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.0.moelayer.experts.0.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.0.moelayer.experts.1.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.0.moelayer.experts.1.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.0.moelayer.experts.1.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.0.moelayer.experts.1.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.0.moelayer.experts.2.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.0.moelayer.experts.2.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.0.moelayer.experts.2.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.0.moelayer.experts.2.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.0.moelayer.experts.3.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.0.moelayer.experts.3.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.0.moelayer.experts.3.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.0.moelayer.experts.3.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.0.moelayer.gate.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.0.self_attn.k_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.0.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.0.self_attn.out_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.0.self_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.0.self_attn.q_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.0.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.0.self_attn.v_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.0.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.1.layer_norm1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.1.layer_norm1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.1.layer_norm2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.1.layer_norm2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.1.moelayer.experts.0.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.1.moelayer.experts.0.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.1.moelayer.experts.0.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.1.moelayer.experts.0.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.1.moelayer.experts.1.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.1.moelayer.experts.1.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.1.moelayer.experts.1.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.1.moelayer.experts.1.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.1.moelayer.experts.2.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.1.moelayer.experts.2.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.1.moelayer.experts.2.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.1.moelayer.experts.2.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.1.moelayer.experts.3.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.1.moelayer.experts.3.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.1.moelayer.experts.3.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.1.moelayer.experts.3.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.1.moelayer.gate.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.1.self_attn.k_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.1.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.1.self_attn.out_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.1.self_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.1.self_attn.q_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.1.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.1.self_attn.v_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.1.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.10.layer_norm1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.10.layer_norm1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.10.layer_norm2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.10.layer_norm2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.10.moelayer.experts.0.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.10.moelayer.experts.0.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.10.moelayer.experts.0.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.10.moelayer.experts.0.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.10.moelayer.experts.1.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.10.moelayer.experts.1.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.10.moelayer.experts.1.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.10.moelayer.experts.1.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.10.moelayer.experts.2.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.10.moelayer.experts.2.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.10.moelayer.experts.2.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.10.moelayer.experts.2.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.10.moelayer.experts.3.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.10.moelayer.experts.3.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.10.moelayer.experts.3.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.10.moelayer.experts.3.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.10.moelayer.gate.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.10.self_attn.k_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.10.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.10.self_attn.out_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.10.self_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.10.self_attn.q_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.10.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.10.self_attn.v_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.10.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.11.layer_norm1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.11.layer_norm1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.11.layer_norm2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.11.layer_norm2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.11.moelayer.experts.0.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.11.moelayer.experts.0.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.11.moelayer.experts.0.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.11.moelayer.experts.0.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.11.moelayer.experts.1.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.11.moelayer.experts.1.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.11.moelayer.experts.1.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.11.moelayer.experts.1.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.11.moelayer.experts.2.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.11.moelayer.experts.2.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.11.moelayer.experts.2.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.11.moelayer.experts.2.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.11.moelayer.experts.3.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.11.moelayer.experts.3.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.11.moelayer.experts.3.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.11.moelayer.experts.3.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.11.moelayer.gate.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.11.self_attn.k_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.11.self_attn.out_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.11.self_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.11.self_attn.q_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.11.self_attn.v_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.12.layer_norm1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.12.layer_norm1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.12.layer_norm2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.12.layer_norm2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.12.moelayer.experts.0.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.12.moelayer.experts.0.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.12.moelayer.experts.0.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.12.moelayer.experts.0.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.12.moelayer.experts.1.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.12.moelayer.experts.1.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.12.moelayer.experts.1.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.12.moelayer.experts.1.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.12.moelayer.experts.2.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.12.moelayer.experts.2.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.12.moelayer.experts.2.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.12.moelayer.experts.2.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.12.moelayer.experts.3.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.12.moelayer.experts.3.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.12.moelayer.experts.3.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.12.moelayer.experts.3.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.12.moelayer.gate.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.12.self_attn.k_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.12.self_attn.out_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.12.self_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.12.self_attn.q_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.12.self_attn.v_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.13.layer_norm1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.13.layer_norm1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.13.layer_norm2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.13.layer_norm2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.13.moelayer.experts.0.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.13.moelayer.experts.0.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.13.moelayer.experts.0.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.13.moelayer.experts.0.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.13.moelayer.experts.1.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.13.moelayer.experts.1.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.13.moelayer.experts.1.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.13.moelayer.experts.1.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.13.moelayer.experts.2.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.13.moelayer.experts.2.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.13.moelayer.experts.2.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.13.moelayer.experts.2.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.13.moelayer.experts.3.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.13.moelayer.experts.3.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.13.moelayer.experts.3.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.13.moelayer.experts.3.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.13.moelayer.gate.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.13.self_attn.k_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.13.self_attn.out_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.13.self_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.13.self_attn.q_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.13.self_attn.v_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.14.layer_norm1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.14.layer_norm1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.14.layer_norm2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.14.layer_norm2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.14.moelayer.experts.0.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.14.moelayer.experts.0.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.14.moelayer.experts.0.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.14.moelayer.experts.0.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.14.moelayer.experts.1.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.14.moelayer.experts.1.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.14.moelayer.experts.1.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.14.moelayer.experts.1.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.14.moelayer.experts.2.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.14.moelayer.experts.2.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.14.moelayer.experts.2.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.14.moelayer.experts.2.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.14.moelayer.experts.3.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.14.moelayer.experts.3.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.14.moelayer.experts.3.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.14.moelayer.experts.3.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.14.moelayer.gate.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.14.self_attn.k_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.14.self_attn.out_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.14.self_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.14.self_attn.q_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.14.self_attn.v_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.15.layer_norm1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.15.layer_norm1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.15.layer_norm2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.15.layer_norm2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.15.moelayer.experts.0.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.15.moelayer.experts.0.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.15.moelayer.experts.0.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.15.moelayer.experts.0.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.15.moelayer.experts.1.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.15.moelayer.experts.1.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.15.moelayer.experts.1.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.15.moelayer.experts.1.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.15.moelayer.experts.2.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.15.moelayer.experts.2.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.15.moelayer.experts.2.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.15.moelayer.experts.2.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.15.moelayer.experts.3.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.15.moelayer.experts.3.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.15.moelayer.experts.3.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.15.moelayer.experts.3.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.15.moelayer.gate.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.15.self_attn.k_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.15.self_attn.out_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.15.self_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.15.self_attn.q_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.15.self_attn.v_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.16.layer_norm1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.16.layer_norm1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.16.layer_norm2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.16.layer_norm2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.16.moelayer.experts.0.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.16.moelayer.experts.0.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.16.moelayer.experts.0.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.16.moelayer.experts.0.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.16.moelayer.experts.1.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.16.moelayer.experts.1.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.16.moelayer.experts.1.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.16.moelayer.experts.1.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.16.moelayer.experts.2.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.16.moelayer.experts.2.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.16.moelayer.experts.2.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.16.moelayer.experts.2.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.16.moelayer.experts.3.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.16.moelayer.experts.3.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.16.moelayer.experts.3.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.16.moelayer.experts.3.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.16.moelayer.gate.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.16.self_attn.k_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.16.self_attn.out_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.16.self_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.16.self_attn.q_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.16.self_attn.v_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.17.layer_norm1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.17.layer_norm1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.17.layer_norm2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.17.layer_norm2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.17.moelayer.experts.0.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.17.moelayer.experts.0.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.17.moelayer.experts.0.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.17.moelayer.experts.0.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.17.moelayer.experts.1.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.17.moelayer.experts.1.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.17.moelayer.experts.1.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.17.moelayer.experts.1.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.17.moelayer.experts.2.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.17.moelayer.experts.2.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.17.moelayer.experts.2.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.17.moelayer.experts.2.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.17.moelayer.experts.3.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.17.moelayer.experts.3.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.17.moelayer.experts.3.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.17.moelayer.experts.3.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.17.moelayer.gate.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.17.self_attn.k_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.17.self_attn.out_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.17.self_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.17.self_attn.q_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.17.self_attn.v_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.18.layer_norm1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.18.layer_norm1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.18.layer_norm2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.18.layer_norm2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.18.moelayer.experts.0.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.18.moelayer.experts.0.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.18.moelayer.experts.0.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.18.moelayer.experts.0.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.18.moelayer.experts.1.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.18.moelayer.experts.1.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.18.moelayer.experts.1.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.18.moelayer.experts.1.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.18.moelayer.experts.2.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.18.moelayer.experts.2.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.18.moelayer.experts.2.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.18.moelayer.experts.2.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.18.moelayer.experts.3.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.18.moelayer.experts.3.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.18.moelayer.experts.3.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.18.moelayer.experts.3.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.18.moelayer.gate.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.18.self_attn.k_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.18.self_attn.out_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.18.self_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.18.self_attn.q_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.18.self_attn.v_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.19.layer_norm1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.19.layer_norm1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.19.layer_norm2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.19.layer_norm2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.19.moelayer.experts.0.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.19.moelayer.experts.0.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.19.moelayer.experts.0.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.19.moelayer.experts.0.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.19.moelayer.experts.1.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.19.moelayer.experts.1.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.19.moelayer.experts.1.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.19.moelayer.experts.1.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.19.moelayer.experts.2.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.19.moelayer.experts.2.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.19.moelayer.experts.2.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.19.moelayer.experts.2.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.19.moelayer.experts.3.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.19.moelayer.experts.3.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.19.moelayer.experts.3.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.19.moelayer.experts.3.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.19.moelayer.gate.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.19.self_attn.k_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.19.self_attn.out_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.19.self_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.19.self_attn.q_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.19.self_attn.v_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.2.layer_norm1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.2.layer_norm1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.2.layer_norm2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.2.layer_norm2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.2.moelayer.experts.0.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.2.moelayer.experts.0.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.2.moelayer.experts.0.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.2.moelayer.experts.0.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.2.moelayer.experts.1.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.2.moelayer.experts.1.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.2.moelayer.experts.1.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.2.moelayer.experts.1.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.2.moelayer.experts.2.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.2.moelayer.experts.2.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.2.moelayer.experts.2.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.2.moelayer.experts.2.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.2.moelayer.experts.3.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.2.moelayer.experts.3.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.2.moelayer.experts.3.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.2.moelayer.experts.3.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.2.moelayer.gate.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.2.self_attn.k_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.2.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.2.self_attn.out_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.2.self_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.2.self_attn.q_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.2.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.2.self_attn.v_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.2.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.20.layer_norm1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.20.layer_norm1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.20.layer_norm2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.20.layer_norm2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.20.moelayer.experts.0.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.20.moelayer.experts.0.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.20.moelayer.experts.0.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.20.moelayer.experts.0.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.20.moelayer.experts.1.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.20.moelayer.experts.1.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.20.moelayer.experts.1.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.20.moelayer.experts.1.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.20.moelayer.experts.2.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.20.moelayer.experts.2.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.20.moelayer.experts.2.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.20.moelayer.experts.2.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.20.moelayer.experts.3.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.20.moelayer.experts.3.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.20.moelayer.experts.3.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.20.moelayer.experts.3.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.20.moelayer.gate.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.20.self_attn.k_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.20.self_attn.out_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.20.self_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.20.self_attn.q_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.20.self_attn.v_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.21.layer_norm1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.21.layer_norm1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.21.layer_norm2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.21.layer_norm2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.21.moelayer.experts.0.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.21.moelayer.experts.0.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.21.moelayer.experts.0.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.21.moelayer.experts.0.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.21.moelayer.experts.1.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.21.moelayer.experts.1.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.21.moelayer.experts.1.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.21.moelayer.experts.1.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.21.moelayer.experts.2.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.21.moelayer.experts.2.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.21.moelayer.experts.2.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.21.moelayer.experts.2.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.21.moelayer.experts.3.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.21.moelayer.experts.3.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.21.moelayer.experts.3.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.21.moelayer.experts.3.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.21.moelayer.gate.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.21.self_attn.k_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.21.self_attn.out_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.21.self_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.21.self_attn.q_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.21.self_attn.v_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.22.layer_norm1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.22.layer_norm1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.22.layer_norm2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.22.layer_norm2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.22.moelayer.experts.0.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.22.moelayer.experts.0.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.22.moelayer.experts.0.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.22.moelayer.experts.0.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.22.moelayer.experts.1.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.22.moelayer.experts.1.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.22.moelayer.experts.1.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.22.moelayer.experts.1.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.22.moelayer.experts.2.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.22.moelayer.experts.2.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.22.moelayer.experts.2.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.22.moelayer.experts.2.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.22.moelayer.experts.3.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.22.moelayer.experts.3.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.22.moelayer.experts.3.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.22.moelayer.experts.3.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.22.moelayer.gate.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.22.self_attn.k_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.22.self_attn.out_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.22.self_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.22.self_attn.q_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.22.self_attn.v_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.23.layer_norm1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.23.layer_norm1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.23.layer_norm2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.23.layer_norm2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.23.moelayer.experts.0.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.23.moelayer.experts.0.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.23.moelayer.experts.0.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.23.moelayer.experts.0.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.23.moelayer.experts.1.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.23.moelayer.experts.1.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.23.moelayer.experts.1.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.23.moelayer.experts.1.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.23.moelayer.experts.2.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.23.moelayer.experts.2.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.23.moelayer.experts.2.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.23.moelayer.experts.2.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.23.moelayer.experts.3.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.23.moelayer.experts.3.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.23.moelayer.experts.3.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.23.moelayer.experts.3.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.23.moelayer.gate.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.23.self_attn.k_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.23.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.23.self_attn.out_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.23.self_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.23.self_attn.q_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.23.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.23.self_attn.v_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.23.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.24.layer_norm1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.24.layer_norm1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.24.layer_norm2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.24.layer_norm2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.24.moelayer.experts.0.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.24.moelayer.experts.0.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.24.moelayer.experts.0.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.24.moelayer.experts.0.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.24.moelayer.experts.1.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.24.moelayer.experts.1.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.24.moelayer.experts.1.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.24.moelayer.experts.1.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.24.moelayer.experts.2.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.24.moelayer.experts.2.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.24.moelayer.experts.2.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.24.moelayer.experts.2.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.24.moelayer.experts.3.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.24.moelayer.experts.3.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.24.moelayer.experts.3.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.24.moelayer.experts.3.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.24.moelayer.gate.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.24.self_attn.k_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.24.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.24.self_attn.out_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.24.self_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.24.self_attn.q_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.24.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.24.self_attn.v_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.24.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.25.layer_norm1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.25.layer_norm1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.25.layer_norm2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.25.layer_norm2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.25.moelayer.experts.0.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.25.moelayer.experts.0.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.25.moelayer.experts.0.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.25.moelayer.experts.0.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.25.moelayer.experts.1.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.25.moelayer.experts.1.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.25.moelayer.experts.1.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.25.moelayer.experts.1.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.25.moelayer.experts.2.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.25.moelayer.experts.2.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.25.moelayer.experts.2.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.25.moelayer.experts.2.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.25.moelayer.experts.3.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.25.moelayer.experts.3.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.25.moelayer.experts.3.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.25.moelayer.experts.3.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.25.moelayer.gate.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.25.self_attn.k_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.25.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.25.self_attn.out_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.25.self_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.25.self_attn.q_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.25.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.25.self_attn.v_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.25.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.26.layer_norm1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.26.layer_norm1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.26.layer_norm2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.26.layer_norm2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.26.moelayer.experts.0.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.26.moelayer.experts.0.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.26.moelayer.experts.0.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.26.moelayer.experts.0.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.26.moelayer.experts.1.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.26.moelayer.experts.1.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.26.moelayer.experts.1.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.26.moelayer.experts.1.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.26.moelayer.experts.2.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.26.moelayer.experts.2.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.26.moelayer.experts.2.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.26.moelayer.experts.2.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.26.moelayer.experts.3.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.26.moelayer.experts.3.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.26.moelayer.experts.3.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.26.moelayer.experts.3.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.26.moelayer.gate.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.26.self_attn.k_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.26.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.26.self_attn.out_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.26.self_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.26.self_attn.q_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.26.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.26.self_attn.v_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.26.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.3.layer_norm1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.3.layer_norm1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.3.layer_norm2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.3.layer_norm2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.3.moelayer.experts.0.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.3.moelayer.experts.0.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.3.moelayer.experts.0.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.3.moelayer.experts.0.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.3.moelayer.experts.1.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.3.moelayer.experts.1.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.3.moelayer.experts.1.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.3.moelayer.experts.1.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.3.moelayer.experts.2.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.3.moelayer.experts.2.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.3.moelayer.experts.2.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.3.moelayer.experts.2.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.3.moelayer.experts.3.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.3.moelayer.experts.3.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.3.moelayer.experts.3.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.3.moelayer.experts.3.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.3.moelayer.gate.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.3.self_attn.k_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.3.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.3.self_attn.out_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.3.self_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.3.self_attn.q_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.3.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.3.self_attn.v_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.3.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.4.layer_norm1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.4.layer_norm1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.4.layer_norm2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.4.layer_norm2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.4.moelayer.experts.0.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.4.moelayer.experts.0.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.4.moelayer.experts.0.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.4.moelayer.experts.0.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.4.moelayer.experts.1.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.4.moelayer.experts.1.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.4.moelayer.experts.1.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.4.moelayer.experts.1.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.4.moelayer.experts.2.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.4.moelayer.experts.2.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.4.moelayer.experts.2.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.4.moelayer.experts.2.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.4.moelayer.experts.3.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.4.moelayer.experts.3.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.4.moelayer.experts.3.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.4.moelayer.experts.3.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.4.moelayer.gate.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.4.self_attn.k_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.4.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.4.self_attn.out_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.4.self_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.4.self_attn.q_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.4.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.4.self_attn.v_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.4.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.5.layer_norm1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.5.layer_norm1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.5.layer_norm2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.5.layer_norm2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.5.moelayer.experts.0.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.5.moelayer.experts.0.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.5.moelayer.experts.0.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.5.moelayer.experts.0.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.5.moelayer.experts.1.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.5.moelayer.experts.1.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.5.moelayer.experts.1.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.5.moelayer.experts.1.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.5.moelayer.experts.2.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.5.moelayer.experts.2.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.5.moelayer.experts.2.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.5.moelayer.experts.2.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.5.moelayer.experts.3.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.5.moelayer.experts.3.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.5.moelayer.experts.3.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.5.moelayer.experts.3.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.5.moelayer.gate.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.5.self_attn.k_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.5.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.5.self_attn.out_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.5.self_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.5.self_attn.q_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.5.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.5.self_attn.v_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.5.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.6.layer_norm1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.6.layer_norm1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.6.layer_norm2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.6.layer_norm2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.6.moelayer.experts.0.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.6.moelayer.experts.0.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.6.moelayer.experts.0.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.6.moelayer.experts.0.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.6.moelayer.experts.1.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.6.moelayer.experts.1.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.6.moelayer.experts.1.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.6.moelayer.experts.1.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.6.moelayer.experts.2.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.6.moelayer.experts.2.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.6.moelayer.experts.2.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.6.moelayer.experts.2.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.6.moelayer.experts.3.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.6.moelayer.experts.3.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.6.moelayer.experts.3.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.6.moelayer.experts.3.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.6.moelayer.gate.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.6.self_attn.k_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.6.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.6.self_attn.out_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.6.self_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.6.self_attn.q_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.6.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.6.self_attn.v_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.6.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.7.layer_norm1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.7.layer_norm1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.7.layer_norm2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.7.layer_norm2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.7.moelayer.experts.0.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.7.moelayer.experts.0.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.7.moelayer.experts.0.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.7.moelayer.experts.0.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.7.moelayer.experts.1.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.7.moelayer.experts.1.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.7.moelayer.experts.1.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.7.moelayer.experts.1.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.7.moelayer.experts.2.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.7.moelayer.experts.2.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.7.moelayer.experts.2.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.7.moelayer.experts.2.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.7.moelayer.experts.3.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.7.moelayer.experts.3.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.7.moelayer.experts.3.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.7.moelayer.experts.3.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.7.moelayer.gate.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.7.self_attn.k_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.7.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.7.self_attn.out_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.7.self_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.7.self_attn.q_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.7.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.7.self_attn.v_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.7.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.8.layer_norm1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.8.layer_norm1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.8.layer_norm2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.8.layer_norm2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.8.moelayer.experts.0.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.8.moelayer.experts.0.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.8.moelayer.experts.0.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.8.moelayer.experts.0.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.8.moelayer.experts.1.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.8.moelayer.experts.1.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.8.moelayer.experts.1.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.8.moelayer.experts.1.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.8.moelayer.experts.2.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.8.moelayer.experts.2.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.8.moelayer.experts.2.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.8.moelayer.experts.2.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.8.moelayer.experts.3.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.8.moelayer.experts.3.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.8.moelayer.experts.3.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.8.moelayer.experts.3.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.8.moelayer.gate.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.8.self_attn.k_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.8.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.8.self_attn.out_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.8.self_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.8.self_attn.q_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.8.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.8.self_attn.v_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.8.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.9.layer_norm1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.9.layer_norm1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.9.layer_norm2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.9.layer_norm2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.9.moelayer.experts.0.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.9.moelayer.experts.0.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.9.moelayer.experts.0.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.9.moelayer.experts.0.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.9.moelayer.experts.1.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.9.moelayer.experts.1.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.9.moelayer.experts.1.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.9.moelayer.experts.1.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.9.moelayer.experts.2.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.9.moelayer.experts.2.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.9.moelayer.experts.2.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.9.moelayer.experts.2.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.9.moelayer.experts.3.fc1.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.9.moelayer.experts.3.fc1.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.9.moelayer.experts.3.fc2.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.9.moelayer.experts.3.fc2.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.9.moelayer.gate.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.9.self_attn.k_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.9.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.9.self_attn.out_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.9.self_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.9.self_attn.q_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.9.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.9.self_attn.v_proj.bias": "model-00002-of-00003.safetensors", + "model.vision_tower.vision_model.encoder.layers.9.self_attn.v_proj.weight": "model-00002-of-00003.safetensors" + } +} diff --git a/sft/1M3/Full_smoe/checkpoint-6893/rng_state_0.pth b/sft/1M3/Full_smoe/checkpoint-6893/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..1e867b5dfb2d90cbb90a88a86ccee52b3f1773d7 --- /dev/null +++ b/sft/1M3/Full_smoe/checkpoint-6893/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d98bcd53a09c8bc1238dda64e113dd6869500cd3502c9a4d572b3f9ffea5dc29 +size 14960 diff --git a/sft/1M3/Full_smoe/checkpoint-6893/rng_state_1.pth b/sft/1M3/Full_smoe/checkpoint-6893/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..219ae96242a1a2df9c7b5d1bf82951eba1bd30f3 --- /dev/null +++ b/sft/1M3/Full_smoe/checkpoint-6893/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f793f5c89da7b5544f5038c073ba86532c704fdd2918ea5c27bfab848c20ad52 +size 14960 diff --git a/sft/1M3/Full_smoe/checkpoint-6893/rng_state_2.pth b/sft/1M3/Full_smoe/checkpoint-6893/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..eefe00c91fede5cf27f4e2fb77afc93f6f856779 --- /dev/null +++ b/sft/1M3/Full_smoe/checkpoint-6893/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cee33fb501fe6367a1ba811f1189b771eecbe30ed36d18486d5aeb9e083e4fdf +size 14960 diff --git a/sft/1M3/Full_smoe/checkpoint-6893/rng_state_3.pth b/sft/1M3/Full_smoe/checkpoint-6893/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..92e494a5b9fac8abd6ffa9c57e3865843d5d9c81 --- /dev/null +++ b/sft/1M3/Full_smoe/checkpoint-6893/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1aa2a4d8920b433e30b4dffe2a189b30027ce66925ec5b6b0ae0f1ccd678ec7 +size 14960 diff --git a/sft/1M3/Full_smoe/checkpoint-6893/special_tokens_map.json b/sft/1M3/Full_smoe/checkpoint-6893/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..3e4d5a5bc1cb51753cc9ae0305ece0da60052b10 --- /dev/null +++ b/sft/1M3/Full_smoe/checkpoint-6893/special_tokens_map.json @@ -0,0 +1,24 @@ +{ + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/sft/1M3/Full_smoe/checkpoint-6893/tokenizer.model b/sft/1M3/Full_smoe/checkpoint-6893/tokenizer.model new file mode 100644 index 0000000000000000000000000000000000000000..6c00c742ce03c627d6cd5b795984876fa49fa899 --- /dev/null +++ b/sft/1M3/Full_smoe/checkpoint-6893/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347 +size 499723 diff --git a/sft/1M3/Full_smoe/checkpoint-6893/tokenizer_config.json b/sft/1M3/Full_smoe/checkpoint-6893/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..d579bb0b91b24b214ea3c2e487e27a65017cdc4a --- /dev/null +++ b/sft/1M3/Full_smoe/checkpoint-6893/tokenizer_config.json @@ -0,0 +1,132 @@ +{ + "add_bos_token": false, + "add_eos_token": false, + "add_prefix_space": true, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": true, + "single_word": false, + "special": false + }, + "32000": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32001": { + "content": "<|assistant|>", + "lstrip": false, + "normalized": false, + "rstrip": true, + "single_word": false, + "special": true + }, + "32002": { + "content": "<|placeholder1|>", + "lstrip": false, + "normalized": false, + "rstrip": true, + "single_word": false, + "special": true + }, + "32003": { + "content": "<|placeholder2|>", + "lstrip": false, + "normalized": false, + "rstrip": true, + "single_word": false, + "special": true + }, + "32004": { + "content": "<|placeholder3|>", + "lstrip": false, + "normalized": false, + "rstrip": true, + "single_word": false, + "special": true + }, + "32005": { + "content": "<|placeholder4|>", + "lstrip": false, + "normalized": false, + "rstrip": true, + "single_word": false, + "special": true + }, + "32006": { + "content": "<|system|>", + "lstrip": false, + "normalized": false, + "rstrip": true, + "single_word": false, + "special": true + }, + "32007": { + "content": "<|end|>", + "lstrip": false, + "normalized": false, + "rstrip": true, + "single_word": false, + "special": true + }, + "32008": { + "content": "<|placeholder5|>", + "lstrip": false, + "normalized": false, + "rstrip": true, + "single_word": false, + "special": true + }, + "32009": { + "content": "<|placeholder6|>", + "lstrip": false, + "normalized": false, + "rstrip": true, + "single_word": false, + "special": true + }, + "32010": { + "content": "<|user|>", + "lstrip": false, + "normalized": false, + "rstrip": true, + "single_word": false, + "special": true + } + }, + "bos_token": "", + "chat_template": "{% for message in messages %}{% if message['role'] == 'system' and message['content'] %}{{'<|system|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'user' %}{{'<|user|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'assistant' %}{{'<|assistant|>\n' + message['content'] + '<|end|>\n'}}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n' }}{% else %}{{ eos_token }}{% endif %}", + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "legacy": false, + "model_max_length": 2048, + "pad_token": "", + "padding_side": "right", + "sp_model_kwargs": {}, + "spaces_between_special_tokens": false, + "tokenizer_class": "LlamaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/sft/1M3/Full_smoe/checkpoint-6893/trainer_state.json b/sft/1M3/Full_smoe/checkpoint-6893/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..7305206fea7cd545859fc0e4f3ae07117a2fa03f --- /dev/null +++ b/sft/1M3/Full_smoe/checkpoint-6893/trainer_state.json @@ -0,0 +1,117197 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.19998839301259358, + "eval_steps": 500, + "global_step": 6892, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "auxiliary_loss_clip": 0.05988652, + "auxiliary_loss_mlp": 0.03116848, + "balance_loss_clip": 2.44030571, + "balance_loss_mlp": 2.75521469, + "epoch": 2.901746851604666e-05, + "flos": 74764816277760.0, + "grad_norm": 2.8551988984568117, + "language_loss": 0.91534865, + "learning_rate": 0.0, + "loss": 0.67869878, + "num_input_tokens_seen": 62630, + "router_z_loss_clip": 35.625, + "router_z_loss_mlp": 3.609375, + "step": 1, + "time_per_iteration": 20.829037189483643 + }, + { + "auxiliary_loss_clip": 0.03534628, + "auxiliary_loss_mlp": 0.01564701, + "balance_loss_clip": 1.65562093, + "balance_loss_mlp": 1.22423863, + "epoch": 5.803493703209332e-05, + "flos": 25223044366080.0, + "grad_norm": 35.58865854882545, + "language_loss": 1.67140377, + "learning_rate": 3.994399663808758e-07, + "loss": 1.72239709, + "num_input_tokens_seen": 76875, + "router_z_loss_clip": 18.796875, + "router_z_loss_mlp": 3.40429688, + "step": 2, + "time_per_iteration": 2.463520050048828 + }, + { + "auxiliary_loss_clip": 0.03515621, + "auxiliary_loss_mlp": 0.01589912, + "balance_loss_clip": 1.63798952, + "balance_loss_mlp": 1.23896027, + "epoch": 8.705240554813998e-05, + "flos": 19855799550720.0, + "grad_norm": 58.094957399366365, + "language_loss": 2.33214188, + "learning_rate": 6.330973680030075e-07, + "loss": 2.38319731, + "num_input_tokens_seen": 88305, + "router_z_loss_clip": 18.765625, + "router_z_loss_mlp": 3.5078125, + "step": 3, + "time_per_iteration": 2.4134585857391357 + }, + { + "auxiliary_loss_clip": 0.03810823, + "auxiliary_loss_mlp": 0.01860688, + "balance_loss_clip": 1.63296986, + "balance_loss_mlp": 1.64401364, + "epoch": 0.00011606987406418665, + "flos": 69995464335360.0, + "grad_norm": 2.563563670588428, + "language_loss": 0.66008765, + "learning_rate": 7.988799327617516e-07, + "loss": 0.71680278, + "num_input_tokens_seen": 152570, + "router_z_loss_clip": 21.75, + "router_z_loss_mlp": 2.171875, + "step": 4, + "time_per_iteration": 3.189251661300659 + }, + { + "auxiliary_loss_clip": 0.03560921, + "auxiliary_loss_mlp": 0.01641322, + "balance_loss_clip": 1.67698658, + "balance_loss_mlp": 1.28598297, + "epoch": 0.0001450873425802333, + "flos": 24746617347840.0, + "grad_norm": 46.66812990013587, + "language_loss": 2.25835276, + "learning_rate": 9.274708801606189e-07, + "loss": 2.31037521, + "num_input_tokens_seen": 166005, + "router_z_loss_clip": 18.828125, + "router_z_loss_mlp": 3.55273438, + "step": 5, + "time_per_iteration": 2.681885242462158 + }, + { + "auxiliary_loss_clip": 0.03500169, + "auxiliary_loss_mlp": 0.01607133, + "balance_loss_clip": 1.64421618, + "balance_loss_mlp": 1.25370073, + "epoch": 0.00017410481109627997, + "flos": 64265626702080.0, + "grad_norm": 23.25004523050376, + "language_loss": 1.52424562, + "learning_rate": 1.0325373343838831e-06, + "loss": 1.57531869, + "num_input_tokens_seen": 189205, + "router_z_loss_clip": 18.578125, + "router_z_loss_mlp": 3.53320312, + "step": 6, + "time_per_iteration": 2.9607090950012207 + }, + { + "auxiliary_loss_clip": 0.03512194, + "auxiliary_loss_mlp": 0.01552827, + "balance_loss_clip": 1.66108572, + "balance_loss_mlp": 1.22628856, + "epoch": 0.00020312227961232662, + "flos": 18729856851840.0, + "grad_norm": 35.684201958565275, + "language_loss": 1.76274145, + "learning_rate": 1.1213697556858757e-06, + "loss": 1.81339157, + "num_input_tokens_seen": 202350, + "router_z_loss_clip": 18.515625, + "router_z_loss_mlp": 3.26757812, + "step": 7, + "time_per_iteration": 5.290036201477051 + }, + { + "auxiliary_loss_clip": 0.0348012, + "auxiliary_loss_mlp": 0.01554654, + "balance_loss_clip": 1.63175058, + "balance_loss_mlp": 1.22487342, + "epoch": 0.0002321397481283733, + "flos": 12852967916160.0, + "grad_norm": 36.4117008376369, + "language_loss": 2.09164047, + "learning_rate": 1.1983198991426273e-06, + "loss": 2.14198828, + "num_input_tokens_seen": 214925, + "router_z_loss_clip": 18.484375, + "router_z_loss_mlp": 3.296875, + "step": 8, + "time_per_iteration": 4.946546792984009 + }, + { + "auxiliary_loss_clip": 0.03517272, + "auxiliary_loss_mlp": 0.01544374, + "balance_loss_clip": 1.65051508, + "balance_loss_mlp": 1.21383095, + "epoch": 0.00026115721664441994, + "flos": 29418595534080.0, + "grad_norm": 45.84079826833446, + "language_loss": 1.90903616, + "learning_rate": 1.266194736006015e-06, + "loss": 1.95965266, + "num_input_tokens_seen": 234220, + "router_z_loss_clip": 18.65625, + "router_z_loss_mlp": 3.30273438, + "step": 9, + "time_per_iteration": 2.7464635372161865 + }, + { + "auxiliary_loss_clip": 0.03725086, + "auxiliary_loss_mlp": 0.01768413, + "balance_loss_clip": 1.63494253, + "balance_loss_mlp": 1.56241918, + "epoch": 0.0002901746851604666, + "flos": 63785970437760.0, + "grad_norm": 2.718768416617142, + "language_loss": 0.66858542, + "learning_rate": 1.326910846541495e-06, + "loss": 0.7235204, + "num_input_tokens_seen": 290580, + "router_z_loss_clip": 20.875, + "router_z_loss_mlp": 2.0625, + "step": 10, + "time_per_iteration": 3.047215461730957 + }, + { + "auxiliary_loss_clip": 0.0341766, + "auxiliary_loss_mlp": 0.01614071, + "balance_loss_clip": 1.64025438, + "balance_loss_mlp": 1.27017605, + "epoch": 0.00031919215367651324, + "flos": 22703443925760.0, + "grad_norm": 10.554621849066764, + "language_loss": 1.5706954, + "learning_rate": 1.3818352494454209e-06, + "loss": 1.62101293, + "num_input_tokens_seen": 305540, + "router_z_loss_clip": 17.78125, + "router_z_loss_mlp": 3.4375, + "step": 11, + "time_per_iteration": 2.714524984359741 + }, + { + "auxiliary_loss_clip": 0.03432441, + "auxiliary_loss_mlp": 0.01587282, + "balance_loss_clip": 1.64628994, + "balance_loss_mlp": 1.24453139, + "epoch": 0.00034820962219255994, + "flos": 15991309059840.0, + "grad_norm": 16.21155587858086, + "language_loss": 1.53470445, + "learning_rate": 1.431977300764759e-06, + "loss": 1.58490181, + "num_input_tokens_seen": 318590, + "router_z_loss_clip": 17.875, + "router_z_loss_mlp": 3.4296875, + "step": 12, + "time_per_iteration": 2.6641924381256104 + }, + { + "auxiliary_loss_clip": 0.03388571, + "auxiliary_loss_mlp": 0.01487386, + "balance_loss_clip": 1.65220523, + "balance_loss_mlp": 1.32182848, + "epoch": 0.0003772270907086066, + "flos": 71012023142400.0, + "grad_norm": 2.5735956366483057, + "language_loss": 0.63743484, + "learning_rate": 1.4781035166087354e-06, + "loss": 0.68619442, + "num_input_tokens_seen": 382525, + "router_z_loss_clip": 17.375, + "router_z_loss_mlp": 1.65625, + "step": 13, + "time_per_iteration": 3.2356152534484863 + }, + { + "auxiliary_loss_clip": 0.03355553, + "auxiliary_loss_mlp": 0.01448368, + "balance_loss_clip": 1.65003037, + "balance_loss_mlp": 1.28662479, + "epoch": 0.00040624455922465323, + "flos": 74777903821440.0, + "grad_norm": 2.4099960549080595, + "language_loss": 0.63405335, + "learning_rate": 1.5208097220667513e-06, + "loss": 0.68209255, + "num_input_tokens_seen": 447495, + "router_z_loss_clip": 17.0, + "router_z_loss_mlp": 1.6171875, + "step": 14, + "time_per_iteration": 3.2094976902008057 + }, + { + "auxiliary_loss_clip": 0.03322019, + "auxiliary_loss_mlp": 0.01449357, + "balance_loss_clip": 1.65238261, + "balance_loss_mlp": 1.29066515, + "epoch": 0.0004352620277406999, + "flos": 74767704359040.0, + "grad_norm": 2.6674168005950407, + "language_loss": 0.64734554, + "learning_rate": 1.5605682481636264e-06, + "loss": 0.6950593, + "num_input_tokens_seen": 503225, + "router_z_loss_clip": 16.75, + "router_z_loss_mlp": 1.5859375, + "step": 15, + "time_per_iteration": 3.0974507331848145 + }, + { + "auxiliary_loss_clip": 0.0335994, + "auxiliary_loss_mlp": 0.01606413, + "balance_loss_clip": 1.64505887, + "balance_loss_mlp": 1.2585125, + "epoch": 0.0004642794962567466, + "flos": 12496160165760.0, + "grad_norm": 15.406101497282318, + "language_loss": 1.69961655, + "learning_rate": 1.5977598655235032e-06, + "loss": 1.74928021, + "num_input_tokens_seen": 515655, + "router_z_loss_clip": 17.140625, + "router_z_loss_mlp": 3.4765625, + "step": 16, + "time_per_iteration": 2.6486966609954834 + }, + { + "auxiliary_loss_clip": 0.03347146, + "auxiliary_loss_mlp": 0.01565346, + "balance_loss_clip": 1.63577557, + "balance_loss_mlp": 1.24662828, + "epoch": 0.0004932969647727932, + "flos": 22302465425280.0, + "grad_norm": 8.36891074208503, + "language_loss": 1.58893371, + "learning_rate": 1.6326960198921147e-06, + "loss": 1.63805866, + "num_input_tokens_seen": 531020, + "router_z_loss_clip": 17.109375, + "router_z_loss_mlp": 3.18554688, + "step": 17, + "time_per_iteration": 2.726891040802002 + }, + { + "auxiliary_loss_clip": 0.0329521, + "auxiliary_loss_mlp": 0.01531019, + "balance_loss_clip": 1.63526392, + "balance_loss_mlp": 1.21993053, + "epoch": 0.0005223144332888399, + "flos": 14862062309760.0, + "grad_norm": 9.396305817769097, + "language_loss": 1.7193464, + "learning_rate": 1.6656347023868906e-06, + "loss": 1.76760864, + "num_input_tokens_seen": 545155, + "router_z_loss_clip": 16.59375, + "router_z_loss_mlp": 3.109375, + "step": 18, + "time_per_iteration": 2.7356998920440674 + }, + { + "auxiliary_loss_clip": 0.03258358, + "auxiliary_loss_mlp": 0.01607444, + "balance_loss_clip": 1.63476837, + "balance_loss_mlp": 1.28853559, + "epoch": 0.0005513319018048866, + "flos": 19092051642240.0, + "grad_norm": 8.366546178751396, + "language_loss": 1.20388103, + "learning_rate": 1.696792023158303e-06, + "loss": 1.25253916, + "num_input_tokens_seen": 559275, + "router_z_loss_clip": 16.203125, + "router_z_loss_mlp": 3.19140625, + "step": 19, + "time_per_iteration": 2.681539297103882 + }, + { + "auxiliary_loss_clip": 0.03133468, + "auxiliary_loss_mlp": 0.01537902, + "balance_loss_clip": 1.62430859, + "balance_loss_mlp": 1.24054646, + "epoch": 0.0005803493703209332, + "flos": 32299457011200.0, + "grad_norm": 7.568056803116112, + "language_loss": 1.45123625, + "learning_rate": 1.7263508129223706e-06, + "loss": 1.49794996, + "num_input_tokens_seen": 574905, + "router_z_loss_clip": 15.1015625, + "router_z_loss_mlp": 2.97167969, + "step": 20, + "time_per_iteration": 2.7750585079193115 + }, + { + "auxiliary_loss_clip": 0.02617599, + "auxiliary_loss_mlp": 0.01441043, + "balance_loss_clip": 1.66590261, + "balance_loss_mlp": 1.34720194, + "epoch": 0.0006093668388369799, + "flos": 60693559810560.0, + "grad_norm": 1.6260202552709413, + "language_loss": 0.64941168, + "learning_rate": 1.754467123688883e-06, + "loss": 0.68999803, + "num_input_tokens_seen": 636915, + "router_z_loss_clip": 9.5, + "router_z_loss_mlp": 0.9375, + "step": 21, + "time_per_iteration": 3.1539041996002197 + }, + { + "auxiliary_loss_clip": 0.03093094, + "auxiliary_loss_mlp": 0.01487697, + "balance_loss_clip": 1.62909436, + "balance_loss_mlp": 1.21742606, + "epoch": 0.0006383843073530265, + "flos": 16320502229760.0, + "grad_norm": 9.443305799551469, + "language_loss": 1.45066643, + "learning_rate": 1.7812752158262967e-06, + "loss": 1.49647439, + "num_input_tokens_seen": 649135, + "router_z_loss_clip": 14.6484375, + "router_z_loss_mlp": 2.703125, + "step": 22, + "time_per_iteration": 2.7303292751312256 + }, + { + "auxiliary_loss_clip": 0.02942134, + "auxiliary_loss_mlp": 0.01710748, + "balance_loss_clip": 1.61982965, + "balance_loss_mlp": 1.40900612, + "epoch": 0.0006674017758690732, + "flos": 13780146706560.0, + "grad_norm": 7.448162063936967, + "language_loss": 1.41412163, + "learning_rate": 1.806891435649222e-06, + "loss": 1.46065044, + "num_input_tokens_seen": 661465, + "router_z_loss_clip": 13.2109375, + "router_z_loss_mlp": 3.01757812, + "step": 23, + "time_per_iteration": 2.60732364654541 + }, + { + "auxiliary_loss_clip": 0.03050678, + "auxiliary_loss_mlp": 0.01632045, + "balance_loss_clip": 1.61976779, + "balance_loss_mlp": 1.33068419, + "epoch": 0.0006964192443851199, + "flos": 40692319113600.0, + "grad_norm": 4.145250142632719, + "language_loss": 1.10089886, + "learning_rate": 1.8314172671456348e-06, + "loss": 1.14772606, + "num_input_tokens_seen": 680760, + "router_z_loss_clip": 14.3203125, + "router_z_loss_mlp": 3.01367188, + "step": 24, + "time_per_iteration": 2.843339681625366 + }, + { + "auxiliary_loss_clip": 0.02973792, + "auxiliary_loss_mlp": 0.01552193, + "balance_loss_clip": 1.62191319, + "balance_loss_mlp": 1.299088, + "epoch": 0.0007254367129011665, + "flos": 16610444812800.0, + "grad_norm": 7.4342161157818625, + "language_loss": 1.79260373, + "learning_rate": 1.8549417603212378e-06, + "loss": 1.83786345, + "num_input_tokens_seen": 693140, + "router_z_loss_clip": 13.515625, + "router_z_loss_mlp": 2.53125, + "step": 25, + "time_per_iteration": 2.606750011444092 + }, + { + "auxiliary_loss_clip": 0.02971645, + "auxiliary_loss_mlp": 0.01592366, + "balance_loss_clip": 1.60962868, + "balance_loss_mlp": 1.3198055, + "epoch": 0.0007544541814172132, + "flos": 11611966976640.0, + "grad_norm": 6.59172015263787, + "language_loss": 1.21843946, + "learning_rate": 1.8775434829896112e-06, + "loss": 1.26407957, + "num_input_tokens_seen": 704545, + "router_z_loss_clip": 13.609375, + "router_z_loss_mlp": 2.7265625, + "step": 26, + "time_per_iteration": 2.6676230430603027 + }, + { + "auxiliary_loss_clip": 0.03030269, + "auxiliary_loss_mlp": 0.01545098, + "balance_loss_clip": 1.62382174, + "balance_loss_mlp": 1.26529002, + "epoch": 0.0007834716499332599, + "flos": 24456243801600.0, + "grad_norm": 3.2280107736824113, + "language_loss": 1.03222024, + "learning_rate": 1.8992921040090223e-06, + "loss": 1.07797384, + "num_input_tokens_seen": 723595, + "router_z_loss_clip": 14.0703125, + "router_z_loss_mlp": 2.796875, + "step": 27, + "time_per_iteration": 2.6925134658813477 + }, + { + "auxiliary_loss_clip": 0.02903467, + "auxiliary_loss_mlp": 0.01496347, + "balance_loss_clip": 1.5909462, + "balance_loss_mlp": 1.24419594, + "epoch": 0.0008124891184493065, + "flos": 31679207936640.0, + "grad_norm": 6.059849135643193, + "language_loss": 1.33854628, + "learning_rate": 1.920249688447627e-06, + "loss": 1.3825444, + "num_input_tokens_seen": 737365, + "router_z_loss_clip": 13.1171875, + "router_z_loss_mlp": 2.52246094, + "step": 28, + "time_per_iteration": 2.7654404640197754 + }, + { + "auxiliary_loss_clip": 0.02321636, + "auxiliary_loss_mlp": 0.01476457, + "balance_loss_clip": 1.65106893, + "balance_loss_mlp": 1.40893674, + "epoch": 0.0008415065869653532, + "flos": 66615086373120.0, + "grad_norm": 1.2730898615889306, + "language_loss": 0.59807932, + "learning_rate": 1.940471765372691e-06, + "loss": 0.63606024, + "num_input_tokens_seen": 804120, + "router_z_loss_clip": 6.6875, + "router_z_loss_mlp": 0.67578125, + "step": 29, + "time_per_iteration": 3.3097267150878906 + }, + { + "auxiliary_loss_clip": 0.02872655, + "auxiliary_loss_mlp": 0.01529767, + "balance_loss_clip": 1.59426725, + "balance_loss_mlp": 1.27017689, + "epoch": 0.0008705240554813998, + "flos": 74734699760640.0, + "grad_norm": 2.443496993535272, + "language_loss": 0.91444242, + "learning_rate": 1.9600082145445022e-06, + "loss": 0.95846665, + "num_input_tokens_seen": 831390, + "router_z_loss_clip": 12.796875, + "router_z_loss_mlp": 2.59570312, + "step": 30, + "time_per_iteration": 3.146343946456909 + }, + { + "auxiliary_loss_clip": 0.02776532, + "auxiliary_loss_mlp": 0.01560222, + "balance_loss_clip": 1.58545804, + "balance_loss_mlp": 1.31560421, + "epoch": 0.0008995415239974465, + "flos": 74735202551040.0, + "grad_norm": 7.2026349951372515, + "language_loss": 1.13319135, + "learning_rate": 1.9789040076651924e-06, + "loss": 1.17655897, + "num_input_tokens_seen": 856215, + "router_z_loss_clip": 11.90625, + "router_z_loss_mlp": 2.44628906, + "step": 31, + "time_per_iteration": 3.0875399112701416 + }, + { + "auxiliary_loss_clip": 0.0274988, + "auxiliary_loss_mlp": 0.01592543, + "balance_loss_clip": 1.57431364, + "balance_loss_mlp": 1.33085489, + "epoch": 0.0009285589925134932, + "flos": 26680586446080.0, + "grad_norm": 3.1288137375420457, + "language_loss": 1.19113946, + "learning_rate": 1.997199831904379e-06, + "loss": 1.23456371, + "num_input_tokens_seen": 870260, + "router_z_loss_clip": 11.765625, + "router_z_loss_mlp": 2.6171875, + "step": 32, + "time_per_iteration": 2.7569713592529297 + }, + { + "auxiliary_loss_clip": 0.02131881, + "auxiliary_loss_mlp": 0.01404769, + "balance_loss_clip": 1.61003017, + "balance_loss_mlp": 1.35689449, + "epoch": 0.0009575764610295398, + "flos": 71602107770880.0, + "grad_norm": 1.4902443617292118, + "language_loss": 0.63757038, + "learning_rate": 2.014932617448428e-06, + "loss": 0.6729368, + "num_input_tokens_seen": 935505, + "router_z_loss_clip": 5.21875, + "router_z_loss_mlp": 0.47851562, + "step": 33, + "time_per_iteration": 3.131701946258545 + }, + { + "auxiliary_loss_clip": 0.02689062, + "auxiliary_loss_mlp": 0.01560037, + "balance_loss_clip": 1.56587934, + "balance_loss_mlp": 1.31952083, + "epoch": 0.0009865939295455864, + "flos": 29200761504000.0, + "grad_norm": 2.8386257644816295, + "language_loss": 1.16805601, + "learning_rate": 2.0321359862729905e-06, + "loss": 1.21054697, + "num_input_tokens_seen": 956585, + "router_z_loss_clip": 11.2265625, + "router_z_loss_mlp": 2.40625, + "step": 34, + "time_per_iteration": 2.750591993331909 + }, + { + "auxiliary_loss_clip": 0.02596135, + "auxiliary_loss_mlp": 0.01475784, + "balance_loss_clip": 1.55488825, + "balance_loss_mlp": 1.27365243, + "epoch": 0.0010156113980616332, + "flos": 32445972587520.0, + "grad_norm": 5.428333352326467, + "language_loss": 1.22372532, + "learning_rate": 2.0488406358464945e-06, + "loss": 1.26444435, + "num_input_tokens_seen": 971900, + "router_z_loss_clip": 10.4140625, + "router_z_loss_mlp": 2.02001953, + "step": 35, + "time_per_iteration": 2.7426109313964844 + }, + { + "auxiliary_loss_clip": 0.02010395, + "auxiliary_loss_mlp": 0.01399516, + "balance_loss_clip": 1.56900597, + "balance_loss_mlp": 1.3604151, + "epoch": 0.0010446288665776798, + "flos": 68933801024640.0, + "grad_norm": 1.405384979794657, + "language_loss": 0.63343883, + "learning_rate": 2.0650746687677663e-06, + "loss": 0.66753793, + "num_input_tokens_seen": 1031825, + "router_z_loss_clip": 4.40625, + "router_z_loss_mlp": 0.390625, + "step": 36, + "time_per_iteration": 3.093555450439453 + }, + { + "auxiliary_loss_clip": 0.01985167, + "auxiliary_loss_mlp": 0.01378659, + "balance_loss_clip": 1.56169283, + "balance_loss_mlp": 1.34165609, + "epoch": 0.0010736463350937264, + "flos": 58464692052480.0, + "grad_norm": 1.3227195981891597, + "language_loss": 0.61510015, + "learning_rate": 2.080863877229568e-06, + "loss": 0.64873838, + "num_input_tokens_seen": 1090910, + "router_z_loss_clip": 4.25, + "router_z_loss_mlp": 0.36914062, + "step": 37, + "time_per_iteration": 2.969282388687134 + }, + { + "auxiliary_loss_clip": 0.02508735, + "auxiliary_loss_mlp": 0.01435152, + "balance_loss_clip": 1.53382587, + "balance_loss_mlp": 1.24727821, + "epoch": 0.0011026638036097732, + "flos": 34963921002240.0, + "grad_norm": 4.23777336232902, + "language_loss": 1.08963597, + "learning_rate": 2.096231989539179e-06, + "loss": 1.12907481, + "num_input_tokens_seen": 1107990, + "router_z_loss_clip": 9.73828125, + "router_z_loss_mlp": 1.87841797, + "step": 38, + "time_per_iteration": 2.7656853199005127 + }, + { + "auxiliary_loss_clip": 0.01941226, + "auxiliary_loss_mlp": 0.0131082, + "balance_loss_clip": 1.54405987, + "balance_loss_mlp": 1.27591598, + "epoch": 0.0011316812721258198, + "flos": 61789697199360.0, + "grad_norm": 1.1180309020440202, + "language_loss": 0.63438296, + "learning_rate": 2.1112008846117425e-06, + "loss": 0.66690344, + "num_input_tokens_seen": 1174060, + "router_z_loss_clip": 3.96875, + "router_z_loss_mlp": 0.34960938, + "step": 39, + "time_per_iteration": 3.3390133380889893 + }, + { + "auxiliary_loss_clip": 0.02449431, + "auxiliary_loss_mlp": 0.013297, + "balance_loss_clip": 1.51004791, + "balance_loss_mlp": 1.16757596, + "epoch": 0.0011606987406418664, + "flos": 19275016544640.0, + "grad_norm": 3.5609524929416736, + "language_loss": 1.30606389, + "learning_rate": 2.1257907793032464e-06, + "loss": 1.34385526, + "num_input_tokens_seen": 1187835, + "router_z_loss_clip": 9.3984375, + "router_z_loss_mlp": 1.62304688, + "step": 40, + "time_per_iteration": 2.8771867752075195 + }, + { + "auxiliary_loss_clip": 0.02459275, + "auxiliary_loss_mlp": 0.01367567, + "balance_loss_clip": 1.52004766, + "balance_loss_mlp": 1.18770397, + "epoch": 0.0011897162091579132, + "flos": 13727320778880.0, + "grad_norm": 5.381045379359313, + "language_loss": 1.43520069, + "learning_rate": 2.140020392608441e-06, + "loss": 1.47346926, + "num_input_tokens_seen": 1198420, + "router_z_loss_clip": 9.4140625, + "router_z_loss_mlp": 1.79785156, + "step": 41, + "time_per_iteration": 2.63822340965271 + }, + { + "auxiliary_loss_clip": 0.01894643, + "auxiliary_loss_mlp": 0.01193891, + "balance_loss_clip": 1.52594519, + "balance_loss_mlp": 1.15765154, + "epoch": 0.0012187336776739598, + "flos": 61382433818880.0, + "grad_norm": 0.926819963431556, + "language_loss": 0.59414029, + "learning_rate": 2.153907090069759e-06, + "loss": 0.62502563, + "num_input_tokens_seen": 1260125, + "router_z_loss_clip": 3.6875, + "router_z_loss_mlp": 0.36328125, + "step": 42, + "time_per_iteration": 3.2144274711608887 + }, + { + "auxiliary_loss_clip": 0.01878823, + "auxiliary_loss_mlp": 0.01136598, + "balance_loss_clip": 1.51800871, + "balance_loss_mlp": 1.09883237, + "epoch": 0.0012477511461900063, + "flos": 70181374152960.0, + "grad_norm": 1.0210799057548863, + "language_loss": 0.62067401, + "learning_rate": 2.167467011191937e-06, + "loss": 0.65082818, + "num_input_tokens_seen": 1323940, + "router_z_loss_clip": 3.609375, + "router_z_loss_mlp": 0.37695312, + "step": 43, + "time_per_iteration": 3.1683871746063232 + }, + { + "auxiliary_loss_clip": 0.01857449, + "auxiliary_loss_mlp": 0.01083189, + "balance_loss_clip": 1.50533891, + "balance_loss_mlp": 1.04313469, + "epoch": 0.001276768614706053, + "flos": 74773127312640.0, + "grad_norm": 1.1863963934610855, + "language_loss": 0.60729605, + "learning_rate": 2.180715182207172e-06, + "loss": 0.63670242, + "num_input_tokens_seen": 1387700, + "router_z_loss_clip": 3.515625, + "router_z_loss_mlp": 0.40039062, + "step": 44, + "time_per_iteration": 3.134892463684082 + }, + { + "auxiliary_loss_clip": 0.02390068, + "auxiliary_loss_mlp": 0.01343527, + "balance_loss_clip": 1.498312, + "balance_loss_mlp": 1.17148435, + "epoch": 0.0013057860832220998, + "flos": 20770049445120.0, + "grad_norm": 2.7897430532427183, + "language_loss": 1.07587957, + "learning_rate": 2.193665616166634e-06, + "loss": 1.11321557, + "num_input_tokens_seen": 1405335, + "router_z_loss_clip": 8.92578125, + "router_z_loss_mlp": 1.72167969, + "step": 45, + "time_per_iteration": 2.7914974689483643 + }, + { + "auxiliary_loss_clip": 0.01812528, + "auxiliary_loss_mlp": 0.01045305, + "balance_loss_clip": 1.47804976, + "balance_loss_mlp": 1.00143552, + "epoch": 0.0013348035517381463, + "flos": 69951437239680.0, + "grad_norm": 1.0310862178590943, + "language_loss": 0.65868759, + "learning_rate": 2.206331402030098e-06, + "loss": 0.68726593, + "num_input_tokens_seen": 1458275, + "router_z_loss_clip": 3.34375, + "router_z_loss_mlp": 0.43945312, + "step": 46, + "time_per_iteration": 3.00219988822937 + }, + { + "auxiliary_loss_clip": 0.02295687, + "auxiliary_loss_mlp": 0.01357863, + "balance_loss_clip": 1.48353601, + "balance_loss_mlp": 1.19259179, + "epoch": 0.001363821020254193, + "flos": 18289699591680.0, + "grad_norm": 2.6654723422391764, + "language_loss": 1.02505708, + "learning_rate": 2.2187247841737033e-06, + "loss": 1.06159258, + "num_input_tokens_seen": 1472945, + "router_z_loss_clip": 8.12890625, + "router_z_loss_mlp": 1.65234375, + "step": 47, + "time_per_iteration": 2.6792972087860107 + }, + { + "auxiliary_loss_clip": 0.01772243, + "auxiliary_loss_mlp": 0.01076297, + "balance_loss_clip": 1.45645642, + "balance_loss_mlp": 1.02937627, + "epoch": 0.0013928384887702398, + "flos": 68029786506240.0, + "grad_norm": 0.9064373205464438, + "language_loss": 0.56066889, + "learning_rate": 2.230857233526511e-06, + "loss": 0.5891543, + "num_input_tokens_seen": 1537970, + "router_z_loss_clip": 3.15625, + "router_z_loss_mlp": 0.46875, + "step": 48, + "time_per_iteration": 3.1709139347076416 + }, + { + "auxiliary_loss_clip": 0.02286339, + "auxiliary_loss_mlp": 0.01313281, + "balance_loss_clip": 1.46808624, + "balance_loss_mlp": 1.16107488, + "epoch": 0.0014218559572862863, + "flos": 20843666369280.0, + "grad_norm": 2.8849348447094108, + "language_loss": 1.00179791, + "learning_rate": 2.2427395113717513e-06, + "loss": 1.03779411, + "num_input_tokens_seen": 1554410, + "router_z_loss_clip": 8.18359375, + "router_z_loss_mlp": 1.52148438, + "step": 49, + "time_per_iteration": 2.619652032852173 + }, + { + "auxiliary_loss_clip": 0.01725369, + "auxiliary_loss_mlp": 0.01128387, + "balance_loss_clip": 1.42655087, + "balance_loss_mlp": 1.07955921, + "epoch": 0.001450873425802333, + "flos": 71730561784320.0, + "grad_norm": 0.958685533680304, + "language_loss": 0.60861075, + "learning_rate": 2.254381726702114e-06, + "loss": 0.63714832, + "num_input_tokens_seen": 1615650, + "router_z_loss_clip": 2.984375, + "router_z_loss_mlp": 0.48828125, + "step": 50, + "time_per_iteration": 3.2864954471588135 + }, + { + "auxiliary_loss_clip": 0.01703616, + "auxiliary_loss_mlp": 0.0113225, + "balance_loss_clip": 1.41208875, + "balance_loss_mlp": 1.08265901, + "epoch": 0.0014798908943183797, + "flos": 74661768172800.0, + "grad_norm": 1.0105245738784314, + "language_loss": 0.6253342, + "learning_rate": 2.265793387895122e-06, + "loss": 0.65369284, + "num_input_tokens_seen": 1679695, + "router_z_loss_clip": 2.90625, + "router_z_loss_mlp": 0.49414062, + "step": 51, + "time_per_iteration": 3.156360149383545 + }, + { + "auxiliary_loss_clip": 0.01676048, + "auxiliary_loss_mlp": 0.01120109, + "balance_loss_clip": 1.39274263, + "balance_loss_mlp": 1.07013607, + "epoch": 0.0015089083628344263, + "flos": 63351092476800.0, + "grad_norm": 0.9999020206511802, + "language_loss": 0.66861886, + "learning_rate": 2.276983449370487e-06, + "loss": 0.69658041, + "num_input_tokens_seen": 1738825, + "router_z_loss_clip": 2.84375, + "router_z_loss_mlp": 0.5, + "step": 52, + "time_per_iteration": 3.095451831817627 + }, + { + "auxiliary_loss_clip": 0.01649763, + "auxiliary_loss_mlp": 0.01100695, + "balance_loss_clip": 1.37332463, + "balance_loss_mlp": 1.05072212, + "epoch": 0.001537925831350473, + "flos": 74773881498240.0, + "grad_norm": 0.8538518014798641, + "language_loss": 0.58392465, + "learning_rate": 2.287960353803055e-06, + "loss": 0.61142921, + "num_input_tokens_seen": 1797415, + "router_z_loss_clip": 2.765625, + "router_z_loss_mlp": 0.5, + "step": 53, + "time_per_iteration": 3.127692461013794 + }, + { + "auxiliary_loss_clip": 0.02114283, + "auxiliary_loss_mlp": 0.01203257, + "balance_loss_clip": 1.42789924, + "balance_loss_mlp": 1.1017381, + "epoch": 0.0015669432998665197, + "flos": 28761358429440.0, + "grad_norm": 2.4337916959848616, + "language_loss": 1.03400385, + "learning_rate": 2.2987320703898984e-06, + "loss": 1.0671792, + "num_input_tokens_seen": 1820515, + "router_z_loss_clip": 6.875, + "router_z_loss_mlp": 1.01513672, + "step": 54, + "time_per_iteration": 2.9016261100769043 + }, + { + "auxiliary_loss_clip": 0.01604019, + "auxiliary_loss_mlp": 0.0107711, + "balance_loss_clip": 1.33673573, + "balance_loss_mlp": 1.02828169, + "epoch": 0.0015959607683825663, + "flos": 61777881624960.0, + "grad_norm": 0.8973764048005921, + "language_loss": 0.54835618, + "learning_rate": 2.30930612960604e-06, + "loss": 0.57516754, + "num_input_tokens_seen": 1875210, + "router_z_loss_clip": 2.671875, + "router_z_loss_mlp": 0.48828125, + "step": 55, + "time_per_iteration": 2.999941825866699 + }, + { + "auxiliary_loss_clip": 0.02115475, + "auxiliary_loss_mlp": 0.01311613, + "balance_loss_clip": 1.43106163, + "balance_loss_mlp": 1.18391621, + "epoch": 0.001624978236898613, + "flos": 13728146791680.0, + "grad_norm": 2.803416428282736, + "language_loss": 1.26835203, + "learning_rate": 2.319689654828503e-06, + "loss": 1.30262303, + "num_input_tokens_seen": 1886820, + "router_z_loss_clip": 6.8359375, + "router_z_loss_mlp": 1.27636719, + "step": 56, + "time_per_iteration": 2.64943265914917 + }, + { + "auxiliary_loss_clip": 0.01568803, + "auxiliary_loss_mlp": 0.01073193, + "balance_loss_clip": 1.31214499, + "balance_loss_mlp": 1.02646339, + "epoch": 0.0016539957054146595, + "flos": 61247231026560.0, + "grad_norm": 0.7852951040556057, + "language_loss": 0.60245371, + "learning_rate": 2.3298893911613107e-06, + "loss": 0.62887365, + "num_input_tokens_seen": 1952960, + "router_z_loss_clip": 2.5625, + "router_z_loss_mlp": 0.46679688, + "step": 57, + "time_per_iteration": 3.191248893737793 + }, + { + "auxiliary_loss_clip": 0.02103617, + "auxiliary_loss_mlp": 0.01282887, + "balance_loss_clip": 1.41705441, + "balance_loss_mlp": 1.14593935, + "epoch": 0.0016830131739307063, + "flos": 20043217739520.0, + "grad_norm": 2.786336399432641, + "language_loss": 1.09452009, + "learning_rate": 2.339911731753567e-06, + "loss": 1.12838519, + "num_input_tokens_seen": 1970440, + "router_z_loss_clip": 6.875, + "router_z_loss_mlp": 1.37011719, + "step": 58, + "time_per_iteration": 2.834606885910034 + }, + { + "auxiliary_loss_clip": 0.02085286, + "auxiliary_loss_mlp": 0.01252769, + "balance_loss_clip": 1.41011095, + "balance_loss_mlp": 1.12574005, + "epoch": 0.001712030642446753, + "flos": 22339058405760.0, + "grad_norm": 2.489813012789095, + "language_loss": 0.81393683, + "learning_rate": 2.3497627418677867e-06, + "loss": 0.84731746, + "num_input_tokens_seen": 1985535, + "router_z_loss_clip": 6.75, + "router_z_loss_mlp": 1.27001953, + "step": 59, + "time_per_iteration": 2.703763961791992 + }, + { + "auxiliary_loss_clip": 0.01521307, + "auxiliary_loss_mlp": 0.01068846, + "balance_loss_clip": 1.28056455, + "balance_loss_mlp": 1.02612138, + "epoch": 0.0017410481109627995, + "flos": 69928311859200.0, + "grad_norm": 0.8400845253069498, + "language_loss": 0.56943107, + "learning_rate": 2.359448180925378e-06, + "loss": 0.59533262, + "num_input_tokens_seen": 2051030, + "router_z_loss_clip": 2.40625, + "router_z_loss_mlp": 0.42773438, + "step": 60, + "time_per_iteration": 3.1575021743774414 + }, + { + "auxiliary_loss_clip": 0.02017033, + "auxiliary_loss_mlp": 0.01259539, + "balance_loss_clip": 1.37044418, + "balance_loss_mlp": 1.14242792, + "epoch": 0.0017700655794788463, + "flos": 12233508940800.0, + "grad_norm": 5.069385659337021, + "language_loss": 1.1187036, + "learning_rate": 2.3689735227299243e-06, + "loss": 1.15146947, + "num_input_tokens_seen": 2063875, + "router_z_loss_clip": 6.46484375, + "router_z_loss_mlp": 1.171875, + "step": 61, + "time_per_iteration": 2.622426748275757 + }, + { + "auxiliary_loss_clip": 0.01496434, + "auxiliary_loss_mlp": 0.01063294, + "balance_loss_clip": 1.27065372, + "balance_loss_mlp": 1.0222863, + "epoch": 0.001799083047994893, + "flos": 68543488863360.0, + "grad_norm": 0.8458606593412125, + "language_loss": 0.62548387, + "learning_rate": 2.3783439740460682e-06, + "loss": 0.6510812, + "num_input_tokens_seen": 2124740, + "router_z_loss_clip": 2.25, + "router_z_loss_mlp": 0.41015625, + "step": 62, + "time_per_iteration": 3.1470892429351807 + }, + { + "auxiliary_loss_clip": 0.01968202, + "auxiliary_loss_mlp": 0.0125059, + "balance_loss_clip": 1.36659741, + "balance_loss_mlp": 1.14129901, + "epoch": 0.0018281005165109395, + "flos": 20733851514240.0, + "grad_norm": 2.719320029220619, + "language_loss": 1.10532355, + "learning_rate": 2.3875644916918902e-06, + "loss": 1.13751149, + "num_input_tokens_seen": 2138655, + "router_z_loss_clip": 6.0234375, + "router_z_loss_mlp": 1.09228516, + "step": 63, + "time_per_iteration": 2.8259031772613525 + }, + { + "auxiliary_loss_clip": 0.02000039, + "auxiliary_loss_mlp": 0.01238764, + "balance_loss_clip": 1.37769055, + "balance_loss_mlp": 1.12174773, + "epoch": 0.0018571179850269863, + "flos": 35769613017600.0, + "grad_norm": 2.9682745538875004, + "language_loss": 1.21956062, + "learning_rate": 2.3966397982852547e-06, + "loss": 1.25194871, + "num_input_tokens_seen": 2156155, + "router_z_loss_clip": 6.234375, + "router_z_loss_mlp": 1.16992188, + "step": 64, + "time_per_iteration": 2.829235792160034 + }, + { + "auxiliary_loss_clip": 0.01985182, + "auxiliary_loss_mlp": 0.01255208, + "balance_loss_clip": 1.36683726, + "balance_loss_mlp": 1.14076662, + "epoch": 0.001886135453543033, + "flos": 29052342506880.0, + "grad_norm": 2.8898063163394387, + "language_loss": 1.25883174, + "learning_rate": 2.4055743967693543e-06, + "loss": 1.29123569, + "num_input_tokens_seen": 2172080, + "router_z_loss_clip": 6.17578125, + "router_z_loss_mlp": 1.14501953, + "step": 65, + "time_per_iteration": 2.7911510467529297 + }, + { + "auxiliary_loss_clip": 0.01947939, + "auxiliary_loss_mlp": 0.01275528, + "balance_loss_clip": 1.37109315, + "balance_loss_mlp": 1.16709495, + "epoch": 0.0019151529220590795, + "flos": 15883074403200.0, + "grad_norm": 2.8390189679114792, + "language_loss": 1.06018925, + "learning_rate": 2.4143725838293036e-06, + "loss": 1.09242392, + "num_input_tokens_seen": 2184295, + "router_z_loss_clip": 5.765625, + "router_z_loss_mlp": 1.08447266, + "step": 66, + "time_per_iteration": 2.7199912071228027 + }, + { + "auxiliary_loss_clip": 0.01458791, + "auxiliary_loss_mlp": 0.01055733, + "balance_loss_clip": 1.25615287, + "balance_loss_mlp": 1.01777673, + "epoch": 0.0019441703905751263, + "flos": 50108889807360.0, + "grad_norm": 0.8893877877935765, + "language_loss": 0.58198804, + "learning_rate": 2.4230384622998466e-06, + "loss": 0.60713333, + "num_input_tokens_seen": 2236095, + "router_z_loss_clip": 2.03125, + "router_z_loss_mlp": 0.37890625, + "step": 67, + "time_per_iteration": 3.046208143234253 + }, + { + "auxiliary_loss_clip": 0.01450683, + "auxiliary_loss_mlp": 0.01053597, + "balance_loss_clip": 1.25078559, + "balance_loss_mlp": 1.01659417, + "epoch": 0.0019731878590911727, + "flos": 59549875793280.0, + "grad_norm": 0.8375268500071488, + "language_loss": 0.57039356, + "learning_rate": 2.4315759526538664e-06, + "loss": 0.59543633, + "num_input_tokens_seen": 2291030, + "router_z_loss_clip": 2.0, + "router_z_loss_mlp": 0.36914062, + "step": 68, + "time_per_iteration": 3.115278720855713 + }, + { + "auxiliary_loss_clip": 0.01877677, + "auxiliary_loss_mlp": 0.01236359, + "balance_loss_clip": 1.34097028, + "balance_loss_mlp": 1.13779688, + "epoch": 0.0020022053276072195, + "flos": 26460238464000.0, + "grad_norm": 3.5310712816138787, + "language_loss": 1.13687372, + "learning_rate": 2.4399888036522294e-06, + "loss": 1.16801417, + "num_input_tokens_seen": 2307790, + "router_z_loss_clip": 5.35742188, + "router_z_loss_mlp": 0.98583984, + "step": 69, + "time_per_iteration": 2.797044038772583 + }, + { + "auxiliary_loss_clip": 0.01876632, + "auxiliary_loss_mlp": 0.01208785, + "balance_loss_clip": 1.33581376, + "balance_loss_mlp": 1.11394191, + "epoch": 0.0020312227961232663, + "flos": 29745095184000.0, + "grad_norm": 6.133649180014404, + "language_loss": 1.14604163, + "learning_rate": 2.4482806022273704e-06, + "loss": 1.17689586, + "num_input_tokens_seen": 2324120, + "router_z_loss_clip": 5.40234375, + "router_z_loss_mlp": 0.94970703, + "step": 70, + "time_per_iteration": 2.7818028926849365 + }, + { + "auxiliary_loss_clip": 0.01845658, + "auxiliary_loss_mlp": 0.01237004, + "balance_loss_clip": 1.33194017, + "balance_loss_mlp": 1.1406827, + "epoch": 0.0020602402646393127, + "flos": 21063224252160.0, + "grad_norm": 2.87058492685793, + "language_loss": 1.17013264, + "learning_rate": 2.456454782665838e-06, + "loss": 1.20095921, + "num_input_tokens_seen": 2339380, + "router_z_loss_clip": 5.140625, + "router_z_loss_mlp": 0.96289062, + "step": 71, + "time_per_iteration": 2.6865081787109375 + }, + { + "auxiliary_loss_clip": 0.01857939, + "auxiliary_loss_mlp": 0.01248598, + "balance_loss_clip": 1.33390379, + "balance_loss_mlp": 1.14989233, + "epoch": 0.0020892577331553595, + "flos": 11904208030080.0, + "grad_norm": 2.783228889014759, + "language_loss": 1.01392865, + "learning_rate": 2.464514635148642e-06, + "loss": 1.04499412, + "num_input_tokens_seen": 2350465, + "router_z_loss_clip": 5.234375, + "router_z_loss_mlp": 0.98681641, + "step": 72, + "time_per_iteration": 2.7027904987335205 + }, + { + "auxiliary_loss_clip": 0.01413193, + "auxiliary_loss_mlp": 0.01039935, + "balance_loss_clip": 1.22346139, + "balance_loss_mlp": 1.00560224, + "epoch": 0.0021182752016714063, + "flos": 57404034322560.0, + "grad_norm": 0.767094431270205, + "language_loss": 0.55268717, + "learning_rate": 2.4724633137025535e-06, + "loss": 0.57721841, + "num_input_tokens_seen": 2411125, + "router_z_loss_clip": 1.8984375, + "router_z_loss_mlp": 0.34375, + "step": 73, + "time_per_iteration": 3.2793147563934326 + }, + { + "auxiliary_loss_clip": 0.01887728, + "auxiliary_loss_mlp": 0.0123407, + "balance_loss_clip": 1.34562743, + "balance_loss_mlp": 1.13245583, + "epoch": 0.0021472926701874527, + "flos": 23105284352640.0, + "grad_norm": 2.3772799658526216, + "language_loss": 1.22434092, + "learning_rate": 2.4803038436104442e-06, + "loss": 1.25555897, + "num_input_tokens_seen": 2426450, + "router_z_loss_clip": 5.421875, + "router_z_loss_mlp": 1.01660156, + "step": 74, + "time_per_iteration": 2.753357172012329 + }, + { + "auxiliary_loss_clip": 0.01826111, + "auxiliary_loss_mlp": 0.0122426, + "balance_loss_clip": 1.32508421, + "balance_loss_mlp": 1.13714159, + "epoch": 0.0021763101387034995, + "flos": 42088023930240.0, + "grad_norm": 2.4586418115214514, + "language_loss": 1.1525743, + "learning_rate": 2.4880391283242453e-06, + "loss": 1.18307805, + "num_input_tokens_seen": 2451490, + "router_z_loss_clip": 5.015625, + "router_z_loss_mlp": 0.87011719, + "step": 75, + "time_per_iteration": 2.9057843685150146 + }, + { + "auxiliary_loss_clip": 0.01397613, + "auxiliary_loss_mlp": 0.01033675, + "balance_loss_clip": 1.21101117, + "balance_loss_mlp": 1.00048721, + "epoch": 0.0022053276072195463, + "flos": 63312847470720.0, + "grad_norm": 0.7887585066205736, + "language_loss": 0.58491492, + "learning_rate": 2.495671955920055e-06, + "loss": 0.60922784, + "num_input_tokens_seen": 2508445, + "router_z_loss_clip": 1.8671875, + "router_z_loss_mlp": 0.33203125, + "step": 76, + "time_per_iteration": 3.0060598850250244 + }, + { + "auxiliary_loss_clip": 0.01831553, + "auxiliary_loss_mlp": 0.01207825, + "balance_loss_clip": 1.31476545, + "balance_loss_mlp": 1.11579561, + "epoch": 0.0022343450757355927, + "flos": 38795122563840.0, + "grad_norm": 2.82803140967443, + "language_loss": 1.10144258, + "learning_rate": 2.5032050051312963e-06, + "loss": 1.13183641, + "num_input_tokens_seen": 2528630, + "router_z_loss_clip": 5.171875, + "router_z_loss_mlp": 0.92041016, + "step": 77, + "time_per_iteration": 2.8830556869506836 + }, + { + "auxiliary_loss_clip": 0.01792819, + "auxiliary_loss_mlp": 0.01229875, + "balance_loss_clip": 1.31045723, + "balance_loss_mlp": 1.1429956, + "epoch": 0.0022633625442516395, + "flos": 30986167950720.0, + "grad_norm": 3.2590703219688657, + "language_loss": 1.33102441, + "learning_rate": 2.5106408509926183e-06, + "loss": 1.36125135, + "num_input_tokens_seen": 2542975, + "router_z_loss_clip": 4.82421875, + "router_z_loss_mlp": 0.86865234, + "step": 78, + "time_per_iteration": 5.100194692611694 + }, + { + "auxiliary_loss_clip": 0.0182564, + "auxiliary_loss_mlp": 0.0122491, + "balance_loss_clip": 1.32661414, + "balance_loss_mlp": 1.13693357, + "epoch": 0.0022923800127676863, + "flos": 11247689197440.0, + "grad_norm": 3.4808668100258306, + "language_loss": 1.09845626, + "learning_rate": 2.517981970124274e-06, + "loss": 1.1289618, + "num_input_tokens_seen": 2554405, + "router_z_loss_clip": 4.9921875, + "router_z_loss_mlp": 0.88037109, + "step": 79, + "time_per_iteration": 6.024174690246582 + }, + { + "auxiliary_loss_clip": 0.0182254, + "auxiliary_loss_mlp": 0.01239837, + "balance_loss_clip": 1.31981802, + "balance_loss_mlp": 1.1519562, + "epoch": 0.0023213974812837327, + "flos": 65902650065280.0, + "grad_norm": 13.05753119064885, + "language_loss": 1.08927202, + "learning_rate": 2.525230745684122e-06, + "loss": 1.1198957, + "num_input_tokens_seen": 2574415, + "router_z_loss_clip": 5.03125, + "router_z_loss_mlp": 0.87890625, + "step": 80, + "time_per_iteration": 2.9899637699127197 + }, + { + "auxiliary_loss_clip": 0.01377188, + "auxiliary_loss_mlp": 0.01033053, + "balance_loss_clip": 1.19436562, + "balance_loss_mlp": 1.00062835, + "epoch": 0.0023504149497997795, + "flos": 59554185425280.0, + "grad_norm": 0.8319774167589701, + "language_loss": 0.5981527, + "learning_rate": 2.53238947201203e-06, + "loss": 0.62225515, + "num_input_tokens_seen": 2634065, + "router_z_loss_clip": 1.828125, + "router_z_loss_mlp": 0.32421875, + "step": 81, + "time_per_iteration": 3.0422801971435547 + }, + { + "auxiliary_loss_clip": 0.01798149, + "auxiliary_loss_mlp": 0.01215055, + "balance_loss_clip": 1.30674171, + "balance_loss_mlp": 1.13227618, + "epoch": 0.0023794324183158263, + "flos": 24235967646720.0, + "grad_norm": 6.08046445886985, + "language_loss": 1.14783573, + "learning_rate": 2.5394603589893167e-06, + "loss": 1.17796779, + "num_input_tokens_seen": 2649080, + "router_z_loss_clip": 4.91796875, + "router_z_loss_mlp": 0.82714844, + "step": 82, + "time_per_iteration": 2.663381338119507 + }, + { + "auxiliary_loss_clip": 0.01757111, + "auxiliary_loss_mlp": 0.01193504, + "balance_loss_clip": 1.29409003, + "balance_loss_mlp": 1.11372912, + "epoch": 0.0024084498868318727, + "flos": 16611234912000.0, + "grad_norm": 2.6891396329427666, + "language_loss": 1.00282395, + "learning_rate": 2.5464455361339734e-06, + "loss": 1.03233004, + "num_input_tokens_seen": 2663825, + "router_z_loss_clip": 4.6328125, + "router_z_loss_mlp": 0.79785156, + "step": 83, + "time_per_iteration": 2.6321189403533936 + }, + { + "auxiliary_loss_clip": 0.01750427, + "auxiliary_loss_mlp": 0.01194681, + "balance_loss_clip": 1.29541624, + "balance_loss_mlp": 1.11876881, + "epoch": 0.0024374673553479195, + "flos": 21571898705280.0, + "grad_norm": 2.7748182108357202, + "language_loss": 1.05312228, + "learning_rate": 2.553347056450635e-06, + "loss": 1.08257318, + "num_input_tokens_seen": 2678215, + "router_z_loss_clip": 4.55859375, + "router_z_loss_mlp": 0.75927734, + "step": 84, + "time_per_iteration": 2.6949379444122314 + }, + { + "auxiliary_loss_clip": 0.01363595, + "auxiliary_loss_mlp": 0.01033345, + "balance_loss_clip": 1.18609881, + "balance_loss_mlp": 1.00149202, + "epoch": 0.0024664848238639663, + "flos": 74776646845440.0, + "grad_norm": 0.7762835771429218, + "language_loss": 0.56631637, + "learning_rate": 2.5601669000527336e-06, + "loss": 0.59028572, + "num_input_tokens_seen": 2744060, + "router_z_loss_clip": 1.7734375, + "router_z_loss_mlp": 0.31835938, + "step": 85, + "time_per_iteration": 3.127362012863159 + }, + { + "auxiliary_loss_clip": 0.0136057, + "auxiliary_loss_mlp": 0.01033927, + "balance_loss_clip": 1.18495309, + "balance_loss_mlp": 1.00188375, + "epoch": 0.0024955022923800127, + "flos": 60214906149120.0, + "grad_norm": 0.7786966521030969, + "language_loss": 0.58142489, + "learning_rate": 2.5669069775728125e-06, + "loss": 0.60536987, + "num_input_tokens_seen": 2807220, + "router_z_loss_clip": 1.7578125, + "router_z_loss_mlp": 0.3203125, + "step": 86, + "time_per_iteration": 3.1196069717407227 + }, + { + "auxiliary_loss_clip": 0.0176379, + "auxiliary_loss_mlp": 0.01216199, + "balance_loss_clip": 1.29284739, + "balance_loss_mlp": 1.13704443, + "epoch": 0.0025245197608960595, + "flos": 13473612040320.0, + "grad_norm": 3.8486948439023205, + "language_loss": 1.23060155, + "learning_rate": 2.5735691333756985e-06, + "loss": 1.26040137, + "num_input_tokens_seen": 2818480, + "router_z_loss_clip": 4.703125, + "router_z_loss_mlp": 0.79199219, + "step": 87, + "time_per_iteration": 2.6650333404541016 + }, + { + "auxiliary_loss_clip": 0.01780095, + "auxiliary_loss_mlp": 0.01188085, + "balance_loss_clip": 1.29807222, + "balance_loss_mlp": 1.11059952, + "epoch": 0.002553537229412106, + "flos": 16281718519680.0, + "grad_norm": 9.108763865230735, + "language_loss": 1.12537825, + "learning_rate": 2.580155148588048e-06, + "loss": 1.15506005, + "num_input_tokens_seen": 2830820, + "router_z_loss_clip": 4.8203125, + "router_z_loss_mlp": 0.77490234, + "step": 88, + "time_per_iteration": 2.6742613315582275 + }, + { + "auxiliary_loss_clip": 0.01786529, + "auxiliary_loss_mlp": 0.01207561, + "balance_loss_clip": 1.31035423, + "balance_loss_mlp": 1.1267848, + "epoch": 0.0025825546979281527, + "flos": 16611055344000.0, + "grad_norm": 2.8316699144042277, + "language_loss": 1.13981795, + "learning_rate": 2.5866667439567312e-06, + "loss": 1.1697588, + "num_input_tokens_seen": 2843205, + "router_z_loss_clip": 4.7578125, + "router_z_loss_mlp": 0.80810547, + "step": 89, + "time_per_iteration": 2.682258367538452 + }, + { + "auxiliary_loss_clip": 0.01713309, + "auxiliary_loss_mlp": 0.01190043, + "balance_loss_clip": 1.28238034, + "balance_loss_mlp": 1.11799312, + "epoch": 0.0026115721664441995, + "flos": 12378659800320.0, + "grad_norm": 2.929139226935787, + "language_loss": 0.90767097, + "learning_rate": 2.5931055825475097e-06, + "loss": 0.93670452, + "num_input_tokens_seen": 2857335, + "router_z_loss_clip": 4.3125, + "router_z_loss_mlp": 0.72070312, + "step": 90, + "time_per_iteration": 2.693840980529785 + }, + { + "auxiliary_loss_clip": 0.01344049, + "auxiliary_loss_mlp": 0.01031609, + "balance_loss_clip": 1.17874169, + "balance_loss_mlp": 1.00109124, + "epoch": 0.002640589634960246, + "flos": 63903865852800.0, + "grad_norm": 1.2060319074233763, + "language_loss": 0.64387941, + "learning_rate": 2.599473272294611e-06, + "loss": 0.66763604, + "num_input_tokens_seen": 2921595, + "router_z_loss_clip": 1.65625, + "router_z_loss_mlp": 0.3046875, + "step": 91, + "time_per_iteration": 3.1734516620635986 + }, + { + "auxiliary_loss_clip": 0.01738188, + "auxiliary_loss_mlp": 0.01166616, + "balance_loss_clip": 1.2908355, + "balance_loss_mlp": 1.09139454, + "epoch": 0.0026696071034762927, + "flos": 31900525585920.0, + "grad_norm": 3.6272262057347704, + "language_loss": 1.15746379, + "learning_rate": 2.605771368410974e-06, + "loss": 1.18651175, + "num_input_tokens_seen": 2937400, + "router_z_loss_clip": 4.47070312, + "router_z_loss_mlp": 0.75219727, + "step": 92, + "time_per_iteration": 2.7666685581207275 + }, + { + "auxiliary_loss_clip": 0.01339798, + "auxiliary_loss_mlp": 0.01029321, + "balance_loss_clip": 1.17727399, + "balance_loss_mlp": 0.99994832, + "epoch": 0.0026986245719923395, + "flos": 74774456115840.0, + "grad_norm": 0.7695339483734882, + "language_loss": 0.58030099, + "learning_rate": 2.6120013756682003e-06, + "loss": 0.60399216, + "num_input_tokens_seen": 3002790, + "router_z_loss_clip": 1.625, + "router_z_loss_mlp": 0.29296875, + "step": 93, + "time_per_iteration": 3.2570180892944336 + }, + { + "auxiliary_loss_clip": 0.01687493, + "auxiliary_loss_mlp": 0.01208816, + "balance_loss_clip": 1.27669263, + "balance_loss_mlp": 1.13485825, + "epoch": 0.002727642040508386, + "flos": 32305059532800.0, + "grad_norm": 2.7247052053383487, + "language_loss": 1.18119383, + "learning_rate": 2.618164750554579e-06, + "loss": 1.21015692, + "num_input_tokens_seen": 3020540, + "router_z_loss_clip": 4.10742188, + "router_z_loss_mlp": 0.73974609, + "step": 94, + "time_per_iteration": 2.8341140747070312 + }, + { + "auxiliary_loss_clip": 0.01704823, + "auxiliary_loss_mlp": 0.01180442, + "balance_loss_clip": 1.27036762, + "balance_loss_mlp": 1.11091924, + "epoch": 0.0027566595090244327, + "flos": 26569837837440.0, + "grad_norm": 5.631082259570134, + "language_loss": 1.25186515, + "learning_rate": 2.624262903318922e-06, + "loss": 1.28071761, + "num_input_tokens_seen": 3033525, + "router_z_loss_clip": 4.34375, + "router_z_loss_mlp": 0.69580078, + "step": 95, + "time_per_iteration": 2.7617619037628174 + }, + { + "auxiliary_loss_clip": 0.01329456, + "auxiliary_loss_mlp": 0.01029826, + "balance_loss_clip": 1.17002964, + "balance_loss_mlp": 1.00121617, + "epoch": 0.0027856769775404795, + "flos": 74784691491840.0, + "grad_norm": 0.7159633907487346, + "language_loss": 0.5843662, + "learning_rate": 2.6302971999073867e-06, + "loss": 0.60795903, + "num_input_tokens_seen": 3107020, + "router_z_loss_clip": 1.59375, + "router_z_loss_mlp": 0.28515625, + "step": 96, + "time_per_iteration": 3.3890562057495117 + }, + { + "auxiliary_loss_clip": 0.01731774, + "auxiliary_loss_mlp": 0.01199245, + "balance_loss_clip": 1.28693414, + "balance_loss_mlp": 1.13043725, + "epoch": 0.002814694446056526, + "flos": 46237321359360.0, + "grad_norm": 2.6213249261132376, + "language_loss": 1.12980855, + "learning_rate": 2.636268963799937e-06, + "loss": 1.15911877, + "num_input_tokens_seen": 3125590, + "router_z_loss_clip": 4.453125, + "router_z_loss_mlp": 0.68847656, + "step": 97, + "time_per_iteration": 2.9248111248016357 + }, + { + "auxiliary_loss_clip": 0.01696824, + "auxiliary_loss_mlp": 0.01161558, + "balance_loss_clip": 1.27481973, + "balance_loss_mlp": 1.09251237, + "epoch": 0.0028437119145725727, + "flos": 11282881547520.0, + "grad_norm": 3.8030096778261697, + "language_loss": 1.14247119, + "learning_rate": 2.642179477752627e-06, + "loss": 1.17105496, + "num_input_tokens_seen": 3135965, + "router_z_loss_clip": 4.21875, + "router_z_loss_mlp": 0.68994141, + "step": 98, + "time_per_iteration": 2.6656229496002197 + }, + { + "auxiliary_loss_clip": 0.01693211, + "auxiliary_loss_mlp": 0.01184595, + "balance_loss_clip": 1.27688301, + "balance_loss_mlp": 1.11545384, + "epoch": 0.0028727293830886195, + "flos": 12524959895040.0, + "grad_norm": 3.2736102218040055, + "language_loss": 1.19347739, + "learning_rate": 2.6480299854514357e-06, + "loss": 1.22225547, + "num_input_tokens_seen": 3147755, + "router_z_loss_clip": 4.1640625, + "router_z_loss_mlp": 0.69091797, + "step": 99, + "time_per_iteration": 2.624107837677002 + }, + { + "auxiliary_loss_clip": 0.01658089, + "auxiliary_loss_mlp": 0.01161818, + "balance_loss_clip": 1.26388204, + "balance_loss_mlp": 1.09825552, + "epoch": 0.002901746851604666, + "flos": 33941472364800.0, + "grad_norm": 2.6368817270538063, + "language_loss": 1.02320158, + "learning_rate": 2.65382169308299e-06, + "loss": 1.05140066, + "num_input_tokens_seen": 3166200, + "router_z_loss_clip": 3.94335938, + "router_z_loss_mlp": 0.63647461, + "step": 100, + "time_per_iteration": 2.858816146850586 + }, + { + "auxiliary_loss_clip": 0.01701228, + "auxiliary_loss_mlp": 0.0116446, + "balance_loss_clip": 1.27423477, + "balance_loss_mlp": 1.09949064, + "epoch": 0.0029307643201207127, + "flos": 27264314367360.0, + "grad_norm": 3.447483880474036, + "language_loss": 1.12209249, + "learning_rate": 2.659555770827138e-06, + "loss": 1.15074933, + "num_input_tokens_seen": 3177955, + "router_z_loss_clip": 4.2734375, + "router_z_loss_mlp": 0.64916992, + "step": 101, + "time_per_iteration": 2.7581913471221924 + }, + { + "auxiliary_loss_clip": 0.01311543, + "auxiliary_loss_mlp": 0.01029052, + "balance_loss_clip": 1.15948534, + "balance_loss_mlp": 1.00215864, + "epoch": 0.0029597817886367595, + "flos": 63945666305280.0, + "grad_norm": 0.7818148649411039, + "language_loss": 0.57701558, + "learning_rate": 2.6652333542759976e-06, + "loss": 0.60042149, + "num_input_tokens_seen": 3238025, + "router_z_loss_clip": 1.5234375, + "router_z_loss_mlp": 0.26953125, + "step": 102, + "time_per_iteration": 3.2399230003356934 + }, + { + "auxiliary_loss_clip": 0.01698985, + "auxiliary_loss_mlp": 0.01184752, + "balance_loss_clip": 1.28644514, + "balance_loss_mlp": 1.11484766, + "epoch": 0.002988799257152806, + "flos": 25000972531200.0, + "grad_norm": 2.569913986924757, + "language_loss": 1.02323294, + "learning_rate": 2.6708555457837733e-06, + "loss": 1.05207038, + "num_input_tokens_seen": 3255000, + "router_z_loss_clip": 4.125, + "router_z_loss_mlp": 0.69921875, + "step": 103, + "time_per_iteration": 2.7631566524505615 + }, + { + "auxiliary_loss_clip": 0.01307732, + "auxiliary_loss_mlp": 0.01027697, + "balance_loss_clip": 1.15643716, + "balance_loss_mlp": 1.00137603, + "epoch": 0.0030178167256688527, + "flos": 69269602296960.0, + "grad_norm": 0.7258269943319984, + "language_loss": 0.58467007, + "learning_rate": 2.676423415751363e-06, + "loss": 0.60802442, + "num_input_tokens_seen": 3319875, + "router_z_loss_clip": 1.515625, + "router_z_loss_mlp": 0.26367188, + "step": 104, + "time_per_iteration": 3.2230396270751953 + }, + { + "auxiliary_loss_clip": 0.01703231, + "auxiliary_loss_mlp": 0.0123614, + "balance_loss_clip": 1.2832613, + "balance_loss_mlp": 1.1643281, + "epoch": 0.0030468341941848995, + "flos": 24747299706240.0, + "grad_norm": 2.983663655625681, + "language_loss": 1.10435104, + "learning_rate": 2.681938003849502e-06, + "loss": 1.13374472, + "num_input_tokens_seen": 3336710, + "router_z_loss_clip": 4.19921875, + "router_z_loss_mlp": 0.71777344, + "step": 105, + "time_per_iteration": 2.701442003250122 + }, + { + "auxiliary_loss_clip": 0.01676202, + "auxiliary_loss_mlp": 0.01163467, + "balance_loss_clip": 1.27175641, + "balance_loss_mlp": 1.0965426, + "epoch": 0.003075851662700946, + "flos": 24783318069120.0, + "grad_norm": 5.191351295001049, + "language_loss": 1.13418984, + "learning_rate": 2.6874003201839304e-06, + "loss": 1.16258645, + "num_input_tokens_seen": 3350890, + "router_z_loss_clip": 4.04296875, + "router_z_loss_mlp": 0.66943359, + "step": 106, + "time_per_iteration": 2.788571834564209 + }, + { + "auxiliary_loss_clip": 0.01654085, + "auxiliary_loss_mlp": 0.01169785, + "balance_loss_clip": 1.2614665, + "balance_loss_mlp": 1.10913134, + "epoch": 0.0031048691312169927, + "flos": 17304813601920.0, + "grad_norm": 3.556303085804161, + "language_loss": 1.14790559, + "learning_rate": 2.692811346405858e-06, + "loss": 1.17614424, + "num_input_tokens_seen": 3361720, + "router_z_loss_clip": 3.92382812, + "router_z_loss_mlp": 0.60693359, + "step": 107, + "time_per_iteration": 2.6733617782592773 + }, + { + "auxiliary_loss_clip": 0.01663632, + "auxiliary_loss_mlp": 0.01173022, + "balance_loss_clip": 1.25918865, + "balance_loss_mlp": 1.11174822, + "epoch": 0.0031338865997330395, + "flos": 67252747587840.0, + "grad_norm": 3.9981377996925107, + "language_loss": 1.20995784, + "learning_rate": 2.698172036770774e-06, + "loss": 1.23832428, + "num_input_tokens_seen": 3383520, + "router_z_loss_clip": 4.03710938, + "router_z_loss_mlp": 0.61279297, + "step": 108, + "time_per_iteration": 3.0134360790252686 + }, + { + "auxiliary_loss_clip": 0.01655935, + "auxiliary_loss_mlp": 0.01193649, + "balance_loss_clip": 1.25039661, + "balance_loss_mlp": 1.1285609, + "epoch": 0.003162904068249086, + "flos": 22376154176640.0, + "grad_norm": 5.391765621321817, + "language_loss": 1.17953837, + "learning_rate": 2.703483319148466e-06, + "loss": 1.20803428, + "num_input_tokens_seen": 3397615, + "router_z_loss_clip": 4.05664062, + "router_z_loss_mlp": 0.65161133, + "step": 109, + "time_per_iteration": 2.66218900680542 + }, + { + "auxiliary_loss_clip": 0.01294247, + "auxiliary_loss_mlp": 0.01026283, + "balance_loss_clip": 1.14781952, + "balance_loss_mlp": 1.0005343, + "epoch": 0.0031919215367651327, + "flos": 72837793497600.0, + "grad_norm": 0.70787402935625, + "language_loss": 0.56342131, + "learning_rate": 2.708746095986916e-06, + "loss": 0.58662665, + "num_input_tokens_seen": 3461295, + "router_z_loss_clip": 1.46875, + "router_z_loss_mlp": 0.2578125, + "step": 110, + "time_per_iteration": 3.29011607170105 + }, + { + "auxiliary_loss_clip": 0.01641208, + "auxiliary_loss_mlp": 0.01185206, + "balance_loss_clip": 1.25353503, + "balance_loss_mlp": 1.12429047, + "epoch": 0.003220939005281179, + "flos": 29052522074880.0, + "grad_norm": 2.800194603361431, + "language_loss": 1.16137564, + "learning_rate": 2.7139612452325754e-06, + "loss": 1.18963981, + "num_input_tokens_seen": 3477540, + "router_z_loss_clip": 3.875, + "router_z_loss_mlp": 0.6081543, + "step": 111, + "time_per_iteration": 2.746112823486328 + }, + { + "auxiliary_loss_clip": 0.01291023, + "auxiliary_loss_mlp": 0.0102648, + "balance_loss_clip": 1.14561605, + "balance_loss_mlp": 1.00092149, + "epoch": 0.003249956473797226, + "flos": 63505652699520.0, + "grad_norm": 0.7089360386540745, + "language_loss": 0.54555178, + "learning_rate": 2.7191296212093786e-06, + "loss": 0.5687269, + "num_input_tokens_seen": 3542620, + "router_z_loss_clip": 1.453125, + "router_z_loss_mlp": 0.25585938, + "step": 112, + "time_per_iteration": 3.167365789413452 + }, + { + "auxiliary_loss_clip": 0.01671891, + "auxiliary_loss_mlp": 0.01171305, + "balance_loss_clip": 1.26515412, + "balance_loss_mlp": 1.10650253, + "epoch": 0.0032789739423132727, + "flos": 23507663483520.0, + "grad_norm": 5.846419020547411, + "language_loss": 0.76810443, + "learning_rate": 2.724252055458679e-06, + "loss": 0.79653633, + "num_input_tokens_seen": 3559185, + "router_z_loss_clip": 4.0625, + "router_z_loss_mlp": 0.6484375, + "step": 113, + "time_per_iteration": 2.780367374420166 + }, + { + "auxiliary_loss_clip": 0.01639524, + "auxiliary_loss_mlp": 0.01142822, + "balance_loss_clip": 1.25557351, + "balance_loss_mlp": 1.08748531, + "epoch": 0.003307991410829319, + "flos": 25514279838720.0, + "grad_norm": 2.4427063459388467, + "language_loss": 0.96135974, + "learning_rate": 2.7293293575421866e-06, + "loss": 0.98918319, + "num_input_tokens_seen": 3576220, + "router_z_loss_clip": 3.8359375, + "router_z_loss_mlp": 0.55322266, + "step": 114, + "time_per_iteration": 2.8273067474365234 + }, + { + "auxiliary_loss_clip": 0.01631297, + "auxiliary_loss_mlp": 0.01169031, + "balance_loss_clip": 1.23856306, + "balance_loss_mlp": 1.10773349, + "epoch": 0.003337008879345366, + "flos": 16137034536960.0, + "grad_norm": 3.053085367583335, + "language_loss": 1.1337316, + "learning_rate": 2.7343623158098412e-06, + "loss": 1.16173494, + "num_input_tokens_seen": 3590170, + "router_z_loss_clip": 3.93164062, + "router_z_loss_mlp": 0.61254883, + "step": 115, + "time_per_iteration": 2.8460710048675537 + }, + { + "auxiliary_loss_clip": 0.01641177, + "auxiliary_loss_mlp": 0.01191148, + "balance_loss_clip": 1.26327658, + "balance_loss_mlp": 1.13140059, + "epoch": 0.0033660263478614127, + "flos": 74734017402240.0, + "grad_norm": 1.7757136602010855, + "language_loss": 0.80068523, + "learning_rate": 2.7393516981344427e-06, + "loss": 0.82900846, + "num_input_tokens_seen": 3619655, + "router_z_loss_clip": 3.78320312, + "router_z_loss_mlp": 0.59716797, + "step": 116, + "time_per_iteration": 3.1337013244628906 + }, + { + "auxiliary_loss_clip": 0.01636093, + "auxiliary_loss_mlp": 0.01168711, + "balance_loss_clip": 1.24774766, + "balance_loss_mlp": 1.10843897, + "epoch": 0.003395043816377459, + "flos": 27703035083520.0, + "grad_norm": 2.6129343064369848, + "language_loss": 1.15629733, + "learning_rate": 2.7442982526147504e-06, + "loss": 1.18434536, + "num_input_tokens_seen": 3636430, + "router_z_loss_clip": 3.88671875, + "router_z_loss_mlp": 0.60253906, + "step": 117, + "time_per_iteration": 2.7650415897369385 + }, + { + "auxiliary_loss_clip": 0.01601749, + "auxiliary_loss_mlp": 0.01154433, + "balance_loss_clip": 1.23977196, + "balance_loss_mlp": 1.09888148, + "epoch": 0.003424061284893506, + "flos": 33828030236160.0, + "grad_norm": 3.488020898685152, + "language_loss": 1.41212785, + "learning_rate": 2.7492027082486626e-06, + "loss": 1.43968964, + "num_input_tokens_seen": 3649100, + "router_z_loss_clip": 3.62109375, + "router_z_loss_mlp": 0.5559082, + "step": 118, + "time_per_iteration": 2.81762957572937 + }, + { + "auxiliary_loss_clip": 0.01279086, + "auxiliary_loss_mlp": 0.01025218, + "balance_loss_clip": 1.13762045, + "balance_loss_mlp": 1.0008992, + "epoch": 0.0034530787534095527, + "flos": 74782069799040.0, + "grad_norm": 0.7659389669162272, + "language_loss": 0.57078534, + "learning_rate": 2.7540657755779904e-06, + "loss": 0.59382838, + "num_input_tokens_seen": 3718090, + "router_z_loss_clip": 1.4140625, + "router_z_loss_mlp": 0.24316406, + "step": 119, + "time_per_iteration": 3.357208251953125 + }, + { + "auxiliary_loss_clip": 0.01629818, + "auxiliary_loss_mlp": 0.01144348, + "balance_loss_clip": 1.24799252, + "balance_loss_mlp": 1.08574438, + "epoch": 0.003482096221925599, + "flos": 27343174677120.0, + "grad_norm": 2.31677616046027, + "language_loss": 1.02661872, + "learning_rate": 2.758888147306254e-06, + "loss": 1.05436039, + "num_input_tokens_seen": 3743795, + "router_z_loss_clip": 3.8125, + "router_z_loss_mlp": 0.58642578, + "step": 120, + "time_per_iteration": 2.950223445892334 + }, + { + "auxiliary_loss_clip": 0.01626181, + "auxiliary_loss_mlp": 0.01131106, + "balance_loss_clip": 1.24218917, + "balance_loss_mlp": 1.07500613, + "epoch": 0.003511113690441646, + "flos": 19530233654400.0, + "grad_norm": 2.551415693670158, + "language_loss": 1.08135939, + "learning_rate": 2.7636704988908417e-06, + "loss": 1.10893226, + "num_input_tokens_seen": 3758265, + "router_z_loss_clip": 3.8359375, + "router_z_loss_mlp": 0.56054688, + "step": 121, + "time_per_iteration": 2.6577024459838867 + }, + { + "auxiliary_loss_clip": 0.0162488, + "auxiliary_loss_mlp": 0.01160895, + "balance_loss_clip": 1.25016475, + "balance_loss_mlp": 1.10224366, + "epoch": 0.0035401311589576927, + "flos": 11904710820480.0, + "grad_norm": 2.789292578581841, + "language_loss": 0.99017304, + "learning_rate": 2.7684134891108e-06, + "loss": 1.01803076, + "num_input_tokens_seen": 3770935, + "router_z_loss_clip": 3.75195312, + "router_z_loss_mlp": 0.58691406, + "step": 122, + "time_per_iteration": 2.6621363162994385 + }, + { + "auxiliary_loss_clip": 0.01612782, + "auxiliary_loss_mlp": 0.01138755, + "balance_loss_clip": 1.24033463, + "balance_loss_mlp": 1.08368015, + "epoch": 0.003569148627473739, + "flos": 23690484731520.0, + "grad_norm": 2.789508597055809, + "language_loss": 0.9587993, + "learning_rate": 2.7731177606114483e-06, + "loss": 0.98631477, + "num_input_tokens_seen": 3786425, + "router_z_loss_clip": 3.7265625, + "router_z_loss_mlp": 0.55102539, + "step": 123, + "time_per_iteration": 2.808793067932129 + }, + { + "auxiliary_loss_clip": 0.01589726, + "auxiliary_loss_mlp": 0.01154883, + "balance_loss_clip": 1.23442996, + "balance_loss_mlp": 1.1011672, + "epoch": 0.003598166095989786, + "flos": 25596264631680.0, + "grad_norm": 2.443413261694475, + "language_loss": 0.91455853, + "learning_rate": 2.777783940426944e-06, + "loss": 0.94200456, + "num_input_tokens_seen": 3805695, + "router_z_loss_clip": 3.55078125, + "router_z_loss_mlp": 0.53710938, + "step": 124, + "time_per_iteration": 2.7416203022003174 + }, + { + "auxiliary_loss_clip": 0.01274473, + "auxiliary_loss_mlp": 0.01025628, + "balance_loss_clip": 1.13490784, + "balance_loss_mlp": 1.00140464, + "epoch": 0.0036271835645058327, + "flos": 71674683200640.0, + "grad_norm": 0.7865674010805416, + "language_loss": 0.53349024, + "learning_rate": 2.782412640481857e-06, + "loss": 0.55649132, + "num_input_tokens_seen": 3870130, + "router_z_loss_clip": 1.390625, + "router_z_loss_mlp": 0.2421875, + "step": 125, + "time_per_iteration": 3.268098831176758 + }, + { + "auxiliary_loss_clip": 0.01595234, + "auxiliary_loss_mlp": 0.01150689, + "balance_loss_clip": 1.24175549, + "balance_loss_mlp": 1.0972836, + "epoch": 0.003656201033021879, + "flos": 15480372049920.0, + "grad_norm": 2.899429205597415, + "language_loss": 1.00163531, + "learning_rate": 2.787004458072766e-06, + "loss": 1.02909458, + "num_input_tokens_seen": 3883485, + "router_z_loss_clip": 3.53320312, + "router_z_loss_mlp": 0.53393555, + "step": 126, + "time_per_iteration": 2.68371319770813 + }, + { + "auxiliary_loss_clip": 0.01272773, + "auxiliary_loss_mlp": 0.0102379, + "balance_loss_clip": 1.13396549, + "balance_loss_mlp": 1.00032961, + "epoch": 0.003685218501537926, + "flos": 61891323753600.0, + "grad_norm": 0.780041169877218, + "language_loss": 0.57610649, + "learning_rate": 2.7915599763308157e-06, + "loss": 0.5990721, + "num_input_tokens_seen": 3943475, + "router_z_loss_clip": 1.390625, + "router_z_loss_mlp": 0.234375, + "step": 127, + "time_per_iteration": 3.072587490081787 + }, + { + "auxiliary_loss_clip": 0.01585647, + "auxiliary_loss_mlp": 0.01139271, + "balance_loss_clip": 1.22579527, + "balance_loss_mlp": 1.08271766, + "epoch": 0.0037142359700539727, + "flos": 10661914200960.0, + "grad_norm": 4.099392532334229, + "language_loss": 1.23350966, + "learning_rate": 2.7960797646661305e-06, + "loss": 1.26075888, + "num_input_tokens_seen": 3953925, + "router_z_loss_clip": 3.6015625, + "router_z_loss_mlp": 0.56542969, + "step": 128, + "time_per_iteration": 2.6676251888275146 + }, + { + "auxiliary_loss_clip": 0.01266993, + "auxiliary_loss_mlp": 0.01025541, + "balance_loss_clip": 1.12933683, + "balance_loss_mlp": 1.00265336, + "epoch": 0.003743253438570019, + "flos": 68942815338240.0, + "grad_norm": 0.7431915384400487, + "language_loss": 0.58946174, + "learning_rate": 2.8005643791949446e-06, + "loss": 0.61238706, + "num_input_tokens_seen": 4021235, + "router_z_loss_clip": 1.375, + "router_z_loss_mlp": 0.22851562, + "step": 129, + "time_per_iteration": 3.2430927753448486 + }, + { + "auxiliary_loss_clip": 0.01574167, + "auxiliary_loss_mlp": 0.01140224, + "balance_loss_clip": 1.22157264, + "balance_loss_mlp": 1.08987045, + "epoch": 0.003772270907086066, + "flos": 22156739948160.0, + "grad_norm": 3.396234342935346, + "language_loss": 1.09106994, + "learning_rate": 2.80501436315023e-06, + "loss": 1.11821377, + "num_input_tokens_seen": 4034220, + "router_z_loss_clip": 3.52734375, + "router_z_loss_mlp": 0.50390625, + "step": 130, + "time_per_iteration": 2.7173986434936523 + }, + { + "auxiliary_loss_clip": 0.01570665, + "auxiliary_loss_mlp": 0.01118181, + "balance_loss_clip": 1.22873962, + "balance_loss_mlp": 1.06684899, + "epoch": 0.0038012883756021127, + "flos": 11941267887360.0, + "grad_norm": 4.127517034301736, + "language_loss": 0.91801655, + "learning_rate": 2.8094302472765976e-06, + "loss": 0.94490504, + "num_input_tokens_seen": 4046425, + "router_z_loss_clip": 3.421875, + "router_z_loss_mlp": 0.51318359, + "step": 131, + "time_per_iteration": 2.767615795135498 + }, + { + "auxiliary_loss_clip": 0.01261128, + "auxiliary_loss_mlp": 0.0102303, + "balance_loss_clip": 1.1250391, + "balance_loss_mlp": 1.00080991, + "epoch": 0.003830305844118159, + "flos": 74765657283840.0, + "grad_norm": 0.7478017352458748, + "language_loss": 0.54986274, + "learning_rate": 2.8138125502101794e-06, + "loss": 0.57270432, + "num_input_tokens_seen": 4107605, + "router_z_loss_clip": 1.359375, + "router_z_loss_mlp": 0.22265625, + "step": 132, + "time_per_iteration": 3.1398227214813232 + }, + { + "auxiliary_loss_clip": 0.01572261, + "auxiliary_loss_mlp": 0.0114881, + "balance_loss_clip": 1.22635436, + "balance_loss_mlp": 1.09251928, + "epoch": 0.003859323312634206, + "flos": 30584327523840.0, + "grad_norm": 3.346961708432993, + "language_loss": 1.19490957, + "learning_rate": 2.818161778844179e-06, + "loss": 1.22212029, + "num_input_tokens_seen": 4121685, + "router_z_loss_clip": 3.45898438, + "router_z_loss_mlp": 0.56298828, + "step": 133, + "time_per_iteration": 2.7836668491363525 + }, + { + "auxiliary_loss_clip": 0.01573323, + "auxiliary_loss_mlp": 0.01138977, + "balance_loss_clip": 1.22127128, + "balance_loss_mlp": 1.08578563, + "epoch": 0.0038883407811502527, + "flos": 16391102411520.0, + "grad_norm": 3.233313398450951, + "language_loss": 0.99345851, + "learning_rate": 2.8224784286807224e-06, + "loss": 1.02058148, + "num_input_tokens_seen": 4134235, + "router_z_loss_clip": 3.51953125, + "router_z_loss_mlp": 0.53222656, + "step": 134, + "time_per_iteration": 2.6952009201049805 + }, + { + "auxiliary_loss_clip": 0.01577817, + "auxiliary_loss_mlp": 0.01133204, + "balance_loss_clip": 1.21962512, + "balance_loss_mlp": 1.0758884, + "epoch": 0.003917358249666299, + "flos": 30766394586240.0, + "grad_norm": 3.4050196606516505, + "language_loss": 1.1337117, + "learning_rate": 2.826762984169642e-06, + "loss": 1.16082191, + "num_input_tokens_seen": 4151530, + "router_z_loss_clip": 3.58203125, + "router_z_loss_mlp": 0.57324219, + "step": 135, + "time_per_iteration": 2.7094943523406982 + }, + { + "auxiliary_loss_clip": 0.01549613, + "auxiliary_loss_mlp": 0.01152526, + "balance_loss_clip": 1.21236229, + "balance_loss_mlp": 1.10238707, + "epoch": 0.003946375718182345, + "flos": 11647841685120.0, + "grad_norm": 3.53359459335293, + "language_loss": 1.07407475, + "learning_rate": 2.8310159190347422e-06, + "loss": 1.10109615, + "num_input_tokens_seen": 4161255, + "router_z_loss_clip": 3.37109375, + "router_z_loss_mlp": 0.50219727, + "step": 136, + "time_per_iteration": 2.681997299194336 + }, + { + "auxiliary_loss_clip": 0.01559283, + "auxiliary_loss_mlp": 0.0112103, + "balance_loss_clip": 1.21193242, + "balance_loss_mlp": 1.06981754, + "epoch": 0.003975393186698393, + "flos": 29015246736000.0, + "grad_norm": 2.83656060028213, + "language_loss": 0.95589298, + "learning_rate": 2.835237696588131e-06, + "loss": 0.98269606, + "num_input_tokens_seen": 4176600, + "router_z_loss_clip": 3.47460938, + "router_z_loss_mlp": 0.51220703, + "step": 137, + "time_per_iteration": 2.7554068565368652 + }, + { + "auxiliary_loss_clip": 0.01566972, + "auxiliary_loss_mlp": 0.01134453, + "balance_loss_clip": 1.22545922, + "balance_loss_mlp": 1.08159554, + "epoch": 0.004004410655214439, + "flos": 20371513069440.0, + "grad_norm": 3.509401714638551, + "language_loss": 1.0641818, + "learning_rate": 2.8394287700331053e-06, + "loss": 1.09119594, + "num_input_tokens_seen": 4189295, + "router_z_loss_clip": 3.4140625, + "router_z_loss_mlp": 0.52880859, + "step": 138, + "time_per_iteration": 2.751598834991455 + }, + { + "auxiliary_loss_clip": 0.01555611, + "auxiliary_loss_mlp": 0.01143147, + "balance_loss_clip": 1.21295524, + "balance_loss_mlp": 1.08871591, + "epoch": 0.004033428123730485, + "flos": 33029951904000.0, + "grad_norm": 7.759601203500636, + "language_loss": 0.96102601, + "learning_rate": 2.8435895827561136e-06, + "loss": 0.98801363, + "num_input_tokens_seen": 4203510, + "router_z_loss_clip": 3.42578125, + "router_z_loss_mlp": 0.54418945, + "step": 139, + "time_per_iteration": 2.822871685028076 + }, + { + "auxiliary_loss_clip": 0.01262564, + "auxiliary_loss_mlp": 0.01021127, + "balance_loss_clip": 1.12464356, + "balance_loss_mlp": 0.99890679, + "epoch": 0.004062445592246533, + "flos": 59585104056960.0, + "grad_norm": 0.7802396713438838, + "language_loss": 0.57611084, + "learning_rate": 2.847720568608246e-06, + "loss": 0.59894776, + "num_input_tokens_seen": 4251885, + "router_z_loss_clip": 1.375, + "router_z_loss_mlp": 0.22265625, + "step": 140, + "time_per_iteration": 3.024404525756836 + }, + { + "auxiliary_loss_clip": 0.01554922, + "auxiliary_loss_mlp": 0.01136095, + "balance_loss_clip": 1.21143138, + "balance_loss_mlp": 1.08342838, + "epoch": 0.004091463060762579, + "flos": 16538228519040.0, + "grad_norm": 5.599942311502046, + "language_loss": 1.08450198, + "learning_rate": 2.8518221521767104e-06, + "loss": 1.11141229, + "num_input_tokens_seen": 4264735, + "router_z_loss_clip": 3.43164062, + "router_z_loss_mlp": 0.52709961, + "step": 141, + "time_per_iteration": 2.66801381111145 + }, + { + "auxiliary_loss_clip": 0.01552132, + "auxiliary_loss_mlp": 0.01134776, + "balance_loss_clip": 1.21651208, + "balance_loss_mlp": 1.08249092, + "epoch": 0.004120480529278625, + "flos": 21026846753280.0, + "grad_norm": 9.935510261620275, + "language_loss": 0.97461057, + "learning_rate": 2.855894749046714e-06, + "loss": 1.00147963, + "num_input_tokens_seen": 4279280, + "router_z_loss_clip": 3.35351562, + "router_z_loss_mlp": 0.52319336, + "step": 142, + "time_per_iteration": 2.729294538497925 + }, + { + "auxiliary_loss_clip": 0.01546849, + "auxiliary_loss_mlp": 0.01128315, + "balance_loss_clip": 1.20650947, + "balance_loss_mlp": 1.07910562, + "epoch": 0.004149497997794673, + "flos": 23653209392640.0, + "grad_norm": 2.3921126577577607, + "language_loss": 1.19780624, + "learning_rate": 2.859938766054156e-06, + "loss": 1.22455788, + "num_input_tokens_seen": 4296225, + "router_z_loss_clip": 3.40039062, + "router_z_loss_mlp": 0.49194336, + "step": 143, + "time_per_iteration": 2.7995858192443848 + }, + { + "auxiliary_loss_clip": 0.01556026, + "auxiliary_loss_mlp": 0.01145706, + "balance_loss_clip": 1.21230984, + "balance_loss_mlp": 1.08958197, + "epoch": 0.004178515466310719, + "flos": 14022147611520.0, + "grad_norm": 4.511096728271764, + "language_loss": 1.12300992, + "learning_rate": 2.863954601529518e-06, + "loss": 1.15002728, + "num_input_tokens_seen": 4309835, + "router_z_loss_clip": 3.4375, + "router_z_loss_mlp": 0.56201172, + "step": 144, + "time_per_iteration": 2.658761501312256 + }, + { + "auxiliary_loss_clip": 0.0156871, + "auxiliary_loss_mlp": 0.01141429, + "balance_loss_clip": 1.21070516, + "balance_loss_mlp": 1.08525801, + "epoch": 0.004207532934826765, + "flos": 35517268995840.0, + "grad_norm": 2.9415750900459887, + "language_loss": 1.02600563, + "learning_rate": 2.86794264553331e-06, + "loss": 1.05310702, + "num_input_tokens_seen": 4328955, + "router_z_loss_clip": 3.578125, + "router_z_loss_mlp": 0.5612793, + "step": 145, + "time_per_iteration": 2.8203094005584717 + }, + { + "auxiliary_loss_clip": 0.01543881, + "auxiliary_loss_mlp": 0.01151721, + "balance_loss_clip": 1.20610797, + "balance_loss_mlp": 1.10191536, + "epoch": 0.004236550403342813, + "flos": 74746874471040.0, + "grad_norm": 2.7685938537443544, + "language_loss": 0.8777343, + "learning_rate": 2.8719032800834294e-06, + "loss": 0.90469027, + "num_input_tokens_seen": 4354660, + "router_z_loss_clip": 3.37890625, + "router_z_loss_mlp": 0.49829102, + "step": 146, + "time_per_iteration": 3.084071159362793 + }, + { + "auxiliary_loss_clip": 0.01560169, + "auxiliary_loss_mlp": 0.01138262, + "balance_loss_clip": 1.19803762, + "balance_loss_mlp": 1.08435571, + "epoch": 0.004265567871858859, + "flos": 27410973598080.0, + "grad_norm": 2.917094327681427, + "language_loss": 1.09599805, + "learning_rate": 2.875836879374759e-06, + "loss": 1.12298226, + "num_input_tokens_seen": 4371170, + "router_z_loss_clip": 3.625, + "router_z_loss_mlp": 0.53881836, + "step": 147, + "time_per_iteration": 2.731842517852783 + }, + { + "auxiliary_loss_clip": 0.01523413, + "auxiliary_loss_mlp": 0.01149572, + "balance_loss_clip": 1.19808388, + "balance_loss_mlp": 1.09826469, + "epoch": 0.004294585340374905, + "flos": 22119967399680.0, + "grad_norm": 5.249766500263327, + "language_loss": 1.01538157, + "learning_rate": 2.8797438099913196e-06, + "loss": 1.0421114, + "num_input_tokens_seen": 4386390, + "router_z_loss_clip": 3.25390625, + "router_z_loss_mlp": 0.51318359, + "step": 148, + "time_per_iteration": 2.7161855697631836 + }, + { + "auxiliary_loss_clip": 0.01516797, + "auxiliary_loss_mlp": 0.01133763, + "balance_loss_clip": 1.18947172, + "balance_loss_mlp": 1.08402872, + "epoch": 0.004323602808890953, + "flos": 12451235230080.0, + "grad_norm": 3.7605976360077777, + "language_loss": 1.09911346, + "learning_rate": 2.8836244311112828e-06, + "loss": 1.12561893, + "num_input_tokens_seen": 4398295, + "router_z_loss_clip": 3.26953125, + "router_z_loss_mlp": 0.49780273, + "step": 149, + "time_per_iteration": 9.705481767654419 + }, + { + "auxiliary_loss_clip": 0.01242001, + "auxiliary_loss_mlp": 0.01036901, + "balance_loss_clip": 1.11448991, + "balance_loss_mlp": 1.01706493, + "epoch": 0.004352620277406999, + "flos": 74788570160640.0, + "grad_norm": 0.7044151802164954, + "language_loss": 0.56531841, + "learning_rate": 2.887479094705121e-06, + "loss": 0.58810741, + "num_input_tokens_seen": 4468280, + "router_z_loss_clip": 1.28125, + "router_z_loss_mlp": 0.19824219, + "step": 150, + "time_per_iteration": 5.955836296081543 + }, + { + "auxiliary_loss_clip": 0.01239886, + "auxiliary_loss_mlp": 0.01031355, + "balance_loss_clip": 1.11292338, + "balance_loss_mlp": 1.01170969, + "epoch": 0.004381637745923045, + "flos": 66422460712320.0, + "grad_norm": 0.6928615246213999, + "language_loss": 0.49986929, + "learning_rate": 2.8913081457271816e-06, + "loss": 0.5225817, + "num_input_tokens_seen": 4530430, + "router_z_loss_clip": 1.265625, + "router_z_loss_mlp": 0.19628906, + "step": 151, + "time_per_iteration": 3.157114267349243 + }, + { + "auxiliary_loss_clip": 0.01547804, + "auxiliary_loss_mlp": 0.01132197, + "balance_loss_clip": 1.19926095, + "balance_loss_mlp": 1.08444166, + "epoch": 0.004410655214439093, + "flos": 32849213644800.0, + "grad_norm": 2.872854259406842, + "language_loss": 1.04939139, + "learning_rate": 2.8951119223009308e-06, + "loss": 1.07619131, + "num_input_tokens_seen": 4549530, + "router_z_loss_clip": 3.48632812, + "router_z_loss_mlp": 0.47753906, + "step": 152, + "time_per_iteration": 2.8210978507995605 + }, + { + "auxiliary_loss_clip": 0.01234546, + "auxiliary_loss_mlp": 0.01018789, + "balance_loss_clip": 1.10688829, + "balance_loss_mlp": 0.99962026, + "epoch": 0.004439672682955139, + "flos": 54744167272320.0, + "grad_norm": 0.9300872511715466, + "language_loss": 0.5660271, + "learning_rate": 2.8988907558981293e-06, + "loss": 0.58856046, + "num_input_tokens_seen": 4605815, + "router_z_loss_clip": 1.28125, + "router_z_loss_mlp": 0.19140625, + "step": 153, + "time_per_iteration": 3.016728162765503 + }, + { + "auxiliary_loss_clip": 0.01528153, + "auxiliary_loss_mlp": 0.01118331, + "balance_loss_clip": 1.20240128, + "balance_loss_mlp": 1.06897795, + "epoch": 0.004468690151471185, + "flos": 19495041304320.0, + "grad_norm": 2.5911925609494397, + "language_loss": 0.95649284, + "learning_rate": 2.902644971512172e-06, + "loss": 0.98295772, + "num_input_tokens_seen": 4623535, + "router_z_loss_clip": 3.26171875, + "router_z_loss_mlp": 0.4934082, + "step": 154, + "time_per_iteration": 2.8436169624328613 + }, + { + "auxiliary_loss_clip": 0.01502296, + "auxiliary_loss_mlp": 0.01124644, + "balance_loss_clip": 1.19009042, + "balance_loss_mlp": 1.07614994, + "epoch": 0.004497707619987233, + "flos": 66706582314240.0, + "grad_norm": 4.04536724112167, + "language_loss": 1.05098438, + "learning_rate": 2.9063748878258113e-06, + "loss": 1.07725382, + "num_input_tokens_seen": 4644870, + "router_z_loss_clip": 3.11914062, + "router_z_loss_mlp": 0.484375, + "step": 155, + "time_per_iteration": 3.062267303466797 + }, + { + "auxiliary_loss_clip": 0.01518821, + "auxiliary_loss_mlp": 0.01128273, + "balance_loss_clip": 1.19390988, + "balance_loss_mlp": 1.07701349, + "epoch": 0.004526725088503279, + "flos": 30620812763520.0, + "grad_norm": 3.5369443607486164, + "language_loss": 1.08214903, + "learning_rate": 2.910080817373494e-06, + "loss": 1.10861993, + "num_input_tokens_seen": 4659890, + "router_z_loss_clip": 3.24609375, + "router_z_loss_mlp": 0.51269531, + "step": 156, + "time_per_iteration": 2.799048662185669 + }, + { + "auxiliary_loss_clip": 0.01522607, + "auxiliary_loss_mlp": 0.01134906, + "balance_loss_clip": 1.1930114, + "balance_loss_mlp": 1.08383679, + "epoch": 0.004555742557019325, + "flos": 36203664965760.0, + "grad_norm": 5.430184062056783, + "language_loss": 1.22823465, + "learning_rate": 2.9137630666985104e-06, + "loss": 1.25480974, + "num_input_tokens_seen": 4673420, + "router_z_loss_clip": 3.29882812, + "router_z_loss_mlp": 0.51098633, + "step": 157, + "time_per_iteration": 2.799896717071533 + }, + { + "auxiliary_loss_clip": 0.01519756, + "auxiliary_loss_mlp": 0.0112357, + "balance_loss_clip": 1.20132875, + "balance_loss_mlp": 1.07214355, + "epoch": 0.004584760025535373, + "flos": 44850128065920.0, + "grad_norm": 3.700334647428312, + "language_loss": 1.06591094, + "learning_rate": 2.91742193650515e-06, + "loss": 1.09234416, + "num_input_tokens_seen": 4689870, + "router_z_loss_clip": 3.18359375, + "router_z_loss_mlp": 0.51464844, + "step": 158, + "time_per_iteration": 2.857442855834961 + }, + { + "auxiliary_loss_clip": 0.01524896, + "auxiliary_loss_mlp": 0.01136277, + "balance_loss_clip": 1.19276071, + "balance_loss_mlp": 1.08768737, + "epoch": 0.004613777494051419, + "flos": 32589866471040.0, + "grad_norm": 2.5498419506644514, + "language_loss": 1.00522184, + "learning_rate": 2.9210577218060625e-06, + "loss": 1.03183365, + "num_input_tokens_seen": 4706790, + "router_z_loss_clip": 3.32226562, + "router_z_loss_mlp": 0.48632812, + "step": 159, + "time_per_iteration": 2.7813827991485596 + }, + { + "auxiliary_loss_clip": 0.01529299, + "auxiliary_loss_mlp": 0.01136741, + "balance_loss_clip": 1.19547129, + "balance_loss_mlp": 1.08796024, + "epoch": 0.004642794962567465, + "flos": 22563213229440.0, + "grad_norm": 2.470214891143402, + "language_loss": 1.09933054, + "learning_rate": 2.9246707120649977e-06, + "loss": 1.12599087, + "num_input_tokens_seen": 4723555, + "router_z_loss_clip": 3.33398438, + "router_z_loss_mlp": 0.48754883, + "step": 160, + "time_per_iteration": 2.6386754512786865 + }, + { + "auxiliary_loss_clip": 0.01515938, + "auxiliary_loss_mlp": 0.0112764, + "balance_loss_clip": 1.19636345, + "balance_loss_mlp": 1.08009982, + "epoch": 0.004671812431083513, + "flos": 11430438618240.0, + "grad_norm": 3.3843064381531374, + "language_loss": 0.94909358, + "learning_rate": 2.928261191335098e-06, + "loss": 0.97552937, + "num_input_tokens_seen": 4737145, + "router_z_loss_clip": 3.19921875, + "router_z_loss_mlp": 0.47509766, + "step": 161, + "time_per_iteration": 2.6627426147460938 + }, + { + "auxiliary_loss_clip": 0.012369, + "auxiliary_loss_mlp": 0.01063791, + "balance_loss_clip": 1.10308194, + "balance_loss_mlp": 1.04357362, + "epoch": 0.004700829899599559, + "flos": 52359302747520.0, + "grad_norm": 0.945336340877093, + "language_loss": 0.62203461, + "learning_rate": 2.9318294383929054e-06, + "loss": 0.64504153, + "num_input_tokens_seen": 4785045, + "router_z_loss_clip": 1.34375, + "router_z_loss_mlp": 0.20214844, + "step": 162, + "time_per_iteration": 2.8997507095336914 + }, + { + "auxiliary_loss_clip": 0.01501741, + "auxiliary_loss_mlp": 0.01151508, + "balance_loss_clip": 1.18569469, + "balance_loss_mlp": 1.10167897, + "epoch": 0.004729847368115605, + "flos": 20586833147520.0, + "grad_norm": 3.2420914203346447, + "language_loss": 1.18228436, + "learning_rate": 2.935375726868257e-06, + "loss": 1.20881677, + "num_input_tokens_seen": 4800570, + "router_z_loss_clip": 3.16210938, + "router_z_loss_mlp": 0.49780273, + "step": 163, + "time_per_iteration": 2.749565839767456 + }, + { + "auxiliary_loss_clip": 0.01491056, + "auxiliary_loss_mlp": 0.01127386, + "balance_loss_clip": 1.1854341, + "balance_loss_mlp": 1.07901144, + "epoch": 0.004758864836631653, + "flos": 22009146963840.0, + "grad_norm": 2.8306640230890237, + "language_loss": 1.01969004, + "learning_rate": 2.9389003253701925e-06, + "loss": 1.0458746, + "num_input_tokens_seen": 4813210, + "router_z_loss_clip": 3.05078125, + "router_z_loss_mlp": 0.4831543, + "step": 164, + "time_per_iteration": 2.737354040145874 + }, + { + "auxiliary_loss_clip": 0.01492725, + "auxiliary_loss_mlp": 0.01117724, + "balance_loss_clip": 1.17919075, + "balance_loss_mlp": 1.07116055, + "epoch": 0.004787882305147699, + "flos": 13734323930880.0, + "grad_norm": 2.651030053867427, + "language_loss": 0.96716678, + "learning_rate": 2.9424034976090475e-06, + "loss": 0.99327123, + "num_input_tokens_seen": 4828395, + "router_z_loss_clip": 3.13671875, + "router_z_loss_mlp": 0.46508789, + "step": 165, + "time_per_iteration": 2.746168613433838 + }, + { + "auxiliary_loss_clip": 0.01487314, + "auxiliary_loss_mlp": 0.01120973, + "balance_loss_clip": 1.1819464, + "balance_loss_mlp": 1.07576919, + "epoch": 0.004816899773663745, + "flos": 23946060977280.0, + "grad_norm": 4.26992238471409, + "language_loss": 1.14641142, + "learning_rate": 2.9458855025148492e-06, + "loss": 1.17249429, + "num_input_tokens_seen": 4841540, + "router_z_loss_clip": 3.0546875, + "router_z_loss_mlp": 0.45239258, + "step": 166, + "time_per_iteration": 2.72293758392334 + }, + { + "auxiliary_loss_clip": 0.01510393, + "auxiliary_loss_mlp": 0.01152368, + "balance_loss_clip": 1.19320798, + "balance_loss_mlp": 1.09743667, + "epoch": 0.004845917242179793, + "flos": 16761808725120.0, + "grad_norm": 3.1210293703975047, + "language_loss": 1.07856297, + "learning_rate": 2.9493465943521642e-06, + "loss": 1.10519052, + "num_input_tokens_seen": 4854770, + "router_z_loss_clip": 3.17382812, + "router_z_loss_mlp": 0.55004883, + "step": 167, + "time_per_iteration": 2.67757248878479 + }, + { + "auxiliary_loss_clip": 0.01513039, + "auxiliary_loss_mlp": 0.01145267, + "balance_loss_clip": 1.18132734, + "balance_loss_mlp": 1.09703517, + "epoch": 0.004874934710695839, + "flos": 32483930284800.0, + "grad_norm": 2.0453477212822206, + "language_loss": 0.83368117, + "learning_rate": 2.9527870228315107e-06, + "loss": 0.86026424, + "num_input_tokens_seen": 4876160, + "router_z_loss_clip": 3.31640625, + "router_z_loss_mlp": 0.48217773, + "step": 168, + "time_per_iteration": 2.775744915008545 + }, + { + "auxiliary_loss_clip": 0.01467992, + "auxiliary_loss_mlp": 0.01099659, + "balance_loss_clip": 1.17766976, + "balance_loss_mlp": 1.05528951, + "epoch": 0.004903952179211885, + "flos": 34343564186880.0, + "grad_norm": 2.4449285715502813, + "language_loss": 0.98892581, + "learning_rate": 2.956207033217471e-06, + "loss": 1.0146023, + "num_input_tokens_seen": 4900820, + "router_z_loss_clip": 2.90234375, + "router_z_loss_mlp": 0.44360352, + "step": 169, + "time_per_iteration": 2.825899124145508 + }, + { + "auxiliary_loss_clip": 0.01220246, + "auxiliary_loss_mlp": 0.01025458, + "balance_loss_clip": 1.09756279, + "balance_loss_mlp": 1.00743341, + "epoch": 0.004932969647727933, + "flos": 67957354730880.0, + "grad_norm": 0.7130425458858981, + "language_loss": 0.61172789, + "learning_rate": 2.9596068664336094e-06, + "loss": 0.6341849, + "num_input_tokens_seen": 4967690, + "router_z_loss_clip": 1.2265625, + "router_z_loss_mlp": 0.18066406, + "step": 170, + "time_per_iteration": 3.239210605621338 + }, + { + "auxiliary_loss_clip": 0.01491485, + "auxiliary_loss_mlp": 0.01129087, + "balance_loss_clip": 1.1812768, + "balance_loss_mlp": 1.08209419, + "epoch": 0.004961987116243979, + "flos": 16355910061440.0, + "grad_norm": 3.146873647143297, + "language_loss": 0.84884369, + "learning_rate": 2.9629867591643182e-06, + "loss": 0.87504935, + "num_input_tokens_seen": 4981430, + "router_z_loss_clip": 3.10351562, + "router_z_loss_mlp": 0.47021484, + "step": 171, + "time_per_iteration": 2.687760829925537 + }, + { + "auxiliary_loss_clip": 0.01501557, + "auxiliary_loss_mlp": 0.01133129, + "balance_loss_clip": 1.18719149, + "balance_loss_mlp": 1.08117723, + "epoch": 0.004991004584760025, + "flos": 40544510647680.0, + "grad_norm": 2.7194163350711116, + "language_loss": 1.02666664, + "learning_rate": 2.9663469439536884e-06, + "loss": 1.05301356, + "num_input_tokens_seen": 4997690, + "router_z_loss_clip": 3.13671875, + "router_z_loss_mlp": 0.51928711, + "step": 172, + "time_per_iteration": 2.816828489303589 + }, + { + "auxiliary_loss_clip": 0.01478509, + "auxiliary_loss_mlp": 0.01115018, + "balance_loss_clip": 1.18108511, + "balance_loss_mlp": 1.06492591, + "epoch": 0.005020022053276072, + "flos": 74732329463040.0, + "grad_norm": 2.8568986434193904, + "language_loss": 0.90388501, + "learning_rate": 2.969687649301524e-06, + "loss": 0.9298203, + "num_input_tokens_seen": 5019750, + "router_z_loss_clip": 2.97265625, + "router_z_loss_mlp": 0.50097656, + "step": 173, + "time_per_iteration": 3.14328670501709 + }, + { + "auxiliary_loss_clip": 0.01217197, + "auxiliary_loss_mlp": 0.01018577, + "balance_loss_clip": 1.09741521, + "balance_loss_mlp": 1.00093353, + "epoch": 0.005049039521792119, + "flos": 73022625907200.0, + "grad_norm": 0.7032498026949618, + "language_loss": 0.56570637, + "learning_rate": 2.9730090997565743e-06, + "loss": 0.58806413, + "num_input_tokens_seen": 5082885, + "router_z_loss_clip": 1.1953125, + "router_z_loss_mlp": 0.17675781, + "step": 174, + "time_per_iteration": 3.21012282371521 + }, + { + "auxiliary_loss_clip": 0.01489208, + "auxiliary_loss_mlp": 0.01126999, + "balance_loss_clip": 1.17543221, + "balance_loss_mlp": 1.07859993, + "epoch": 0.005078056990308165, + "flos": 28073633656320.0, + "grad_norm": 2.2770919745416616, + "language_loss": 0.9726789, + "learning_rate": 2.976311516007114e-06, + "loss": 0.99884093, + "num_input_tokens_seen": 5100670, + "router_z_loss_clip": 3.13671875, + "router_z_loss_mlp": 0.484375, + "step": 175, + "time_per_iteration": 2.910430908203125 + }, + { + "auxiliary_loss_clip": 0.01214494, + "auxiliary_loss_mlp": 0.01015152, + "balance_loss_clip": 1.09481871, + "balance_loss_mlp": 0.99827212, + "epoch": 0.005107074458824212, + "flos": 74758510477440.0, + "grad_norm": 0.755964981896061, + "language_loss": 0.53526759, + "learning_rate": 2.9795951149689236e-06, + "loss": 0.55756402, + "num_input_tokens_seen": 5153055, + "router_z_loss_clip": 1.1953125, + "router_z_loss_mlp": 0.16894531, + "step": 176, + "time_per_iteration": 3.168511390686035 + }, + { + "auxiliary_loss_clip": 0.01493863, + "auxiliary_loss_mlp": 0.01104293, + "balance_loss_clip": 1.17975664, + "balance_loss_mlp": 1.05618, + "epoch": 0.005136091927340259, + "flos": 12824742804480.0, + "grad_norm": 2.625428044746406, + "language_loss": 0.82910562, + "learning_rate": 2.982860109870794e-06, + "loss": 0.85508716, + "num_input_tokens_seen": 5167845, + "router_z_loss_clip": 3.14453125, + "router_z_loss_mlp": 0.48120117, + "step": 177, + "time_per_iteration": 2.7017292976379395 + }, + { + "auxiliary_loss_clip": 0.01211323, + "auxiliary_loss_mlp": 0.01015148, + "balance_loss_clip": 1.09435511, + "balance_loss_mlp": 0.99807739, + "epoch": 0.005165109395856305, + "flos": 63469418855040.0, + "grad_norm": 0.7375585105413205, + "language_loss": 0.58468819, + "learning_rate": 2.986106710337607e-06, + "loss": 0.60695291, + "num_input_tokens_seen": 5226630, + "router_z_loss_clip": 1.171875, + "router_z_loss_mlp": 0.17089844, + "step": 178, + "time_per_iteration": 3.1215672492980957 + }, + { + "auxiliary_loss_clip": 0.0120752, + "auxiliary_loss_mlp": 0.01016253, + "balance_loss_clip": 1.09169579, + "balance_loss_mlp": 0.99956417, + "epoch": 0.005194126864372352, + "flos": 72661472611200.0, + "grad_norm": 0.7568956044842681, + "language_loss": 0.65133262, + "learning_rate": 2.9893351224711024e-06, + "loss": 0.67357039, + "num_input_tokens_seen": 5290860, + "router_z_loss_clip": 1.15625, + "router_z_loss_mlp": 0.16699219, + "step": 179, + "time_per_iteration": 3.171156644821167 + }, + { + "auxiliary_loss_clip": 0.01476488, + "auxiliary_loss_mlp": 0.01117045, + "balance_loss_clip": 1.17412806, + "balance_loss_mlp": 1.06714451, + "epoch": 0.005223144332888399, + "flos": 19166458665600.0, + "grad_norm": 2.524673742289009, + "language_loss": 0.98210764, + "learning_rate": 2.9925455489283856e-06, + "loss": 1.00804305, + "num_input_tokens_seen": 5305520, + "router_z_loss_clip": 3.02734375, + "router_z_loss_mlp": 0.49853516, + "step": 180, + "time_per_iteration": 2.710643768310547 + }, + { + "auxiliary_loss_clip": 0.01492492, + "auxiliary_loss_mlp": 0.01113196, + "balance_loss_clip": 1.17944789, + "balance_loss_mlp": 1.06100595, + "epoch": 0.005252161801404445, + "flos": 22305015290880.0, + "grad_norm": 1.9647053011640354, + "language_loss": 0.88965815, + "learning_rate": 2.9957381889982656e-06, + "loss": 0.91571504, + "num_input_tokens_seen": 5323950, + "router_z_loss_clip": 3.13085938, + "router_z_loss_mlp": 0.5222168, + "step": 181, + "time_per_iteration": 2.6576290130615234 + }, + { + "auxiliary_loss_clip": 0.01200327, + "auxiliary_loss_mlp": 0.01017697, + "balance_loss_clip": 1.08587742, + "balance_loss_mlp": 1.00091195, + "epoch": 0.005281179269920492, + "flos": 55143493747200.0, + "grad_norm": 0.7237948618139682, + "language_loss": 0.54708469, + "learning_rate": 2.998913238675487e-06, + "loss": 0.56926495, + "num_input_tokens_seen": 5385430, + "router_z_loss_clip": 1.140625, + "router_z_loss_mlp": 0.16796875, + "step": 182, + "time_per_iteration": 3.1502084732055664 + }, + { + "auxiliary_loss_clip": 0.01476607, + "auxiliary_loss_mlp": 0.01128436, + "balance_loss_clip": 1.17367435, + "balance_loss_mlp": 1.07722354, + "epoch": 0.005310196738436539, + "flos": 10334839933440.0, + "grad_norm": 3.2812155633860414, + "language_loss": 0.89665508, + "learning_rate": 3.0020708907329318e-06, + "loss": 0.92270553, + "num_input_tokens_seen": 5396130, + "router_z_loss_clip": 3.03125, + "router_z_loss_mlp": 0.51269531, + "step": 183, + "time_per_iteration": 2.6400351524353027 + }, + { + "auxiliary_loss_clip": 0.01480462, + "auxiliary_loss_mlp": 0.01119438, + "balance_loss_clip": 1.17637873, + "balance_loss_mlp": 1.07273149, + "epoch": 0.005339214206952585, + "flos": 34853854752000.0, + "grad_norm": 2.784607714867241, + "language_loss": 1.10914826, + "learning_rate": 3.00521133479185e-06, + "loss": 1.13514721, + "num_input_tokens_seen": 5410825, + "router_z_loss_clip": 3.04101562, + "router_z_loss_mlp": 0.46728516, + "step": 184, + "time_per_iteration": 2.765960931777954 + }, + { + "auxiliary_loss_clip": 0.01488564, + "auxiliary_loss_mlp": 0.01126315, + "balance_loss_clip": 1.17235482, + "balance_loss_mlp": 1.07457817, + "epoch": 0.005368231675468632, + "flos": 21609461352960.0, + "grad_norm": 3.9711385702387343, + "language_loss": 1.01097798, + "learning_rate": 3.008334757390187e-06, + "loss": 1.03712678, + "num_input_tokens_seen": 5424460, + "router_z_loss_clip": 3.16015625, + "router_z_loss_mlp": 0.5168457, + "step": 185, + "time_per_iteration": 2.701887845993042 + }, + { + "auxiliary_loss_clip": 0.01485069, + "auxiliary_loss_mlp": 0.01131184, + "balance_loss_clip": 1.18008208, + "balance_loss_mlp": 1.08259451, + "epoch": 0.005397249143984679, + "flos": 21317687176320.0, + "grad_norm": 6.4055770916365775, + "language_loss": 0.97144723, + "learning_rate": 3.011441342049076e-06, + "loss": 0.99760979, + "num_input_tokens_seen": 5438690, + "router_z_loss_clip": 3.05078125, + "router_z_loss_mlp": 0.48632812, + "step": 186, + "time_per_iteration": 2.719025135040283 + }, + { + "auxiliary_loss_clip": 0.01467212, + "auxiliary_loss_mlp": 0.01128446, + "balance_loss_clip": 1.16682696, + "balance_loss_mlp": 1.0791409, + "epoch": 0.005426266612500725, + "flos": 74731647104640.0, + "grad_norm": 2.084154355133478, + "language_loss": 0.89162582, + "learning_rate": 3.0145312693375354e-06, + "loss": 0.91758239, + "num_input_tokens_seen": 5469720, + "router_z_loss_clip": 3.00195312, + "router_z_loss_mlp": 0.49267578, + "step": 187, + "time_per_iteration": 3.20255708694458 + }, + { + "auxiliary_loss_clip": 0.01467352, + "auxiliary_loss_mlp": 0.01136762, + "balance_loss_clip": 1.16705596, + "balance_loss_mlp": 1.09096193, + "epoch": 0.005455284081016772, + "flos": 30401254880640.0, + "grad_norm": 4.158535080308381, + "language_loss": 1.0464232, + "learning_rate": 3.017604716935455e-06, + "loss": 1.07246435, + "num_input_tokens_seen": 5485260, + "router_z_loss_clip": 3.00390625, + "router_z_loss_mlp": 0.45825195, + "step": 188, + "time_per_iteration": 2.719259738922119 + }, + { + "auxiliary_loss_clip": 0.01475163, + "auxiliary_loss_mlp": 0.01106815, + "balance_loss_clip": 1.1701293, + "balance_loss_mlp": 1.05908394, + "epoch": 0.005484301549532819, + "flos": 24929797731840.0, + "grad_norm": 4.630405795977377, + "language_loss": 1.18258286, + "learning_rate": 3.020661859694898e-06, + "loss": 1.20840263, + "num_input_tokens_seen": 5498850, + "router_z_loss_clip": 3.05273438, + "router_z_loss_mlp": 0.47680664, + "step": 189, + "time_per_iteration": 2.7359111309051514 + }, + { + "auxiliary_loss_clip": 0.01195765, + "auxiliary_loss_mlp": 0.01022077, + "balance_loss_clip": 1.07990766, + "balance_loss_mlp": 1.00567424, + "epoch": 0.005513319018048865, + "flos": 71532477256320.0, + "grad_norm": 0.782594708064869, + "language_loss": 0.52287996, + "learning_rate": 3.023702869699798e-06, + "loss": 0.54505837, + "num_input_tokens_seen": 5564350, + "router_z_loss_clip": 1.15625, + "router_z_loss_mlp": 0.1640625, + "step": 190, + "time_per_iteration": 3.225161552429199 + }, + { + "auxiliary_loss_clip": 0.01194932, + "auxiliary_loss_mlp": 0.01022654, + "balance_loss_clip": 1.07987452, + "balance_loss_mlp": 1.00691891, + "epoch": 0.005542336486564912, + "flos": 72594822925440.0, + "grad_norm": 0.7070836520261836, + "language_loss": 0.54708391, + "learning_rate": 3.0267279163240784e-06, + "loss": 0.56925976, + "num_input_tokens_seen": 5633550, + "router_z_loss_clip": 1.1484375, + "router_z_loss_mlp": 0.15722656, + "step": 191, + "time_per_iteration": 3.264491558074951 + }, + { + "auxiliary_loss_clip": 0.01460238, + "auxiliary_loss_mlp": 0.01101112, + "balance_loss_clip": 1.15997553, + "balance_loss_mlp": 1.05519295, + "epoch": 0.005571353955080959, + "flos": 11430079482240.0, + "grad_norm": 3.1176592734010176, + "language_loss": 1.02594399, + "learning_rate": 3.0297371662882626e-06, + "loss": 1.05155742, + "num_input_tokens_seen": 5645160, + "router_z_loss_clip": 3.0, + "router_z_loss_mlp": 0.45922852, + "step": 192, + "time_per_iteration": 2.614100456237793 + }, + { + "auxiliary_loss_clip": 0.01474198, + "auxiliary_loss_mlp": 0.01138797, + "balance_loss_clip": 1.17109954, + "balance_loss_mlp": 1.08987367, + "epoch": 0.005600371423597005, + "flos": 17157902976000.0, + "grad_norm": 3.0502387307347147, + "language_loss": 0.87105185, + "learning_rate": 3.032730783714606e-06, + "loss": 0.89718175, + "num_input_tokens_seen": 5659580, + "router_z_loss_clip": 3.03710938, + "router_z_loss_mlp": 0.48901367, + "step": 193, + "time_per_iteration": 2.7088050842285156 + }, + { + "auxiliary_loss_clip": 0.01450904, + "auxiliary_loss_mlp": 0.0111846, + "balance_loss_clip": 1.16430819, + "balance_loss_mlp": 1.07060945, + "epoch": 0.005629388892113052, + "flos": 26968338299520.0, + "grad_norm": 2.951656113485853, + "language_loss": 1.09297013, + "learning_rate": 3.0357089301808127e-06, + "loss": 1.11866367, + "num_input_tokens_seen": 5673350, + "router_z_loss_clip": 2.86523438, + "router_z_loss_mlp": 0.47875977, + "step": 194, + "time_per_iteration": 2.8061459064483643 + }, + { + "auxiliary_loss_clip": 0.01442112, + "auxiliary_loss_mlp": 0.01107181, + "balance_loss_clip": 1.15696275, + "balance_loss_mlp": 1.06197739, + "epoch": 0.005658406360629099, + "flos": 12195658984320.0, + "grad_norm": 3.6272253192111417, + "language_loss": 1.2313776, + "learning_rate": 3.038671764772362e-06, + "loss": 1.25687051, + "num_input_tokens_seen": 5684200, + "router_z_loss_clip": 2.84960938, + "router_z_loss_mlp": 0.45239258, + "step": 195, + "time_per_iteration": 2.5831639766693115 + }, + { + "auxiliary_loss_clip": 0.01190484, + "auxiliary_loss_mlp": 0.01014726, + "balance_loss_clip": 1.07580304, + "balance_loss_mlp": 1.00042057, + "epoch": 0.005687423829145145, + "flos": 56461200180480.0, + "grad_norm": 0.8452184454922833, + "language_loss": 0.5078733, + "learning_rate": 3.0416194441335026e-06, + "loss": 0.52992541, + "num_input_tokens_seen": 5738855, + "router_z_loss_clip": 1.1484375, + "router_z_loss_mlp": 0.14257812, + "step": 196, + "time_per_iteration": 3.136431932449341 + }, + { + "auxiliary_loss_clip": 0.01436756, + "auxiliary_loss_mlp": 0.01114187, + "balance_loss_clip": 1.16064072, + "balance_loss_mlp": 1.07129574, + "epoch": 0.005716441297661192, + "flos": 34597560234240.0, + "grad_norm": 3.322084075784323, + "language_loss": 1.21802282, + "learning_rate": 3.0445521225169482e-06, + "loss": 1.24353242, + "num_input_tokens_seen": 5753825, + "router_z_loss_clip": 2.75976562, + "router_z_loss_mlp": 0.42871094, + "step": 197, + "time_per_iteration": 2.8464717864990234 + }, + { + "auxiliary_loss_clip": 0.01458324, + "auxiliary_loss_mlp": 0.01136446, + "balance_loss_clip": 1.16325748, + "balance_loss_mlp": 1.08566296, + "epoch": 0.005745458766177239, + "flos": 15116130184320.0, + "grad_norm": 3.3371247200632563, + "language_loss": 1.17094326, + "learning_rate": 3.0474699518323115e-06, + "loss": 1.19689107, + "num_input_tokens_seen": 5767625, + "router_z_loss_clip": 2.94921875, + "router_z_loss_mlp": 0.5078125, + "step": 198, + "time_per_iteration": 2.6567773818969727 + }, + { + "auxiliary_loss_clip": 0.01183799, + "auxiliary_loss_mlp": 0.01014231, + "balance_loss_clip": 1.07320631, + "balance_loss_mlp": 1.00078416, + "epoch": 0.005774476234693285, + "flos": 62034283883520.0, + "grad_norm": 0.7664952048683493, + "language_loss": 0.6051119, + "learning_rate": 3.0503730816933237e-06, + "loss": 0.62709224, + "num_input_tokens_seen": 5822440, + "router_z_loss_clip": 1.109375, + "router_z_loss_mlp": 0.13476562, + "step": 199, + "time_per_iteration": 3.0378611087799072 + }, + { + "auxiliary_loss_clip": 0.01446633, + "auxiliary_loss_mlp": 0.01120426, + "balance_loss_clip": 1.16260695, + "balance_loss_mlp": 1.07491159, + "epoch": 0.005803493703209332, + "flos": 24201709050240.0, + "grad_norm": 2.322342639692562, + "language_loss": 0.93008149, + "learning_rate": 3.0532616594638653e-06, + "loss": 0.95575213, + "num_input_tokens_seen": 5836670, + "router_z_loss_clip": 2.8359375, + "router_z_loss_mlp": 0.45507812, + "step": 200, + "time_per_iteration": 2.650373935699463 + }, + { + "auxiliary_loss_clip": 0.01446047, + "auxiliary_loss_mlp": 0.01125015, + "balance_loss_clip": 1.16349292, + "balance_loss_mlp": 1.07706904, + "epoch": 0.005832511171725379, + "flos": 39232981353600.0, + "grad_norm": 2.457240292781804, + "language_loss": 0.97544354, + "learning_rate": 3.056135830302854e-06, + "loss": 1.00115418, + "num_input_tokens_seen": 5856935, + "router_z_loss_clip": 2.82617188, + "router_z_loss_mlp": 0.47973633, + "step": 201, + "time_per_iteration": 2.780911684036255 + }, + { + "auxiliary_loss_clip": 0.0144104, + "auxiliary_loss_mlp": 0.01116386, + "balance_loss_clip": 1.15994883, + "balance_loss_mlp": 1.06998944, + "epoch": 0.005861528640241425, + "flos": 32774627053440.0, + "grad_norm": 2.584287576787482, + "language_loss": 1.14339864, + "learning_rate": 3.058995737208014e-06, + "loss": 1.16897285, + "num_input_tokens_seen": 5872555, + "router_z_loss_clip": 2.8125, + "router_z_loss_mlp": 0.46411133, + "step": 202, + "time_per_iteration": 2.7464871406555176 + }, + { + "auxiliary_loss_clip": 0.01457042, + "auxiliary_loss_mlp": 0.01123071, + "balance_loss_clip": 1.16331339, + "balance_loss_mlp": 1.0771991, + "epoch": 0.005890546108757472, + "flos": 34097792353920.0, + "grad_norm": 2.7759626827490385, + "language_loss": 1.07982159, + "learning_rate": 3.0618415210585666e-06, + "loss": 1.10562277, + "num_input_tokens_seen": 5891840, + "router_z_loss_clip": 2.93554688, + "router_z_loss_mlp": 0.45898438, + "step": 203, + "time_per_iteration": 2.778264045715332 + }, + { + "auxiliary_loss_clip": 0.01441884, + "auxiliary_loss_mlp": 0.01117234, + "balance_loss_clip": 1.16247129, + "balance_loss_mlp": 1.0680964, + "epoch": 0.005919563577273519, + "flos": 20150841864960.0, + "grad_norm": 3.965347330002513, + "language_loss": 1.14547694, + "learning_rate": 3.064673320656874e-06, + "loss": 1.17106819, + "num_input_tokens_seen": 5906690, + "router_z_loss_clip": 2.796875, + "router_z_loss_mlp": 0.49145508, + "step": 204, + "time_per_iteration": 2.5965094566345215 + }, + { + "auxiliary_loss_clip": 0.01450043, + "auxiliary_loss_mlp": 0.01115115, + "balance_loss_clip": 1.16657376, + "balance_loss_mlp": 1.07093644, + "epoch": 0.005948581045789565, + "flos": 28725304152960.0, + "grad_norm": 3.1592633757830852, + "language_loss": 0.99710613, + "learning_rate": 3.0674912727690606e-06, + "loss": 1.02275765, + "num_input_tokens_seen": 5923535, + "router_z_loss_clip": 2.83984375, + "router_z_loss_mlp": 0.44189453, + "step": 205, + "time_per_iteration": 2.737786293029785 + }, + { + "auxiliary_loss_clip": 0.01179803, + "auxiliary_loss_mlp": 0.01015562, + "balance_loss_clip": 1.07270861, + "balance_loss_mlp": 1.00259233, + "epoch": 0.005977598514305612, + "flos": 67441784866560.0, + "grad_norm": 0.6361262820025216, + "language_loss": 0.52634943, + "learning_rate": 3.070295512164649e-06, + "loss": 0.54830307, + "num_input_tokens_seen": 5987620, + "router_z_loss_clip": 1.0703125, + "router_z_loss_mlp": 0.12988281, + "step": 206, + "time_per_iteration": 3.19638991355896 + }, + { + "auxiliary_loss_clip": 0.0145203, + "auxiliary_loss_mlp": 0.01117378, + "balance_loss_clip": 1.15862942, + "balance_loss_mlp": 1.07183981, + "epoch": 0.006006615982821659, + "flos": 34714773290880.0, + "grad_norm": 2.663316801681807, + "language_loss": 0.98382056, + "learning_rate": 3.073086171655237e-06, + "loss": 1.00951469, + "num_input_tokens_seen": 6003980, + "router_z_loss_clip": 2.9375, + "router_z_loss_mlp": 0.45556641, + "step": 207, + "time_per_iteration": 2.837791919708252 + }, + { + "auxiliary_loss_clip": 0.01177487, + "auxiliary_loss_mlp": 0.01012314, + "balance_loss_clip": 1.07052922, + "balance_loss_mlp": 0.99905825, + "epoch": 0.006035633451337705, + "flos": 57447774109440.0, + "grad_norm": 0.7923291671639728, + "language_loss": 0.4927094, + "learning_rate": 3.0758633821322388e-06, + "loss": 0.51460743, + "num_input_tokens_seen": 6058280, + "router_z_loss_clip": 1.0703125, + "router_z_loss_mlp": 0.1328125, + "step": 208, + "time_per_iteration": 3.0421054363250732 + }, + { + "auxiliary_loss_clip": 0.01440211, + "auxiliary_loss_mlp": 0.01107383, + "balance_loss_clip": 1.15042257, + "balance_loss_mlp": 1.06215549, + "epoch": 0.006064650919853752, + "flos": 23734511827200.0, + "grad_norm": 3.0562323504104083, + "language_loss": 1.16293836, + "learning_rate": 3.078627272603724e-06, + "loss": 1.18841422, + "num_input_tokens_seen": 6076740, + "router_z_loss_clip": 2.89648438, + "router_z_loss_mlp": 0.4519043, + "step": 209, + "time_per_iteration": 2.679053544998169 + }, + { + "auxiliary_loss_clip": 0.01174691, + "auxiliary_loss_mlp": 0.01014682, + "balance_loss_clip": 1.06848526, + "balance_loss_mlp": 1.00133061, + "epoch": 0.006093668388369799, + "flos": 62191286231040.0, + "grad_norm": 0.7932439224687058, + "language_loss": 0.58162439, + "learning_rate": 3.081377970230378e-06, + "loss": 0.60351813, + "num_input_tokens_seen": 6136335, + "router_z_loss_clip": 1.0625, + "router_z_loss_mlp": 0.13378906, + "step": 210, + "time_per_iteration": 3.2400519847869873 + }, + { + "auxiliary_loss_clip": 0.01432245, + "auxiliary_loss_mlp": 0.01128967, + "balance_loss_clip": 1.15152764, + "balance_loss_mlp": 1.07949555, + "epoch": 0.006122685856885845, + "flos": 40107980661120.0, + "grad_norm": 2.851566269271029, + "language_loss": 1.01624107, + "learning_rate": 3.0841156003606057e-06, + "loss": 1.04185319, + "num_input_tokens_seen": 6153170, + "router_z_loss_clip": 2.8046875, + "router_z_loss_mlp": 0.49487305, + "step": 211, + "time_per_iteration": 2.8820645809173584 + }, + { + "auxiliary_loss_clip": 0.01441271, + "auxiliary_loss_mlp": 0.0111752, + "balance_loss_clip": 1.1526593, + "balance_loss_mlp": 1.07310307, + "epoch": 0.006151703325401892, + "flos": 74732939994240.0, + "grad_norm": 2.8021308184533114, + "language_loss": 0.9184016, + "learning_rate": 3.0868402865648067e-06, + "loss": 0.94398952, + "num_input_tokens_seen": 6175540, + "router_z_loss_clip": 2.88671875, + "router_z_loss_mlp": 0.44384766, + "step": 212, + "time_per_iteration": 3.1315813064575195 + }, + { + "auxiliary_loss_clip": 0.01432173, + "auxiliary_loss_mlp": 0.01105379, + "balance_loss_clip": 1.1583451, + "balance_loss_mlp": 1.06456161, + "epoch": 0.006180720793917939, + "flos": 74736387699840.0, + "grad_norm": 2.341378679648656, + "language_loss": 1.00449681, + "learning_rate": 3.0895521506688455e-06, + "loss": 1.0298723, + "num_input_tokens_seen": 6209680, + "router_z_loss_clip": 2.7421875, + "router_z_loss_mlp": 0.40844727, + "step": 213, + "time_per_iteration": 3.1421010494232178 + }, + { + "auxiliary_loss_clip": 0.01167769, + "auxiliary_loss_mlp": 0.01013442, + "balance_loss_clip": 1.0628823, + "balance_loss_mlp": 1.00128233, + "epoch": 0.006209738262433985, + "flos": 69663685386240.0, + "grad_norm": 0.7038181275235352, + "language_loss": 0.56834507, + "learning_rate": 3.092251312786734e-06, + "loss": 0.59015715, + "num_input_tokens_seen": 6273105, + "router_z_loss_clip": 1.046875, + "router_z_loss_mlp": 0.12158203, + "step": 214, + "time_per_iteration": 3.2137203216552734 + }, + { + "auxiliary_loss_clip": 0.01417221, + "auxiliary_loss_mlp": 0.01095679, + "balance_loss_clip": 1.14570904, + "balance_loss_mlp": 1.05402768, + "epoch": 0.006238755730950032, + "flos": 14239873900800.0, + "grad_norm": 5.697274891999129, + "language_loss": 1.03179944, + "learning_rate": 3.094937891352556e-06, + "loss": 1.0569284, + "num_input_tokens_seen": 6287415, + "router_z_loss_clip": 2.71679688, + "router_z_loss_mlp": 0.41674805, + "step": 215, + "time_per_iteration": 2.6453168392181396 + }, + { + "auxiliary_loss_clip": 0.01165571, + "auxiliary_loss_mlp": 0.01013341, + "balance_loss_clip": 1.06126928, + "balance_loss_mlp": 1.00118124, + "epoch": 0.006267773199466079, + "flos": 74778083389440.0, + "grad_norm": 0.8267158758128312, + "language_loss": 0.57180744, + "learning_rate": 3.09761200315165e-06, + "loss": 0.59359652, + "num_input_tokens_seen": 6350530, + "router_z_loss_clip": 1.0390625, + "router_z_loss_mlp": 0.12158203, + "step": 216, + "time_per_iteration": 3.201176404953003 + }, + { + "auxiliary_loss_clip": 0.01424338, + "auxiliary_loss_mlp": 0.01111735, + "balance_loss_clip": 1.15389585, + "balance_loss_mlp": 1.06409919, + "epoch": 0.006296790667982125, + "flos": 23799868623360.0, + "grad_norm": 2.4381919432122263, + "language_loss": 0.86565584, + "learning_rate": 3.100273763351068e-06, + "loss": 0.8910166, + "num_input_tokens_seen": 6366080, + "router_z_loss_clip": 2.70507812, + "router_z_loss_mlp": 0.47583008, + "step": 217, + "time_per_iteration": 2.6250226497650146 + }, + { + "auxiliary_loss_clip": 0.01163283, + "auxiliary_loss_mlp": 0.01011437, + "balance_loss_clip": 1.06022787, + "balance_loss_mlp": 0.99913484, + "epoch": 0.006325808136498172, + "flos": 74773019571840.0, + "grad_norm": 0.7130051909858278, + "language_loss": 0.59348536, + "learning_rate": 3.102923285529342e-06, + "loss": 0.61523259, + "num_input_tokens_seen": 6426405, + "router_z_loss_clip": 1.03125, + "router_z_loss_mlp": 0.12304688, + "step": 218, + "time_per_iteration": 3.1381947994232178 + }, + { + "auxiliary_loss_clip": 0.01162802, + "auxiliary_loss_mlp": 0.01014516, + "balance_loss_clip": 1.05983567, + "balance_loss_mlp": 1.00230885, + "epoch": 0.006354825605014219, + "flos": 74782967639040.0, + "grad_norm": 0.686831405986638, + "language_loss": 0.5391469, + "learning_rate": 3.105560681705561e-06, + "loss": 0.56092012, + "num_input_tokens_seen": 6492415, + "router_z_loss_clip": 1.03125, + "router_z_loss_mlp": 0.12207031, + "step": 219, + "time_per_iteration": 3.212846040725708 + }, + { + "auxiliary_loss_clip": 0.01429138, + "auxiliary_loss_mlp": 0.01105628, + "balance_loss_clip": 1.14673972, + "balance_loss_mlp": 1.05939853, + "epoch": 0.006383843073530265, + "flos": 23908498329600.0, + "grad_norm": 2.6286560083223445, + "language_loss": 0.94658816, + "learning_rate": 3.1081860623677917e-06, + "loss": 0.97193575, + "num_input_tokens_seen": 6506585, + "router_z_loss_clip": 2.828125, + "router_z_loss_mlp": 0.46264648, + "step": 220, + "time_per_iteration": 7.304349184036255 + }, + { + "auxiliary_loss_clip": 0.01159631, + "auxiliary_loss_mlp": 0.01011163, + "balance_loss_clip": 1.05728424, + "balance_loss_mlp": 0.99909937, + "epoch": 0.006412860542046312, + "flos": 70562600173440.0, + "grad_norm": 0.7876519390339636, + "language_loss": 0.54014361, + "learning_rate": 3.11079953650085e-06, + "loss": 0.56185162, + "num_input_tokens_seen": 6560050, + "router_z_loss_clip": 1.0234375, + "router_z_loss_mlp": 0.12060547, + "step": 221, + "time_per_iteration": 5.748308181762695 + }, + { + "auxiliary_loss_clip": 0.01428075, + "auxiliary_loss_mlp": 0.01107792, + "balance_loss_clip": 1.1509043, + "balance_loss_mlp": 1.0654254, + "epoch": 0.006441878010562358, + "flos": 10406122473600.0, + "grad_norm": 5.628989619864438, + "language_loss": 0.99670196, + "learning_rate": 3.1134012116134513e-06, + "loss": 1.02206063, + "num_input_tokens_seen": 6569480, + "router_z_loss_clip": 2.77148438, + "router_z_loss_mlp": 0.42358398, + "step": 222, + "time_per_iteration": 2.5462896823883057 + }, + { + "auxiliary_loss_clip": 0.01435631, + "auxiliary_loss_mlp": 0.01137444, + "balance_loss_clip": 1.15723181, + "balance_loss_mlp": 1.09092867, + "epoch": 0.006470895479078405, + "flos": 15077992919040.0, + "grad_norm": 2.64810809753763, + "language_loss": 0.93379498, + "learning_rate": 3.1159911937647437e-06, + "loss": 0.95952576, + "num_input_tokens_seen": 6584705, + "router_z_loss_clip": 2.78320312, + "router_z_loss_mlp": 0.46557617, + "step": 223, + "time_per_iteration": 2.6138038635253906 + }, + { + "auxiliary_loss_clip": 0.01421008, + "auxiliary_loss_mlp": 0.0111781, + "balance_loss_clip": 1.14949727, + "balance_loss_mlp": 1.07503819, + "epoch": 0.006499912947594452, + "flos": 24528460095360.0, + "grad_norm": 2.5230820346490455, + "language_loss": 1.0177145, + "learning_rate": 3.1185695875902545e-06, + "loss": 1.04310262, + "num_input_tokens_seen": 6601500, + "router_z_loss_clip": 2.71289062, + "router_z_loss_mlp": 0.42773438, + "step": 224, + "time_per_iteration": 2.791318655014038 + }, + { + "auxiliary_loss_clip": 0.01440395, + "auxiliary_loss_mlp": 0.01125887, + "balance_loss_clip": 1.15029728, + "balance_loss_mlp": 1.08099246, + "epoch": 0.006528930416110498, + "flos": 16283801508480.0, + "grad_norm": 2.620863874831698, + "language_loss": 0.93547672, + "learning_rate": 3.1211364963272528e-06, + "loss": 0.9611395, + "num_input_tokens_seen": 6615935, + "router_z_loss_clip": 2.90234375, + "router_z_loss_mlp": 0.44921875, + "step": 225, + "time_per_iteration": 2.572956085205078 + }, + { + "auxiliary_loss_clip": 0.0142366, + "auxiliary_loss_mlp": 0.01114972, + "balance_loss_clip": 1.14806259, + "balance_loss_mlp": 1.06833732, + "epoch": 0.006557947884626545, + "flos": 74732006240640.0, + "grad_norm": 3.312572615310535, + "language_loss": 1.03657711, + "learning_rate": 3.123692021839555e-06, + "loss": 1.06196356, + "num_input_tokens_seen": 6639770, + "router_z_loss_clip": 2.75195312, + "router_z_loss_mlp": 0.46679688, + "step": 226, + "time_per_iteration": 3.0675222873687744 + }, + { + "auxiliary_loss_clip": 0.01161893, + "auxiliary_loss_mlp": 0.01015478, + "balance_loss_clip": 1.05586219, + "balance_loss_mlp": 1.0022223, + "epoch": 0.006586965353142592, + "flos": 63941105278080.0, + "grad_norm": 0.7496300718589692, + "language_loss": 0.56033921, + "learning_rate": 3.126236264641778e-06, + "loss": 0.58211291, + "num_input_tokens_seen": 6699950, + "router_z_loss_clip": 1.0625, + "router_z_loss_mlp": 0.1328125, + "step": 227, + "time_per_iteration": 3.0768556594848633 + }, + { + "auxiliary_loss_clip": 0.01410813, + "auxiliary_loss_mlp": 0.0109834, + "balance_loss_clip": 1.13896155, + "balance_loss_mlp": 1.05382681, + "epoch": 0.006615982821658638, + "flos": 31316797664640.0, + "grad_norm": 2.8038159331852737, + "language_loss": 1.05246568, + "learning_rate": 3.1287693239230624e-06, + "loss": 1.07755721, + "num_input_tokens_seen": 6716875, + "router_z_loss_clip": 2.71679688, + "router_z_loss_mlp": 0.4453125, + "step": 228, + "time_per_iteration": 2.756025552749634 + }, + { + "auxiliary_loss_clip": 0.0142266, + "auxiliary_loss_mlp": 0.01110576, + "balance_loss_clip": 1.15269494, + "balance_loss_mlp": 1.06899631, + "epoch": 0.006645000290174685, + "flos": 14823565908480.0, + "grad_norm": 2.578451657473668, + "language_loss": 0.87998539, + "learning_rate": 3.1312912975702777e-06, + "loss": 0.90531772, + "num_input_tokens_seen": 6729765, + "router_z_loss_clip": 2.70117188, + "router_z_loss_mlp": 0.41601562, + "step": 229, + "time_per_iteration": 2.6023483276367188 + }, + { + "auxiliary_loss_clip": 0.01166457, + "auxiliary_loss_mlp": 0.01015938, + "balance_loss_clip": 1.05521119, + "balance_loss_mlp": 1.00029778, + "epoch": 0.006674017758690732, + "flos": 53025230943360.0, + "grad_norm": 0.7301836461177259, + "language_loss": 0.51846266, + "learning_rate": 3.133802282190717e-06, + "loss": 0.54028654, + "num_input_tokens_seen": 6789420, + "router_z_loss_clip": 1.109375, + "router_z_loss_mlp": 0.15625, + "step": 230, + "time_per_iteration": 3.079266309738159 + }, + { + "auxiliary_loss_clip": 0.01423351, + "auxiliary_loss_mlp": 0.01120778, + "balance_loss_clip": 1.14055729, + "balance_loss_mlp": 1.07566953, + "epoch": 0.006703035227206778, + "flos": 14494480479360.0, + "grad_norm": 3.809652608478116, + "language_loss": 1.13165402, + "learning_rate": 3.1363023731343034e-06, + "loss": 1.15709531, + "num_input_tokens_seen": 6801630, + "router_z_loss_clip": 2.82617188, + "router_z_loss_mlp": 0.45141602, + "step": 231, + "time_per_iteration": 2.6054136753082275 + }, + { + "auxiliary_loss_clip": 0.01418816, + "auxiliary_loss_mlp": 0.01117883, + "balance_loss_clip": 1.14392614, + "balance_loss_mlp": 1.07089055, + "epoch": 0.006732052695722825, + "flos": 11359156078080.0, + "grad_norm": 2.646533657254097, + "language_loss": 0.87907994, + "learning_rate": 3.1387916645153185e-06, + "loss": 0.90444696, + "num_input_tokens_seen": 6812580, + "router_z_loss_clip": 2.74609375, + "router_z_loss_mlp": 0.4699707, + "step": 232, + "time_per_iteration": 2.613832950592041 + }, + { + "auxiliary_loss_clip": 0.01412652, + "auxiliary_loss_mlp": 0.0112774, + "balance_loss_clip": 1.13555241, + "balance_loss_mlp": 1.07931733, + "epoch": 0.006761070164238872, + "flos": 18219889509120.0, + "grad_norm": 5.007109906319075, + "language_loss": 1.08371234, + "learning_rate": 3.1412702492336547e-06, + "loss": 1.10911632, + "num_input_tokens_seen": 6827190, + "router_z_loss_clip": 2.77148438, + "router_z_loss_mlp": 0.484375, + "step": 233, + "time_per_iteration": 2.595782995223999 + }, + { + "auxiliary_loss_clip": 0.01399925, + "auxiliary_loss_mlp": 0.01127459, + "balance_loss_clip": 1.14076853, + "balance_loss_mlp": 1.08549738, + "epoch": 0.006790087632754918, + "flos": 23360465548800.0, + "grad_norm": 2.64206643475225, + "language_loss": 1.05125356, + "learning_rate": 3.1437382189956262e-06, + "loss": 1.07652736, + "num_input_tokens_seen": 6840370, + "router_z_loss_clip": 2.59179688, + "router_z_loss_mlp": 0.41967773, + "step": 234, + "time_per_iteration": 2.587033748626709 + }, + { + "auxiliary_loss_clip": 0.01409837, + "auxiliary_loss_mlp": 0.01105185, + "balance_loss_clip": 1.13781929, + "balance_loss_mlp": 1.058002, + "epoch": 0.006819105101270965, + "flos": 29711339377920.0, + "grad_norm": 2.3107554175543954, + "language_loss": 0.94590509, + "learning_rate": 3.146195664334322e-06, + "loss": 0.97105527, + "num_input_tokens_seen": 6857495, + "router_z_loss_clip": 2.72265625, + "router_z_loss_mlp": 0.47167969, + "step": 235, + "time_per_iteration": 2.672821521759033 + }, + { + "auxiliary_loss_clip": 0.0140248, + "auxiliary_loss_mlp": 0.01107505, + "balance_loss_clip": 1.13438451, + "balance_loss_mlp": 1.06079936, + "epoch": 0.006848122569787012, + "flos": 31787550334080.0, + "grad_norm": 2.8070468431191147, + "language_loss": 1.02912104, + "learning_rate": 3.1486426746295384e-06, + "loss": 1.05422091, + "num_input_tokens_seen": 6874455, + "router_z_loss_clip": 2.68164062, + "router_z_loss_mlp": 0.4675293, + "step": 236, + "time_per_iteration": 2.6547598838806152 + }, + { + "auxiliary_loss_clip": 0.01409413, + "auxiliary_loss_mlp": 0.01095199, + "balance_loss_clip": 1.13965273, + "balance_loss_mlp": 1.04839706, + "epoch": 0.006877140038303058, + "flos": 23031523774080.0, + "grad_norm": 3.5090887762235985, + "language_loss": 1.11253846, + "learning_rate": 3.151079338127282e-06, + "loss": 1.13758445, + "num_input_tokens_seen": 6887410, + "router_z_loss_clip": 2.70117188, + "router_z_loss_mlp": 0.4675293, + "step": 237, + "time_per_iteration": 2.6683616638183594 + }, + { + "auxiliary_loss_clip": 0.01400418, + "auxiliary_loss_mlp": 0.01093966, + "balance_loss_clip": 1.14454687, + "balance_loss_mlp": 1.05295813, + "epoch": 0.006906157506819105, + "flos": 37954310025600.0, + "grad_norm": 2.5979003595075576, + "language_loss": 1.10200548, + "learning_rate": 3.1535057419588662e-06, + "loss": 1.12694931, + "num_input_tokens_seen": 6907445, + "router_z_loss_clip": 2.56054688, + "router_z_loss_mlp": 0.40991211, + "step": 238, + "time_per_iteration": 2.7418127059936523 + }, + { + "auxiliary_loss_clip": 0.01395941, + "auxiliary_loss_mlp": 0.01105451, + "balance_loss_clip": 1.13521957, + "balance_loss_mlp": 1.06489587, + "epoch": 0.006935174975335152, + "flos": 40107729265920.0, + "grad_norm": 3.5994897592977173, + "language_loss": 0.89635921, + "learning_rate": 3.155921972159608e-06, + "loss": 0.92137313, + "num_input_tokens_seen": 6922595, + "router_z_loss_clip": 2.61132812, + "router_z_loss_mlp": 0.40551758, + "step": 239, + "time_per_iteration": 2.6510391235351562 + }, + { + "auxiliary_loss_clip": 0.01385516, + "auxiliary_loss_mlp": 0.01104395, + "balance_loss_clip": 1.13343942, + "balance_loss_mlp": 1.06591415, + "epoch": 0.006964192443851198, + "flos": 43428604348800.0, + "grad_norm": 2.5084552618332974, + "language_loss": 0.85538471, + "learning_rate": 3.1583281136871298e-06, + "loss": 0.88028383, + "num_input_tokens_seen": 6943745, + "router_z_loss_clip": 2.51953125, + "router_z_loss_mlp": 0.38500977, + "step": 240, + "time_per_iteration": 2.7657859325408936 + }, + { + "auxiliary_loss_clip": 0.01382978, + "auxiliary_loss_mlp": 0.01092493, + "balance_loss_clip": 1.13759851, + "balance_loss_mlp": 1.0551331, + "epoch": 0.006993209912367245, + "flos": 18947152177920.0, + "grad_norm": 2.9122272628242576, + "language_loss": 0.94236213, + "learning_rate": 3.1607242504392867e-06, + "loss": 0.96711689, + "num_input_tokens_seen": 6957530, + "router_z_loss_clip": 2.453125, + "router_z_loss_mlp": 0.37353516, + "step": 241, + "time_per_iteration": 2.588900566101074 + }, + { + "auxiliary_loss_clip": 0.01155915, + "auxiliary_loss_mlp": 0.0103367, + "balance_loss_clip": 1.04722595, + "balance_loss_mlp": 1.02117717, + "epoch": 0.007022227380883292, + "flos": 74785804813440.0, + "grad_norm": 0.7621012319277067, + "language_loss": 0.62361574, + "learning_rate": 3.1631104652717176e-06, + "loss": 0.64551157, + "num_input_tokens_seen": 7022510, + "router_z_loss_clip": 1.0859375, + "router_z_loss_mlp": 0.125, + "step": 242, + "time_per_iteration": 3.259478807449341 + }, + { + "auxiliary_loss_clip": 0.0140536, + "auxiliary_loss_mlp": 0.0109921, + "balance_loss_clip": 1.13358855, + "balance_loss_mlp": 1.05362451, + "epoch": 0.007051244849399338, + "flos": 28505530788480.0, + "grad_norm": 3.580113208884119, + "language_loss": 1.17286849, + "learning_rate": 3.1654868400150375e-06, + "loss": 1.19791424, + "num_input_tokens_seen": 7036600, + "router_z_loss_clip": 2.72265625, + "router_z_loss_mlp": 0.45605469, + "step": 243, + "time_per_iteration": 2.61830997467041 + }, + { + "auxiliary_loss_clip": 0.01153293, + "auxiliary_loss_mlp": 0.01019826, + "balance_loss_clip": 1.04815888, + "balance_loss_mlp": 1.00771463, + "epoch": 0.007080262317915385, + "flos": 72258087899520.0, + "grad_norm": 0.7078239816031288, + "language_loss": 0.5857079, + "learning_rate": 3.167853455491676e-06, + "loss": 0.60743904, + "num_input_tokens_seen": 7099645, + "router_z_loss_clip": 1.046875, + "router_z_loss_mlp": 0.12109375, + "step": 244, + "time_per_iteration": 3.137233257293701 + }, + { + "auxiliary_loss_clip": 0.01406307, + "auxiliary_loss_mlp": 0.01101884, + "balance_loss_clip": 1.13707054, + "balance_loss_mlp": 1.0604465, + "epoch": 0.007109279786431432, + "flos": 25659538439040.0, + "grad_norm": 4.4696304190581255, + "language_loss": 0.86883956, + "learning_rate": 3.1702103915323702e-06, + "loss": 0.89392138, + "num_input_tokens_seen": 7119230, + "router_z_loss_clip": 2.69140625, + "router_z_loss_mlp": 0.41430664, + "step": 245, + "time_per_iteration": 2.627448081970215 + }, + { + "auxiliary_loss_clip": 0.01151291, + "auxiliary_loss_mlp": 0.0101342, + "balance_loss_clip": 1.04735434, + "balance_loss_mlp": 1.00149882, + "epoch": 0.007138297254947478, + "flos": 51784301831040.0, + "grad_norm": 0.7989613592550834, + "language_loss": 0.54240084, + "learning_rate": 3.172557726992324e-06, + "loss": 0.56404799, + "num_input_tokens_seen": 7171815, + "router_z_loss_clip": 1.0390625, + "router_z_loss_mlp": 0.11914062, + "step": 246, + "time_per_iteration": 2.932478427886963 + }, + { + "auxiliary_loss_clip": 0.01389688, + "auxiliary_loss_mlp": 0.0111732, + "balance_loss_clip": 1.12829554, + "balance_loss_mlp": 1.07435703, + "epoch": 0.007167314723463525, + "flos": 11868261494400.0, + "grad_norm": 3.4231271089665793, + "language_loss": 1.13667965, + "learning_rate": 3.1748955397670386e-06, + "loss": 1.16174972, + "num_input_tokens_seen": 7183960, + "router_z_loss_clip": 2.61328125, + "router_z_loss_mlp": 0.4296875, + "step": 247, + "time_per_iteration": 2.5296471118927 + }, + { + "auxiliary_loss_clip": 0.01391383, + "auxiliary_loss_mlp": 0.01102608, + "balance_loss_clip": 1.12848699, + "balance_loss_mlp": 1.06272078, + "epoch": 0.007196332191979572, + "flos": 12778920028800.0, + "grad_norm": 3.681500394160386, + "language_loss": 1.07676804, + "learning_rate": 3.17722390680782e-06, + "loss": 1.10170794, + "num_input_tokens_seen": 7195335, + "router_z_loss_clip": 2.62695312, + "router_z_loss_mlp": 0.39892578, + "step": 248, + "time_per_iteration": 2.6279375553131104 + }, + { + "auxiliary_loss_clip": 0.01409589, + "auxiliary_loss_mlp": 0.01117944, + "balance_loss_clip": 1.14075053, + "balance_loss_mlp": 1.07574391, + "epoch": 0.007225349660495618, + "flos": 31104458415360.0, + "grad_norm": 2.8268698212571786, + "language_loss": 0.99522716, + "learning_rate": 3.1795429041369805e-06, + "loss": 1.02050233, + "num_input_tokens_seen": 7216575, + "router_z_loss_clip": 2.68945312, + "router_z_loss_mlp": 0.42236328, + "step": 249, + "time_per_iteration": 2.6878273487091064 + }, + { + "auxiliary_loss_clip": 0.01396664, + "auxiliary_loss_mlp": 0.01096417, + "balance_loss_clip": 1.13692987, + "balance_loss_mlp": 1.05719686, + "epoch": 0.007254367129011665, + "flos": 14204897032320.0, + "grad_norm": 3.159664553747116, + "language_loss": 1.10278201, + "learning_rate": 3.1818526068627325e-06, + "loss": 1.12771285, + "num_input_tokens_seen": 7235890, + "router_z_loss_clip": 2.59570312, + "router_z_loss_mlp": 0.39233398, + "step": 250, + "time_per_iteration": 2.7364470958709717 + }, + { + "auxiliary_loss_clip": 0.0139863, + "auxiliary_loss_mlp": 0.01114792, + "balance_loss_clip": 1.13472915, + "balance_loss_mlp": 1.07416534, + "epoch": 0.007283384597527712, + "flos": 27123724535040.0, + "grad_norm": 2.4000663743064212, + "language_loss": 0.83176672, + "learning_rate": 3.1841530891937837e-06, + "loss": 0.85690093, + "num_input_tokens_seen": 7257230, + "router_z_loss_clip": 2.63671875, + "router_z_loss_mlp": 0.40673828, + "step": 251, + "time_per_iteration": 2.6564102172851562 + }, + { + "auxiliary_loss_clip": 0.01407628, + "auxiliary_loss_mlp": 0.01150641, + "balance_loss_clip": 1.13468444, + "balance_loss_mlp": 1.10429204, + "epoch": 0.007312402066043758, + "flos": 28769043939840.0, + "grad_norm": 2.5703134523818787, + "language_loss": 0.94181454, + "learning_rate": 3.186444424453642e-06, + "loss": 0.96739721, + "num_input_tokens_seen": 7276215, + "router_z_loss_clip": 2.73046875, + "router_z_loss_mlp": 0.46337891, + "step": 252, + "time_per_iteration": 2.7245566844940186 + }, + { + "auxiliary_loss_clip": 0.01396367, + "auxiliary_loss_mlp": 0.01109501, + "balance_loss_clip": 1.13265991, + "balance_loss_mlp": 1.06804061, + "epoch": 0.007341419534559805, + "flos": 17303017921920.0, + "grad_norm": 2.5224105032692425, + "language_loss": 0.9315784, + "learning_rate": 3.188726685094643e-06, + "loss": 0.95663702, + "num_input_tokens_seen": 7289725, + "router_z_loss_clip": 2.63476562, + "router_z_loss_mlp": 0.41455078, + "step": 253, + "time_per_iteration": 2.609553813934326 + }, + { + "auxiliary_loss_clip": 0.01409459, + "auxiliary_loss_mlp": 0.01135682, + "balance_loss_clip": 1.13720047, + "balance_loss_mlp": 1.09174109, + "epoch": 0.007370437003075852, + "flos": 36750799906560.0, + "grad_norm": 2.2782237352626944, + "language_loss": 1.12718809, + "learning_rate": 3.1909999427116915e-06, + "loss": 1.15263951, + "num_input_tokens_seen": 7311345, + "router_z_loss_clip": 2.72265625, + "router_z_loss_mlp": 0.43920898, + "step": 254, + "time_per_iteration": 2.738415479660034 + }, + { + "auxiliary_loss_clip": 0.01374387, + "auxiliary_loss_mlp": 0.01112475, + "balance_loss_clip": 1.12630379, + "balance_loss_mlp": 1.07048976, + "epoch": 0.007399454471591898, + "flos": 17449102535040.0, + "grad_norm": 3.3555788113979212, + "language_loss": 1.09659386, + "learning_rate": 3.193264268055741e-06, + "loss": 1.12146258, + "num_input_tokens_seen": 7324090, + "router_z_loss_clip": 2.48242188, + "router_z_loss_mlp": 0.41967773, + "step": 255, + "time_per_iteration": 2.6069235801696777 + }, + { + "auxiliary_loss_clip": 0.01145898, + "auxiliary_loss_mlp": 0.01035777, + "balance_loss_clip": 1.04621625, + "balance_loss_mlp": 1.02509582, + "epoch": 0.007428471940107945, + "flos": 67924353110400.0, + "grad_norm": 0.7452968491232065, + "language_loss": 0.59055316, + "learning_rate": 3.1955197310470064e-06, + "loss": 0.61236989, + "num_input_tokens_seen": 7388060, + "router_z_loss_clip": 0.99609375, + "router_z_loss_mlp": 0.10693359, + "step": 256, + "time_per_iteration": 3.1716012954711914 + }, + { + "auxiliary_loss_clip": 0.01398596, + "auxiliary_loss_mlp": 0.01113615, + "balance_loss_clip": 1.13809848, + "balance_loss_mlp": 1.07003236, + "epoch": 0.007457489408623992, + "flos": 26388668615040.0, + "grad_norm": 3.5115508219203857, + "language_loss": 1.02457476, + "learning_rate": 3.197766400787917e-06, + "loss": 1.04969692, + "num_input_tokens_seen": 7402750, + "router_z_loss_clip": 2.6015625, + "router_z_loss_mlp": 0.43554688, + "step": 257, + "time_per_iteration": 2.6257574558258057 + }, + { + "auxiliary_loss_clip": 0.013785, + "auxiliary_loss_mlp": 0.0110209, + "balance_loss_clip": 1.13331962, + "balance_loss_mlp": 1.0636332, + "epoch": 0.007486506877140038, + "flos": 28906581116160.0, + "grad_norm": 4.7883166734053075, + "language_loss": 1.0880686, + "learning_rate": 3.2000043455758205e-06, + "loss": 1.11287451, + "num_input_tokens_seen": 7417125, + "router_z_loss_clip": 2.44921875, + "router_z_loss_mlp": 0.38427734, + "step": 258, + "time_per_iteration": 2.622642755508423 + }, + { + "auxiliary_loss_clip": 0.01389441, + "auxiliary_loss_mlp": 0.01112878, + "balance_loss_clip": 1.13198137, + "balance_loss_mlp": 1.06753063, + "epoch": 0.007515524345656085, + "flos": 24856863166080.0, + "grad_norm": 6.154255529815866, + "language_loss": 1.03048658, + "learning_rate": 3.2022336329154436e-06, + "loss": 1.05550981, + "num_input_tokens_seen": 7430290, + "router_z_loss_clip": 2.57421875, + "router_z_loss_mlp": 0.45361328, + "step": 259, + "time_per_iteration": 2.630505084991455 + }, + { + "auxiliary_loss_clip": 0.01375065, + "auxiliary_loss_mlp": 0.01103997, + "balance_loss_clip": 1.12666762, + "balance_loss_mlp": 1.06143928, + "epoch": 0.007544541814172132, + "flos": 22884218098560.0, + "grad_norm": 4.008195944307945, + "language_loss": 0.98943162, + "learning_rate": 3.204454329531106e-06, + "loss": 1.01422238, + "num_input_tokens_seen": 7441050, + "router_z_loss_clip": 2.484375, + "router_z_loss_mlp": 0.42529297, + "step": 260, + "time_per_iteration": 2.5706348419189453 + }, + { + "auxiliary_loss_clip": 0.01381911, + "auxiliary_loss_mlp": 0.01100804, + "balance_loss_clip": 1.12796044, + "balance_loss_mlp": 1.06108379, + "epoch": 0.007573559282688178, + "flos": 53774290189440.0, + "grad_norm": 2.2903399377098665, + "language_loss": 0.76678491, + "learning_rate": 3.2066665013787064e-06, + "loss": 0.79161203, + "num_input_tokens_seen": 7464410, + "router_z_loss_clip": 2.54101562, + "router_z_loss_mlp": 0.39697266, + "step": 261, + "time_per_iteration": 2.8131892681121826 + }, + { + "auxiliary_loss_clip": 0.01387476, + "auxiliary_loss_mlp": 0.01102486, + "balance_loss_clip": 1.12926483, + "balance_loss_mlp": 1.06004715, + "epoch": 0.007602576751204225, + "flos": 23836533431040.0, + "grad_norm": 2.6713645430403163, + "language_loss": 1.00384855, + "learning_rate": 3.2088702136574735e-06, + "loss": 1.02874827, + "num_input_tokens_seen": 7478955, + "router_z_loss_clip": 2.58203125, + "router_z_loss_mlp": 0.42431641, + "step": 262, + "time_per_iteration": 2.5876076221466064 + }, + { + "auxiliary_loss_clip": 0.01138658, + "auxiliary_loss_mlp": 0.01025906, + "balance_loss_clip": 1.04174924, + "balance_loss_mlp": 1.01574922, + "epoch": 0.007631594219720272, + "flos": 63515923989120.0, + "grad_norm": 0.7320784877054474, + "language_loss": 0.58065045, + "learning_rate": 3.2110655308215014e-06, + "loss": 0.60229605, + "num_input_tokens_seen": 7542680, + "router_z_loss_clip": 0.96875, + "router_z_loss_mlp": 0.1015625, + "step": 263, + "time_per_iteration": 3.256380796432495 + }, + { + "auxiliary_loss_clip": 0.01369242, + "auxiliary_loss_mlp": 0.0110856, + "balance_loss_clip": 1.11667013, + "balance_loss_mlp": 1.06697965, + "epoch": 0.007660611688236318, + "flos": 27447710232960.0, + "grad_norm": 4.153082607519633, + "language_loss": 1.09041286, + "learning_rate": 3.2132525165910553e-06, + "loss": 1.11519086, + "num_input_tokens_seen": 7558260, + "router_z_loss_clip": 2.52636719, + "router_z_loss_mlp": 0.41577148, + "step": 264, + "time_per_iteration": 2.65812087059021 + }, + { + "auxiliary_loss_clip": 0.01136264, + "auxiliary_loss_mlp": 0.0101422, + "balance_loss_clip": 1.03978395, + "balance_loss_mlp": 1.00415897, + "epoch": 0.007689629156752365, + "flos": 57002050241280.0, + "grad_norm": 0.7863625623193251, + "language_loss": 0.55799538, + "learning_rate": 3.2154312339636743e-06, + "loss": 0.5795002, + "num_input_tokens_seen": 7609175, + "router_z_loss_clip": 0.96484375, + "router_z_loss_mlp": 0.10058594, + "step": 265, + "time_per_iteration": 2.916006326675415 + }, + { + "auxiliary_loss_clip": 0.01373546, + "auxiliary_loss_mlp": 0.01112741, + "balance_loss_clip": 1.12349582, + "balance_loss_mlp": 1.07349765, + "epoch": 0.007718646625268412, + "flos": 21576100596480.0, + "grad_norm": 2.976107937127973, + "language_loss": 0.95896137, + "learning_rate": 3.2176017452250547e-06, + "loss": 0.98382425, + "num_input_tokens_seen": 7625380, + "router_z_loss_clip": 2.5, + "router_z_loss_mlp": 0.39233398, + "step": 266, + "time_per_iteration": 2.7254693508148193 + }, + { + "auxiliary_loss_clip": 0.01386075, + "auxiliary_loss_mlp": 0.01111986, + "balance_loss_clip": 1.12468922, + "balance_loss_mlp": 1.06573224, + "epoch": 0.007747664093784458, + "flos": 21827510864640.0, + "grad_norm": 3.5092258112801864, + "language_loss": 0.98589259, + "learning_rate": 3.219764111959739e-06, + "loss": 1.0108732, + "num_input_tokens_seen": 7639375, + "router_z_loss_clip": 2.61328125, + "router_z_loss_mlp": 0.46264648, + "step": 267, + "time_per_iteration": 2.625382423400879 + }, + { + "auxiliary_loss_clip": 0.01134167, + "auxiliary_loss_mlp": 0.01014656, + "balance_loss_clip": 1.03868163, + "balance_loss_mlp": 1.00473821, + "epoch": 0.007776681562300505, + "flos": 60252001920000.0, + "grad_norm": 0.7189945277953248, + "language_loss": 0.60332328, + "learning_rate": 3.2219183950615983e-06, + "loss": 0.62481153, + "num_input_tokens_seen": 7700035, + "router_z_loss_clip": 0.953125, + "router_z_loss_mlp": 0.09912109, + "step": 268, + "time_per_iteration": 3.049086809158325 + }, + { + "auxiliary_loss_clip": 0.01391866, + "auxiliary_loss_mlp": 0.01117145, + "balance_loss_clip": 1.12310624, + "balance_loss_mlp": 1.07322824, + "epoch": 0.007805699030816552, + "flos": 13874662368000.0, + "grad_norm": 4.58156322280313, + "language_loss": 1.09850562, + "learning_rate": 3.2240646547441223e-06, + "loss": 1.12359571, + "num_input_tokens_seen": 7712075, + "router_z_loss_clip": 2.6875, + "router_z_loss_mlp": 0.43896484, + "step": 269, + "time_per_iteration": 2.5337719917297363 + }, + { + "auxiliary_loss_clip": 0.01384355, + "auxiliary_loss_mlp": 0.01101904, + "balance_loss_clip": 1.12981582, + "balance_loss_mlp": 1.06094408, + "epoch": 0.007834716499332598, + "flos": 36529697738880.0, + "grad_norm": 3.0096780436481514, + "language_loss": 0.86742485, + "learning_rate": 3.2262029505505177e-06, + "loss": 0.89228743, + "num_input_tokens_seen": 7725860, + "router_z_loss_clip": 2.54296875, + "router_z_loss_mlp": 0.40942383, + "step": 270, + "time_per_iteration": 2.7166645526885986 + }, + { + "auxiliary_loss_clip": 0.01133161, + "auxiliary_loss_mlp": 0.01016805, + "balance_loss_clip": 1.03809357, + "balance_loss_mlp": 1.00698233, + "epoch": 0.007863733967848644, + "flos": 68388318109440.0, + "grad_norm": 0.7348169206749412, + "language_loss": 0.55359006, + "learning_rate": 3.2283333413636183e-06, + "loss": 0.57508981, + "num_input_tokens_seen": 7790930, + "router_z_loss_clip": 0.953125, + "router_z_loss_mlp": 0.09814453, + "step": 271, + "time_per_iteration": 3.3052639961242676 + }, + { + "auxiliary_loss_clip": 0.01364696, + "auxiliary_loss_mlp": 0.01090162, + "balance_loss_clip": 1.11670411, + "balance_loss_mlp": 1.04929757, + "epoch": 0.00789275143636469, + "flos": 11870452224000.0, + "grad_norm": 2.8899322481638823, + "language_loss": 0.95322955, + "learning_rate": 3.230455885415618e-06, + "loss": 0.97777814, + "num_input_tokens_seen": 7803035, + "router_z_loss_clip": 2.48046875, + "router_z_loss_mlp": 0.40869141, + "step": 272, + "time_per_iteration": 2.5374269485473633 + }, + { + "auxiliary_loss_clip": 0.01130908, + "auxiliary_loss_mlp": 0.01010103, + "balance_loss_clip": 1.03631616, + "balance_loss_mlp": 1.00051868, + "epoch": 0.007921768904880739, + "flos": 74783326775040.0, + "grad_norm": 0.6981007206388958, + "language_loss": 0.56731492, + "learning_rate": 3.232570640297618e-06, + "loss": 0.58872497, + "num_input_tokens_seen": 7872035, + "router_z_loss_clip": 0.9453125, + "router_z_loss_mlp": 0.09570312, + "step": 273, + "time_per_iteration": 3.25598406791687 + }, + { + "auxiliary_loss_clip": 0.01398368, + "auxiliary_loss_mlp": 0.01102017, + "balance_loss_clip": 1.1298666, + "balance_loss_mlp": 1.05779076, + "epoch": 0.007950786373396785, + "flos": 74732939994240.0, + "grad_norm": 2.6394927318162305, + "language_loss": 0.94540346, + "learning_rate": 3.2346776629690067e-06, + "loss": 0.97040737, + "num_input_tokens_seen": 7898235, + "router_z_loss_clip": 2.68164062, + "router_z_loss_mlp": 0.44238281, + "step": 274, + "time_per_iteration": 3.0003819465637207 + }, + { + "auxiliary_loss_clip": 0.01381781, + "auxiliary_loss_mlp": 0.01109428, + "balance_loss_clip": 1.12262464, + "balance_loss_mlp": 1.06870592, + "epoch": 0.007979803841912832, + "flos": 19971145100160.0, + "grad_norm": 2.826958058179837, + "language_loss": 0.92638171, + "learning_rate": 3.236777009766659e-06, + "loss": 0.95129383, + "num_input_tokens_seen": 7913715, + "router_z_loss_clip": 2.59179688, + "router_z_loss_mlp": 0.40722656, + "step": 275, + "time_per_iteration": 2.6528563499450684 + }, + { + "auxiliary_loss_clip": 0.01392274, + "auxiliary_loss_mlp": 0.01101623, + "balance_loss_clip": 1.12943888, + "balance_loss_mlp": 1.05963743, + "epoch": 0.008008821310428878, + "flos": 35881869997440.0, + "grad_norm": 11.541142751821164, + "language_loss": 0.99352562, + "learning_rate": 3.2388687364139807e-06, + "loss": 1.01846457, + "num_input_tokens_seen": 7933165, + "router_z_loss_clip": 2.62890625, + "router_z_loss_mlp": 0.41992188, + "step": 276, + "time_per_iteration": 2.6919939517974854 + }, + { + "auxiliary_loss_clip": 0.01132587, + "auxiliary_loss_mlp": 0.01018833, + "balance_loss_clip": 1.04077935, + "balance_loss_mlp": 1.00924873, + "epoch": 0.008037838778944924, + "flos": 58204231557120.0, + "grad_norm": 0.6909369723281147, + "language_loss": 0.54634619, + "learning_rate": 3.2409528980297825e-06, + "loss": 0.56786036, + "num_input_tokens_seen": 7991740, + "router_z_loss_clip": 0.91796875, + "router_z_loss_mlp": 0.09570312, + "step": 277, + "time_per_iteration": 3.044372320175171 + }, + { + "auxiliary_loss_clip": 0.01133028, + "auxiliary_loss_mlp": 0.01019805, + "balance_loss_clip": 1.04196596, + "balance_loss_mlp": 1.01017332, + "epoch": 0.00806685624746097, + "flos": 70139896922880.0, + "grad_norm": 0.6708947438159981, + "language_loss": 0.47862309, + "learning_rate": 3.2430295491369894e-06, + "loss": 0.5001514, + "num_input_tokens_seen": 8051905, + "router_z_loss_clip": 0.91015625, + "router_z_loss_mlp": 0.09619141, + "step": 278, + "time_per_iteration": 3.1240580081939697 + }, + { + "auxiliary_loss_clip": 0.01375913, + "auxiliary_loss_mlp": 0.01104486, + "balance_loss_clip": 1.12392628, + "balance_loss_mlp": 1.06480169, + "epoch": 0.008095873715977019, + "flos": 37923678702720.0, + "grad_norm": 2.227831197687187, + "language_loss": 0.94556689, + "learning_rate": 3.245098743671207e-06, + "loss": 0.97037089, + "num_input_tokens_seen": 8068575, + "router_z_loss_clip": 2.51953125, + "router_z_loss_mlp": 0.39685059, + "step": 279, + "time_per_iteration": 2.7118828296661377 + }, + { + "auxiliary_loss_clip": 0.01375207, + "auxiliary_loss_mlp": 0.01096548, + "balance_loss_clip": 1.11848259, + "balance_loss_mlp": 1.05382347, + "epoch": 0.008124891184493065, + "flos": 25923662121600.0, + "grad_norm": 3.0572284147786375, + "language_loss": 1.03254366, + "learning_rate": 3.2471605349891217e-06, + "loss": 1.05726123, + "num_input_tokens_seen": 8082865, + "router_z_loss_clip": 2.56445312, + "router_z_loss_mlp": 0.42700195, + "step": 280, + "time_per_iteration": 2.608429431915283 + }, + { + "auxiliary_loss_clip": 0.01364715, + "auxiliary_loss_mlp": 0.01102839, + "balance_loss_clip": 1.11927199, + "balance_loss_mlp": 1.0644536, + "epoch": 0.008153908653009112, + "flos": 24894389900160.0, + "grad_norm": 2.818785526232021, + "language_loss": 1.0410316, + "learning_rate": 3.249214975876758e-06, + "loss": 1.06570709, + "num_input_tokens_seen": 8099145, + "router_z_loss_clip": 2.45703125, + "router_z_loss_mlp": 0.3840332, + "step": 281, + "time_per_iteration": 2.6804051399230957 + }, + { + "auxiliary_loss_clip": 0.01363541, + "auxiliary_loss_mlp": 0.01094768, + "balance_loss_clip": 1.118047, + "balance_loss_mlp": 1.05564344, + "epoch": 0.008182926121525158, + "flos": 21573191594880.0, + "grad_norm": 3.312697072491523, + "language_loss": 1.21375096, + "learning_rate": 3.2512621185575862e-06, + "loss": 1.23833406, + "num_input_tokens_seen": 8113860, + "router_z_loss_clip": 2.45507812, + "router_z_loss_mlp": 0.39160156, + "step": 282, + "time_per_iteration": 2.6806867122650146 + }, + { + "auxiliary_loss_clip": 0.01373733, + "auxiliary_loss_mlp": 0.01090061, + "balance_loss_clip": 1.11781573, + "balance_loss_mlp": 1.04919648, + "epoch": 0.008211943590041204, + "flos": 11902699658880.0, + "grad_norm": 3.7579985658052997, + "language_loss": 1.02002895, + "learning_rate": 3.25330201470049e-06, + "loss": 1.04466701, + "num_input_tokens_seen": 8125035, + "router_z_loss_clip": 2.55859375, + "router_z_loss_mlp": 0.40869141, + "step": 283, + "time_per_iteration": 2.610884666442871 + }, + { + "auxiliary_loss_clip": 0.01120847, + "auxiliary_loss_mlp": 0.01012562, + "balance_loss_clip": 1.03246832, + "balance_loss_mlp": 1.0033114, + "epoch": 0.00824096105855725, + "flos": 72655367299200.0, + "grad_norm": 0.7856468694048994, + "language_loss": 0.60141623, + "learning_rate": 3.2553347154275897e-06, + "loss": 0.6227504, + "num_input_tokens_seen": 8186125, + "router_z_loss_clip": 0.8828125, + "router_z_loss_mlp": 0.09228516, + "step": 284, + "time_per_iteration": 3.1514129638671875 + }, + { + "auxiliary_loss_clip": 0.01378441, + "auxiliary_loss_mlp": 0.01104046, + "balance_loss_clip": 1.1176157, + "balance_loss_mlp": 1.060606, + "epoch": 0.008269978527073299, + "flos": 23895716065920.0, + "grad_norm": 2.746281966471232, + "language_loss": 0.93739957, + "learning_rate": 3.25736027132193e-06, + "loss": 0.96222436, + "num_input_tokens_seen": 8207535, + "router_z_loss_clip": 2.60742188, + "router_z_loss_mlp": 0.43457031, + "step": 285, + "time_per_iteration": 2.749629497528076 + }, + { + "auxiliary_loss_clip": 0.0135822, + "auxiliary_loss_mlp": 0.01100139, + "balance_loss_clip": 1.12043452, + "balance_loss_mlp": 1.06261218, + "epoch": 0.008298995995589345, + "flos": 40776027759360.0, + "grad_norm": 1.9975004976292814, + "language_loss": 0.73205101, + "learning_rate": 3.259378732435032e-06, + "loss": 0.75663459, + "num_input_tokens_seen": 8229160, + "router_z_loss_clip": 2.37304688, + "router_z_loss_mlp": 0.375, + "step": 286, + "time_per_iteration": 2.7423455715179443 + }, + { + "auxiliary_loss_clip": 0.01371248, + "auxiliary_loss_mlp": 0.01115263, + "balance_loss_clip": 1.11881995, + "balance_loss_mlp": 1.07604301, + "epoch": 0.008328013464105392, + "flos": 41203184296320.0, + "grad_norm": 3.0884372583815916, + "language_loss": 1.12387967, + "learning_rate": 3.2613901482943165e-06, + "loss": 1.14874482, + "num_input_tokens_seen": 8250280, + "router_z_loss_clip": 2.5234375, + "router_z_loss_mlp": 0.39208984, + "step": 287, + "time_per_iteration": 2.7644875049591064 + }, + { + "auxiliary_loss_clip": 0.01352014, + "auxiliary_loss_mlp": 0.01102864, + "balance_loss_clip": 1.11798346, + "balance_loss_mlp": 1.06395423, + "epoch": 0.008357030932621438, + "flos": 35327552336640.0, + "grad_norm": 2.7847709381743018, + "language_loss": 0.95855433, + "learning_rate": 3.263394567910394e-06, + "loss": 0.9831031, + "num_input_tokens_seen": 8266780, + "router_z_loss_clip": 2.34179688, + "router_z_loss_mlp": 0.38891602, + "step": 288, + "time_per_iteration": 2.663325071334839 + }, + { + "auxiliary_loss_clip": 0.01370257, + "auxiliary_loss_mlp": 0.01103951, + "balance_loss_clip": 1.12223005, + "balance_loss_mlp": 1.0633961, + "epoch": 0.008386048401137484, + "flos": 24533164776960.0, + "grad_norm": 2.4031608558774753, + "language_loss": 0.9207145, + "learning_rate": 3.2653920397842294e-06, + "loss": 0.94545656, + "num_input_tokens_seen": 8288370, + "router_z_loss_clip": 2.48046875, + "router_z_loss_mlp": 0.40551758, + "step": 289, + "time_per_iteration": 2.79732084274292 + }, + { + "auxiliary_loss_clip": 0.01118101, + "auxiliary_loss_mlp": 0.01007065, + "balance_loss_clip": 1.03129411, + "balance_loss_mlp": 0.99795705, + "epoch": 0.00841506586965353, + "flos": 62629647811200.0, + "grad_norm": 0.7355428774816334, + "language_loss": 0.56213558, + "learning_rate": 3.2673826119141857e-06, + "loss": 0.58338726, + "num_input_tokens_seen": 8352020, + "router_z_loss_clip": 0.8671875, + "router_z_loss_mlp": 0.09130859, + "step": 290, + "time_per_iteration": 3.1160619258880615 + }, + { + "auxiliary_loss_clip": 0.01370292, + "auxiliary_loss_mlp": 0.01102611, + "balance_loss_clip": 1.12132823, + "balance_loss_mlp": 1.06270003, + "epoch": 0.008444083338169579, + "flos": 17086081731840.0, + "grad_norm": 4.7463154191334125, + "language_loss": 1.05642509, + "learning_rate": 3.2693663318029444e-06, + "loss": 1.08115411, + "num_input_tokens_seen": 8365070, + "router_z_loss_clip": 2.49023438, + "router_z_loss_mlp": 0.39916992, + "step": 291, + "time_per_iteration": 7.272295713424683 + }, + { + "auxiliary_loss_clip": 0.0136002, + "auxiliary_loss_mlp": 0.01113959, + "balance_loss_clip": 1.11741519, + "balance_loss_mlp": 1.06861222, + "epoch": 0.008473100806685625, + "flos": 7703952180480.0, + "grad_norm": 5.19033898491664, + "language_loss": 1.13063288, + "learning_rate": 3.2713432464643052e-06, + "loss": 1.1553725, + "num_input_tokens_seen": 8372095, + "router_z_loss_clip": 2.42578125, + "router_z_loss_mlp": 0.45336914, + "step": 292, + "time_per_iteration": 4.818337917327881 + }, + { + "auxiliary_loss_clip": 0.01117421, + "auxiliary_loss_mlp": 0.01011828, + "balance_loss_clip": 1.03095078, + "balance_loss_mlp": 1.00181413, + "epoch": 0.008502118275201672, + "flos": 74780202291840.0, + "grad_norm": 0.760399296951391, + "language_loss": 0.46079642, + "learning_rate": 3.2733134024298745e-06, + "loss": 0.48208892, + "num_input_tokens_seen": 8430365, + "router_z_loss_clip": 0.8671875, + "router_z_loss_mlp": 0.10009766, + "step": 293, + "time_per_iteration": 5.674290418624878 + }, + { + "auxiliary_loss_clip": 0.01355564, + "auxiliary_loss_mlp": 0.01099081, + "balance_loss_clip": 1.11460567, + "balance_loss_mlp": 1.06284118, + "epoch": 0.008531135743717718, + "flos": 17127127998720.0, + "grad_norm": 3.2888975379154055, + "language_loss": 0.98854053, + "learning_rate": 3.2752768457556347e-06, + "loss": 1.01308703, + "num_input_tokens_seen": 8443410, + "router_z_loss_clip": 2.41015625, + "router_z_loss_mlp": 0.36230469, + "step": 294, + "time_per_iteration": 2.5483198165893555 + }, + { + "auxiliary_loss_clip": 0.01382701, + "auxiliary_loss_mlp": 0.01103334, + "balance_loss_clip": 1.12144017, + "balance_loss_mlp": 1.05965543, + "epoch": 0.008560153212233764, + "flos": 45544496855040.0, + "grad_norm": 2.3580624821143017, + "language_loss": 0.85802746, + "learning_rate": 3.2772336220284056e-06, + "loss": 0.88288784, + "num_input_tokens_seen": 8462000, + "router_z_loss_clip": 2.61328125, + "router_z_loss_mlp": 0.4362793, + "step": 295, + "time_per_iteration": 2.778754472732544 + }, + { + "auxiliary_loss_clip": 0.01361134, + "auxiliary_loss_mlp": 0.01104807, + "balance_loss_clip": 1.12373042, + "balance_loss_mlp": 1.06642199, + "epoch": 0.00858917068074981, + "flos": 74732113981440.0, + "grad_norm": 4.739130216322988, + "language_loss": 0.79500639, + "learning_rate": 3.2791837763721955e-06, + "loss": 0.81966573, + "num_input_tokens_seen": 8484775, + "router_z_loss_clip": 2.375, + "router_z_loss_mlp": 0.38354492, + "step": 296, + "time_per_iteration": 2.975083589553833 + }, + { + "auxiliary_loss_clip": 0.01359689, + "auxiliary_loss_mlp": 0.01110996, + "balance_loss_clip": 1.11987305, + "balance_loss_mlp": 1.07244408, + "epoch": 0.008618188149265857, + "flos": 11284749054720.0, + "grad_norm": 3.4433271424199625, + "language_loss": 0.98196715, + "learning_rate": 3.2811273534544436e-06, + "loss": 1.00667393, + "num_input_tokens_seen": 8496330, + "router_z_loss_clip": 2.3984375, + "router_z_loss_mlp": 0.38574219, + "step": 297, + "time_per_iteration": 2.4750494956970215 + }, + { + "auxiliary_loss_clip": 0.01352335, + "auxiliary_loss_mlp": 0.01094372, + "balance_loss_clip": 1.11196113, + "balance_loss_mlp": 1.05362654, + "epoch": 0.008647205617781905, + "flos": 21900660912000.0, + "grad_norm": 3.485197576750252, + "language_loss": 0.97993702, + "learning_rate": 3.2830643974921586e-06, + "loss": 1.00440407, + "num_input_tokens_seen": 8509655, + "router_z_loss_clip": 2.40234375, + "router_z_loss_mlp": 0.40771484, + "step": 298, + "time_per_iteration": 2.556751251220703 + }, + { + "auxiliary_loss_clip": 0.01377187, + "auxiliary_loss_mlp": 0.01103294, + "balance_loss_clip": 1.12174702, + "balance_loss_mlp": 1.06083214, + "epoch": 0.008676223086297952, + "flos": 21537460540800.0, + "grad_norm": 2.852966736451804, + "language_loss": 0.82194221, + "learning_rate": 3.2849949522579577e-06, + "loss": 0.84674704, + "num_input_tokens_seen": 8525945, + "router_z_loss_clip": 2.55273438, + "router_z_loss_mlp": 0.42480469, + "step": 299, + "time_per_iteration": 2.4944045543670654 + }, + { + "auxiliary_loss_clip": 0.01344194, + "auxiliary_loss_mlp": 0.01078262, + "balance_loss_clip": 1.112432, + "balance_loss_mlp": 1.04118776, + "epoch": 0.008705240554813998, + "flos": 63130705603200.0, + "grad_norm": 2.399790630165257, + "language_loss": 1.00480914, + "learning_rate": 3.286919061085997e-06, + "loss": 1.02903354, + "num_input_tokens_seen": 8546645, + "router_z_loss_clip": 2.31835938, + "router_z_loss_mlp": 0.37060547, + "step": 300, + "time_per_iteration": 2.904325246810913 + }, + { + "auxiliary_loss_clip": 0.01357553, + "auxiliary_loss_mlp": 0.01094859, + "balance_loss_clip": 1.11306179, + "balance_loss_mlp": 1.0567832, + "epoch": 0.008734258023330044, + "flos": 48482673459840.0, + "grad_norm": 1.7609938189291747, + "language_loss": 0.92112744, + "learning_rate": 3.2888367668778124e-06, + "loss": 0.94565153, + "num_input_tokens_seen": 8582450, + "router_z_loss_clip": 2.44335938, + "router_z_loss_mlp": 0.38061523, + "step": 301, + "time_per_iteration": 3.038166046142578 + }, + { + "auxiliary_loss_clip": 0.01367282, + "auxiliary_loss_mlp": 0.01098828, + "balance_loss_clip": 1.11604643, + "balance_loss_mlp": 1.05903614, + "epoch": 0.00876327549184609, + "flos": 28943820541440.0, + "grad_norm": 7.5503832920654945, + "language_loss": 0.9310866, + "learning_rate": 3.2907481121080574e-06, + "loss": 0.95574772, + "num_input_tokens_seen": 8599775, + "router_z_loss_clip": 2.515625, + "router_z_loss_mlp": 0.39794922, + "step": 302, + "time_per_iteration": 2.655099391937256 + }, + { + "auxiliary_loss_clip": 0.01355173, + "auxiliary_loss_mlp": 0.01091161, + "balance_loss_clip": 1.11551583, + "balance_loss_mlp": 1.04874671, + "epoch": 0.008792292960362137, + "flos": 16067439936000.0, + "grad_norm": 2.6389265981747183, + "language_loss": 0.96183193, + "learning_rate": 3.2926531388301455e-06, + "loss": 0.98629522, + "num_input_tokens_seen": 8612710, + "router_z_loss_clip": 2.39453125, + "router_z_loss_mlp": 0.42382812, + "step": 303, + "time_per_iteration": 2.491086959838867 + }, + { + "auxiliary_loss_clip": 0.01343939, + "auxiliary_loss_mlp": 0.01101119, + "balance_loss_clip": 1.10754943, + "balance_loss_mlp": 1.06324649, + "epoch": 0.008821310428878185, + "flos": 74731323882240.0, + "grad_norm": 2.1592568120052418, + "language_loss": 0.82758904, + "learning_rate": 3.2945518886818066e-06, + "loss": 0.85203969, + "num_input_tokens_seen": 8634650, + "router_z_loss_clip": 2.36328125, + "router_z_loss_mlp": 0.37866211, + "step": 304, + "time_per_iteration": 2.9470953941345215 + }, + { + "auxiliary_loss_clip": 0.01342946, + "auxiliary_loss_mlp": 0.01092641, + "balance_loss_clip": 1.11167097, + "balance_loss_mlp": 1.0526104, + "epoch": 0.008850327897394232, + "flos": 33212593584000.0, + "grad_norm": 2.719652103112711, + "language_loss": 1.20110738, + "learning_rate": 3.296444402890543e-06, + "loss": 1.22546327, + "num_input_tokens_seen": 8651660, + "router_z_loss_clip": 2.30859375, + "router_z_loss_mlp": 0.40014648, + "step": 305, + "time_per_iteration": 2.6472229957580566 + }, + { + "auxiliary_loss_clip": 0.01343105, + "auxiliary_loss_mlp": 0.01096084, + "balance_loss_clip": 1.11441004, + "balance_loss_mlp": 1.05741262, + "epoch": 0.008879345365910278, + "flos": 14386317649920.0, + "grad_norm": 3.12190983858185, + "language_loss": 0.79707742, + "learning_rate": 3.298330722279005e-06, + "loss": 0.82146931, + "num_input_tokens_seen": 8665185, + "router_z_loss_clip": 2.28710938, + "router_z_loss_mlp": 0.38696289, + "step": 306, + "time_per_iteration": 2.5609610080718994 + }, + { + "auxiliary_loss_clip": 0.01352984, + "auxiliary_loss_mlp": 0.01088768, + "balance_loss_clip": 1.11305308, + "balance_loss_mlp": 1.05176568, + "epoch": 0.008908362834426324, + "flos": 15954787906560.0, + "grad_norm": 4.381374254613734, + "language_loss": 0.89238137, + "learning_rate": 3.3002108872702717e-06, + "loss": 0.91679895, + "num_input_tokens_seen": 8678150, + "router_z_loss_clip": 2.39648438, + "router_z_loss_mlp": 0.37036133, + "step": 307, + "time_per_iteration": 2.5478217601776123 + }, + { + "auxiliary_loss_clip": 0.011155, + "auxiliary_loss_mlp": 0.01007572, + "balance_loss_clip": 1.03043437, + "balance_loss_mlp": 0.99965686, + "epoch": 0.00893738030294237, + "flos": 63860126065920.0, + "grad_norm": 0.6446246984536392, + "language_loss": 0.45959085, + "learning_rate": 3.3020849378930476e-06, + "loss": 0.48082161, + "num_input_tokens_seen": 8741240, + "router_z_loss_clip": 0.8515625, + "router_z_loss_mlp": 0.07910156, + "step": 308, + "time_per_iteration": 3.195348024368286 + }, + { + "auxiliary_loss_clip": 0.0111261, + "auxiliary_loss_mlp": 0.0100778, + "balance_loss_clip": 1.02762151, + "balance_loss_mlp": 0.999865, + "epoch": 0.008966397771458417, + "flos": 69554086012800.0, + "grad_norm": 0.730291149473373, + "language_loss": 0.55399394, + "learning_rate": 3.303952913786781e-06, + "loss": 0.57519794, + "num_input_tokens_seen": 8799210, + "router_z_loss_clip": 0.8515625, + "router_z_loss_mlp": 0.07910156, + "step": 309, + "time_per_iteration": 3.0924017429351807 + }, + { + "auxiliary_loss_clip": 0.01345247, + "auxiliary_loss_mlp": 0.01093691, + "balance_loss_clip": 1.11210084, + "balance_loss_mlp": 1.05523384, + "epoch": 0.008995415239974465, + "flos": 21387102209280.0, + "grad_norm": 3.10234983005725, + "language_loss": 1.0103693, + "learning_rate": 3.305814854206687e-06, + "loss": 1.03475881, + "num_input_tokens_seen": 8811205, + "router_z_loss_clip": 2.32910156, + "router_z_loss_mlp": 0.38452148, + "step": 310, + "time_per_iteration": 2.626711130142212 + }, + { + "auxiliary_loss_clip": 0.0134522, + "auxiliary_loss_mlp": 0.01083951, + "balance_loss_clip": 1.10243869, + "balance_loss_mlp": 1.04380107, + "epoch": 0.009024432708490512, + "flos": 19894870569600.0, + "grad_norm": 3.0170574847696363, + "language_loss": 0.947191, + "learning_rate": 3.307670798028707e-06, + "loss": 0.97148275, + "num_input_tokens_seen": 8824250, + "router_z_loss_clip": 2.43164062, + "router_z_loss_mlp": 0.40136719, + "step": 311, + "time_per_iteration": 2.644301414489746 + }, + { + "auxiliary_loss_clip": 0.01349668, + "auxiliary_loss_mlp": 0.01118437, + "balance_loss_clip": 1.1111064, + "balance_loss_mlp": 1.08095813, + "epoch": 0.009053450177006558, + "flos": 35926723105920.0, + "grad_norm": 2.7515915470044408, + "language_loss": 1.01958299, + "learning_rate": 3.30952078375437e-06, + "loss": 1.04426408, + "num_input_tokens_seen": 8842500, + "router_z_loss_clip": 2.38476562, + "router_z_loss_mlp": 0.37475586, + "step": 312, + "time_per_iteration": 2.6998512744903564 + }, + { + "auxiliary_loss_clip": 0.01348345, + "auxiliary_loss_mlp": 0.01110282, + "balance_loss_clip": 1.11644256, + "balance_loss_mlp": 1.07172966, + "epoch": 0.009082467645522604, + "flos": 17596911000960.0, + "grad_norm": 3.940804292071401, + "language_loss": 0.94585812, + "learning_rate": 3.3113648495155915e-06, + "loss": 0.97044444, + "num_input_tokens_seen": 8858640, + "router_z_loss_clip": 2.3203125, + "router_z_loss_mlp": 0.38574219, + "step": 313, + "time_per_iteration": 2.5884909629821777 + }, + { + "auxiliary_loss_clip": 0.01107993, + "auxiliary_loss_mlp": 0.01008273, + "balance_loss_clip": 1.02264071, + "balance_loss_mlp": 0.99992794, + "epoch": 0.00911148511403865, + "flos": 74774492029440.0, + "grad_norm": 0.7298268130179368, + "language_loss": 0.54665542, + "learning_rate": 3.3132030330793862e-06, + "loss": 0.56781805, + "num_input_tokens_seen": 8920080, + "router_z_loss_clip": 0.8515625, + "router_z_loss_mlp": 0.08349609, + "step": 314, + "time_per_iteration": 3.117180585861206 + }, + { + "auxiliary_loss_clip": 0.01106927, + "auxiliary_loss_mlp": 0.01008319, + "balance_loss_clip": 1.02222276, + "balance_loss_mlp": 0.9998793, + "epoch": 0.009140502582554697, + "flos": 65292064727040.0, + "grad_norm": 0.7348020938944783, + "language_loss": 0.53809589, + "learning_rate": 3.3150353718525096e-06, + "loss": 0.55924839, + "num_input_tokens_seen": 8979035, + "router_z_loss_clip": 0.84765625, + "router_z_loss_mlp": 0.08447266, + "step": 315, + "time_per_iteration": 3.093806743621826 + }, + { + "auxiliary_loss_clip": 0.01349517, + "auxiliary_loss_mlp": 0.01095562, + "balance_loss_clip": 1.10650003, + "balance_loss_mlp": 1.05379152, + "epoch": 0.009169520051070745, + "flos": 16390384139520.0, + "grad_norm": 3.133856078917295, + "language_loss": 0.86984813, + "learning_rate": 3.3168619028860257e-06, + "loss": 0.89429891, + "num_input_tokens_seen": 8992800, + "router_z_loss_clip": 2.42773438, + "router_z_loss_mlp": 0.41748047, + "step": 316, + "time_per_iteration": 2.5269908905029297 + }, + { + "auxiliary_loss_clip": 0.01327988, + "auxiliary_loss_mlp": 0.01087272, + "balance_loss_clip": 1.10844004, + "balance_loss_mlp": 1.05155718, + "epoch": 0.009198537519586792, + "flos": 15624876464640.0, + "grad_norm": 7.786513138087846, + "language_loss": 1.1742866, + "learning_rate": 3.3186826628798026e-06, + "loss": 1.19843924, + "num_input_tokens_seen": 9004985, + "router_z_loss_clip": 2.1953125, + "router_z_loss_mlp": 0.35693359, + "step": 317, + "time_per_iteration": 2.4790050983428955 + }, + { + "auxiliary_loss_clip": 0.01340833, + "auxiliary_loss_mlp": 0.01095829, + "balance_loss_clip": 1.10631132, + "balance_loss_mlp": 1.05728841, + "epoch": 0.009227554988102838, + "flos": 29745669801600.0, + "grad_norm": 3.569057773760449, + "language_loss": 1.19772589, + "learning_rate": 3.3204976881869384e-06, + "loss": 1.22209239, + "num_input_tokens_seen": 9020355, + "router_z_loss_clip": 2.34570312, + "router_z_loss_mlp": 0.38574219, + "step": 318, + "time_per_iteration": 2.6503231525421143 + }, + { + "auxiliary_loss_clip": 0.01359133, + "auxiliary_loss_mlp": 0.01106739, + "balance_loss_clip": 1.11161017, + "balance_loss_mlp": 1.06649399, + "epoch": 0.009256572456618884, + "flos": 28579039971840.0, + "grad_norm": 2.6158517824194396, + "language_loss": 0.92354918, + "learning_rate": 3.3223070148181116e-06, + "loss": 0.94820791, + "num_input_tokens_seen": 9037890, + "router_z_loss_clip": 2.47460938, + "router_z_loss_mlp": 0.40234375, + "step": 319, + "time_per_iteration": 2.626044988632202 + }, + { + "auxiliary_loss_clip": 0.01326374, + "auxiliary_loss_mlp": 0.01091976, + "balance_loss_clip": 1.10331941, + "balance_loss_mlp": 1.05697632, + "epoch": 0.00928558992513493, + "flos": 21027026321280.0, + "grad_norm": 3.8248961966198998, + "language_loss": 0.94418877, + "learning_rate": 3.3241106784458735e-06, + "loss": 0.96837229, + "num_input_tokens_seen": 9052035, + "router_z_loss_clip": 2.23046875, + "router_z_loss_mlp": 0.35009766, + "step": 320, + "time_per_iteration": 2.608366012573242 + }, + { + "auxiliary_loss_clip": 0.01324947, + "auxiliary_loss_mlp": 0.01081066, + "balance_loss_clip": 1.10219324, + "balance_loss_mlp": 1.04706717, + "epoch": 0.009314607393650977, + "flos": 17084393792640.0, + "grad_norm": 3.590842736010261, + "language_loss": 0.88287294, + "learning_rate": 3.3259087144088656e-06, + "loss": 0.90693313, + "num_input_tokens_seen": 9064630, + "router_z_loss_clip": 2.2265625, + "router_z_loss_mlp": 0.33984375, + "step": 321, + "time_per_iteration": 2.4597623348236084 + }, + { + "auxiliary_loss_clip": 0.0110752, + "auxiliary_loss_mlp": 0.01018152, + "balance_loss_clip": 1.0254755, + "balance_loss_mlp": 1.00999856, + "epoch": 0.009343624862167025, + "flos": 62037947070720.0, + "grad_norm": 0.696645116699763, + "language_loss": 0.57650828, + "learning_rate": 3.327701157715974e-06, + "loss": 0.59776497, + "num_input_tokens_seen": 9122435, + "router_z_loss_clip": 0.8203125, + "router_z_loss_mlp": 0.08154297, + "step": 322, + "time_per_iteration": 3.0021724700927734 + }, + { + "auxiliary_loss_clip": 0.01345552, + "auxiliary_loss_mlp": 0.01106588, + "balance_loss_clip": 1.11434758, + "balance_loss_mlp": 1.06915641, + "epoch": 0.009372642330683072, + "flos": 32882287092480.0, + "grad_norm": 3.024870634981815, + "language_loss": 0.82727736, + "learning_rate": 3.329488043050418e-06, + "loss": 0.85179877, + "num_input_tokens_seen": 9141455, + "router_z_loss_clip": 2.3125, + "router_z_loss_mlp": 0.37426758, + "step": 323, + "time_per_iteration": 2.691697597503662 + }, + { + "auxiliary_loss_clip": 0.01349148, + "auxiliary_loss_mlp": 0.01113826, + "balance_loss_clip": 1.11112928, + "balance_loss_mlp": 1.0720787, + "epoch": 0.009401659799199118, + "flos": 31353677953920.0, + "grad_norm": 4.106591864797272, + "language_loss": 1.18143129, + "learning_rate": 3.3312694047737813e-06, + "loss": 1.20606112, + "num_input_tokens_seen": 9157615, + "router_z_loss_clip": 2.37695312, + "router_z_loss_mlp": 0.41772461, + "step": 324, + "time_per_iteration": 2.6544182300567627 + }, + { + "auxiliary_loss_clip": 0.01340678, + "auxiliary_loss_mlp": 0.01101554, + "balance_loss_clip": 1.105896, + "balance_loss_mlp": 1.06369328, + "epoch": 0.009430677267715164, + "flos": 13727464433280.0, + "grad_norm": 2.6829564062538447, + "language_loss": 0.95295197, + "learning_rate": 3.333045276929973e-06, + "loss": 0.97737432, + "num_input_tokens_seen": 9169345, + "router_z_loss_clip": 2.34960938, + "router_z_loss_mlp": 0.37890625, + "step": 325, + "time_per_iteration": 2.5263116359710693 + }, + { + "auxiliary_loss_clip": 0.01346095, + "auxiliary_loss_mlp": 0.01102717, + "balance_loss_clip": 1.1092217, + "balance_loss_mlp": 1.06325841, + "epoch": 0.00945969473623121, + "flos": 34858559433600.0, + "grad_norm": 2.474561747058606, + "language_loss": 1.08836555, + "learning_rate": 3.334815693249133e-06, + "loss": 1.11285365, + "num_input_tokens_seen": 9194125, + "router_z_loss_clip": 2.36914062, + "router_z_loss_mlp": 0.39453125, + "step": 326, + "time_per_iteration": 2.762877941131592 + }, + { + "auxiliary_loss_clip": 0.01103591, + "auxiliary_loss_mlp": 0.01006928, + "balance_loss_clip": 1.0230267, + "balance_loss_mlp": 0.99925083, + "epoch": 0.009488712204747257, + "flos": 62944080491520.0, + "grad_norm": 0.6991473687772032, + "language_loss": 0.52254784, + "learning_rate": 3.3365806871514735e-06, + "loss": 0.54365301, + "num_input_tokens_seen": 9249005, + "router_z_loss_clip": 0.8046875, + "router_z_loss_mlp": 0.07666016, + "step": 327, + "time_per_iteration": 2.9793527126312256 + }, + { + "auxiliary_loss_clip": 0.01336578, + "auxiliary_loss_mlp": 0.01088912, + "balance_loss_clip": 1.10650373, + "balance_loss_mlp": 1.04919171, + "epoch": 0.009517729673263305, + "flos": 32480303011200.0, + "grad_norm": 32.57485994385985, + "language_loss": 0.96011949, + "learning_rate": 3.3383402917510684e-06, + "loss": 0.98437434, + "num_input_tokens_seen": 9263710, + "router_z_loss_clip": 2.29882812, + "router_z_loss_mlp": 0.39770508, + "step": 328, + "time_per_iteration": 2.8875811100006104 + }, + { + "auxiliary_loss_clip": 0.01103241, + "auxiliary_loss_mlp": 0.01006043, + "balance_loss_clip": 1.02251577, + "balance_loss_mlp": 0.9982233, + "epoch": 0.009546747141779352, + "flos": 70617149953920.0, + "grad_norm": 0.7001908081866421, + "language_loss": 0.56861246, + "learning_rate": 3.340094539859579e-06, + "loss": 0.58970523, + "num_input_tokens_seen": 9328025, + "router_z_loss_clip": 0.8046875, + "router_z_loss_mlp": 0.078125, + "step": 329, + "time_per_iteration": 3.1727511882781982 + }, + { + "auxiliary_loss_clip": 0.01102508, + "auxiliary_loss_mlp": 0.01008428, + "balance_loss_clip": 1.02180028, + "balance_loss_mlp": 1.00027359, + "epoch": 0.009575764610295398, + "flos": 69599118689280.0, + "grad_norm": 0.7301938104493142, + "language_loss": 0.58849645, + "learning_rate": 3.3418434639899233e-06, + "loss": 0.60960579, + "num_input_tokens_seen": 9393160, + "router_z_loss_clip": 0.80859375, + "router_z_loss_mlp": 0.08154297, + "step": 330, + "time_per_iteration": 3.1196670532226562 + }, + { + "auxiliary_loss_clip": 0.01101973, + "auxiliary_loss_mlp": 0.01011254, + "balance_loss_clip": 1.02175522, + "balance_loss_mlp": 1.00357711, + "epoch": 0.009604782078811444, + "flos": 74778478439040.0, + "grad_norm": 0.7592126882122489, + "language_loss": 0.57570875, + "learning_rate": 3.3435870963598952e-06, + "loss": 0.59684098, + "num_input_tokens_seen": 9459550, + "router_z_loss_clip": 0.80078125, + "router_z_loss_mlp": 0.07666016, + "step": 331, + "time_per_iteration": 3.1397595405578613 + }, + { + "auxiliary_loss_clip": 0.01353675, + "auxiliary_loss_mlp": 0.01090212, + "balance_loss_clip": 1.10362244, + "balance_loss_mlp": 1.0489893, + "epoch": 0.00963379954732749, + "flos": 26536943957760.0, + "grad_norm": 7.581767134017988, + "language_loss": 0.86542475, + "learning_rate": 3.3453254688957247e-06, + "loss": 0.88986373, + "num_input_tokens_seen": 9475915, + "router_z_loss_clip": 2.50195312, + "router_z_loss_mlp": 0.41259766, + "step": 332, + "time_per_iteration": 2.6268582344055176 + }, + { + "auxiliary_loss_clip": 0.01335139, + "auxiliary_loss_mlp": 0.01093028, + "balance_loss_clip": 1.10734344, + "balance_loss_mlp": 1.06017375, + "epoch": 0.009662817015843537, + "flos": 15230254671360.0, + "grad_norm": 2.5319580550909815, + "language_loss": 0.82428461, + "learning_rate": 3.347058613235583e-06, + "loss": 0.84856629, + "num_input_tokens_seen": 9491815, + "router_z_loss_clip": 2.27539062, + "router_z_loss_mlp": 0.328125, + "step": 333, + "time_per_iteration": 2.6162936687469482 + }, + { + "auxiliary_loss_clip": 0.0131858, + "auxiliary_loss_mlp": 0.0107706, + "balance_loss_clip": 1.10070086, + "balance_loss_mlp": 1.04315662, + "epoch": 0.009691834484359585, + "flos": 38612014007040.0, + "grad_norm": 2.971617884933602, + "language_loss": 1.02023053, + "learning_rate": 3.34878656073304e-06, + "loss": 1.04418695, + "num_input_tokens_seen": 9509620, + "router_z_loss_clip": 2.1796875, + "router_z_loss_mlp": 0.33898926, + "step": 334, + "time_per_iteration": 2.7252461910247803 + }, + { + "auxiliary_loss_clip": 0.01100493, + "auxiliary_loss_mlp": 0.01009179, + "balance_loss_clip": 1.02192831, + "balance_loss_mlp": 1.00126362, + "epoch": 0.009720851952875632, + "flos": 74783973219840.0, + "grad_norm": 0.7300424757700379, + "language_loss": 0.54139864, + "learning_rate": 3.350509342460466e-06, + "loss": 0.56249535, + "num_input_tokens_seen": 9573665, + "router_z_loss_clip": 0.78515625, + "router_z_loss_mlp": 0.07910156, + "step": 335, + "time_per_iteration": 3.2537388801574707 + }, + { + "auxiliary_loss_clip": 0.01099278, + "auxiliary_loss_mlp": 0.01007466, + "balance_loss_clip": 1.02126718, + "balance_loss_mlp": 0.99964583, + "epoch": 0.009749869421391678, + "flos": 60835514359680.0, + "grad_norm": 0.7081201656461071, + "language_loss": 0.53178436, + "learning_rate": 3.3522269892123866e-06, + "loss": 0.5528518, + "num_input_tokens_seen": 9635490, + "router_z_loss_clip": 0.78125, + "router_z_loss_mlp": 0.078125, + "step": 336, + "time_per_iteration": 3.0568125247955322 + }, + { + "auxiliary_loss_clip": 0.01334168, + "auxiliary_loss_mlp": 0.01084277, + "balance_loss_clip": 1.10907531, + "balance_loss_mlp": 1.04910994, + "epoch": 0.009778886889907724, + "flos": 15808595552640.0, + "grad_norm": 7.620409390581945, + "language_loss": 0.96436971, + "learning_rate": 3.3539395315087827e-06, + "loss": 0.98855412, + "num_input_tokens_seen": 9649630, + "router_z_loss_clip": 2.25195312, + "router_z_loss_mlp": 0.3515625, + "step": 337, + "time_per_iteration": 2.56781005859375 + }, + { + "auxiliary_loss_clip": 0.01331994, + "auxiliary_loss_mlp": 0.0110578, + "balance_loss_clip": 1.10304546, + "balance_loss_mlp": 1.06656027, + "epoch": 0.00980790435842377, + "flos": 25074948591360.0, + "grad_norm": 3.1176215665037446, + "language_loss": 0.94064689, + "learning_rate": 3.3556469995983466e-06, + "loss": 0.96502465, + "num_input_tokens_seen": 9662900, + "router_z_loss_clip": 2.28515625, + "router_z_loss_mlp": 0.39208984, + "step": 338, + "time_per_iteration": 2.6532533168792725 + }, + { + "auxiliary_loss_clip": 0.01354742, + "auxiliary_loss_mlp": 0.01106527, + "balance_loss_clip": 1.10548031, + "balance_loss_mlp": 1.06203842, + "epoch": 0.009836921826939817, + "flos": 13035358200960.0, + "grad_norm": 3.3010315930914285, + "language_loss": 1.05408359, + "learning_rate": 3.357349423461686e-06, + "loss": 1.07869625, + "num_input_tokens_seen": 9674755, + "router_z_loss_clip": 2.49414062, + "router_z_loss_mlp": 0.44506836, + "step": 339, + "time_per_iteration": 2.54465913772583 + }, + { + "auxiliary_loss_clip": 0.01346962, + "auxiliary_loss_mlp": 0.0110176, + "balance_loss_clip": 1.10586321, + "balance_loss_mlp": 1.06304121, + "epoch": 0.009865939295455865, + "flos": 31061185505280.0, + "grad_norm": 2.995166717905246, + "language_loss": 1.07339811, + "learning_rate": 3.3590468328144853e-06, + "loss": 1.09788537, + "num_input_tokens_seen": 9691705, + "router_z_loss_clip": 2.41210938, + "router_z_loss_mlp": 0.38745117, + "step": 340, + "time_per_iteration": 2.6234848499298096 + }, + { + "auxiliary_loss_clip": 0.01335798, + "auxiliary_loss_mlp": 0.01079572, + "balance_loss_clip": 1.11176968, + "balance_loss_mlp": 1.04240298, + "epoch": 0.009894956763971912, + "flos": 16499049759360.0, + "grad_norm": 4.836370259594257, + "language_loss": 0.8402611, + "learning_rate": 3.360739257110613e-06, + "loss": 0.86441481, + "num_input_tokens_seen": 9702375, + "router_z_loss_clip": 2.24414062, + "router_z_loss_mlp": 0.37158203, + "step": 341, + "time_per_iteration": 2.487189769744873 + }, + { + "auxiliary_loss_clip": 0.01344294, + "auxiliary_loss_mlp": 0.0109761, + "balance_loss_clip": 1.10618591, + "balance_loss_mlp": 1.05653048, + "epoch": 0.009923974232487958, + "flos": 28870814148480.0, + "grad_norm": 2.4322421017279563, + "language_loss": 0.90934086, + "learning_rate": 3.3624267255451937e-06, + "loss": 0.93375987, + "num_input_tokens_seen": 9718690, + "router_z_loss_clip": 2.38085938, + "router_z_loss_mlp": 0.41088867, + "step": 342, + "time_per_iteration": 2.624307155609131 + }, + { + "auxiliary_loss_clip": 0.0133234, + "auxiliary_loss_mlp": 0.01103977, + "balance_loss_clip": 1.10710156, + "balance_loss_mlp": 1.06608009, + "epoch": 0.009952991701004004, + "flos": 74055055547520.0, + "grad_norm": 5.863519786021692, + "language_loss": 1.1102047, + "learning_rate": 3.3641092670576266e-06, + "loss": 1.13456786, + "num_input_tokens_seen": 9751970, + "router_z_loss_clip": 2.25585938, + "router_z_loss_mlp": 0.37902832, + "step": 343, + "time_per_iteration": 3.0156912803649902 + }, + { + "auxiliary_loss_clip": 0.01353895, + "auxiliary_loss_mlp": 0.01101928, + "balance_loss_clip": 1.11330807, + "balance_loss_mlp": 1.06151652, + "epoch": 0.00998200916952005, + "flos": 29876063149440.0, + "grad_norm": 2.0814187168990337, + "language_loss": 0.8486231, + "learning_rate": 3.3657869103345642e-06, + "loss": 0.87318134, + "num_input_tokens_seen": 9775060, + "router_z_loss_clip": 2.40234375, + "router_z_loss_mlp": 0.40405273, + "step": 344, + "time_per_iteration": 2.698378562927246 + }, + { + "auxiliary_loss_clip": 0.01098607, + "auxiliary_loss_mlp": 0.01009343, + "balance_loss_clip": 1.02147079, + "balance_loss_mlp": 1.00095081, + "epoch": 0.010011026638036097, + "flos": 64846197204480.0, + "grad_norm": 0.7608059825676318, + "language_loss": 0.55136716, + "learning_rate": 3.3674596838128487e-06, + "loss": 0.5724467, + "num_input_tokens_seen": 9829910, + "router_z_loss_clip": 0.7734375, + "router_z_loss_mlp": 0.08398438, + "step": 345, + "time_per_iteration": 3.009338140487671 + }, + { + "auxiliary_loss_clip": 0.01340945, + "auxiliary_loss_mlp": 0.0110771, + "balance_loss_clip": 1.11223149, + "balance_loss_mlp": 1.06891942, + "epoch": 0.010040044106552144, + "flos": 12670864940160.0, + "grad_norm": 3.2779223984654804, + "language_loss": 0.93217373, + "learning_rate": 3.3691276156823998e-06, + "loss": 0.95666027, + "num_input_tokens_seen": 9841350, + "router_z_loss_clip": 2.2890625, + "router_z_loss_mlp": 0.38793945, + "step": 346, + "time_per_iteration": 2.5235023498535156 + }, + { + "auxiliary_loss_clip": 0.01337747, + "auxiliary_loss_mlp": 0.01097459, + "balance_loss_clip": 1.11011767, + "balance_loss_mlp": 1.06203055, + "epoch": 0.010069061575068192, + "flos": 11794105866240.0, + "grad_norm": 2.9168521347652763, + "language_loss": 0.88999903, + "learning_rate": 3.3707907338890692e-06, + "loss": 0.91435111, + "num_input_tokens_seen": 9853295, + "router_z_loss_clip": 2.27734375, + "router_z_loss_mlp": 0.35400391, + "step": 347, + "time_per_iteration": 2.5275509357452393 + }, + { + "auxiliary_loss_clip": 0.01336952, + "auxiliary_loss_mlp": 0.01080874, + "balance_loss_clip": 1.10792518, + "balance_loss_mlp": 1.04456258, + "epoch": 0.010098079043584238, + "flos": 25989234399360.0, + "grad_norm": 2.7403418951938785, + "language_loss": 0.84689748, + "learning_rate": 3.37244906613745e-06, + "loss": 0.87107569, + "num_input_tokens_seen": 9867620, + "router_z_loss_clip": 2.28710938, + "router_z_loss_mlp": 0.36303711, + "step": 348, + "time_per_iteration": 2.5009305477142334 + }, + { + "auxiliary_loss_clip": 0.01098491, + "auxiliary_loss_mlp": 0.01008528, + "balance_loss_clip": 1.02184784, + "balance_loss_mlp": 1.00065982, + "epoch": 0.010127096512100284, + "flos": 64154701503360.0, + "grad_norm": 0.7679105455332236, + "language_loss": 0.56578636, + "learning_rate": 3.3741026398936434e-06, + "loss": 0.58685654, + "num_input_tokens_seen": 9925330, + "router_z_loss_clip": 0.765625, + "router_z_loss_mlp": 0.07861328, + "step": 349, + "time_per_iteration": 3.0100955963134766 + }, + { + "auxiliary_loss_clip": 0.01339122, + "auxiliary_loss_mlp": 0.01102904, + "balance_loss_clip": 1.11060059, + "balance_loss_mlp": 1.06759453, + "epoch": 0.01015611398061633, + "flos": 74736279959040.0, + "grad_norm": 1.8342662484698842, + "language_loss": 0.87943923, + "learning_rate": 3.3757514823879893e-06, + "loss": 0.9038595, + "num_input_tokens_seen": 9955195, + "router_z_loss_clip": 2.28710938, + "router_z_loss_mlp": 0.35302734, + "step": 350, + "time_per_iteration": 2.936582088470459 + }, + { + "auxiliary_loss_clip": 0.01333169, + "auxiliary_loss_mlp": 0.01083435, + "balance_loss_clip": 1.10327625, + "balance_loss_mlp": 1.04726672, + "epoch": 0.010185131449132377, + "flos": 49960324177920.0, + "grad_norm": 2.724358599235992, + "language_loss": 0.81054699, + "learning_rate": 3.3773956206177575e-06, + "loss": 0.8347131, + "num_input_tokens_seen": 9974750, + "router_z_loss_clip": 2.30078125, + "router_z_loss_mlp": 0.36193848, + "step": 351, + "time_per_iteration": 2.791544198989868 + }, + { + "auxiliary_loss_clip": 0.01328977, + "auxiliary_loss_mlp": 0.0109359, + "balance_loss_clip": 1.09919465, + "balance_loss_mlp": 1.0535121, + "epoch": 0.010214148917648424, + "flos": 43902014624640.0, + "grad_norm": 4.036566348050595, + "language_loss": 1.24599457, + "learning_rate": 3.3790350813497995e-06, + "loss": 1.27022016, + "num_input_tokens_seen": 9995975, + "router_z_loss_clip": 2.29882812, + "router_z_loss_mlp": 0.40039062, + "step": 352, + "time_per_iteration": 2.762678861618042 + }, + { + "auxiliary_loss_clip": 0.01321761, + "auxiliary_loss_mlp": 0.01091773, + "balance_loss_clip": 1.10611033, + "balance_loss_mlp": 1.05787039, + "epoch": 0.010243166386164472, + "flos": 28687956986880.0, + "grad_norm": 2.4226502917659274, + "language_loss": 0.88895595, + "learning_rate": 3.380669891123163e-06, + "loss": 0.9130913, + "num_input_tokens_seen": 10011615, + "router_z_loss_clip": 2.15527344, + "router_z_loss_mlp": 0.33886719, + "step": 353, + "time_per_iteration": 2.6478772163391113 + }, + { + "auxiliary_loss_clip": 0.01315226, + "auxiliary_loss_mlp": 0.01076555, + "balance_loss_clip": 1.10237825, + "balance_loss_mlp": 1.04222298, + "epoch": 0.010272183854680518, + "flos": 25989054831360.0, + "grad_norm": 2.645788964141364, + "language_loss": 0.98312312, + "learning_rate": 3.3823000762516696e-06, + "loss": 1.00704098, + "num_input_tokens_seen": 10027920, + "router_z_loss_clip": 2.12988281, + "router_z_loss_mlp": 0.34338379, + "step": 354, + "time_per_iteration": 2.583465576171875 + }, + { + "auxiliary_loss_clip": 0.0109967, + "auxiliary_loss_mlp": 0.01008073, + "balance_loss_clip": 1.02282107, + "balance_loss_mlp": 0.99982399, + "epoch": 0.010301201323196564, + "flos": 62406247173120.0, + "grad_norm": 0.7916566864697773, + "language_loss": 0.52139354, + "learning_rate": 3.3839256628264573e-06, + "loss": 0.54247093, + "num_input_tokens_seen": 10090375, + "router_z_loss_clip": 0.76953125, + "router_z_loss_mlp": 0.08251953, + "step": 355, + "time_per_iteration": 3.074944257736206 + }, + { + "auxiliary_loss_clip": 0.01099665, + "auxiliary_loss_mlp": 0.01010773, + "balance_loss_clip": 1.0229243, + "balance_loss_mlp": 1.00238037, + "epoch": 0.01033021879171261, + "flos": 68577172842240.0, + "grad_norm": 0.679804537916637, + "language_loss": 0.55384851, + "learning_rate": 3.385546676718483e-06, + "loss": 0.57495296, + "num_input_tokens_seen": 10156230, + "router_z_loss_clip": 0.765625, + "router_z_loss_mlp": 0.08398438, + "step": 356, + "time_per_iteration": 3.307912588119507 + }, + { + "auxiliary_loss_clip": 0.01342308, + "auxiliary_loss_mlp": 0.01110603, + "balance_loss_clip": 1.10393667, + "balance_loss_mlp": 1.06885564, + "epoch": 0.010359236260228657, + "flos": 32080832881920.0, + "grad_norm": 4.464767960241058, + "language_loss": 1.08439612, + "learning_rate": 3.387163143580998e-06, + "loss": 1.1089251, + "num_input_tokens_seen": 10172395, + "router_z_loss_clip": 2.3828125, + "router_z_loss_mlp": 0.41772461, + "step": 357, + "time_per_iteration": 2.6817991733551025 + }, + { + "auxiliary_loss_clip": 0.01099168, + "auxiliary_loss_mlp": 0.01008925, + "balance_loss_clip": 1.02281797, + "balance_loss_mlp": 1.00058031, + "epoch": 0.010388253728744704, + "flos": 74777329203840.0, + "grad_norm": 0.7760503366667062, + "language_loss": 0.51966745, + "learning_rate": 3.3887750888519783e-06, + "loss": 0.54074836, + "num_input_tokens_seen": 10234795, + "router_z_loss_clip": 0.765625, + "router_z_loss_mlp": 0.08349609, + "step": 358, + "time_per_iteration": 3.150313138961792 + }, + { + "auxiliary_loss_clip": 0.01332341, + "auxiliary_loss_mlp": 0.01098515, + "balance_loss_clip": 1.09910059, + "balance_loss_mlp": 1.05788839, + "epoch": 0.010417271197260752, + "flos": 14349796496640.0, + "grad_norm": 2.9918609963280276, + "language_loss": 0.91537774, + "learning_rate": 3.3903825377565315e-06, + "loss": 0.9396863, + "num_input_tokens_seen": 10247845, + "router_z_loss_clip": 2.33203125, + "router_z_loss_mlp": 0.40625, + "step": 359, + "time_per_iteration": 2.5220296382904053 + }, + { + "auxiliary_loss_clip": 0.01321455, + "auxiliary_loss_mlp": 0.01098879, + "balance_loss_clip": 1.10847235, + "balance_loss_mlp": 1.06314015, + "epoch": 0.010446288665776798, + "flos": 74734017402240.0, + "grad_norm": 2.299570296621784, + "language_loss": 0.87520456, + "learning_rate": 3.3919855153092614e-06, + "loss": 0.89940786, + "num_input_tokens_seen": 10271540, + "router_z_loss_clip": 2.12695312, + "router_z_loss_mlp": 0.35742188, + "step": 360, + "time_per_iteration": 2.948045253753662 + }, + { + "auxiliary_loss_clip": 0.01331979, + "auxiliary_loss_mlp": 0.01112534, + "balance_loss_clip": 1.10522187, + "balance_loss_mlp": 1.07359993, + "epoch": 0.010475306134292844, + "flos": 21646233901440.0, + "grad_norm": 2.255278525834134, + "language_loss": 0.95881617, + "learning_rate": 3.393584046316606e-06, + "loss": 0.98326123, + "num_input_tokens_seen": 10284925, + "router_z_loss_clip": 2.26367188, + "router_z_loss_mlp": 0.3894043, + "step": 361, + "time_per_iteration": 2.5769541263580322 + }, + { + "auxiliary_loss_clip": 0.01315818, + "auxiliary_loss_mlp": 0.01088924, + "balance_loss_clip": 1.09979117, + "balance_loss_mlp": 1.05471075, + "epoch": 0.01050432360280889, + "flos": 16210184584320.0, + "grad_norm": 3.0175443995563684, + "language_loss": 0.91882342, + "learning_rate": 3.3951781553791414e-06, + "loss": 0.94287086, + "num_input_tokens_seen": 10298005, + "router_z_loss_clip": 2.16015625, + "router_z_loss_mlp": 0.3425293, + "step": 362, + "time_per_iteration": 9.43061876296997 + }, + { + "auxiliary_loss_clip": 0.01096822, + "auxiliary_loss_mlp": 0.01009232, + "balance_loss_clip": 1.02091265, + "balance_loss_mlp": 1.00145912, + "epoch": 0.010533341071324937, + "flos": 58857805474560.0, + "grad_norm": 0.7586956190782234, + "language_loss": 0.55879802, + "learning_rate": 3.396767866893849e-06, + "loss": 0.57985854, + "num_input_tokens_seen": 10348085, + "router_z_loss_clip": 0.7578125, + "router_z_loss_mlp": 0.07763672, + "step": 363, + "time_per_iteration": 5.474961280822754 + }, + { + "auxiliary_loss_clip": 0.01327227, + "auxiliary_loss_mlp": 0.01088873, + "balance_loss_clip": 1.10232663, + "balance_loss_mlp": 1.05101192, + "epoch": 0.010562358539840984, + "flos": 14093394238080.0, + "grad_norm": 3.051198052138523, + "language_loss": 0.78211516, + "learning_rate": 3.3983532050563628e-06, + "loss": 0.8062762, + "num_input_tokens_seen": 10360400, + "router_z_loss_clip": 2.24414062, + "router_z_loss_mlp": 0.37890625, + "step": 364, + "time_per_iteration": 2.493088722229004 + }, + { + "auxiliary_loss_clip": 0.01325033, + "auxiliary_loss_mlp": 0.01093362, + "balance_loss_clip": 1.09799922, + "balance_loss_mlp": 1.05564427, + "epoch": 0.010591376008357032, + "flos": 36463083966720.0, + "grad_norm": 2.6262136870210635, + "language_loss": 1.00951028, + "learning_rate": 3.3999341938631724e-06, + "loss": 1.03369415, + "num_input_tokens_seen": 10379230, + "router_z_loss_clip": 2.27148438, + "router_z_loss_mlp": 0.37744141, + "step": 365, + "time_per_iteration": 2.7128865718841553 + }, + { + "auxiliary_loss_clip": 0.01330117, + "auxiliary_loss_mlp": 0.01115089, + "balance_loss_clip": 1.10512519, + "balance_loss_mlp": 1.07427144, + "epoch": 0.010620393476873078, + "flos": 37225072108800.0, + "grad_norm": 3.744927263839661, + "language_loss": 1.01431, + "learning_rate": 3.401510857113807e-06, + "loss": 1.03876209, + "num_input_tokens_seen": 10395170, + "router_z_loss_clip": 2.25, + "router_z_loss_mlp": 0.40820312, + "step": 366, + "time_per_iteration": 2.6711437702178955 + }, + { + "auxiliary_loss_clip": 0.01324797, + "auxiliary_loss_mlp": 0.01078987, + "balance_loss_clip": 1.10852408, + "balance_loss_mlp": 1.04670525, + "epoch": 0.010649410945389124, + "flos": 37116047352960.0, + "grad_norm": 4.0157687181314765, + "language_loss": 0.8305344, + "learning_rate": 3.4030832184129836e-06, + "loss": 0.85457224, + "num_input_tokens_seen": 10410605, + "router_z_loss_clip": 2.16699219, + "router_z_loss_mlp": 0.32275391, + "step": 367, + "time_per_iteration": 2.683499336242676 + }, + { + "auxiliary_loss_clip": 0.0132045, + "auxiliary_loss_mlp": 0.01091478, + "balance_loss_clip": 1.10412288, + "balance_loss_mlp": 1.05399847, + "epoch": 0.01067842841390517, + "flos": 38247628487040.0, + "grad_norm": 2.6236790094824984, + "language_loss": 0.84123313, + "learning_rate": 3.4046513011727257e-06, + "loss": 0.86535245, + "num_input_tokens_seen": 10429875, + "router_z_loss_clip": 2.16210938, + "router_z_loss_mlp": 0.37524414, + "step": 368, + "time_per_iteration": 2.700082302093506 + }, + { + "auxiliary_loss_clip": 0.01333105, + "auxiliary_loss_mlp": 0.01091032, + "balance_loss_clip": 1.10205007, + "balance_loss_mlp": 1.05388618, + "epoch": 0.010707445882421217, + "flos": 12414929558400.0, + "grad_norm": 3.6774113515571885, + "language_loss": 0.75997925, + "learning_rate": 3.406215128614456e-06, + "loss": 0.78422058, + "num_input_tokens_seen": 10442110, + "router_z_loss_clip": 2.30859375, + "router_z_loss_mlp": 0.37133789, + "step": 369, + "time_per_iteration": 2.5422728061676025 + }, + { + "auxiliary_loss_clip": 0.01323797, + "auxiliary_loss_mlp": 0.01082451, + "balance_loss_clip": 1.0998292, + "balance_loss_mlp": 1.0469501, + "epoch": 0.010736463350937264, + "flos": 28469907475200.0, + "grad_norm": 11.855583815604602, + "language_loss": 1.02752757, + "learning_rate": 3.4077747237710627e-06, + "loss": 1.05159009, + "num_input_tokens_seen": 10458400, + "router_z_loss_clip": 2.24023438, + "router_z_loss_mlp": 0.35498047, + "step": 370, + "time_per_iteration": 2.632398843765259 + }, + { + "auxiliary_loss_clip": 0.01099258, + "auxiliary_loss_mlp": 0.01012155, + "balance_loss_clip": 1.02268839, + "balance_loss_mlp": 1.00433469, + "epoch": 0.010765480819453312, + "flos": 72942580448640.0, + "grad_norm": 0.7449200550081297, + "language_loss": 0.50984204, + "learning_rate": 3.4093301094889307e-06, + "loss": 0.53095615, + "num_input_tokens_seen": 10517930, + "router_z_loss_clip": 0.765625, + "router_z_loss_mlp": 0.078125, + "step": 371, + "time_per_iteration": 3.1762804985046387 + }, + { + "auxiliary_loss_clip": 0.01324631, + "auxiliary_loss_mlp": 0.01103791, + "balance_loss_clip": 1.10424328, + "balance_loss_mlp": 1.06509566, + "epoch": 0.010794498287969358, + "flos": 23504754481920.0, + "grad_norm": 3.74573657779749, + "language_loss": 1.0530467, + "learning_rate": 3.410881308429951e-06, + "loss": 1.07733095, + "num_input_tokens_seen": 10529360, + "router_z_loss_clip": 2.20117188, + "router_z_loss_mlp": 0.38696289, + "step": 372, + "time_per_iteration": 2.5339465141296387 + }, + { + "auxiliary_loss_clip": 0.01099785, + "auxiliary_loss_mlp": 0.01008864, + "balance_loss_clip": 1.0219655, + "balance_loss_mlp": 1.00123501, + "epoch": 0.010823515756485404, + "flos": 66675953969280.0, + "grad_norm": 0.7634651445365125, + "language_loss": 0.54546821, + "learning_rate": 3.412428343073505e-06, + "loss": 0.56655467, + "num_input_tokens_seen": 10589220, + "router_z_loss_clip": 0.78125, + "router_z_loss_mlp": 0.07617188, + "step": 373, + "time_per_iteration": 3.0513556003570557 + }, + { + "auxiliary_loss_clip": 0.01319264, + "auxiliary_loss_mlp": 0.01081164, + "balance_loss_clip": 1.10129285, + "balance_loss_mlp": 1.04709411, + "epoch": 0.01085253322500145, + "flos": 43540645847040.0, + "grad_norm": 2.473211020185792, + "language_loss": 0.8625443, + "learning_rate": 3.413971235718411e-06, + "loss": 0.88654852, + "num_input_tokens_seen": 10607085, + "router_z_loss_clip": 2.18261719, + "router_z_loss_mlp": 0.34082031, + "step": 374, + "time_per_iteration": 2.8074569702148438 + }, + { + "auxiliary_loss_clip": 0.0133341, + "auxiliary_loss_mlp": 0.01099578, + "balance_loss_clip": 1.10190225, + "balance_loss_mlp": 1.0615263, + "epoch": 0.010881550693517497, + "flos": 23800550981760.0, + "grad_norm": 2.521771576933621, + "language_loss": 1.16592407, + "learning_rate": 3.4155100084848646e-06, + "loss": 1.19025397, + "num_input_tokens_seen": 10625140, + "router_z_loss_clip": 2.31542969, + "router_z_loss_mlp": 0.38012695, + "step": 375, + "time_per_iteration": 2.6130058765411377 + }, + { + "auxiliary_loss_clip": 0.01319282, + "auxiliary_loss_mlp": 0.01095017, + "balance_loss_clip": 1.10437369, + "balance_loss_mlp": 1.05768085, + "epoch": 0.010910568162033544, + "flos": 22120972980480.0, + "grad_norm": 2.988950973042031, + "language_loss": 0.91216761, + "learning_rate": 3.417044683316331e-06, + "loss": 0.93631065, + "num_input_tokens_seen": 10640310, + "router_z_loss_clip": 2.14648438, + "router_z_loss_mlp": 0.37353516, + "step": 376, + "time_per_iteration": 2.576792001724243 + }, + { + "auxiliary_loss_clip": 0.01313901, + "auxiliary_loss_mlp": 0.01082141, + "balance_loss_clip": 1.100384, + "balance_loss_mlp": 1.04892957, + "epoch": 0.010939585630549592, + "flos": 32737962245760.0, + "grad_norm": 2.556448832595328, + "language_loss": 0.85074508, + "learning_rate": 3.4185752819814268e-06, + "loss": 0.87470549, + "num_input_tokens_seen": 10659645, + "router_z_loss_clip": 2.13378906, + "router_z_loss_mlp": 0.33251953, + "step": 377, + "time_per_iteration": 2.6569199562072754 + }, + { + "auxiliary_loss_clip": 0.01316273, + "auxiliary_loss_mlp": 0.0110029, + "balance_loss_clip": 1.10277092, + "balance_loss_mlp": 1.06710243, + "epoch": 0.010968603099065638, + "flos": 30512362625280.0, + "grad_norm": 3.078149366996999, + "language_loss": 0.97027904, + "learning_rate": 3.420101826075774e-06, + "loss": 0.99444467, + "num_input_tokens_seen": 10675925, + "router_z_loss_clip": 2.13476562, + "router_z_loss_mlp": 0.33178711, + "step": 378, + "time_per_iteration": 2.701831340789795 + }, + { + "auxiliary_loss_clip": 0.01322167, + "auxiliary_loss_mlp": 0.01079675, + "balance_loss_clip": 1.10273051, + "balance_loss_mlp": 1.04579592, + "epoch": 0.010997620567581684, + "flos": 36896812692480.0, + "grad_norm": 2.9558497379381445, + "language_loss": 0.96707249, + "learning_rate": 3.4216243370238263e-06, + "loss": 0.99109089, + "num_input_tokens_seen": 10691055, + "router_z_loss_clip": 2.1953125, + "router_z_loss_mlp": 0.33886719, + "step": 379, + "time_per_iteration": 2.6710779666900635 + }, + { + "auxiliary_loss_clip": 0.01323043, + "auxiliary_loss_mlp": 0.01087392, + "balance_loss_clip": 1.09928489, + "balance_loss_mlp": 1.05077052, + "epoch": 0.01102663803609773, + "flos": 32520810574080.0, + "grad_norm": 2.511088426841464, + "language_loss": 0.9016766, + "learning_rate": 3.423142836080674e-06, + "loss": 0.92578101, + "num_input_tokens_seen": 10710200, + "router_z_loss_clip": 2.24023438, + "router_z_loss_mlp": 0.36645508, + "step": 380, + "time_per_iteration": 2.6873366832733154 + }, + { + "auxiliary_loss_clip": 0.01305979, + "auxiliary_loss_mlp": 0.01095639, + "balance_loss_clip": 1.09037232, + "balance_loss_mlp": 1.06026971, + "epoch": 0.011055655504613777, + "flos": 42515647344000.0, + "grad_norm": 3.674689915291902, + "language_loss": 1.18864, + "learning_rate": 3.4246573443338227e-06, + "loss": 1.21265626, + "num_input_tokens_seen": 10725840, + "router_z_loss_clip": 2.15332031, + "router_z_loss_mlp": 0.35339355, + "step": 381, + "time_per_iteration": 2.6395137310028076 + }, + { + "auxiliary_loss_clip": 0.0131366, + "auxiliary_loss_mlp": 0.01079194, + "balance_loss_clip": 1.10095036, + "balance_loss_mlp": 1.04276347, + "epoch": 0.011084672973129824, + "flos": 15625558823040.0, + "grad_norm": 3.514608968729682, + "language_loss": 1.0225755, + "learning_rate": 3.4261678827049543e-06, + "loss": 1.04650402, + "num_input_tokens_seen": 10738165, + "router_z_loss_clip": 2.12597656, + "router_z_loss_mlp": 0.36474609, + "step": 382, + "time_per_iteration": 2.544663429260254 + }, + { + "auxiliary_loss_clip": 0.01326239, + "auxiliary_loss_mlp": 0.01085434, + "balance_loss_clip": 1.10574102, + "balance_loss_mlp": 1.0480504, + "epoch": 0.011113690441645872, + "flos": 15735337764480.0, + "grad_norm": 3.22437888708682, + "language_loss": 1.01178682, + "learning_rate": 3.4276744719516564e-06, + "loss": 1.03590357, + "num_input_tokens_seen": 10753495, + "router_z_loss_clip": 2.20507812, + "router_z_loss_mlp": 0.37390137, + "step": 383, + "time_per_iteration": 2.5892751216888428 + }, + { + "auxiliary_loss_clip": 0.01323904, + "auxiliary_loss_mlp": 0.01095387, + "balance_loss_clip": 1.10706937, + "balance_loss_mlp": 1.05967164, + "epoch": 0.011142707910161918, + "flos": 39487372450560.0, + "grad_norm": 2.2669132769431486, + "language_loss": 1.08073115, + "learning_rate": 3.4291771326691384e-06, + "loss": 1.10492396, + "num_input_tokens_seen": 10772245, + "router_z_loss_clip": 2.16992188, + "router_z_loss_mlp": 0.35742188, + "step": 384, + "time_per_iteration": 2.748900890350342 + }, + { + "auxiliary_loss_clip": 0.01308091, + "auxiliary_loss_mlp": 0.01086541, + "balance_loss_clip": 1.09620333, + "balance_loss_mlp": 1.05337751, + "epoch": 0.011171725378677964, + "flos": 32628722008320.0, + "grad_norm": 3.172911392629614, + "language_loss": 0.76222962, + "learning_rate": 3.4306758852919156e-06, + "loss": 0.78617591, + "num_input_tokens_seen": 10788255, + "router_z_loss_clip": 2.11914062, + "router_z_loss_mlp": 0.33154297, + "step": 385, + "time_per_iteration": 2.5791120529174805 + }, + { + "auxiliary_loss_clip": 0.01315903, + "auxiliary_loss_mlp": 0.01087682, + "balance_loss_clip": 1.10093379, + "balance_loss_mlp": 1.05117989, + "epoch": 0.01120074284719401, + "flos": 29745095184000.0, + "grad_norm": 2.6736201697429647, + "language_loss": 0.89352548, + "learning_rate": 3.4321707500954817e-06, + "loss": 0.91756129, + "num_input_tokens_seen": 10806540, + "router_z_loss_clip": 2.1484375, + "router_z_loss_mlp": 0.36499023, + "step": 386, + "time_per_iteration": 2.7255382537841797 + }, + { + "auxiliary_loss_clip": 0.01330425, + "auxiliary_loss_mlp": 0.01083926, + "balance_loss_clip": 1.10392976, + "balance_loss_mlp": 1.04701865, + "epoch": 0.011229760315710057, + "flos": 30553983509760.0, + "grad_norm": 2.5765488049515772, + "language_loss": 0.86346418, + "learning_rate": 3.433661747197952e-06, + "loss": 0.88760769, + "num_input_tokens_seen": 10828090, + "router_z_loss_clip": 2.26757812, + "router_z_loss_mlp": 0.36914062, + "step": 387, + "time_per_iteration": 2.6215367317199707 + }, + { + "auxiliary_loss_clip": 0.01100029, + "auxiliary_loss_mlp": 0.01031638, + "balance_loss_clip": 1.02679038, + "balance_loss_mlp": 1.02429509, + "epoch": 0.011258777784226104, + "flos": 62696082015360.0, + "grad_norm": 0.7131582145646421, + "language_loss": 0.5348587, + "learning_rate": 3.4351488965616886e-06, + "loss": 0.55617535, + "num_input_tokens_seen": 10887500, + "router_z_loss_clip": 0.734375, + "router_z_loss_mlp": 0.07324219, + "step": 388, + "time_per_iteration": 3.078666925430298 + }, + { + "auxiliary_loss_clip": 0.01314369, + "auxiliary_loss_mlp": 0.01095422, + "balance_loss_clip": 1.10121298, + "balance_loss_mlp": 1.06347346, + "epoch": 0.011287795252742152, + "flos": 15407760706560.0, + "grad_norm": 2.4852787157300353, + "language_loss": 0.83029717, + "learning_rate": 3.4366322179949013e-06, + "loss": 0.85439509, + "num_input_tokens_seen": 10903135, + "router_z_loss_clip": 2.12890625, + "router_z_loss_mlp": 0.31933594, + "step": 389, + "time_per_iteration": 2.6003823280334473 + }, + { + "auxiliary_loss_clip": 0.01325931, + "auxiliary_loss_mlp": 0.01108561, + "balance_loss_clip": 1.10329342, + "balance_loss_mlp": 1.07379973, + "epoch": 0.011316812721258198, + "flos": 30987101704320.0, + "grad_norm": 2.4720341095933622, + "language_loss": 1.05350018, + "learning_rate": 3.438111731153238e-06, + "loss": 1.07784498, + "num_input_tokens_seen": 10922040, + "router_z_loss_clip": 2.22460938, + "router_z_loss_mlp": 0.34765625, + "step": 390, + "time_per_iteration": 2.650697708129883 + }, + { + "auxiliary_loss_clip": 0.01308277, + "auxiliary_loss_mlp": 0.01091707, + "balance_loss_clip": 1.09683704, + "balance_loss_mlp": 1.0575422, + "epoch": 0.011345830189774244, + "flos": 23365349798400.0, + "grad_norm": 3.3291190666012, + "language_loss": 0.90073466, + "learning_rate": 3.439587455541337e-06, + "loss": 0.92473447, + "num_input_tokens_seen": 10937670, + "router_z_loss_clip": 2.11621094, + "router_z_loss_mlp": 0.34155273, + "step": 391, + "time_per_iteration": 2.616384267807007 + }, + { + "auxiliary_loss_clip": 0.01316838, + "auxiliary_loss_mlp": 0.01081222, + "balance_loss_clip": 1.09684801, + "balance_loss_mlp": 1.04392171, + "epoch": 0.01137484765829029, + "flos": 34710643226880.0, + "grad_norm": 3.1324888575924894, + "language_loss": 1.01897025, + "learning_rate": 3.4410594105143784e-06, + "loss": 1.04295087, + "num_input_tokens_seen": 10958640, + "router_z_loss_clip": 2.19726562, + "router_z_loss_mlp": 0.37280273, + "step": 392, + "time_per_iteration": 2.7008745670318604 + }, + { + "auxiliary_loss_clip": 0.01311548, + "auxiliary_loss_mlp": 0.01092384, + "balance_loss_clip": 1.09327412, + "balance_loss_mlp": 1.05497622, + "epoch": 0.011403865126806337, + "flos": 25623807384960.0, + "grad_norm": 2.2209810354583333, + "language_loss": 1.00377858, + "learning_rate": 3.442527615279605e-06, + "loss": 1.02781796, + "num_input_tokens_seen": 10977625, + "router_z_loss_clip": 2.18164062, + "router_z_loss_mlp": 0.37402344, + "step": 393, + "time_per_iteration": 2.7826356887817383 + }, + { + "auxiliary_loss_clip": 0.01316769, + "auxiliary_loss_mlp": 0.01086577, + "balance_loss_clip": 1.09852707, + "balance_loss_mlp": 1.0532465, + "epoch": 0.011432882595322384, + "flos": 31462343573760.0, + "grad_norm": 2.6816827425558976, + "language_loss": 0.8451587, + "learning_rate": 3.443992088897824e-06, + "loss": 0.86919218, + "num_input_tokens_seen": 11004130, + "router_z_loss_clip": 2.1796875, + "router_z_loss_mlp": 0.33349609, + "step": 394, + "time_per_iteration": 2.9477739334106445 + }, + { + "auxiliary_loss_clip": 0.01107874, + "auxiliary_loss_mlp": 0.01031487, + "balance_loss_clip": 1.0278039, + "balance_loss_mlp": 1.02352381, + "epoch": 0.01146190006383843, + "flos": 74770864755840.0, + "grad_norm": 0.6873104672552192, + "language_loss": 0.51574558, + "learning_rate": 3.4454528502848933e-06, + "loss": 0.53713918, + "num_input_tokens_seen": 11068260, + "router_z_loss_clip": 0.796875, + "router_z_loss_mlp": 0.07958984, + "step": 395, + "time_per_iteration": 3.184182643890381 + }, + { + "auxiliary_loss_clip": 0.01110437, + "auxiliary_loss_mlp": 0.01017988, + "balance_loss_clip": 1.02887964, + "balance_loss_mlp": 1.01002526, + "epoch": 0.011490917532354478, + "flos": 63249250440960.0, + "grad_norm": 0.6887623298590515, + "language_loss": 0.51883048, + "learning_rate": 3.4469099182131874e-06, + "loss": 0.5401147, + "num_input_tokens_seen": 11135770, + "router_z_loss_clip": 0.81640625, + "router_z_loss_mlp": 0.07958984, + "step": 396, + "time_per_iteration": 3.304978132247925 + }, + { + "auxiliary_loss_clip": 0.01306646, + "auxiliary_loss_mlp": 0.01091826, + "balance_loss_clip": 1.09539664, + "balance_loss_mlp": 1.05844748, + "epoch": 0.011519935000870524, + "flos": 36022388002560.0, + "grad_norm": 2.743980667224717, + "language_loss": 0.98562878, + "learning_rate": 3.4483633113130455e-06, + "loss": 1.00961351, + "num_input_tokens_seen": 11152745, + "router_z_loss_clip": 2.11230469, + "router_z_loss_mlp": 0.33398438, + "step": 397, + "time_per_iteration": 2.6853010654449463 + }, + { + "auxiliary_loss_clip": 0.01109713, + "auxiliary_loss_mlp": 0.01014014, + "balance_loss_clip": 1.02934968, + "balance_loss_mlp": 1.00595593, + "epoch": 0.01154895246938657, + "flos": 65350992988800.0, + "grad_norm": 0.7023398366291074, + "language_loss": 0.50545532, + "learning_rate": 3.4498130480741995e-06, + "loss": 0.52669263, + "num_input_tokens_seen": 11204975, + "router_z_loss_clip": 0.8046875, + "router_z_loss_mlp": 0.08056641, + "step": 398, + "time_per_iteration": 2.9680516719818115 + }, + { + "auxiliary_loss_clip": 0.01317925, + "auxiliary_loss_mlp": 0.01098799, + "balance_loss_clip": 1.10171115, + "balance_loss_mlp": 1.06375182, + "epoch": 0.011577969937902617, + "flos": 17708270140800.0, + "grad_norm": 3.0177626823643924, + "language_loss": 0.8474074, + "learning_rate": 3.4512591468471864e-06, + "loss": 0.87157458, + "num_input_tokens_seen": 11218985, + "router_z_loss_clip": 2.1640625, + "router_z_loss_mlp": 0.35009766, + "step": 399, + "time_per_iteration": 2.5236728191375732 + }, + { + "auxiliary_loss_clip": 0.01312321, + "auxiliary_loss_mlp": 0.01095688, + "balance_loss_clip": 1.09570789, + "balance_loss_mlp": 1.06137991, + "epoch": 0.011606987406418664, + "flos": 30917291621760.0, + "grad_norm": 7.641500203857065, + "language_loss": 1.11217058, + "learning_rate": 3.452701625844741e-06, + "loss": 1.13625073, + "num_input_tokens_seen": 11239685, + "router_z_loss_clip": 2.16503906, + "router_z_loss_mlp": 0.34301758, + "step": 400, + "time_per_iteration": 2.6431429386138916 + }, + { + "auxiliary_loss_clip": 0.01320283, + "auxiliary_loss_mlp": 0.01091528, + "balance_loss_clip": 1.09928119, + "balance_loss_mlp": 1.05691004, + "epoch": 0.01163600487493471, + "flos": 29891969896320.0, + "grad_norm": 3.4240688461353708, + "language_loss": 1.13923597, + "learning_rate": 3.4541405031431746e-06, + "loss": 1.16335416, + "num_input_tokens_seen": 11256765, + "router_z_loss_clip": 2.20507812, + "router_z_loss_mlp": 0.34594727, + "step": 401, + "time_per_iteration": 2.607975959777832 + }, + { + "auxiliary_loss_clip": 0.01293897, + "auxiliary_loss_mlp": 0.01073525, + "balance_loss_clip": 1.09546137, + "balance_loss_mlp": 1.04264975, + "epoch": 0.011665022343450758, + "flos": 32703344513280.0, + "grad_norm": 2.1701012204762113, + "language_loss": 0.81593144, + "learning_rate": 3.45557579668373e-06, + "loss": 0.83960569, + "num_input_tokens_seen": 11274260, + "router_z_loss_clip": 1.984375, + "router_z_loss_mlp": 0.30883789, + "step": 402, + "time_per_iteration": 2.716440439224243 + }, + { + "auxiliary_loss_clip": 0.01110928, + "auxiliary_loss_mlp": 0.01051047, + "balance_loss_clip": 1.03346276, + "balance_loss_mlp": 1.04265499, + "epoch": 0.011694039811966804, + "flos": 57465943413120.0, + "grad_norm": 0.8101117381577724, + "language_loss": 0.55296481, + "learning_rate": 3.4570075242739278e-06, + "loss": 0.57458448, + "num_input_tokens_seen": 11319090, + "router_z_loss_clip": 0.7734375, + "router_z_loss_mlp": 0.08398438, + "step": 403, + "time_per_iteration": 2.8104918003082275 + }, + { + "auxiliary_loss_clip": 0.01310713, + "auxiliary_loss_mlp": 0.01077879, + "balance_loss_clip": 1.09876931, + "balance_loss_mlp": 1.04577613, + "epoch": 0.01172305728048285, + "flos": 32483247926400.0, + "grad_norm": 2.392102325616908, + "language_loss": 1.08746231, + "learning_rate": 3.4584357035888897e-06, + "loss": 1.11134815, + "num_input_tokens_seen": 11345600, + "router_z_loss_clip": 2.11523438, + "router_z_loss_mlp": 0.32104492, + "step": 404, + "time_per_iteration": 2.9067375659942627 + }, + { + "auxiliary_loss_clip": 0.0132139, + "auxiliary_loss_mlp": 0.01094607, + "balance_loss_clip": 1.10429585, + "balance_loss_mlp": 1.06039357, + "epoch": 0.011752074748998897, + "flos": 35184340811520.0, + "grad_norm": 2.261085870271482, + "language_loss": 0.80502081, + "learning_rate": 3.4598603521726485e-06, + "loss": 0.82918084, + "num_input_tokens_seen": 11366800, + "router_z_loss_clip": 2.171875, + "router_z_loss_mlp": 0.34179688, + "step": 405, + "time_per_iteration": 2.6165990829467773 + }, + { + "auxiliary_loss_clip": 0.01302321, + "auxiliary_loss_mlp": 0.01099486, + "balance_loss_clip": 1.09190857, + "balance_loss_mlp": 1.06476068, + "epoch": 0.011781092217514944, + "flos": 31059353911680.0, + "grad_norm": 2.6865935073599356, + "language_loss": 0.98698473, + "learning_rate": 3.4612814874394425e-06, + "loss": 1.01100278, + "num_input_tokens_seen": 11382550, + "router_z_loss_clip": 2.10546875, + "router_z_loss_mlp": 0.34729004, + "step": 406, + "time_per_iteration": 2.6797566413879395 + }, + { + "auxiliary_loss_clip": 0.01316121, + "auxiliary_loss_mlp": 0.01079532, + "balance_loss_clip": 1.10366523, + "balance_loss_mlp": 1.04744077, + "epoch": 0.01181010968603099, + "flos": 24309081780480.0, + "grad_norm": 4.786462352006664, + "language_loss": 0.91958839, + "learning_rate": 3.4626991266749886e-06, + "loss": 0.94354486, + "num_input_tokens_seen": 11397580, + "router_z_loss_clip": 2.12402344, + "router_z_loss_mlp": 0.32080078, + "step": 407, + "time_per_iteration": 2.6146926879882812 + }, + { + "auxiliary_loss_clip": 0.01310676, + "auxiliary_loss_mlp": 0.01093216, + "balance_loss_clip": 1.09763265, + "balance_loss_mlp": 1.05971849, + "epoch": 0.011839127154547038, + "flos": 7194559455360.0, + "grad_norm": 5.2559719141860795, + "language_loss": 0.96316326, + "learning_rate": 3.4641132870377497e-06, + "loss": 0.98720223, + "num_input_tokens_seen": 11405950, + "router_z_loss_clip": 2.12890625, + "router_z_loss_mlp": 0.33520508, + "step": 408, + "time_per_iteration": 2.562509059906006 + }, + { + "auxiliary_loss_clip": 0.0110174, + "auxiliary_loss_mlp": 0.01013916, + "balance_loss_clip": 1.02571416, + "balance_loss_mlp": 1.00700176, + "epoch": 0.011868144623063084, + "flos": 67377397737600.0, + "grad_norm": 0.7084467565862828, + "language_loss": 0.51358163, + "learning_rate": 3.4655239855601753e-06, + "loss": 0.53473818, + "num_input_tokens_seen": 11471845, + "router_z_loss_clip": 0.7578125, + "router_z_loss_mlp": 0.06933594, + "step": 409, + "time_per_iteration": 3.1808931827545166 + }, + { + "auxiliary_loss_clip": 0.01307443, + "auxiliary_loss_mlp": 0.01081617, + "balance_loss_clip": 1.09826076, + "balance_loss_mlp": 1.05063415, + "epoch": 0.01189716209157913, + "flos": 22084846876800.0, + "grad_norm": 2.413487335331761, + "language_loss": 0.67463058, + "learning_rate": 3.4669312391499364e-06, + "loss": 0.6985212, + "num_input_tokens_seen": 11485650, + "router_z_loss_clip": 2.09277344, + "router_z_loss_mlp": 0.30993652, + "step": 410, + "time_per_iteration": 2.529475450515747 + }, + { + "auxiliary_loss_clip": 0.01318386, + "auxiliary_loss_mlp": 0.0108775, + "balance_loss_clip": 1.10557151, + "balance_loss_mlp": 1.05625474, + "epoch": 0.011926179560095177, + "flos": 74734089229440.0, + "grad_norm": 2.358512003816413, + "language_loss": 0.75491697, + "learning_rate": 3.468335064591138e-06, + "loss": 0.77897835, + "num_input_tokens_seen": 11512540, + "router_z_loss_clip": 2.12695312, + "router_z_loss_mlp": 0.31494141, + "step": 411, + "time_per_iteration": 3.1185390949249268 + }, + { + "auxiliary_loss_clip": 0.01307743, + "auxiliary_loss_mlp": 0.01102303, + "balance_loss_clip": 1.09496772, + "balance_loss_mlp": 1.06856668, + "epoch": 0.011955197028611224, + "flos": 18771369995520.0, + "grad_norm": 3.7242455819171454, + "language_loss": 1.00518787, + "learning_rate": 3.469735478545525e-06, + "loss": 1.02928829, + "num_input_tokens_seen": 11526705, + "router_z_loss_clip": 2.125, + "router_z_loss_mlp": 0.3371582, + "step": 412, + "time_per_iteration": 2.5611696243286133 + }, + { + "auxiliary_loss_clip": 0.01294916, + "auxiliary_loss_mlp": 0.0110067, + "balance_loss_clip": 1.09539688, + "balance_loss_mlp": 1.06912696, + "epoch": 0.01198421449712727, + "flos": 24236111301120.0, + "grad_norm": 2.965273872691427, + "language_loss": 0.97105992, + "learning_rate": 3.4711324975536624e-06, + "loss": 0.99501574, + "num_input_tokens_seen": 11542650, + "router_z_loss_clip": 1.99414062, + "router_z_loss_mlp": 0.31518555, + "step": 413, + "time_per_iteration": 2.572779655456543 + }, + { + "auxiliary_loss_clip": 0.01310538, + "auxiliary_loss_mlp": 0.01090773, + "balance_loss_clip": 1.09907389, + "balance_loss_mlp": 1.05970716, + "epoch": 0.012013231965643318, + "flos": 47587957585920.0, + "grad_norm": 3.136616151705879, + "language_loss": 0.68838716, + "learning_rate": 3.4725261380361128e-06, + "loss": 0.71240032, + "num_input_tokens_seen": 11561345, + "router_z_loss_clip": 2.11425781, + "router_z_loss_mlp": 0.31103516, + "step": 414, + "time_per_iteration": 2.8273346424102783 + }, + { + "auxiliary_loss_clip": 0.01302417, + "auxiliary_loss_mlp": 0.01080097, + "balance_loss_clip": 1.09800792, + "balance_loss_mlp": 1.04974627, + "epoch": 0.012042249434159364, + "flos": 15152364028800.0, + "grad_norm": 2.7928097203970834, + "language_loss": 0.82845038, + "learning_rate": 3.473916416294592e-06, + "loss": 0.85227549, + "num_input_tokens_seen": 11574225, + "router_z_loss_clip": 2.04394531, + "router_z_loss_mlp": 0.30371094, + "step": 415, + "time_per_iteration": 2.50691556930542 + }, + { + "auxiliary_loss_clip": 0.01313117, + "auxiliary_loss_mlp": 0.01086922, + "balance_loss_clip": 1.10129905, + "balance_loss_mlp": 1.05280423, + "epoch": 0.01207126690267541, + "flos": 19201866497280.0, + "grad_norm": 3.714959401308144, + "language_loss": 0.88886243, + "learning_rate": 3.4753033485131146e-06, + "loss": 0.91286278, + "num_input_tokens_seen": 11595270, + "router_z_loss_clip": 2.12207031, + "router_z_loss_mlp": 0.34082031, + "step": 416, + "time_per_iteration": 2.933443546295166 + }, + { + "auxiliary_loss_clip": 0.01305791, + "auxiliary_loss_mlp": 0.0107865, + "balance_loss_clip": 1.09832799, + "balance_loss_mlp": 1.04659438, + "epoch": 0.012100284371191457, + "flos": 16536289184640.0, + "grad_norm": 3.469304941157518, + "language_loss": 0.96267754, + "learning_rate": 3.4766869507591215e-06, + "loss": 0.98652202, + "num_input_tokens_seen": 11608715, + "router_z_loss_clip": 2.078125, + "router_z_loss_mlp": 0.32043457, + "step": 417, + "time_per_iteration": 2.5029051303863525 + }, + { + "auxiliary_loss_clip": 0.01305409, + "auxiliary_loss_mlp": 0.01104978, + "balance_loss_clip": 1.09628439, + "balance_loss_mlp": 1.07105064, + "epoch": 0.012129301839707504, + "flos": 57803645128320.0, + "grad_norm": 2.577953108296549, + "language_loss": 0.76576972, + "learning_rate": 3.4780672389845997e-06, + "loss": 0.7898736, + "num_input_tokens_seen": 11632190, + "router_z_loss_clip": 2.09472656, + "router_z_loss_mlp": 0.33947754, + "step": 418, + "time_per_iteration": 2.8248167037963867 + }, + { + "auxiliary_loss_clip": 0.01307606, + "auxiliary_loss_mlp": 0.01087838, + "balance_loss_clip": 1.09731352, + "balance_loss_mlp": 1.05455446, + "epoch": 0.01215831930822355, + "flos": 28651184438400.0, + "grad_norm": 2.8711000449933533, + "language_loss": 0.72344023, + "learning_rate": 3.4794442290271854e-06, + "loss": 0.74739468, + "num_input_tokens_seen": 11649890, + "router_z_loss_clip": 2.10253906, + "router_z_loss_mlp": 0.33239746, + "step": 419, + "time_per_iteration": 2.665510416030884 + }, + { + "auxiliary_loss_clip": 0.01312184, + "auxiliary_loss_mlp": 0.01095809, + "balance_loss_clip": 1.09760094, + "balance_loss_mlp": 1.06284761, + "epoch": 0.012187336776739598, + "flos": 11428176061440.0, + "grad_norm": 7.778494919939288, + "language_loss": 0.96799791, + "learning_rate": 3.4808179366112537e-06, + "loss": 0.99207783, + "num_input_tokens_seen": 11662145, + "router_z_loss_clip": 2.14746094, + "router_z_loss_mlp": 0.32958984, + "step": 420, + "time_per_iteration": 2.5265557765960693 + }, + { + "auxiliary_loss_clip": 0.01304316, + "auxiliary_loss_mlp": 0.0108753, + "balance_loss_clip": 1.09649193, + "balance_loss_mlp": 1.05593956, + "epoch": 0.012216354245255644, + "flos": 31782094444800.0, + "grad_norm": 2.243266898507432, + "language_loss": 0.94154894, + "learning_rate": 3.482188377348995e-06, + "loss": 0.96546745, + "num_input_tokens_seen": 11685255, + "router_z_loss_clip": 2.078125, + "router_z_loss_mlp": 0.3157959, + "step": 421, + "time_per_iteration": 2.7325878143310547 + }, + { + "auxiliary_loss_clip": 0.01295044, + "auxiliary_loss_mlp": 0.01079373, + "balance_loss_clip": 1.09301734, + "balance_loss_mlp": 1.04699576, + "epoch": 0.01224537171377169, + "flos": 27557704656000.0, + "grad_norm": 2.5684681483739187, + "language_loss": 0.91192186, + "learning_rate": 3.4835555667414816e-06, + "loss": 0.93566608, + "num_input_tokens_seen": 11701360, + "router_z_loss_clip": 2.01953125, + "router_z_loss_mlp": 0.32373047, + "step": 422, + "time_per_iteration": 2.645716428756714 + }, + { + "auxiliary_loss_clip": 0.01306439, + "auxiliary_loss_mlp": 0.01091571, + "balance_loss_clip": 1.09651542, + "balance_loss_mlp": 1.05835915, + "epoch": 0.012274389182287737, + "flos": 25475855264640.0, + "grad_norm": 2.8499004839543374, + "language_loss": 1.00535464, + "learning_rate": 3.484919520179718e-06, + "loss": 1.02933478, + "num_input_tokens_seen": 11716840, + "router_z_loss_clip": 2.09765625, + "router_z_loss_mlp": 0.33190918, + "step": 423, + "time_per_iteration": 2.5567164421081543 + }, + { + "auxiliary_loss_clip": 0.01092067, + "auxiliary_loss_mlp": 0.01007248, + "balance_loss_clip": 1.02187431, + "balance_loss_mlp": 1.00071549, + "epoch": 0.012303406650803783, + "flos": 57665033521920.0, + "grad_norm": 0.7256627429760897, + "language_loss": 0.54968381, + "learning_rate": 3.4862802529456826e-06, + "loss": 0.57067698, + "num_input_tokens_seen": 11775725, + "router_z_loss_clip": 0.703125, + "router_z_loss_mlp": 0.06542969, + "step": 424, + "time_per_iteration": 3.0793750286102295 + }, + { + "auxiliary_loss_clip": 0.0131027, + "auxiliary_loss_mlp": 0.01094113, + "balance_loss_clip": 1.09707141, + "balance_loss_mlp": 1.06137776, + "epoch": 0.01233242411931983, + "flos": 27557345520000.0, + "grad_norm": 3.3835002729045263, + "language_loss": 0.93564588, + "learning_rate": 3.487637780213353e-06, + "loss": 0.95968974, + "num_input_tokens_seen": 11792910, + "router_z_loss_clip": 2.13183594, + "router_z_loss_mlp": 0.32714844, + "step": 425, + "time_per_iteration": 2.600395441055298 + }, + { + "auxiliary_loss_clip": 0.01314527, + "auxiliary_loss_mlp": 0.01082387, + "balance_loss_clip": 1.10195577, + "balance_loss_mlp": 1.04862654, + "epoch": 0.012361441587835878, + "flos": 15811324986240.0, + "grad_norm": 2.8831970015235098, + "language_loss": 0.85287559, + "learning_rate": 3.4889921170497213e-06, + "loss": 0.87684476, + "num_input_tokens_seen": 11805285, + "router_z_loss_clip": 2.12304688, + "router_z_loss_mlp": 0.33764648, + "step": 426, + "time_per_iteration": 2.5100302696228027 + }, + { + "auxiliary_loss_clip": 0.01089814, + "auxiliary_loss_mlp": 0.01009728, + "balance_loss_clip": 1.02154791, + "balance_loss_mlp": 1.00276589, + "epoch": 0.012390459056351924, + "flos": 60625653148800.0, + "grad_norm": 0.6866784127823171, + "language_loss": 0.5924899, + "learning_rate": 3.4903432784158e-06, + "loss": 0.61348534, + "num_input_tokens_seen": 11868365, + "router_z_loss_clip": 0.68359375, + "router_z_loss_mlp": 0.06982422, + "step": 427, + "time_per_iteration": 3.0768966674804688 + }, + { + "auxiliary_loss_clip": 0.01317778, + "auxiliary_loss_mlp": 0.01095677, + "balance_loss_clip": 1.09724021, + "balance_loss_mlp": 1.0614996, + "epoch": 0.01241947652486797, + "flos": 25079940581760.0, + "grad_norm": 2.5921111623246156, + "language_loss": 1.04732561, + "learning_rate": 3.49169127916761e-06, + "loss": 1.07146013, + "num_input_tokens_seen": 11889040, + "router_z_loss_clip": 2.203125, + "router_z_loss_mlp": 0.34191895, + "step": 428, + "time_per_iteration": 2.7251923084259033 + }, + { + "auxiliary_loss_clip": 0.01293361, + "auxiliary_loss_mlp": 0.01067498, + "balance_loss_clip": 1.09376144, + "balance_loss_mlp": 1.03750467, + "epoch": 0.012448493993384017, + "flos": 22518467861760.0, + "grad_norm": 3.2867745314491854, + "language_loss": 0.92901933, + "learning_rate": 3.4930361340571636e-06, + "loss": 0.9526279, + "num_input_tokens_seen": 11901770, + "router_z_loss_clip": 1.99609375, + "router_z_loss_mlp": 0.29956055, + "step": 429, + "time_per_iteration": 2.546724319458008 + }, + { + "auxiliary_loss_clip": 0.01302473, + "auxiliary_loss_mlp": 0.01084304, + "balance_loss_clip": 1.09413791, + "balance_loss_mlp": 1.05216479, + "epoch": 0.012477511461900063, + "flos": 26208971850240.0, + "grad_norm": 3.4031575962022873, + "language_loss": 0.97996026, + "learning_rate": 3.494377857733432e-06, + "loss": 1.00382805, + "num_input_tokens_seen": 11918465, + "router_z_loss_clip": 2.08203125, + "router_z_loss_mlp": 0.3215332, + "step": 430, + "time_per_iteration": 2.582951784133911 + }, + { + "auxiliary_loss_clip": 0.01308423, + "auxiliary_loss_mlp": 0.01082784, + "balance_loss_clip": 1.09575224, + "balance_loss_mlp": 1.04876161, + "epoch": 0.01250652893041611, + "flos": 13912476410880.0, + "grad_norm": 3.7774777095107734, + "language_loss": 0.97363985, + "learning_rate": 3.4957164647433026e-06, + "loss": 0.99755192, + "num_input_tokens_seen": 11931705, + "router_z_loss_clip": 2.12890625, + "router_z_loss_mlp": 0.34033203, + "step": 431, + "time_per_iteration": 2.6421921253204346 + }, + { + "auxiliary_loss_clip": 0.01297056, + "auxiliary_loss_mlp": 0.01085905, + "balance_loss_clip": 1.09216189, + "balance_loss_mlp": 1.05073833, + "epoch": 0.012535546398932158, + "flos": 23945019482880.0, + "grad_norm": 2.385630223541665, + "language_loss": 0.9303115, + "learning_rate": 3.497051969532526e-06, + "loss": 0.95414108, + "num_input_tokens_seen": 11947765, + "router_z_loss_clip": 2.04882812, + "router_z_loss_mlp": 0.35131836, + "step": 432, + "time_per_iteration": 2.5868797302246094 + }, + { + "auxiliary_loss_clip": 0.01090008, + "auxiliary_loss_mlp": 0.01015799, + "balance_loss_clip": 1.02255416, + "balance_loss_mlp": 1.00859916, + "epoch": 0.012564563867448204, + "flos": 62878615954560.0, + "grad_norm": 0.6886588827088117, + "language_loss": 0.54485887, + "learning_rate": 3.498384386446649e-06, + "loss": 0.56591696, + "num_input_tokens_seen": 12009485, + "router_z_loss_clip": 0.671875, + "router_z_loss_mlp": 0.07177734, + "step": 433, + "time_per_iteration": 7.7564451694488525 + }, + { + "auxiliary_loss_clip": 0.0129647, + "auxiliary_loss_mlp": 0.01089285, + "balance_loss_clip": 1.09548831, + "balance_loss_mlp": 1.05738485, + "epoch": 0.01259358133596425, + "flos": 17044460847360.0, + "grad_norm": 4.260431680947252, + "language_loss": 0.99233556, + "learning_rate": 3.499713729731944e-06, + "loss": 1.01619303, + "num_input_tokens_seen": 12020175, + "router_z_loss_clip": 2.00976562, + "router_z_loss_mlp": 0.31896973, + "step": 434, + "time_per_iteration": 7.3047239780426025 + }, + { + "auxiliary_loss_clip": 0.01296727, + "auxiliary_loss_mlp": 0.01071793, + "balance_loss_clip": 1.09364367, + "balance_loss_mlp": 1.04189515, + "epoch": 0.012622598804480297, + "flos": 74731826672640.0, + "grad_norm": 4.074681411467061, + "language_loss": 0.92734957, + "learning_rate": 3.5010400135363173e-06, + "loss": 0.95103478, + "num_input_tokens_seen": 12041690, + "router_z_loss_clip": 2.03125, + "router_z_loss_mlp": 0.29882812, + "step": 435, + "time_per_iteration": 2.9783904552459717 + }, + { + "auxiliary_loss_clip": 0.01089776, + "auxiliary_loss_mlp": 0.01014335, + "balance_loss_clip": 1.022686, + "balance_loss_mlp": 1.00746858, + "epoch": 0.012651616272996343, + "flos": 74777472858240.0, + "grad_norm": 0.6881987063121897, + "language_loss": 0.56303811, + "learning_rate": 3.5023632519102177e-06, + "loss": 0.58407915, + "num_input_tokens_seen": 12108800, + "router_z_loss_clip": 0.671875, + "router_z_loss_mlp": 0.06884766, + "step": 436, + "time_per_iteration": 3.2211544513702393 + }, + { + "auxiliary_loss_clip": 0.0129791, + "auxiliary_loss_mlp": 0.01077859, + "balance_loss_clip": 1.09408033, + "balance_loss_mlp": 1.04431319, + "epoch": 0.01268063374151239, + "flos": 15698672956800.0, + "grad_norm": 3.2432146180520998, + "language_loss": 1.0182656, + "learning_rate": 3.503683458807525e-06, + "loss": 1.0420233, + "num_input_tokens_seen": 12120415, + "router_z_loss_clip": 2.0390625, + "router_z_loss_mlp": 0.33520508, + "step": 437, + "time_per_iteration": 2.518847942352295 + }, + { + "auxiliary_loss_clip": 0.01305552, + "auxiliary_loss_mlp": 0.01087375, + "balance_loss_clip": 1.10085142, + "balance_loss_mlp": 1.05691683, + "epoch": 0.012709651210028438, + "flos": 34598960864640.0, + "grad_norm": 2.749108209312204, + "language_loss": 0.93896616, + "learning_rate": 3.505000648086437e-06, + "loss": 0.96289539, + "num_input_tokens_seen": 12139055, + "router_z_loss_clip": 2.046875, + "router_z_loss_mlp": 0.3046875, + "step": 438, + "time_per_iteration": 2.6664927005767822 + }, + { + "auxiliary_loss_clip": 0.01298358, + "auxiliary_loss_mlp": 0.0107639, + "balance_loss_clip": 1.10067463, + "balance_loss_mlp": 1.04539514, + "epoch": 0.012738668678544484, + "flos": 28287086227200.0, + "grad_norm": 2.337096741560056, + "language_loss": 0.99150348, + "learning_rate": 3.5063148335103383e-06, + "loss": 1.01525092, + "num_input_tokens_seen": 12156030, + "router_z_loss_clip": 1.97753906, + "router_z_loss_mlp": 0.31030273, + "step": 439, + "time_per_iteration": 2.627196788787842 + }, + { + "auxiliary_loss_clip": 0.01312534, + "auxiliary_loss_mlp": 0.01088141, + "balance_loss_clip": 1.09775984, + "balance_loss_mlp": 1.05533504, + "epoch": 0.01276768614706053, + "flos": 24952100077440.0, + "grad_norm": 2.792781954058606, + "language_loss": 1.08647466, + "learning_rate": 3.507626028748667e-06, + "loss": 1.11048126, + "num_input_tokens_seen": 12175310, + "router_z_loss_clip": 2.14746094, + "router_z_loss_mlp": 0.32800293, + "step": 440, + "time_per_iteration": 2.6686415672302246 + }, + { + "auxiliary_loss_clip": 0.01088494, + "auxiliary_loss_mlp": 0.01005806, + "balance_loss_clip": 1.02193856, + "balance_loss_mlp": 0.99922562, + "epoch": 0.012796703615576577, + "flos": 68354131340160.0, + "grad_norm": 0.7920112859153342, + "language_loss": 0.50281036, + "learning_rate": 3.508934247377766e-06, + "loss": 0.52375335, + "num_input_tokens_seen": 12236095, + "router_z_loss_clip": 0.6640625, + "router_z_loss_mlp": 0.06591797, + "step": 441, + "time_per_iteration": 3.0653226375579834 + }, + { + "auxiliary_loss_clip": 0.01088877, + "auxiliary_loss_mlp": 0.01005846, + "balance_loss_clip": 1.02248859, + "balance_loss_mlp": 0.99931383, + "epoch": 0.012825721084092623, + "flos": 66712941999360.0, + "grad_norm": 4.55118971483166, + "language_loss": 0.56292373, + "learning_rate": 3.510239502881726e-06, + "loss": 0.58387095, + "num_input_tokens_seen": 12292810, + "router_z_loss_clip": 0.6640625, + "router_z_loss_mlp": 0.06542969, + "step": 442, + "time_per_iteration": 3.030334234237671 + }, + { + "auxiliary_loss_clip": 0.01301411, + "auxiliary_loss_mlp": 0.01080769, + "balance_loss_clip": 1.0967263, + "balance_loss_mlp": 1.04739022, + "epoch": 0.01285473855260867, + "flos": 27668309610240.0, + "grad_norm": 2.306386888091903, + "language_loss": 0.89091921, + "learning_rate": 3.5115418086532197e-06, + "loss": 0.9147411, + "num_input_tokens_seen": 12312710, + "router_z_loss_clip": 2.04589844, + "router_z_loss_mlp": 0.33374023, + "step": 443, + "time_per_iteration": 2.600262403488159 + }, + { + "auxiliary_loss_clip": 0.01310182, + "auxiliary_loss_mlp": 0.01106045, + "balance_loss_clip": 1.09534621, + "balance_loss_mlp": 1.06801701, + "epoch": 0.012883756021124716, + "flos": 26211844938240.0, + "grad_norm": 6.662439984694716, + "language_loss": 0.77766335, + "learning_rate": 3.512841177994327e-06, + "loss": 0.80182558, + "num_input_tokens_seen": 12333435, + "router_z_loss_clip": 2.14550781, + "router_z_loss_mlp": 0.38049316, + "step": 444, + "time_per_iteration": 2.863276720046997 + }, + { + "auxiliary_loss_clip": 0.01312865, + "auxiliary_loss_mlp": 0.01087714, + "balance_loss_clip": 1.09002686, + "balance_loss_mlp": 1.0524044, + "epoch": 0.012912773489640764, + "flos": 16939961205120.0, + "grad_norm": 5.425816566782864, + "language_loss": 0.98516816, + "learning_rate": 3.5141376241173505e-06, + "loss": 1.00917399, + "num_input_tokens_seen": 12346170, + "router_z_loss_clip": 2.2265625, + "router_z_loss_mlp": 0.35302734, + "step": 445, + "time_per_iteration": 2.449223756790161 + }, + { + "auxiliary_loss_clip": 0.01304098, + "auxiliary_loss_mlp": 0.01076854, + "balance_loss_clip": 1.09895551, + "balance_loss_mlp": 1.04352355, + "epoch": 0.01294179095815681, + "flos": 22850929169280.0, + "grad_norm": 4.470110368004726, + "language_loss": 1.01792264, + "learning_rate": 3.5154311601456196e-06, + "loss": 1.04173231, + "num_input_tokens_seen": 12359885, + "router_z_loss_clip": 2.05273438, + "router_z_loss_mlp": 0.33300781, + "step": 446, + "time_per_iteration": 2.578244686126709 + }, + { + "auxiliary_loss_clip": 0.01280794, + "auxiliary_loss_mlp": 0.01073657, + "balance_loss_clip": 1.09203136, + "balance_loss_mlp": 1.0434258, + "epoch": 0.012970808426672857, + "flos": 31642579042560.0, + "grad_norm": 3.7228972090247385, + "language_loss": 1.12970781, + "learning_rate": 3.5167217991142907e-06, + "loss": 1.15325236, + "num_input_tokens_seen": 12373035, + "router_z_loss_clip": 1.88769531, + "router_z_loss_mlp": 0.30249023, + "step": 447, + "time_per_iteration": 2.6396453380584717 + }, + { + "auxiliary_loss_clip": 0.01087844, + "auxiliary_loss_mlp": 0.01005145, + "balance_loss_clip": 1.02208161, + "balance_loss_mlp": 0.99920815, + "epoch": 0.012999825895188903, + "flos": 66639360988800.0, + "grad_norm": 0.7362263116722976, + "language_loss": 0.59154367, + "learning_rate": 3.5180095539711303e-06, + "loss": 0.61247361, + "num_input_tokens_seen": 12431200, + "router_z_loss_clip": 0.65625, + "router_z_loss_mlp": 0.05932617, + "step": 448, + "time_per_iteration": 3.0551483631134033 + }, + { + "auxiliary_loss_clip": 0.0129559, + "auxiliary_loss_mlp": 0.01070117, + "balance_loss_clip": 1.09697771, + "balance_loss_mlp": 1.0396117, + "epoch": 0.01302884336370495, + "flos": 14495091010560.0, + "grad_norm": 3.684402945524637, + "language_loss": 0.98529935, + "learning_rate": 3.5192944375773016e-06, + "loss": 1.00895643, + "num_input_tokens_seen": 12443425, + "router_z_loss_clip": 1.984375, + "router_z_loss_mlp": 0.30517578, + "step": 449, + "time_per_iteration": 2.526200532913208 + }, + { + "auxiliary_loss_clip": 0.01306429, + "auxiliary_loss_mlp": 0.0109331, + "balance_loss_clip": 1.0940696, + "balance_loss_mlp": 1.05893052, + "epoch": 0.013057860832220996, + "flos": 30849133564800.0, + "grad_norm": 2.079640983904993, + "language_loss": 0.99178916, + "learning_rate": 3.5205764627081286e-06, + "loss": 1.01578641, + "num_input_tokens_seen": 12463685, + "router_z_loss_clip": 2.12695312, + "router_z_loss_mlp": 0.34362793, + "step": 450, + "time_per_iteration": 2.6531381607055664 + }, + { + "auxiliary_loss_clip": 0.0129382, + "auxiliary_loss_mlp": 0.010735, + "balance_loss_clip": 1.08863795, + "balance_loss_mlp": 1.04157555, + "epoch": 0.013086878300737044, + "flos": 23399213345280.0, + "grad_norm": 2.788451119858122, + "language_loss": 1.06867492, + "learning_rate": 3.521855642053862e-06, + "loss": 1.0923481, + "num_input_tokens_seen": 12481025, + "router_z_loss_clip": 2.05175781, + "router_z_loss_mlp": 0.31933594, + "step": 451, + "time_per_iteration": 2.5718047618865967 + }, + { + "auxiliary_loss_clip": 0.01295597, + "auxiliary_loss_mlp": 0.01082498, + "balance_loss_clip": 1.0934428, + "balance_loss_mlp": 1.0514555, + "epoch": 0.01311589576925309, + "flos": 17303125662720.0, + "grad_norm": 3.1126600573379686, + "language_loss": 0.92723465, + "learning_rate": 3.5231319882204308e-06, + "loss": 0.95101553, + "num_input_tokens_seen": 12492870, + "router_z_loss_clip": 2.02246094, + "router_z_loss_mlp": 0.31030273, + "step": 452, + "time_per_iteration": 2.596883773803711 + }, + { + "auxiliary_loss_clip": 0.01297628, + "auxiliary_loss_mlp": 0.01113505, + "balance_loss_clip": 1.09310865, + "balance_loss_mlp": 1.08197427, + "epoch": 0.013144913237769137, + "flos": 34523548260480.0, + "grad_norm": 2.9799351446749376, + "language_loss": 0.88727713, + "learning_rate": 3.524405513730189e-06, + "loss": 0.91138852, + "num_input_tokens_seen": 12506590, + "router_z_loss_clip": 2.04492188, + "router_z_loss_mlp": 0.31542969, + "step": 453, + "time_per_iteration": 2.650315761566162 + }, + { + "auxiliary_loss_clip": 0.01302034, + "auxiliary_loss_mlp": 0.01075868, + "balance_loss_clip": 1.09654808, + "balance_loss_mlp": 1.04258442, + "epoch": 0.013173930706285183, + "flos": 16757104043520.0, + "grad_norm": 3.7507864124834565, + "language_loss": 0.81616282, + "learning_rate": 3.5256762310226537e-06, + "loss": 0.83994186, + "num_input_tokens_seen": 12519175, + "router_z_loss_clip": 2.05761719, + "router_z_loss_mlp": 0.33300781, + "step": 454, + "time_per_iteration": 2.5639894008636475 + }, + { + "auxiliary_loss_clip": 0.01292505, + "auxiliary_loss_mlp": 0.01071987, + "balance_loss_clip": 1.09310365, + "balance_loss_mlp": 1.04065931, + "epoch": 0.01320294817480123, + "flos": 11319654096000.0, + "grad_norm": 5.374456331441165, + "language_loss": 0.94946492, + "learning_rate": 3.52694415245523e-06, + "loss": 0.97310978, + "num_input_tokens_seen": 12531215, + "router_z_loss_clip": 1.99316406, + "router_z_loss_mlp": 0.31298828, + "step": 455, + "time_per_iteration": 2.562181234359741 + }, + { + "auxiliary_loss_clip": 0.0130127, + "auxiliary_loss_mlp": 0.01071935, + "balance_loss_clip": 1.09987211, + "balance_loss_mlp": 1.04145312, + "epoch": 0.013231965643317276, + "flos": 41063564131200.0, + "grad_norm": 2.0358798238949087, + "language_loss": 0.95573258, + "learning_rate": 3.5282092903039383e-06, + "loss": 0.97946465, + "num_input_tokens_seen": 12552195, + "router_z_loss_clip": 2.01171875, + "router_z_loss_mlp": 0.30493164, + "step": 456, + "time_per_iteration": 2.682570457458496 + }, + { + "auxiliary_loss_clip": 0.01309365, + "auxiliary_loss_mlp": 0.01097762, + "balance_loss_clip": 1.09453154, + "balance_loss_mlp": 1.06424069, + "epoch": 0.013260983111833324, + "flos": 14786721532800.0, + "grad_norm": 3.7590762184756503, + "language_loss": 0.90128195, + "learning_rate": 3.529471656764121e-06, + "loss": 0.92535317, + "num_input_tokens_seen": 12567845, + "router_z_loss_clip": 2.1484375, + "router_z_loss_mlp": 0.33544922, + "step": 457, + "time_per_iteration": 2.5510692596435547 + }, + { + "auxiliary_loss_clip": 0.01317808, + "auxiliary_loss_mlp": 0.01089924, + "balance_loss_clip": 1.09686768, + "balance_loss_mlp": 1.05540097, + "epoch": 0.01329000058034937, + "flos": 74735346205440.0, + "grad_norm": 2.0557661570247605, + "language_loss": 0.96399063, + "learning_rate": 3.5307312639511536e-06, + "loss": 0.98806798, + "num_input_tokens_seen": 12594020, + "router_z_loss_clip": 2.20898438, + "router_z_loss_mlp": 0.34509277, + "step": 458, + "time_per_iteration": 2.8998594284057617 + }, + { + "auxiliary_loss_clip": 0.010906, + "auxiliary_loss_mlp": 0.01005484, + "balance_loss_clip": 1.0243187, + "balance_loss_mlp": 1.0, + "epoch": 0.013319018048865417, + "flos": 59850987505920.0, + "grad_norm": 0.7894302745797163, + "language_loss": 0.58731896, + "learning_rate": 3.531988123901137e-06, + "loss": 0.60827976, + "num_input_tokens_seen": 12657785, + "router_z_loss_clip": 0.6640625, + "router_z_loss_mlp": 0.05493164, + "step": 459, + "time_per_iteration": 3.2224273681640625 + }, + { + "auxiliary_loss_clip": 0.01283676, + "auxiliary_loss_mlp": 0.01079483, + "balance_loss_clip": 1.08767259, + "balance_loss_mlp": 1.04913259, + "epoch": 0.013348035517381463, + "flos": 39456094682880.0, + "grad_norm": 2.4814249246096245, + "language_loss": 0.93075413, + "learning_rate": 3.533242248571593e-06, + "loss": 0.95438576, + "num_input_tokens_seen": 12673185, + "router_z_loss_clip": 1.9609375, + "router_z_loss_mlp": 0.3034668, + "step": 460, + "time_per_iteration": 2.6837680339813232 + }, + { + "auxiliary_loss_clip": 0.01090766, + "auxiliary_loss_mlp": 0.01003985, + "balance_loss_clip": 1.02379346, + "balance_loss_mlp": 0.99873936, + "epoch": 0.01337705298589751, + "flos": 68037507930240.0, + "grad_norm": 0.7236362070619151, + "language_loss": 0.57632965, + "learning_rate": 3.5344936498421413e-06, + "loss": 0.59727716, + "num_input_tokens_seen": 12736475, + "router_z_loss_clip": 0.66796875, + "router_z_loss_mlp": 0.05249023, + "step": 461, + "time_per_iteration": 3.2070131301879883 + }, + { + "auxiliary_loss_clip": 0.01309328, + "auxiliary_loss_mlp": 0.01091876, + "balance_loss_clip": 1.0982213, + "balance_loss_mlp": 1.05914068, + "epoch": 0.013406070454413556, + "flos": 28873723150080.0, + "grad_norm": 2.4123417811924672, + "language_loss": 0.82045245, + "learning_rate": 3.5357423395151797e-06, + "loss": 0.84446442, + "num_input_tokens_seen": 12757220, + "router_z_loss_clip": 2.11132812, + "router_z_loss_mlp": 0.32714844, + "step": 462, + "time_per_iteration": 2.6530590057373047 + }, + { + "auxiliary_loss_clip": 0.01301382, + "auxiliary_loss_mlp": 0.01087133, + "balance_loss_clip": 1.09527814, + "balance_loss_mlp": 1.05567372, + "epoch": 0.013435087922929604, + "flos": 24818977296000.0, + "grad_norm": 5.03817407205054, + "language_loss": 0.82631814, + "learning_rate": 3.536988329316549e-06, + "loss": 0.85020334, + "num_input_tokens_seen": 12772730, + "router_z_loss_clip": 2.06152344, + "router_z_loss_mlp": 0.31433105, + "step": 463, + "time_per_iteration": 2.5452065467834473 + }, + { + "auxiliary_loss_clip": 0.01303101, + "auxiliary_loss_mlp": 0.0108289, + "balance_loss_clip": 1.09371686, + "balance_loss_mlp": 1.04564881, + "epoch": 0.01346410539144565, + "flos": 26644424428800.0, + "grad_norm": 2.228442957079701, + "language_loss": 0.96896899, + "learning_rate": 3.5382316308961943e-06, + "loss": 0.99282897, + "num_input_tokens_seen": 12785895, + "router_z_loss_clip": 2.09667969, + "router_z_loss_mlp": 0.37255859, + "step": 464, + "time_per_iteration": 2.5050952434539795 + }, + { + "auxiliary_loss_clip": 0.01290251, + "auxiliary_loss_mlp": 0.01079498, + "balance_loss_clip": 1.09349227, + "balance_loss_mlp": 1.04801512, + "epoch": 0.013493122859961697, + "flos": 30225472698240.0, + "grad_norm": 2.1027216985490895, + "language_loss": 0.9038524, + "learning_rate": 3.5394722558288188e-06, + "loss": 0.92754996, + "num_input_tokens_seen": 12809695, + "router_z_loss_clip": 1.96875, + "router_z_loss_mlp": 0.31481934, + "step": 465, + "time_per_iteration": 2.643104314804077 + }, + { + "auxiliary_loss_clip": 0.01290277, + "auxiliary_loss_mlp": 0.01080949, + "balance_loss_clip": 1.09018564, + "balance_loss_mlp": 1.04726088, + "epoch": 0.013522140328477743, + "flos": 31242139246080.0, + "grad_norm": 2.583015790632826, + "language_loss": 0.87775934, + "learning_rate": 3.5407102156145306e-06, + "loss": 0.90147161, + "num_input_tokens_seen": 12827885, + "router_z_loss_clip": 2.00390625, + "router_z_loss_mlp": 0.33654785, + "step": 466, + "time_per_iteration": 2.740981101989746 + }, + { + "auxiliary_loss_clip": 0.01089199, + "auxiliary_loss_mlp": 0.0100929, + "balance_loss_clip": 1.02076602, + "balance_loss_mlp": 1.00397301, + "epoch": 0.01355115779699379, + "flos": 74783649997440.0, + "grad_norm": 0.7297034913800302, + "language_loss": 0.59561121, + "learning_rate": 3.5419455216794824e-06, + "loss": 0.6165961, + "num_input_tokens_seen": 12887480, + "router_z_loss_clip": 0.68359375, + "router_z_loss_mlp": 0.05322266, + "step": 467, + "time_per_iteration": 3.214641809463501 + }, + { + "auxiliary_loss_clip": 0.01089169, + "auxiliary_loss_mlp": 0.01006874, + "balance_loss_clip": 1.02020395, + "balance_loss_mlp": 1.00162828, + "epoch": 0.013580175265509836, + "flos": 63325560885120.0, + "grad_norm": 0.6518524434595517, + "language_loss": 0.59545726, + "learning_rate": 3.543178185376502e-06, + "loss": 0.61641765, + "num_input_tokens_seen": 12950825, + "router_z_loss_clip": 0.6875, + "router_z_loss_mlp": 0.05249023, + "step": 468, + "time_per_iteration": 3.1247990131378174 + }, + { + "auxiliary_loss_clip": 0.01308178, + "auxiliary_loss_mlp": 0.01095491, + "balance_loss_clip": 1.10001874, + "balance_loss_mlp": 1.06213665, + "epoch": 0.013609192734025884, + "flos": 11393666069760.0, + "grad_norm": 3.0244511959164053, + "language_loss": 0.80161214, + "learning_rate": 3.5444082179857223e-06, + "loss": 0.82564884, + "num_input_tokens_seen": 12964055, + "router_z_loss_clip": 2.07910156, + "router_z_loss_mlp": 0.33374023, + "step": 469, + "time_per_iteration": 2.542414665222168 + }, + { + "auxiliary_loss_clip": 0.01299365, + "auxiliary_loss_mlp": 0.01076482, + "balance_loss_clip": 1.09082198, + "balance_loss_mlp": 1.04160154, + "epoch": 0.01363821020254193, + "flos": 22376477399040.0, + "grad_norm": 4.072664599446992, + "language_loss": 1.14489985, + "learning_rate": 3.545635630715198e-06, + "loss": 1.16865826, + "num_input_tokens_seen": 12977770, + "router_z_loss_clip": 2.0859375, + "router_z_loss_mlp": 0.34887695, + "step": 470, + "time_per_iteration": 2.5487020015716553 + }, + { + "auxiliary_loss_clip": 0.0129901, + "auxiliary_loss_mlp": 0.01084254, + "balance_loss_clip": 1.09506917, + "balance_loss_mlp": 1.05132806, + "epoch": 0.013667227671057977, + "flos": 42880463827200.0, + "grad_norm": 3.0541698432413593, + "language_loss": 0.85983026, + "learning_rate": 3.546860434701518e-06, + "loss": 0.88366294, + "num_input_tokens_seen": 12995525, + "router_z_loss_clip": 2.04101562, + "router_z_loss_mlp": 0.32958984, + "step": 471, + "time_per_iteration": 2.703856945037842 + }, + { + "auxiliary_loss_clip": 0.01296817, + "auxiliary_loss_mlp": 0.01089064, + "balance_loss_clip": 1.08900797, + "balance_loss_mlp": 1.05630493, + "epoch": 0.013696245139574023, + "flos": 21136338385920.0, + "grad_norm": 3.442011954348205, + "language_loss": 0.88022351, + "learning_rate": 3.548082641010414e-06, + "loss": 0.9040823, + "num_input_tokens_seen": 13010905, + "router_z_loss_clip": 2.078125, + "router_z_loss_mlp": 0.32763672, + "step": 472, + "time_per_iteration": 2.5169801712036133 + }, + { + "auxiliary_loss_clip": 0.01091102, + "auxiliary_loss_mlp": 0.01009854, + "balance_loss_clip": 1.02160525, + "balance_loss_mlp": 1.00475216, + "epoch": 0.01372526260809007, + "flos": 74790581322240.0, + "grad_norm": 0.949228923393677, + "language_loss": 0.54106575, + "learning_rate": 3.5493022606373578e-06, + "loss": 0.56207532, + "num_input_tokens_seen": 13080835, + "router_z_loss_clip": 0.6953125, + "router_z_loss_mlp": 0.05102539, + "step": 473, + "time_per_iteration": 3.280669927597046 + }, + { + "auxiliary_loss_clip": 0.01283722, + "auxiliary_loss_mlp": 0.01074593, + "balance_loss_clip": 1.08824158, + "balance_loss_mlp": 1.04545832, + "epoch": 0.013754280076606116, + "flos": 16509967493760.0, + "grad_norm": 3.643817328743856, + "language_loss": 0.88495821, + "learning_rate": 3.550519304508158e-06, + "loss": 0.90854138, + "num_input_tokens_seen": 13095550, + "router_z_loss_clip": 1.95410156, + "router_z_loss_mlp": 0.29077148, + "step": 474, + "time_per_iteration": 2.475400447845459 + }, + { + "auxiliary_loss_clip": 0.01297854, + "auxiliary_loss_mlp": 0.01097458, + "balance_loss_clip": 1.09479451, + "balance_loss_mlp": 1.06252956, + "epoch": 0.013783297545122164, + "flos": 12488726050560.0, + "grad_norm": 2.8943494464216646, + "language_loss": 0.90261388, + "learning_rate": 3.551733783479541e-06, + "loss": 0.92656702, + "num_input_tokens_seen": 13108305, + "router_z_loss_clip": 2.02929688, + "router_z_loss_mlp": 0.3494873, + "step": 475, + "time_per_iteration": 2.5120646953582764 + }, + { + "auxiliary_loss_clip": 0.012907, + "auxiliary_loss_mlp": 0.01066957, + "balance_loss_clip": 1.0926224, + "balance_loss_mlp": 1.03624892, + "epoch": 0.01381231501363821, + "flos": 56964484615680.0, + "grad_norm": 4.626539849413453, + "language_loss": 0.94451368, + "learning_rate": 3.552945708339742e-06, + "loss": 0.9680903, + "num_input_tokens_seen": 13128735, + "router_z_loss_clip": 1.97753906, + "router_z_loss_mlp": 0.30712891, + "step": 476, + "time_per_iteration": 2.9068586826324463 + }, + { + "auxiliary_loss_clip": 0.01292007, + "auxiliary_loss_mlp": 0.01079688, + "balance_loss_clip": 1.08860695, + "balance_loss_mlp": 1.04542708, + "epoch": 0.013841332482154257, + "flos": 16207491064320.0, + "grad_norm": 4.321550563842357, + "language_loss": 0.86250663, + "learning_rate": 3.5541550898090704e-06, + "loss": 0.88622355, + "num_input_tokens_seen": 13139930, + "router_z_loss_clip": 2.03222656, + "router_z_loss_mlp": 0.34277344, + "step": 477, + "time_per_iteration": 2.4741764068603516 + }, + { + "auxiliary_loss_clip": 0.01300363, + "auxiliary_loss_mlp": 0.01083508, + "balance_loss_clip": 1.09380579, + "balance_loss_mlp": 1.0519886, + "epoch": 0.013870349950670303, + "flos": 55735083768960.0, + "grad_norm": 2.481072717094662, + "language_loss": 0.78981566, + "learning_rate": 3.5553619385404838e-06, + "loss": 0.81365436, + "num_input_tokens_seen": 13160645, + "router_z_loss_clip": 2.06445312, + "router_z_loss_mlp": 0.31567383, + "step": 478, + "time_per_iteration": 2.7209692001342773 + }, + { + "auxiliary_loss_clip": 0.01280242, + "auxiliary_loss_mlp": 0.01067036, + "balance_loss_clip": 1.08672023, + "balance_loss_mlp": 1.03878331, + "epoch": 0.01389936741918635, + "flos": 11648488129920.0, + "grad_norm": 3.3891012132309766, + "language_loss": 0.74203587, + "learning_rate": 3.5565662651201502e-06, + "loss": 0.76550865, + "num_input_tokens_seen": 13172570, + "router_z_loss_clip": 1.93652344, + "router_z_loss_mlp": 0.28259277, + "step": 479, + "time_per_iteration": 2.484178304672241 + }, + { + "auxiliary_loss_clip": 0.01291551, + "auxiliary_loss_mlp": 0.01085792, + "balance_loss_clip": 1.09233737, + "balance_loss_mlp": 1.05294991, + "epoch": 0.013928384887702396, + "flos": 37371551771520.0, + "grad_norm": 2.3907806765119717, + "language_loss": 0.90967333, + "learning_rate": 3.5577680800680056e-06, + "loss": 0.93344676, + "num_input_tokens_seen": 13190760, + "router_z_loss_clip": 1.99121094, + "router_z_loss_mlp": 0.32824707, + "step": 480, + "time_per_iteration": 2.590579032897949 + }, + { + "auxiliary_loss_clip": 0.01296859, + "auxiliary_loss_mlp": 0.01077067, + "balance_loss_clip": 1.09478033, + "balance_loss_mlp": 1.04631114, + "epoch": 0.013957402356218444, + "flos": 37736116859520.0, + "grad_norm": 2.599250989111084, + "language_loss": 0.8440187, + "learning_rate": 3.5589673938383033e-06, + "loss": 0.86775792, + "num_input_tokens_seen": 13212150, + "router_z_loss_clip": 2.02050781, + "router_z_loss_mlp": 0.30761719, + "step": 481, + "time_per_iteration": 2.8782100677490234 + }, + { + "auxiliary_loss_clip": 0.01291911, + "auxiliary_loss_mlp": 0.01074242, + "balance_loss_clip": 1.08946705, + "balance_loss_mlp": 1.04272354, + "epoch": 0.01398641982473449, + "flos": 32122417852800.0, + "grad_norm": 1.877081245488087, + "language_loss": 1.0985359, + "learning_rate": 3.5601642168201625e-06, + "loss": 1.12219739, + "num_input_tokens_seen": 13235835, + "router_z_loss_clip": 2.02441406, + "router_z_loss_mlp": 0.31542969, + "step": 482, + "time_per_iteration": 2.645198106765747 + }, + { + "auxiliary_loss_clip": 0.01285676, + "auxiliary_loss_mlp": 0.01084677, + "balance_loss_clip": 1.08665037, + "balance_loss_mlp": 1.05246639, + "epoch": 0.014015437293250537, + "flos": 33909260843520.0, + "grad_norm": 2.048105673341298, + "language_loss": 1.05304146, + "learning_rate": 3.5613585593381047e-06, + "loss": 1.07674503, + "num_input_tokens_seen": 13255755, + "router_z_loss_clip": 1.98925781, + "router_z_loss_mlp": 0.32214355, + "step": 483, + "time_per_iteration": 2.664055109024048 + }, + { + "auxiliary_loss_clip": 0.01293287, + "auxiliary_loss_mlp": 0.01078345, + "balance_loss_clip": 1.09201431, + "balance_loss_mlp": 1.0472548, + "epoch": 0.014044454761766583, + "flos": 16033720043520.0, + "grad_norm": 3.4430100819596956, + "language_loss": 0.87265617, + "learning_rate": 3.5625504316525934e-06, + "loss": 0.89637244, + "num_input_tokens_seen": 13269210, + "router_z_loss_clip": 2.01269531, + "router_z_loss_mlp": 0.31079102, + "step": 484, + "time_per_iteration": 2.569953680038452 + }, + { + "auxiliary_loss_clip": 0.01288208, + "auxiliary_loss_mlp": 0.01073749, + "balance_loss_clip": 1.08553529, + "balance_loss_mlp": 1.04215848, + "epoch": 0.01407347223028263, + "flos": 22634172547200.0, + "grad_norm": 2.315637396453595, + "language_loss": 0.76075172, + "learning_rate": 3.5637398439605558e-06, + "loss": 0.78437126, + "num_input_tokens_seen": 13283685, + "router_z_loss_clip": 2.02636719, + "router_z_loss_mlp": 0.31591797, + "step": 485, + "time_per_iteration": 2.5894439220428467 + }, + { + "auxiliary_loss_clip": 0.01297387, + "auxiliary_loss_mlp": 0.01079571, + "balance_loss_clip": 1.09154606, + "balance_loss_mlp": 1.04752779, + "epoch": 0.014102489698798676, + "flos": 19565892881280.0, + "grad_norm": 3.0430333833292504, + "language_loss": 0.82494617, + "learning_rate": 3.5649268063959134e-06, + "loss": 0.84871578, + "num_input_tokens_seen": 13297210, + "router_z_loss_clip": 2.05859375, + "router_z_loss_mlp": 0.32055664, + "step": 486, + "time_per_iteration": 2.525233745574951 + }, + { + "auxiliary_loss_clip": 0.01279904, + "auxiliary_loss_mlp": 0.01090924, + "balance_loss_clip": 1.09446263, + "balance_loss_mlp": 1.06081116, + "epoch": 0.014131507167314724, + "flos": 15078818931840.0, + "grad_norm": 3.173661259601389, + "language_loss": 0.84719259, + "learning_rate": 3.566111329030094e-06, + "loss": 0.87090087, + "num_input_tokens_seen": 13311410, + "router_z_loss_clip": 1.85351562, + "router_z_loss_mlp": 0.30126953, + "step": 487, + "time_per_iteration": 2.542419910430908 + }, + { + "auxiliary_loss_clip": 0.01086183, + "auxiliary_loss_mlp": 0.01010606, + "balance_loss_clip": 1.02058864, + "balance_loss_mlp": 1.00533724, + "epoch": 0.01416052463583077, + "flos": 64227133278720.0, + "grad_norm": 0.7394106016135872, + "language_loss": 0.54053599, + "learning_rate": 3.567293421872552e-06, + "loss": 0.56150389, + "num_input_tokens_seen": 13372965, + "router_z_loss_clip": 0.65625, + "router_z_loss_mlp": 0.05273438, + "step": 488, + "time_per_iteration": 3.1479763984680176 + }, + { + "auxiliary_loss_clip": 0.01277507, + "auxiliary_loss_mlp": 0.01068274, + "balance_loss_clip": 1.08750641, + "balance_loss_mlp": 1.03797126, + "epoch": 0.014189542104346817, + "flos": 28688244295680.0, + "grad_norm": 2.592113915789499, + "language_loss": 0.91131914, + "learning_rate": 3.568473094871265e-06, + "loss": 0.9347769, + "num_input_tokens_seen": 13388965, + "router_z_loss_clip": 1.90136719, + "router_z_loss_mlp": 0.30297852, + "step": 489, + "time_per_iteration": 2.5881757736206055 + }, + { + "auxiliary_loss_clip": 0.01287941, + "auxiliary_loss_mlp": 0.01069126, + "balance_loss_clip": 1.09146833, + "balance_loss_mlp": 1.03853655, + "epoch": 0.014218559572862863, + "flos": 38537427415680.0, + "grad_norm": 2.2419060933770547, + "language_loss": 1.06713009, + "learning_rate": 3.5696503579132456e-06, + "loss": 1.09070086, + "num_input_tokens_seen": 13405945, + "router_z_loss_clip": 1.96679688, + "router_z_loss_mlp": 0.30578613, + "step": 490, + "time_per_iteration": 2.7104201316833496 + }, + { + "auxiliary_loss_clip": 0.01085997, + "auxiliary_loss_mlp": 0.01008072, + "balance_loss_clip": 1.02121687, + "balance_loss_mlp": 1.00308895, + "epoch": 0.01424757704137891, + "flos": 67193894131200.0, + "grad_norm": 0.710181577075479, + "language_loss": 0.49053657, + "learning_rate": 3.570825220825037e-06, + "loss": 0.51147723, + "num_input_tokens_seen": 13462405, + "router_z_loss_clip": 0.6484375, + "router_z_loss_mlp": 0.04980469, + "step": 491, + "time_per_iteration": 3.102816104888916 + }, + { + "auxiliary_loss_clip": 0.01291123, + "auxiliary_loss_mlp": 0.01091293, + "balance_loss_clip": 1.09337592, + "balance_loss_mlp": 1.05966687, + "epoch": 0.014276594509894956, + "flos": 21829126976640.0, + "grad_norm": 2.5314969623608263, + "language_loss": 0.7601285, + "learning_rate": 3.5719976933732e-06, + "loss": 0.78395265, + "num_input_tokens_seen": 13477160, + "router_z_loss_clip": 1.97460938, + "router_z_loss_mlp": 0.31628418, + "step": 492, + "time_per_iteration": 2.5309455394744873 + }, + { + "auxiliary_loss_clip": 0.01295768, + "auxiliary_loss_mlp": 0.01089105, + "balance_loss_clip": 1.09663105, + "balance_loss_mlp": 1.05929065, + "epoch": 0.014305611978411003, + "flos": 17997386711040.0, + "grad_norm": 3.5844269093411003, + "language_loss": 0.93503058, + "learning_rate": 3.5731677852648057e-06, + "loss": 0.95887929, + "num_input_tokens_seen": 13491465, + "router_z_loss_clip": 1.9921875, + "router_z_loss_mlp": 0.29797363, + "step": 493, + "time_per_iteration": 2.522329092025757 + }, + { + "auxiliary_loss_clip": 0.01295871, + "auxiliary_loss_mlp": 0.01084443, + "balance_loss_clip": 1.091254, + "balance_loss_mlp": 1.05208921, + "epoch": 0.01433462944692705, + "flos": 27666190707840.0, + "grad_norm": 2.5227266335716454, + "language_loss": 0.79735863, + "learning_rate": 3.5743355061479145e-06, + "loss": 0.82116175, + "num_input_tokens_seen": 13506200, + "router_z_loss_clip": 2.04296875, + "router_z_loss_mlp": 0.32348633, + "step": 494, + "time_per_iteration": 2.6038401126861572 + }, + { + "auxiliary_loss_clip": 0.01084419, + "auxiliary_loss_mlp": 0.01016594, + "balance_loss_clip": 1.02091849, + "balance_loss_mlp": 1.01182544, + "epoch": 0.014363646915443097, + "flos": 74777616512640.0, + "grad_norm": 0.6823604191852135, + "language_loss": 0.54704946, + "learning_rate": 3.5755008656120545e-06, + "loss": 0.56805956, + "num_input_tokens_seen": 13572210, + "router_z_loss_clip": 0.6328125, + "router_z_loss_mlp": 0.04760742, + "step": 495, + "time_per_iteration": 3.242713212966919 + }, + { + "auxiliary_loss_clip": 0.01286974, + "auxiliary_loss_mlp": 0.01081731, + "balance_loss_clip": 1.09041452, + "balance_loss_mlp": 1.04957986, + "epoch": 0.014392664383959143, + "flos": 35291713541760.0, + "grad_norm": 2.492133073360625, + "language_loss": 1.16260481, + "learning_rate": 3.5766638731886958e-06, + "loss": 1.18629181, + "num_input_tokens_seen": 13592030, + "router_z_loss_clip": 1.96386719, + "router_z_loss_mlp": 0.32128906, + "step": 496, + "time_per_iteration": 2.737586259841919 + }, + { + "auxiliary_loss_clip": 0.01294418, + "auxiliary_loss_mlp": 0.0108634, + "balance_loss_clip": 1.09653878, + "balance_loss_mlp": 1.05461836, + "epoch": 0.01442168185247519, + "flos": 31971592644480.0, + "grad_norm": 2.021212958472341, + "language_loss": 0.947065, + "learning_rate": 3.5778245383517136e-06, + "loss": 0.97087258, + "num_input_tokens_seen": 13615635, + "router_z_loss_clip": 1.97949219, + "router_z_loss_mlp": 0.31726074, + "step": 497, + "time_per_iteration": 2.7496449947357178 + }, + { + "auxiliary_loss_clip": 0.01282362, + "auxiliary_loss_mlp": 0.01081267, + "balance_loss_clip": 1.08836043, + "balance_loss_mlp": 1.05036807, + "epoch": 0.014450699320991236, + "flos": 26680263223680.0, + "grad_norm": 4.317168893052861, + "language_loss": 0.9711206, + "learning_rate": 3.5789828705178567e-06, + "loss": 0.99475688, + "num_input_tokens_seen": 13630230, + "router_z_loss_clip": 1.94140625, + "router_z_loss_mlp": 0.30883789, + "step": 498, + "time_per_iteration": 2.5581445693969727 + }, + { + "auxiliary_loss_clip": 0.01085079, + "auxiliary_loss_mlp": 0.01010926, + "balance_loss_clip": 1.02284849, + "balance_loss_mlp": 1.00637233, + "epoch": 0.014479716789507283, + "flos": 74773809671040.0, + "grad_norm": 0.6839182993929157, + "language_loss": 0.52907765, + "learning_rate": 3.5801388790472013e-06, + "loss": 0.55003768, + "num_input_tokens_seen": 13693550, + "router_z_loss_clip": 0.625, + "router_z_loss_mlp": 0.0456543, + "step": 499, + "time_per_iteration": 3.1735291481018066 + }, + { + "auxiliary_loss_clip": 0.0129504, + "auxiliary_loss_mlp": 0.01082934, + "balance_loss_clip": 1.09839642, + "balance_loss_mlp": 1.04907823, + "epoch": 0.01450873425802333, + "flos": 27447889800960.0, + "grad_norm": 2.605145324265331, + "language_loss": 0.88434505, + "learning_rate": 3.5812925732436083e-06, + "loss": 0.90812474, + "num_input_tokens_seen": 13711270, + "router_z_loss_clip": 1.96679688, + "router_z_loss_mlp": 0.33862305, + "step": 500, + "time_per_iteration": 2.588348150253296 + }, + { + "auxiliary_loss_clip": 0.01290248, + "auxiliary_loss_mlp": 0.01091755, + "balance_loss_clip": 1.09515548, + "balance_loss_mlp": 1.0602119, + "epoch": 0.014537751726539377, + "flos": 29528589957120.0, + "grad_norm": 2.7637565878680537, + "language_loss": 0.76258403, + "learning_rate": 3.582443962355171e-06, + "loss": 0.78640413, + "num_input_tokens_seen": 13727425, + "router_z_loss_clip": 1.953125, + "router_z_loss_mlp": 0.31542969, + "step": 501, + "time_per_iteration": 2.5822999477386475 + }, + { + "auxiliary_loss_clip": 0.01289072, + "auxiliary_loss_mlp": 0.01097697, + "balance_loss_clip": 1.0933975, + "balance_loss_mlp": 1.06447351, + "epoch": 0.014566769195055423, + "flos": 15990590787840.0, + "grad_norm": 2.8016026338328937, + "language_loss": 0.96851885, + "learning_rate": 3.5835930555746595e-06, + "loss": 0.99238658, + "num_input_tokens_seen": 13739685, + "router_z_loss_clip": 1.95898438, + "router_z_loss_mlp": 0.3326416, + "step": 502, + "time_per_iteration": 2.483388662338257 + }, + { + "auxiliary_loss_clip": 0.01083276, + "auxiliary_loss_mlp": 0.01004642, + "balance_loss_clip": 1.02208424, + "balance_loss_mlp": 0.99980241, + "epoch": 0.01459578666357147, + "flos": 60916565399040.0, + "grad_norm": 0.7502220338920699, + "language_loss": 0.59664828, + "learning_rate": 3.584739862039961e-06, + "loss": 0.61752748, + "num_input_tokens_seen": 13804690, + "router_z_loss_clip": 0.61328125, + "router_z_loss_mlp": 0.04833984, + "step": 503, + "time_per_iteration": 3.158979892730713 + }, + { + "auxiliary_loss_clip": 0.01286754, + "auxiliary_loss_mlp": 0.010929, + "balance_loss_clip": 1.08859015, + "balance_loss_mlp": 1.05817413, + "epoch": 0.014624804132087516, + "flos": 27488469191040.0, + "grad_norm": 2.6604354826401653, + "language_loss": 1.11114776, + "learning_rate": 3.5858843908345178e-06, + "loss": 1.1349442, + "num_input_tokens_seen": 13823020, + "router_z_loss_clip": 1.98242188, + "router_z_loss_mlp": 0.34729004, + "step": 504, + "time_per_iteration": 9.557486534118652 + }, + { + "auxiliary_loss_clip": 0.01285624, + "auxiliary_loss_mlp": 0.0109114, + "balance_loss_clip": 1.08970332, + "balance_loss_mlp": 1.05687881, + "epoch": 0.014653821600603563, + "flos": 12487828210560.0, + "grad_norm": 4.123952983845743, + "language_loss": 0.99316162, + "learning_rate": 3.5870266509877573e-06, + "loss": 1.01692927, + "num_input_tokens_seen": 13834385, + "router_z_loss_clip": 1.95996094, + "router_z_loss_mlp": 0.34301758, + "step": 505, + "time_per_iteration": 2.4836559295654297 + }, + { + "auxiliary_loss_clip": 0.01280831, + "auxiliary_loss_mlp": 0.01074017, + "balance_loss_clip": 1.09503293, + "balance_loss_mlp": 1.04557335, + "epoch": 0.01468283906911961, + "flos": 18582407521920.0, + "grad_norm": 3.5730246617125796, + "language_loss": 1.02675617, + "learning_rate": 3.588166651475519e-06, + "loss": 1.05030465, + "num_input_tokens_seen": 13846355, + "router_z_loss_clip": 1.85839844, + "router_z_loss_mlp": 0.28442383, + "step": 506, + "time_per_iteration": 5.051361560821533 + }, + { + "auxiliary_loss_clip": 0.01271403, + "auxiliary_loss_mlp": 0.0107171, + "balance_loss_clip": 1.08957696, + "balance_loss_mlp": 1.04255176, + "epoch": 0.014711856537635657, + "flos": 24784682785920.0, + "grad_norm": 2.690044299236268, + "language_loss": 1.00173485, + "learning_rate": 3.5893044012204783e-06, + "loss": 1.02516603, + "num_input_tokens_seen": 13860710, + "router_z_loss_clip": 1.8203125, + "router_z_loss_mlp": 0.29162598, + "step": 507, + "time_per_iteration": 2.578547954559326 + }, + { + "auxiliary_loss_clip": 0.01282836, + "auxiliary_loss_mlp": 0.01068686, + "balance_loss_clip": 1.09215784, + "balance_loss_mlp": 1.03714299, + "epoch": 0.014740874006151703, + "flos": 18144297336960.0, + "grad_norm": 4.168571837767483, + "language_loss": 0.88490021, + "learning_rate": 3.5904399090925674e-06, + "loss": 0.90841538, + "num_input_tokens_seen": 13875745, + "router_z_loss_clip": 1.90820312, + "router_z_loss_mlp": 0.31567383, + "step": 508, + "time_per_iteration": 2.4880919456481934 + }, + { + "auxiliary_loss_clip": 0.01278665, + "auxiliary_loss_mlp": 0.010755, + "balance_loss_clip": 1.09087133, + "balance_loss_mlp": 1.04544711, + "epoch": 0.01476989147466775, + "flos": 29928419222400.0, + "grad_norm": 3.4754299006323564, + "language_loss": 0.88679427, + "learning_rate": 3.5915731839093863e-06, + "loss": 0.9103359, + "num_input_tokens_seen": 13891275, + "router_z_loss_clip": 1.87695312, + "router_z_loss_mlp": 0.30041504, + "step": 509, + "time_per_iteration": 2.6086134910583496 + }, + { + "auxiliary_loss_clip": 0.01294098, + "auxiliary_loss_mlp": 0.01090976, + "balance_loss_clip": 1.08691335, + "balance_loss_mlp": 1.05461681, + "epoch": 0.014798908943183796, + "flos": 36935524575360.0, + "grad_norm": 3.7725961095427034, + "language_loss": 0.97280705, + "learning_rate": 3.592704234436617e-06, + "loss": 0.99665773, + "num_input_tokens_seen": 13908485, + "router_z_loss_clip": 2.07128906, + "router_z_loss_mlp": 0.3638916, + "step": 510, + "time_per_iteration": 2.6725292205810547 + }, + { + "auxiliary_loss_clip": 0.01278827, + "auxiliary_loss_mlp": 0.01072267, + "balance_loss_clip": 1.09156203, + "balance_loss_mlp": 1.04525447, + "epoch": 0.014827926411699843, + "flos": 26689026142080.0, + "grad_norm": 1.8677110012264757, + "language_loss": 0.94962662, + "learning_rate": 3.593833069388429e-06, + "loss": 0.97313762, + "num_input_tokens_seen": 13928705, + "router_z_loss_clip": 1.87402344, + "router_z_loss_mlp": 0.2701416, + "step": 511, + "time_per_iteration": 2.6360154151916504 + }, + { + "auxiliary_loss_clip": 0.01283055, + "auxiliary_loss_mlp": 0.01076466, + "balance_loss_clip": 1.09289074, + "balance_loss_mlp": 1.04528117, + "epoch": 0.01485694388021589, + "flos": 33031460275200.0, + "grad_norm": 2.6115018555503973, + "language_loss": 0.94695568, + "learning_rate": 3.594959697427882e-06, + "loss": 0.97055089, + "num_input_tokens_seen": 13947700, + "router_z_loss_clip": 1.90039062, + "router_z_loss_mlp": 0.31176758, + "step": 512, + "time_per_iteration": 2.5908195972442627 + }, + { + "auxiliary_loss_clip": 0.01280213, + "auxiliary_loss_mlp": 0.01077899, + "balance_loss_clip": 1.0902288, + "balance_loss_mlp": 1.05027843, + "epoch": 0.014885961348731937, + "flos": 13765996748160.0, + "grad_norm": 2.084546353395382, + "language_loss": 0.81566507, + "learning_rate": 3.5960841271673257e-06, + "loss": 0.83924615, + "num_input_tokens_seen": 13960880, + "router_z_loss_clip": 1.89941406, + "router_z_loss_mlp": 0.27624512, + "step": 513, + "time_per_iteration": 2.4946377277374268 + }, + { + "auxiliary_loss_clip": 0.012783, + "auxiliary_loss_mlp": 0.01069374, + "balance_loss_clip": 1.08692718, + "balance_loss_mlp": 1.0387727, + "epoch": 0.014914978817247983, + "flos": 74736710922240.0, + "grad_norm": 2.3516917749373287, + "language_loss": 0.97345626, + "learning_rate": 3.597206367168793e-06, + "loss": 0.9969331, + "num_input_tokens_seen": 13987420, + "router_z_loss_clip": 1.91308594, + "router_z_loss_mlp": 0.30578613, + "step": 514, + "time_per_iteration": 2.9634275436401367 + }, + { + "auxiliary_loss_clip": 0.01089013, + "auxiliary_loss_mlp": 0.01007687, + "balance_loss_clip": 1.02820814, + "balance_loss_mlp": 1.00222719, + "epoch": 0.01494399628576403, + "flos": 56785473187200.0, + "grad_norm": 0.7563668743556464, + "language_loss": 0.57471555, + "learning_rate": 3.598326425944392e-06, + "loss": 0.59568256, + "num_input_tokens_seen": 14040220, + "router_z_loss_clip": 0.609375, + "router_z_loss_mlp": 0.0546875, + "step": 515, + "time_per_iteration": 2.9729769229888916 + }, + { + "auxiliary_loss_clip": 0.01281724, + "auxiliary_loss_mlp": 0.01091408, + "balance_loss_clip": 1.09521759, + "balance_loss_mlp": 1.06162965, + "epoch": 0.014973013754280076, + "flos": 74733658266240.0, + "grad_norm": 2.606245784120068, + "language_loss": 0.6949321, + "learning_rate": 3.5994443119566963e-06, + "loss": 0.71866345, + "num_input_tokens_seen": 14064065, + "router_z_loss_clip": 1.86523438, + "router_z_loss_mlp": 0.29785156, + "step": 516, + "time_per_iteration": 2.9761133193969727 + }, + { + "auxiliary_loss_clip": 0.01085695, + "auxiliary_loss_mlp": 0.01005046, + "balance_loss_clip": 1.02594209, + "balance_loss_mlp": 0.99989599, + "epoch": 0.015002031222796123, + "flos": 56821383809280.0, + "grad_norm": 0.7392128574199037, + "language_loss": 0.55134487, + "learning_rate": 3.600560033619124e-06, + "loss": 0.57225227, + "num_input_tokens_seen": 14119065, + "router_z_loss_clip": 0.59375, + "router_z_loss_mlp": 0.05151367, + "step": 517, + "time_per_iteration": 2.9403958320617676 + }, + { + "auxiliary_loss_clip": 0.0127901, + "auxiliary_loss_mlp": 0.0108734, + "balance_loss_clip": 1.09073567, + "balance_loss_mlp": 1.05574906, + "epoch": 0.01503104869131217, + "flos": 22119967399680.0, + "grad_norm": 2.6094671316543114, + "language_loss": 0.85144717, + "learning_rate": 3.6016735992963195e-06, + "loss": 0.87511063, + "num_input_tokens_seen": 14137580, + "router_z_loss_clip": 1.8828125, + "router_z_loss_mlp": 0.31567383, + "step": 518, + "time_per_iteration": 2.6099021434783936 + }, + { + "auxiliary_loss_clip": 0.01287105, + "auxiliary_loss_mlp": 0.01081973, + "balance_loss_clip": 1.09145212, + "balance_loss_mlp": 1.05298114, + "epoch": 0.015060066159828217, + "flos": 24492621300480.0, + "grad_norm": 2.3957137292782504, + "language_loss": 1.01962638, + "learning_rate": 3.602785017304531e-06, + "loss": 1.04331732, + "num_input_tokens_seen": 14154380, + "router_z_loss_clip": 1.95507812, + "router_z_loss_mlp": 0.29016113, + "step": 519, + "time_per_iteration": 2.5382044315338135 + }, + { + "auxiliary_loss_clip": 0.01084122, + "auxiliary_loss_mlp": 0.01005336, + "balance_loss_clip": 1.02461004, + "balance_loss_mlp": 1.00023353, + "epoch": 0.015089083628344263, + "flos": 64078211491200.0, + "grad_norm": 0.7293543005076253, + "language_loss": 0.56629866, + "learning_rate": 3.603894295911982e-06, + "loss": 0.58719319, + "num_input_tokens_seen": 14211185, + "router_z_loss_clip": 0.59375, + "router_z_loss_mlp": 0.05102539, + "step": 520, + "time_per_iteration": 2.975405216217041 + }, + { + "auxiliary_loss_clip": 0.01084209, + "auxiliary_loss_mlp": 0.01005244, + "balance_loss_clip": 1.02446747, + "balance_loss_mlp": 1.00030923, + "epoch": 0.01511810109686031, + "flos": 56572236097920.0, + "grad_norm": 0.7230744309591668, + "language_loss": 0.55408657, + "learning_rate": 3.6050014433392397e-06, + "loss": 0.57498109, + "num_input_tokens_seen": 14271735, + "router_z_loss_clip": 0.59765625, + "router_z_loss_mlp": 0.04931641, + "step": 521, + "time_per_iteration": 3.0146894454956055 + }, + { + "auxiliary_loss_clip": 0.01083935, + "auxiliary_loss_mlp": 0.01005341, + "balance_loss_clip": 1.02460456, + "balance_loss_mlp": 1.00031078, + "epoch": 0.015147118565376356, + "flos": 61775331759360.0, + "grad_norm": 0.7671959019823568, + "language_loss": 0.53818125, + "learning_rate": 3.6061064677595822e-06, + "loss": 0.55907404, + "num_input_tokens_seen": 14327630, + "router_z_loss_clip": 0.59375, + "router_z_loss_mlp": 0.05029297, + "step": 522, + "time_per_iteration": 2.970808982849121 + }, + { + "auxiliary_loss_clip": 0.01278658, + "auxiliary_loss_mlp": 0.01086735, + "balance_loss_clip": 1.08829713, + "balance_loss_mlp": 1.05404735, + "epoch": 0.015176136033892403, + "flos": 37407965184000.0, + "grad_norm": 2.7859665200814256, + "language_loss": 0.89004588, + "learning_rate": 3.6072093772993584e-06, + "loss": 0.91369981, + "num_input_tokens_seen": 14343715, + "router_z_loss_clip": 1.90332031, + "router_z_loss_mlp": 0.3269043, + "step": 523, + "time_per_iteration": 2.678586721420288 + }, + { + "auxiliary_loss_clip": 0.01274191, + "auxiliary_loss_mlp": 0.0107127, + "balance_loss_clip": 1.08648384, + "balance_loss_mlp": 1.04070473, + "epoch": 0.01520515350240845, + "flos": 31971879953280.0, + "grad_norm": 2.7717851636500543, + "language_loss": 0.83148885, + "learning_rate": 3.6083101800383493e-06, + "loss": 0.85494345, + "num_input_tokens_seen": 14359915, + "router_z_loss_clip": 1.87792969, + "router_z_loss_mlp": 0.30554199, + "step": 524, + "time_per_iteration": 2.64270281791687 + }, + { + "auxiliary_loss_clip": 0.01272626, + "auxiliary_loss_mlp": 0.01076105, + "balance_loss_clip": 1.08530641, + "balance_loss_mlp": 1.04577816, + "epoch": 0.015234170970924497, + "flos": 32372176095360.0, + "grad_norm": 2.115565464026922, + "language_loss": 0.99304223, + "learning_rate": 3.609408884010121e-06, + "loss": 1.01652956, + "num_input_tokens_seen": 14380945, + "router_z_loss_clip": 1.87402344, + "router_z_loss_mlp": 0.30334473, + "step": 525, + "time_per_iteration": 2.7173173427581787 + }, + { + "auxiliary_loss_clip": 0.01278854, + "auxiliary_loss_mlp": 0.01078885, + "balance_loss_clip": 1.09057784, + "balance_loss_mlp": 1.0477953, + "epoch": 0.015263188439440543, + "flos": 11612685248640.0, + "grad_norm": 3.599097943677085, + "language_loss": 0.86646056, + "learning_rate": 3.6105054972023773e-06, + "loss": 0.89003801, + "num_input_tokens_seen": 14392745, + "router_z_loss_clip": 1.88378906, + "router_z_loss_mlp": 0.31054688, + "step": 526, + "time_per_iteration": 2.5127158164978027 + }, + { + "auxiliary_loss_clip": 0.01081836, + "auxiliary_loss_mlp": 0.01008905, + "balance_loss_clip": 1.02361774, + "balance_loss_mlp": 1.00456548, + "epoch": 0.01529220590795659, + "flos": 74780848736640.0, + "grad_norm": 0.6678650987259067, + "language_loss": 0.50329536, + "learning_rate": 3.611600027557307e-06, + "loss": 0.52420276, + "num_input_tokens_seen": 14459565, + "router_z_loss_clip": 0.58203125, + "router_z_loss_mlp": 0.04345703, + "step": 527, + "time_per_iteration": 3.2011654376983643 + }, + { + "auxiliary_loss_clip": 0.01295348, + "auxiliary_loss_mlp": 0.01089573, + "balance_loss_clip": 1.09697855, + "balance_loss_mlp": 1.05636144, + "epoch": 0.015321223376472636, + "flos": 23214775985280.0, + "grad_norm": 2.8016161150524925, + "language_loss": 0.91894114, + "learning_rate": 3.6126924829719315e-06, + "loss": 0.94279039, + "num_input_tokens_seen": 14477405, + "router_z_loss_clip": 1.98339844, + "router_z_loss_mlp": 0.33203125, + "step": 528, + "time_per_iteration": 2.5906858444213867 + }, + { + "auxiliary_loss_clip": 0.01282337, + "auxiliary_loss_mlp": 0.01078319, + "balance_loss_clip": 1.08991337, + "balance_loss_mlp": 1.04726493, + "epoch": 0.015350240844988683, + "flos": 20884533068160.0, + "grad_norm": 3.3516856317481665, + "language_loss": 1.02336729, + "learning_rate": 3.613782871298444e-06, + "loss": 1.04697382, + "num_input_tokens_seen": 14490740, + "router_z_loss_clip": 1.92480469, + "router_z_loss_mlp": 0.31066895, + "step": 529, + "time_per_iteration": 2.587672233581543 + }, + { + "auxiliary_loss_clip": 0.01290192, + "auxiliary_loss_mlp": 0.01088981, + "balance_loss_clip": 1.09433174, + "balance_loss_mlp": 1.05866623, + "epoch": 0.01537925831350473, + "flos": 16323842194560.0, + "grad_norm": 2.617897518269383, + "language_loss": 0.65774822, + "learning_rate": 3.61487120034455e-06, + "loss": 0.68153989, + "num_input_tokens_seen": 14505210, + "router_z_loss_clip": 1.95898438, + "router_z_loss_mlp": 0.3034668, + "step": 530, + "time_per_iteration": 2.4770328998565674 + }, + { + "auxiliary_loss_clip": 0.01270537, + "auxiliary_loss_mlp": 0.01072846, + "balance_loss_clip": 1.09047747, + "balance_loss_mlp": 1.04491568, + "epoch": 0.015408275782020777, + "flos": 13728865063680.0, + "grad_norm": 3.022971599485943, + "language_loss": 0.91779459, + "learning_rate": 3.6159574778738017e-06, + "loss": 0.94122845, + "num_input_tokens_seen": 14516860, + "router_z_loss_clip": 1.80078125, + "router_z_loss_mlp": 0.27893066, + "step": 531, + "time_per_iteration": 2.5016958713531494 + }, + { + "auxiliary_loss_clip": 0.01282431, + "auxiliary_loss_mlp": 0.0107141, + "balance_loss_clip": 1.09210181, + "balance_loss_mlp": 1.04209614, + "epoch": 0.015437293250536823, + "flos": 32265342069120.0, + "grad_norm": 3.5969708285979767, + "language_loss": 0.87366271, + "learning_rate": 3.6170417116059306e-06, + "loss": 0.89720118, + "num_input_tokens_seen": 14535365, + "router_z_loss_clip": 1.90527344, + "router_z_loss_mlp": 0.29345703, + "step": 532, + "time_per_iteration": 2.538902997970581 + }, + { + "auxiliary_loss_clip": 0.012766, + "auxiliary_loss_mlp": 0.01088182, + "balance_loss_clip": 1.09069216, + "balance_loss_mlp": 1.05660343, + "epoch": 0.01546631071905287, + "flos": 16499301154560.0, + "grad_norm": 4.391422115493587, + "language_loss": 0.98104501, + "learning_rate": 3.6181239092171762e-06, + "loss": 1.00469291, + "num_input_tokens_seen": 14547670, + "router_z_loss_clip": 1.85839844, + "router_z_loss_mlp": 0.3157959, + "step": 533, + "time_per_iteration": 2.5027687549591064 + }, + { + "auxiliary_loss_clip": 0.01291252, + "auxiliary_loss_mlp": 0.01085685, + "balance_loss_clip": 1.09565687, + "balance_loss_mlp": 1.05101943, + "epoch": 0.015495328187568916, + "flos": 46130774641920.0, + "grad_norm": 7.1678755265451315, + "language_loss": 1.03691947, + "learning_rate": 3.619204078340615e-06, + "loss": 1.06068885, + "num_input_tokens_seen": 14567995, + "router_z_loss_clip": 1.9609375, + "router_z_loss_mlp": 0.34655762, + "step": 534, + "time_per_iteration": 2.722449779510498 + }, + { + "auxiliary_loss_clip": 0.01079629, + "auxiliary_loss_mlp": 0.01005459, + "balance_loss_clip": 1.02247775, + "balance_loss_mlp": 1.00104856, + "epoch": 0.015524345656084963, + "flos": 74775174387840.0, + "grad_norm": 0.7212228243258912, + "language_loss": 0.52175438, + "learning_rate": 3.620282226566477e-06, + "loss": 0.54260528, + "num_input_tokens_seen": 14630350, + "router_z_loss_clip": 0.5703125, + "router_z_loss_mlp": 0.04418945, + "step": 535, + "time_per_iteration": 3.143406867980957 + }, + { + "auxiliary_loss_clip": 0.01277686, + "auxiliary_loss_mlp": 0.01089104, + "balance_loss_clip": 1.09345829, + "balance_loss_mlp": 1.05942106, + "epoch": 0.01555336312460101, + "flos": 10918567854720.0, + "grad_norm": 2.651823998595307, + "language_loss": 0.89458996, + "learning_rate": 3.621358361442474e-06, + "loss": 0.91825789, + "num_input_tokens_seen": 14642155, + "router_z_loss_clip": 1.84472656, + "router_z_loss_mlp": 0.29711914, + "step": 536, + "time_per_iteration": 2.5515456199645996 + }, + { + "auxiliary_loss_clip": 0.01275678, + "auxiliary_loss_mlp": 0.01067404, + "balance_loss_clip": 1.08929741, + "balance_loss_mlp": 1.03819728, + "epoch": 0.015582380593117057, + "flos": 25623807384960.0, + "grad_norm": 3.0309695959502765, + "language_loss": 0.82158804, + "learning_rate": 3.62243249047411e-06, + "loss": 0.84501886, + "num_input_tokens_seen": 14655125, + "router_z_loss_clip": 1.86328125, + "router_z_loss_mlp": 0.29223633, + "step": 537, + "time_per_iteration": 2.581376075744629 + }, + { + "auxiliary_loss_clip": 0.01279269, + "auxiliary_loss_mlp": 0.01080903, + "balance_loss_clip": 1.09465122, + "balance_loss_mlp": 1.05187559, + "epoch": 0.015611398061633103, + "flos": 22630868496000.0, + "grad_norm": 3.4013263541223484, + "language_loss": 0.83228636, + "learning_rate": 3.623504621124998e-06, + "loss": 0.85588813, + "num_input_tokens_seen": 14669050, + "router_z_loss_clip": 1.84667969, + "router_z_loss_mlp": 0.2902832, + "step": 538, + "time_per_iteration": 2.553567409515381 + }, + { + "auxiliary_loss_clip": 0.01274952, + "auxiliary_loss_mlp": 0.01079329, + "balance_loss_clip": 1.09362888, + "balance_loss_mlp": 1.04949093, + "epoch": 0.015640415530149148, + "flos": 39930797848320.0, + "grad_norm": 2.3351559404241327, + "language_loss": 0.76024747, + "learning_rate": 3.624574760817172e-06, + "loss": 0.78379023, + "num_input_tokens_seen": 14691675, + "router_z_loss_clip": 1.8125, + "router_z_loss_mlp": 0.29858398, + "step": 539, + "time_per_iteration": 2.7007758617401123 + }, + { + "auxiliary_loss_clip": 0.01279425, + "auxiliary_loss_mlp": 0.01094426, + "balance_loss_clip": 1.09376359, + "balance_loss_mlp": 1.06448019, + "epoch": 0.015669432998665196, + "flos": 18292033975680.0, + "grad_norm": 2.9916474627064082, + "language_loss": 0.88873732, + "learning_rate": 3.6256429169313935e-06, + "loss": 0.91247582, + "num_input_tokens_seen": 14706105, + "router_z_loss_clip": 1.85742188, + "router_z_loss_mlp": 0.29943848, + "step": 540, + "time_per_iteration": 2.5666213035583496 + }, + { + "auxiliary_loss_clip": 0.01272943, + "auxiliary_loss_mlp": 0.01079098, + "balance_loss_clip": 1.08593071, + "balance_loss_mlp": 1.04726875, + "epoch": 0.015698450467181244, + "flos": 15954033720960.0, + "grad_norm": 3.402792928802378, + "language_loss": 0.91637993, + "learning_rate": 3.626709096807456e-06, + "loss": 0.9399004, + "num_input_tokens_seen": 14720355, + "router_z_loss_clip": 1.87109375, + "router_z_loss_mlp": 0.31787109, + "step": 541, + "time_per_iteration": 2.4895260334014893 + }, + { + "auxiliary_loss_clip": 0.01263907, + "auxiliary_loss_mlp": 0.01080029, + "balance_loss_clip": 1.09146571, + "balance_loss_mlp": 1.05233681, + "epoch": 0.01572746793569729, + "flos": 20331831519360.0, + "grad_norm": 2.521633034275504, + "language_loss": 0.89864063, + "learning_rate": 3.627773307744494e-06, + "loss": 0.92208004, + "num_input_tokens_seen": 14733005, + "router_z_loss_clip": 1.72460938, + "router_z_loss_mlp": 0.2767334, + "step": 542, + "time_per_iteration": 2.581289529800415 + }, + { + "auxiliary_loss_clip": 0.01267347, + "auxiliary_loss_mlp": 0.01073893, + "balance_loss_clip": 1.08546734, + "balance_loss_mlp": 1.04537845, + "epoch": 0.015756485404213337, + "flos": 30840909350400.0, + "grad_norm": 3.104213816833183, + "language_loss": 0.97810346, + "learning_rate": 3.6288355570012727e-06, + "loss": 1.00151587, + "num_input_tokens_seen": 14749155, + "router_z_loss_clip": 1.8203125, + "router_z_loss_mlp": 0.28527832, + "step": 543, + "time_per_iteration": 2.6460297107696533 + }, + { + "auxiliary_loss_clip": 0.01267013, + "auxiliary_loss_mlp": 0.01085192, + "balance_loss_clip": 1.08743572, + "balance_loss_mlp": 1.05641508, + "epoch": 0.01578550287272938, + "flos": 11357073089280.0, + "grad_norm": 2.894204611074797, + "language_loss": 0.92845559, + "learning_rate": 3.6298958517964935e-06, + "loss": 0.95197761, + "num_input_tokens_seen": 14760760, + "router_z_loss_clip": 1.79589844, + "router_z_loss_mlp": 0.28759766, + "step": 544, + "time_per_iteration": 2.480856418609619 + }, + { + "auxiliary_loss_clip": 0.01253173, + "auxiliary_loss_mlp": 0.01065436, + "balance_loss_clip": 1.0821538, + "balance_loss_mlp": 1.03878081, + "epoch": 0.01581452034124543, + "flos": 16318419240960.0, + "grad_norm": 3.186266851815329, + "language_loss": 0.92735058, + "learning_rate": 3.630954199309085e-06, + "loss": 0.95053673, + "num_input_tokens_seen": 14773135, + "router_z_loss_clip": 1.7109375, + "router_z_loss_mlp": 0.26647949, + "step": 545, + "time_per_iteration": 2.4398293495178223 + }, + { + "auxiliary_loss_clip": 0.01082187, + "auxiliary_loss_mlp": 0.01009412, + "balance_loss_clip": 1.02547836, + "balance_loss_mlp": 1.00442874, + "epoch": 0.015843537809761478, + "flos": 64743996032640.0, + "grad_norm": 0.820264793872741, + "language_loss": 0.53578526, + "learning_rate": 3.632010606678494e-06, + "loss": 0.55670124, + "num_input_tokens_seen": 14835015, + "router_z_loss_clip": 0.56640625, + "router_z_loss_mlp": 0.04980469, + "step": 546, + "time_per_iteration": 3.130859136581421 + }, + { + "auxiliary_loss_clip": 0.01269796, + "auxiliary_loss_mlp": 0.01073741, + "balance_loss_clip": 1.08468556, + "balance_loss_mlp": 1.04262757, + "epoch": 0.015872555278277523, + "flos": 27047665486080.0, + "grad_norm": 3.8947763789183005, + "language_loss": 0.98148841, + "learning_rate": 3.6330650810049766e-06, + "loss": 1.00492382, + "num_input_tokens_seen": 14851830, + "router_z_loss_clip": 1.85546875, + "router_z_loss_mlp": 0.3112793, + "step": 547, + "time_per_iteration": 2.5863513946533203 + }, + { + "auxiliary_loss_clip": 0.01270052, + "auxiliary_loss_mlp": 0.01080643, + "balance_loss_clip": 1.09349561, + "balance_loss_mlp": 1.05264091, + "epoch": 0.01590157274679357, + "flos": 31462343573760.0, + "grad_norm": 3.0185338281688927, + "language_loss": 0.7732293, + "learning_rate": 3.6341176293498826e-06, + "loss": 0.79673624, + "num_input_tokens_seen": 14869145, + "router_z_loss_clip": 1.765625, + "router_z_loss_mlp": 0.27978516, + "step": 548, + "time_per_iteration": 2.6303200721740723 + }, + { + "auxiliary_loss_clip": 0.01266723, + "auxiliary_loss_mlp": 0.01075572, + "balance_loss_clip": 1.08815002, + "balance_loss_mlp": 1.04526925, + "epoch": 0.015930590215309615, + "flos": 12525570426240.0, + "grad_norm": 4.215496431877273, + "language_loss": 0.84847152, + "learning_rate": 3.635168258735939e-06, + "loss": 0.87189448, + "num_input_tokens_seen": 14881820, + "router_z_loss_clip": 1.78515625, + "router_z_loss_mlp": 0.30273438, + "step": 549, + "time_per_iteration": 2.479729413986206 + }, + { + "auxiliary_loss_clip": 0.01287024, + "auxiliary_loss_mlp": 0.01083818, + "balance_loss_clip": 1.08954966, + "balance_loss_mlp": 1.04813826, + "epoch": 0.015959607683825663, + "flos": 24600784129920.0, + "grad_norm": 2.4282527086159087, + "language_loss": 0.97048903, + "learning_rate": 3.6362169761475343e-06, + "loss": 0.99419737, + "num_input_tokens_seen": 14896690, + "router_z_loss_clip": 1.97460938, + "router_z_loss_mlp": 0.35681152, + "step": 550, + "time_per_iteration": 2.570289134979248 + }, + { + "auxiliary_loss_clip": 0.01282841, + "auxiliary_loss_mlp": 0.01074285, + "balance_loss_clip": 1.094872, + "balance_loss_mlp": 1.04451847, + "epoch": 0.015988625152341708, + "flos": 10444331566080.0, + "grad_norm": 3.1514071776815387, + "language_loss": 0.90604877, + "learning_rate": 3.6372637885309946e-06, + "loss": 0.92962003, + "num_input_tokens_seen": 14907260, + "router_z_loss_clip": 1.87792969, + "router_z_loss_mlp": 0.29748535, + "step": 551, + "time_per_iteration": 2.4707448482513428 + }, + { + "auxiliary_loss_clip": 0.01084247, + "auxiliary_loss_mlp": 0.01005603, + "balance_loss_clip": 1.02794158, + "balance_loss_mlp": 1.00059652, + "epoch": 0.016017642620857756, + "flos": 70510387754880.0, + "grad_norm": 0.7933308054490612, + "language_loss": 0.56209928, + "learning_rate": 3.6383087027948565e-06, + "loss": 0.5829978, + "num_input_tokens_seen": 14972845, + "router_z_loss_clip": 0.5625, + "router_z_loss_mlp": 0.05004883, + "step": 552, + "time_per_iteration": 3.1979050636291504 + }, + { + "auxiliary_loss_clip": 0.01272346, + "auxiliary_loss_mlp": 0.01074362, + "balance_loss_clip": 1.08866227, + "balance_loss_mlp": 1.04062605, + "epoch": 0.016046660089373804, + "flos": 31896826485120.0, + "grad_norm": 2.4361038516336198, + "language_loss": 0.87659717, + "learning_rate": 3.6393517258101497e-06, + "loss": 0.90006429, + "num_input_tokens_seen": 14989565, + "router_z_loss_clip": 1.83496094, + "router_z_loss_mlp": 0.33740234, + "step": 553, + "time_per_iteration": 2.672544002532959 + }, + { + "auxiliary_loss_clip": 0.01270333, + "auxiliary_loss_mlp": 0.01085788, + "balance_loss_clip": 1.08798492, + "balance_loss_mlp": 1.05627203, + "epoch": 0.01607567755788985, + "flos": 10374844705920.0, + "grad_norm": 2.784486996426869, + "language_loss": 0.86093581, + "learning_rate": 3.6403928644106584e-06, + "loss": 0.88449705, + "num_input_tokens_seen": 14999900, + "router_z_loss_clip": 1.82421875, + "router_z_loss_mlp": 0.29541016, + "step": 554, + "time_per_iteration": 2.4744396209716797 + }, + { + "auxiliary_loss_clip": 0.01285145, + "auxiliary_loss_mlp": 0.01080411, + "balance_loss_clip": 1.09220338, + "balance_loss_mlp": 1.04886794, + "epoch": 0.016104695026405897, + "flos": 23873270065920.0, + "grad_norm": 2.6528484012378755, + "language_loss": 0.986301, + "learning_rate": 3.6414321253931943e-06, + "loss": 1.0099566, + "num_input_tokens_seen": 15017125, + "router_z_loss_clip": 1.92773438, + "router_z_loss_mlp": 0.31542969, + "step": 555, + "time_per_iteration": 2.5475733280181885 + }, + { + "auxiliary_loss_clip": 0.01279276, + "auxiliary_loss_mlp": 0.01075679, + "balance_loss_clip": 1.08941293, + "balance_loss_mlp": 1.04270601, + "epoch": 0.01613371249492194, + "flos": 13075578455040.0, + "grad_norm": 3.773132241964171, + "language_loss": 1.16345167, + "learning_rate": 3.6424695155178653e-06, + "loss": 1.18700123, + "num_input_tokens_seen": 15030265, + "router_z_loss_clip": 1.8984375, + "router_z_loss_mlp": 0.32983398, + "step": 556, + "time_per_iteration": 2.5166494846343994 + }, + { + "auxiliary_loss_clip": 0.01259129, + "auxiliary_loss_mlp": 0.01082544, + "balance_loss_clip": 1.08541715, + "balance_loss_mlp": 1.05493534, + "epoch": 0.01616272996343799, + "flos": 14020675153920.0, + "grad_norm": 3.052218413236908, + "language_loss": 0.73083282, + "learning_rate": 3.643505041508334e-06, + "loss": 0.75424957, + "num_input_tokens_seen": 15043175, + "router_z_loss_clip": 1.73925781, + "router_z_loss_mlp": 0.27563477, + "step": 557, + "time_per_iteration": 2.5222108364105225 + }, + { + "auxiliary_loss_clip": 0.01084278, + "auxiliary_loss_mlp": 0.01005944, + "balance_loss_clip": 1.02789998, + "balance_loss_mlp": 1.00119996, + "epoch": 0.016191747431954038, + "flos": 63866339118720.0, + "grad_norm": 0.7458972826855246, + "language_loss": 0.54451615, + "learning_rate": 3.644538710052083e-06, + "loss": 0.56541836, + "num_input_tokens_seen": 15102595, + "router_z_loss_clip": 0.5625, + "router_z_loss_mlp": 0.04736328, + "step": 558, + "time_per_iteration": 3.097581624984741 + }, + { + "auxiliary_loss_clip": 0.01083748, + "auxiliary_loss_mlp": 0.01004977, + "balance_loss_clip": 1.02732408, + "balance_loss_mlp": 1.00030422, + "epoch": 0.016220764900470083, + "flos": 67033695473280.0, + "grad_norm": 0.7463040744210614, + "language_loss": 0.52472341, + "learning_rate": 3.6455705278006725e-06, + "loss": 0.54561067, + "num_input_tokens_seen": 15151495, + "router_z_loss_clip": 0.5625, + "router_z_loss_mlp": 0.04663086, + "step": 559, + "time_per_iteration": 3.0076277256011963 + }, + { + "auxiliary_loss_clip": 0.01261941, + "auxiliary_loss_mlp": 0.01059741, + "balance_loss_clip": 1.08821821, + "balance_loss_mlp": 1.03319275, + "epoch": 0.01624978236898613, + "flos": 14969075904000.0, + "grad_norm": 3.3229468086584992, + "language_loss": 1.07818675, + "learning_rate": 3.6466005013699975e-06, + "loss": 1.10140359, + "num_input_tokens_seen": 15163330, + "router_z_loss_clip": 1.73632812, + "router_z_loss_mlp": 0.26538086, + "step": 560, + "time_per_iteration": 2.5822536945343018 + }, + { + "auxiliary_loss_clip": 0.01083579, + "auxiliary_loss_mlp": 0.0100588, + "balance_loss_clip": 1.02727914, + "balance_loss_mlp": 1.00125432, + "epoch": 0.016278799837502175, + "flos": 74773486448640.0, + "grad_norm": 0.7127575889305064, + "language_loss": 0.52669597, + "learning_rate": 3.6476286373405424e-06, + "loss": 0.54759049, + "num_input_tokens_seen": 15225765, + "router_z_loss_clip": 0.5625, + "router_z_loss_mlp": 0.04614258, + "step": 561, + "time_per_iteration": 3.126417875289917 + }, + { + "auxiliary_loss_clip": 0.01264746, + "auxiliary_loss_mlp": 0.01070697, + "balance_loss_clip": 1.08857489, + "balance_loss_mlp": 1.04227805, + "epoch": 0.016307817306018223, + "flos": 10406948486400.0, + "grad_norm": 3.9714728124108087, + "language_loss": 1.12176013, + "learning_rate": 3.6486549422576337e-06, + "loss": 1.14511454, + "num_input_tokens_seen": 15234910, + "router_z_loss_clip": 1.75878906, + "router_z_loss_mlp": 0.28417969, + "step": 562, + "time_per_iteration": 2.449082136154175 + }, + { + "auxiliary_loss_clip": 0.01082361, + "auxiliary_loss_mlp": 0.01005619, + "balance_loss_clip": 1.02623224, + "balance_loss_mlp": 1.0013752, + "epoch": 0.016336834774534268, + "flos": 57880281772800.0, + "grad_norm": 0.7301069079142786, + "language_loss": 0.54480416, + "learning_rate": 3.649679422631688e-06, + "loss": 0.56568396, + "num_input_tokens_seen": 15291310, + "router_z_loss_clip": 0.5625, + "router_z_loss_mlp": 0.04248047, + "step": 563, + "time_per_iteration": 2.9006943702697754 + }, + { + "auxiliary_loss_clip": 0.01272304, + "auxiliary_loss_mlp": 0.01082403, + "balance_loss_clip": 1.08905435, + "balance_loss_mlp": 1.05226707, + "epoch": 0.016365852243050316, + "flos": 74734340624640.0, + "grad_norm": 6.197943994904777, + "language_loss": 0.89897931, + "learning_rate": 3.650702084938462e-06, + "loss": 0.92252636, + "num_input_tokens_seen": 15313550, + "router_z_loss_clip": 1.83300781, + "router_z_loss_mlp": 0.3013916, + "step": 564, + "time_per_iteration": 2.9278876781463623 + }, + { + "auxiliary_loss_clip": 0.01272136, + "auxiliary_loss_mlp": 0.01092005, + "balance_loss_clip": 1.09560668, + "balance_loss_mlp": 1.06416976, + "epoch": 0.016394869711566364, + "flos": 10480314015360.0, + "grad_norm": 2.7764979182045693, + "language_loss": 0.69736123, + "learning_rate": 3.6517229356192984e-06, + "loss": 0.7210027, + "num_input_tokens_seen": 15325600, + "router_z_loss_clip": 1.765625, + "router_z_loss_mlp": 0.27783203, + "step": 565, + "time_per_iteration": 2.4891016483306885 + }, + { + "auxiliary_loss_clip": 0.0129029, + "auxiliary_loss_mlp": 0.01088142, + "balance_loss_clip": 1.09591413, + "balance_loss_mlp": 1.05532312, + "epoch": 0.01642388718008241, + "flos": 16319173426560.0, + "grad_norm": 3.6344059506305446, + "language_loss": 0.96683896, + "learning_rate": 3.652741981081366e-06, + "loss": 0.99062335, + "num_input_tokens_seen": 15337850, + "router_z_loss_clip": 1.94140625, + "router_z_loss_mlp": 0.32824707, + "step": 566, + "time_per_iteration": 2.480208158493042 + }, + { + "auxiliary_loss_clip": 0.01081381, + "auxiliary_loss_mlp": 0.01005385, + "balance_loss_clip": 1.02544141, + "balance_loss_mlp": 1.00126064, + "epoch": 0.016452904648598457, + "flos": 55693106726400.0, + "grad_norm": 0.716660525275876, + "language_loss": 0.53820366, + "learning_rate": 3.6537592276979053e-06, + "loss": 0.55907136, + "num_input_tokens_seen": 15393285, + "router_z_loss_clip": 0.55859375, + "router_z_loss_mlp": 0.04125977, + "step": 567, + "time_per_iteration": 3.0247504711151123 + }, + { + "auxiliary_loss_clip": 0.01283399, + "auxiliary_loss_mlp": 0.01096465, + "balance_loss_clip": 1.09016049, + "balance_loss_mlp": 1.06277609, + "epoch": 0.0164819221171145, + "flos": 16902470384640.0, + "grad_norm": 3.327121948024119, + "language_loss": 0.99571478, + "learning_rate": 3.6547746818084655e-06, + "loss": 1.01951337, + "num_input_tokens_seen": 15406580, + "router_z_loss_clip": 1.93359375, + "router_z_loss_mlp": 0.33691406, + "step": 568, + "time_per_iteration": 2.493711233139038 + }, + { + "auxiliary_loss_clip": 0.01079485, + "auxiliary_loss_mlp": 0.01004505, + "balance_loss_clip": 1.02407312, + "balance_loss_mlp": 1.00071371, + "epoch": 0.01651093958563055, + "flos": 66571885290240.0, + "grad_norm": 0.6441058809059415, + "language_loss": 0.59681475, + "learning_rate": 3.6557883497191405e-06, + "loss": 0.61765468, + "num_input_tokens_seen": 15473025, + "router_z_loss_clip": 0.5546875, + "router_z_loss_mlp": 0.0378418, + "step": 569, + "time_per_iteration": 3.1960337162017822 + }, + { + "auxiliary_loss_clip": 0.01079024, + "auxiliary_loss_mlp": 0.01005076, + "balance_loss_clip": 1.02417576, + "balance_loss_mlp": 1.00130868, + "epoch": 0.016539957054146598, + "flos": 70719494780160.0, + "grad_norm": 0.6942931838704514, + "language_loss": 0.55621088, + "learning_rate": 3.656800237702806e-06, + "loss": 0.57705188, + "num_input_tokens_seen": 15528640, + "router_z_loss_clip": 0.546875, + "router_z_loss_mlp": 0.03759766, + "step": 570, + "time_per_iteration": 3.0360772609710693 + }, + { + "auxiliary_loss_clip": 0.01272105, + "auxiliary_loss_mlp": 0.01071215, + "balance_loss_clip": 1.08947968, + "balance_loss_mlp": 1.04090011, + "epoch": 0.016568974522662643, + "flos": 24383273322240.0, + "grad_norm": 2.7910750866324254, + "language_loss": 1.00101829, + "learning_rate": 3.65781035199935e-06, + "loss": 1.02445161, + "num_input_tokens_seen": 15544405, + "router_z_loss_clip": 1.82714844, + "router_z_loss_mlp": 0.30334473, + "step": 571, + "time_per_iteration": 2.5852956771850586 + }, + { + "auxiliary_loss_clip": 0.01255922, + "auxiliary_loss_mlp": 0.01069626, + "balance_loss_clip": 1.08191872, + "balance_loss_mlp": 1.03833377, + "epoch": 0.01659799199117869, + "flos": 28100960928000.0, + "grad_norm": 2.6086553266997394, + "language_loss": 1.08791399, + "learning_rate": 3.6588186988159077e-06, + "loss": 1.11116958, + "num_input_tokens_seen": 15558795, + "router_z_loss_clip": 1.74023438, + "router_z_loss_mlp": 0.31274414, + "step": 572, + "time_per_iteration": 2.5691559314727783 + }, + { + "auxiliary_loss_clip": 0.01270194, + "auxiliary_loss_mlp": 0.01070603, + "balance_loss_clip": 1.0915463, + "balance_loss_mlp": 1.04282689, + "epoch": 0.016627009459694735, + "flos": 16319029772160.0, + "grad_norm": 3.7345936023670783, + "language_loss": 0.82049763, + "learning_rate": 3.6598252843270863e-06, + "loss": 0.84390557, + "num_input_tokens_seen": 15571720, + "router_z_loss_clip": 1.78613281, + "router_z_loss_mlp": 0.27758789, + "step": 573, + "time_per_iteration": 2.4566125869750977 + }, + { + "auxiliary_loss_clip": 0.01267031, + "auxiliary_loss_mlp": 0.01092312, + "balance_loss_clip": 1.08914995, + "balance_loss_mlp": 1.06242609, + "epoch": 0.016656026928210783, + "flos": 26985792309120.0, + "grad_norm": 2.0937674633129872, + "language_loss": 0.78500086, + "learning_rate": 3.6608301146751923e-06, + "loss": 0.80859435, + "num_input_tokens_seen": 15590020, + "router_z_loss_clip": 1.77929688, + "router_z_loss_mlp": 0.29882812, + "step": 574, + "time_per_iteration": 2.6253879070281982 + }, + { + "auxiliary_loss_clip": 0.0127977, + "auxiliary_loss_mlp": 0.01083664, + "balance_loss_clip": 1.09598756, + "balance_loss_mlp": 1.0531944, + "epoch": 0.016685044396726828, + "flos": 16359178199040.0, + "grad_norm": 3.5729839031489785, + "language_loss": 0.91377044, + "learning_rate": 3.66183319597046e-06, + "loss": 0.93740487, + "num_input_tokens_seen": 15604860, + "router_z_loss_clip": 1.83886719, + "router_z_loss_mlp": 0.30432129, + "step": 575, + "time_per_iteration": 7.1615684032440186 + }, + { + "auxiliary_loss_clip": 0.01275642, + "auxiliary_loss_mlp": 0.01094837, + "balance_loss_clip": 1.09006882, + "balance_loss_mlp": 1.06380665, + "epoch": 0.016714061865242876, + "flos": 27848724647040.0, + "grad_norm": 2.5663125212048654, + "language_loss": 0.80251294, + "learning_rate": 3.6628345342912697e-06, + "loss": 0.82621777, + "num_input_tokens_seen": 15621045, + "router_z_loss_clip": 1.85742188, + "router_z_loss_mlp": 0.31030273, + "step": 576, + "time_per_iteration": 7.3756561279296875 + }, + { + "auxiliary_loss_clip": 0.01080871, + "auxiliary_loss_mlp": 0.01017259, + "balance_loss_clip": 1.02679896, + "balance_loss_mlp": 1.01325309, + "epoch": 0.016743079333758924, + "flos": 74777005981440.0, + "grad_norm": 0.7228194364274252, + "language_loss": 0.60065651, + "learning_rate": 3.663834135684372e-06, + "loss": 0.62163782, + "num_input_tokens_seen": 15691015, + "router_z_loss_clip": 0.5390625, + "router_z_loss_mlp": 0.04003906, + "step": 577, + "time_per_iteration": 3.253126382827759 + }, + { + "auxiliary_loss_clip": 0.01292527, + "auxiliary_loss_mlp": 0.01087535, + "balance_loss_clip": 1.09625745, + "balance_loss_mlp": 1.05437076, + "epoch": 0.01677209680227497, + "flos": 21498605003520.0, + "grad_norm": 2.428184255761008, + "language_loss": 0.93162763, + "learning_rate": 3.6648320061651052e-06, + "loss": 0.95542824, + "num_input_tokens_seen": 15709155, + "router_z_loss_clip": 1.96289062, + "router_z_loss_mlp": 0.33178711, + "step": 578, + "time_per_iteration": 2.56661319732666 + }, + { + "auxiliary_loss_clip": 0.01077865, + "auxiliary_loss_mlp": 0.01009754, + "balance_loss_clip": 1.02439177, + "balance_loss_mlp": 1.00579667, + "epoch": 0.016801114270791017, + "flos": 74780669168640.0, + "grad_norm": 0.7227542998371962, + "language_loss": 0.56635624, + "learning_rate": 3.665828151717614e-06, + "loss": 0.58723247, + "num_input_tokens_seen": 15771180, + "router_z_loss_clip": 0.53125, + "router_z_loss_mlp": 0.03955078, + "step": 579, + "time_per_iteration": 3.1713855266571045 + }, + { + "auxiliary_loss_clip": 0.01266197, + "auxiliary_loss_mlp": 0.01075787, + "balance_loss_clip": 1.08839369, + "balance_loss_mlp": 1.0455792, + "epoch": 0.01683013173930706, + "flos": 13655319966720.0, + "grad_norm": 3.579471188512206, + "language_loss": 0.82444608, + "learning_rate": 3.6668225782950615e-06, + "loss": 0.84786594, + "num_input_tokens_seen": 15783845, + "router_z_loss_clip": 1.77734375, + "router_z_loss_mlp": 0.30212402, + "step": 580, + "time_per_iteration": 2.5000534057617188 + }, + { + "auxiliary_loss_clip": 0.01076157, + "auxiliary_loss_mlp": 0.01009592, + "balance_loss_clip": 1.02289045, + "balance_loss_mlp": 1.00570619, + "epoch": 0.01685914920782311, + "flos": 60695140008960.0, + "grad_norm": 0.6827747292790131, + "language_loss": 0.54629904, + "learning_rate": 3.6678152918198486e-06, + "loss": 0.56715649, + "num_input_tokens_seen": 15846885, + "router_z_loss_clip": 0.53125, + "router_z_loss_mlp": 0.03881836, + "step": 581, + "time_per_iteration": 3.0721466541290283 + }, + { + "auxiliary_loss_clip": 0.01075113, + "auxiliary_loss_mlp": 0.01008149, + "balance_loss_clip": 1.02190924, + "balance_loss_mlp": 1.00435817, + "epoch": 0.016888166676339158, + "flos": 74771942163840.0, + "grad_norm": 0.6561376300899914, + "language_loss": 0.5225023, + "learning_rate": 3.6688062981838202e-06, + "loss": 0.5433349, + "num_input_tokens_seen": 15914395, + "router_z_loss_clip": 0.53125, + "router_z_loss_mlp": 0.0378418, + "step": 582, + "time_per_iteration": 3.2394485473632812 + }, + { + "auxiliary_loss_clip": 0.01257611, + "auxiliary_loss_mlp": 0.01074594, + "balance_loss_clip": 1.08633184, + "balance_loss_mlp": 1.04489946, + "epoch": 0.016917184144855203, + "flos": 12452384465280.0, + "grad_norm": 3.560106068536052, + "language_loss": 0.93126512, + "learning_rate": 3.6697956032484757e-06, + "loss": 0.95458722, + "num_input_tokens_seen": 15925365, + "router_z_loss_clip": 1.7109375, + "router_z_loss_mlp": 0.29724121, + "step": 583, + "time_per_iteration": 2.507654905319214 + }, + { + "auxiliary_loss_clip": 0.01270267, + "auxiliary_loss_mlp": 0.01076086, + "balance_loss_clip": 1.08669662, + "balance_loss_mlp": 1.04476964, + "epoch": 0.01694620161337125, + "flos": 29178964926720.0, + "grad_norm": 3.4836971326433703, + "language_loss": 0.9882555, + "learning_rate": 3.670783212845181e-06, + "loss": 1.01171899, + "num_input_tokens_seen": 15947345, + "router_z_loss_clip": 1.83398438, + "router_z_loss_mlp": 0.31311035, + "step": 584, + "time_per_iteration": 2.6737000942230225 + }, + { + "auxiliary_loss_clip": 0.012604, + "auxiliary_loss_mlp": 0.01090864, + "balance_loss_clip": 1.08322525, + "balance_loss_mlp": 1.05929708, + "epoch": 0.016975219081887295, + "flos": 29391914707200.0, + "grad_norm": 1.6154884787526282, + "language_loss": 0.73429382, + "learning_rate": 3.6717691327753693e-06, + "loss": 0.75780642, + "num_input_tokens_seen": 15973560, + "router_z_loss_clip": 1.77246094, + "router_z_loss_mlp": 0.31591797, + "step": 585, + "time_per_iteration": 2.8334262371063232 + }, + { + "auxiliary_loss_clip": 0.01073939, + "auxiliary_loss_mlp": 0.01014938, + "balance_loss_clip": 1.02042389, + "balance_loss_mlp": 1.0111233, + "epoch": 0.017004236550403343, + "flos": 74630274923520.0, + "grad_norm": 0.6688468252309698, + "language_loss": 0.54401046, + "learning_rate": 3.67275336881075e-06, + "loss": 0.56489921, + "num_input_tokens_seen": 16036605, + "router_z_loss_clip": 0.53515625, + "router_z_loss_mlp": 0.03808594, + "step": 586, + "time_per_iteration": 3.1285037994384766 + }, + { + "auxiliary_loss_clip": 0.01073077, + "auxiliary_loss_mlp": 0.01011011, + "balance_loss_clip": 1.02006269, + "balance_loss_mlp": 1.00741088, + "epoch": 0.017033254018919388, + "flos": 67446309980160.0, + "grad_norm": 0.712811212816914, + "language_loss": 0.55615687, + "learning_rate": 3.6737359266935092e-06, + "loss": 0.5769977, + "num_input_tokens_seen": 16102765, + "router_z_loss_clip": 0.53125, + "router_z_loss_mlp": 0.03588867, + "step": 587, + "time_per_iteration": 3.1421401500701904 + }, + { + "auxiliary_loss_clip": 0.01260257, + "auxiliary_loss_mlp": 0.01084547, + "balance_loss_clip": 1.08747458, + "balance_loss_mlp": 1.05448222, + "epoch": 0.017062271487435436, + "flos": 28834257081600.0, + "grad_norm": 2.2784937042271305, + "language_loss": 0.75069773, + "learning_rate": 3.6747168121365105e-06, + "loss": 0.77414578, + "num_input_tokens_seen": 16120360, + "router_z_loss_clip": 1.72851562, + "router_z_loss_mlp": 0.30078125, + "step": 588, + "time_per_iteration": 2.614506244659424 + }, + { + "auxiliary_loss_clip": 0.01261856, + "auxiliary_loss_mlp": 0.01055715, + "balance_loss_clip": 1.09047198, + "balance_loss_mlp": 1.02746272, + "epoch": 0.017091288955951484, + "flos": 12604753958400.0, + "grad_norm": 2.9417773444651014, + "language_loss": 0.99906504, + "learning_rate": 3.6756960308234956e-06, + "loss": 1.02224076, + "num_input_tokens_seen": 16133410, + "router_z_loss_clip": 1.71289062, + "router_z_loss_mlp": 0.28234863, + "step": 589, + "time_per_iteration": 2.5457851886749268 + }, + { + "auxiliary_loss_clip": 0.01276544, + "auxiliary_loss_mlp": 0.01094318, + "balance_loss_clip": 1.09763849, + "balance_loss_mlp": 1.06513596, + "epoch": 0.01712030642446753, + "flos": 12159820189440.0, + "grad_norm": 2.746067839598561, + "language_loss": 0.8563453, + "learning_rate": 3.676673588409281e-06, + "loss": 0.88005388, + "num_input_tokens_seen": 16144475, + "router_z_loss_clip": 1.78710938, + "router_z_loss_mlp": 0.29150391, + "step": 590, + "time_per_iteration": 2.495243549346924 + }, + { + "auxiliary_loss_clip": 0.01270474, + "auxiliary_loss_mlp": 0.01074551, + "balance_loss_clip": 1.08657742, + "balance_loss_mlp": 1.04019475, + "epoch": 0.017149323892983577, + "flos": 35547684837120.0, + "grad_norm": 1.9315427446931364, + "language_loss": 0.94467103, + "learning_rate": 3.6776494905199557e-06, + "loss": 0.96812135, + "num_input_tokens_seen": 16163055, + "router_z_loss_clip": 1.83886719, + "router_z_loss_mlp": 0.34338379, + "step": 591, + "time_per_iteration": 2.6331217288970947 + }, + { + "auxiliary_loss_clip": 0.01279979, + "auxiliary_loss_mlp": 0.01093358, + "balance_loss_clip": 1.09224534, + "balance_loss_mlp": 1.06002665, + "epoch": 0.01717834136149962, + "flos": 52585106618880.0, + "grad_norm": 2.975865588104254, + "language_loss": 0.89277852, + "learning_rate": 3.6786237427530713e-06, + "loss": 0.91651183, + "num_input_tokens_seen": 16183490, + "router_z_loss_clip": 1.87792969, + "router_z_loss_mlp": 0.33312988, + "step": 592, + "time_per_iteration": 2.805225133895874 + }, + { + "auxiliary_loss_clip": 0.0126331, + "auxiliary_loss_mlp": 0.01073694, + "balance_loss_clip": 1.09088159, + "balance_loss_mlp": 1.04678845, + "epoch": 0.01720735883001567, + "flos": 19563486670080.0, + "grad_norm": 3.4697716453823366, + "language_loss": 0.95326871, + "learning_rate": 3.679596350677839e-06, + "loss": 0.97663873, + "num_input_tokens_seen": 16194475, + "router_z_loss_clip": 1.72558594, + "router_z_loss_mlp": 0.26904297, + "step": 593, + "time_per_iteration": 2.4965038299560547 + }, + { + "auxiliary_loss_clip": 0.01273783, + "auxiliary_loss_mlp": 0.01097468, + "balance_loss_clip": 1.09072804, + "balance_loss_mlp": 1.0650903, + "epoch": 0.017236376298531714, + "flos": 30739390536960.0, + "grad_norm": 2.702494489866647, + "language_loss": 0.91737992, + "learning_rate": 3.6805673198353194e-06, + "loss": 0.94109237, + "num_input_tokens_seen": 16211245, + "router_z_loss_clip": 1.83203125, + "router_z_loss_mlp": 0.32397461, + "step": 594, + "time_per_iteration": 2.6366000175476074 + }, + { + "auxiliary_loss_clip": 0.01084979, + "auxiliary_loss_mlp": 0.01040889, + "balance_loss_clip": 1.03162658, + "balance_loss_mlp": 1.03724146, + "epoch": 0.017265393767047763, + "flos": 57139946553600.0, + "grad_norm": 0.8362194971499834, + "language_loss": 0.56147164, + "learning_rate": 3.6815366557386092e-06, + "loss": 0.58273023, + "num_input_tokens_seen": 16260285, + "router_z_loss_clip": 0.53125, + "router_z_loss_mlp": 0.03637695, + "step": 595, + "time_per_iteration": 2.8772664070129395 + }, + { + "auxiliary_loss_clip": 0.0126593, + "auxiliary_loss_mlp": 0.01079269, + "balance_loss_clip": 1.08503163, + "balance_loss_mlp": 1.04863262, + "epoch": 0.01729441123556381, + "flos": 24674867930880.0, + "grad_norm": 3.4148916303399344, + "language_loss": 0.91479874, + "learning_rate": 3.6825043638730345e-06, + "loss": 0.93825072, + "num_input_tokens_seen": 16276780, + "router_z_loss_clip": 1.80664062, + "router_z_loss_mlp": 0.30639648, + "step": 596, + "time_per_iteration": 2.6774349212646484 + }, + { + "auxiliary_loss_clip": 0.0125919, + "auxiliary_loss_mlp": 0.0107259, + "balance_loss_clip": 1.08368337, + "balance_loss_mlp": 1.04419422, + "epoch": 0.017323428704079855, + "flos": 40836248910720.0, + "grad_norm": 3.608753655253272, + "language_loss": 1.0399282, + "learning_rate": 3.6834704496963308e-06, + "loss": 1.06324601, + "num_input_tokens_seen": 16294420, + "router_z_loss_clip": 1.75585938, + "router_z_loss_mlp": 0.28393555, + "step": 597, + "time_per_iteration": 2.6924445629119873 + }, + { + "auxiliary_loss_clip": 0.01074016, + "auxiliary_loss_mlp": 0.01007318, + "balance_loss_clip": 1.02132988, + "balance_loss_mlp": 1.00383663, + "epoch": 0.017352446172595903, + "flos": 71598300929280.0, + "grad_norm": 0.7062902538218568, + "language_loss": 0.52348685, + "learning_rate": 3.6844349186388327e-06, + "loss": 0.5443002, + "num_input_tokens_seen": 16357610, + "router_z_loss_clip": 0.52734375, + "router_z_loss_mlp": 0.03491211, + "step": 598, + "time_per_iteration": 3.237166166305542 + }, + { + "auxiliary_loss_clip": 0.01269731, + "auxiliary_loss_mlp": 0.01074048, + "balance_loss_clip": 1.08337402, + "balance_loss_mlp": 1.04391146, + "epoch": 0.017381463641111948, + "flos": 22193835719040.0, + "grad_norm": 2.151726435036707, + "language_loss": 0.75931811, + "learning_rate": 3.685397776103655e-06, + "loss": 0.78275585, + "num_input_tokens_seen": 16375270, + "router_z_loss_clip": 1.86328125, + "router_z_loss_mlp": 0.3013916, + "step": 599, + "time_per_iteration": 2.5318853855133057 + }, + { + "auxiliary_loss_clip": 0.01265326, + "auxiliary_loss_mlp": 0.01077294, + "balance_loss_clip": 1.08661914, + "balance_loss_mlp": 1.04712224, + "epoch": 0.017410481109627996, + "flos": 20916385453440.0, + "grad_norm": 2.784261647268414, + "language_loss": 0.90052372, + "learning_rate": 3.686359027466873e-06, + "loss": 0.9239499, + "num_input_tokens_seen": 16389455, + "router_z_loss_clip": 1.78613281, + "router_z_loss_mlp": 0.30163574, + "step": 600, + "time_per_iteration": 2.497224807739258 + }, + { + "auxiliary_loss_clip": 0.01263131, + "auxiliary_loss_mlp": 0.01074205, + "balance_loss_clip": 1.08754647, + "balance_loss_mlp": 1.04713225, + "epoch": 0.017439498578144044, + "flos": 18291028394880.0, + "grad_norm": 2.753159012941114, + "language_loss": 0.92500186, + "learning_rate": 3.6873186780777043e-06, + "loss": 0.94837523, + "num_input_tokens_seen": 16403635, + "router_z_loss_clip": 1.75683594, + "router_z_loss_mlp": 0.27087402, + "step": 601, + "time_per_iteration": 2.53521466255188 + }, + { + "auxiliary_loss_clip": 0.01274764, + "auxiliary_loss_mlp": 0.01077969, + "balance_loss_clip": 1.09127307, + "balance_loss_mlp": 1.04733229, + "epoch": 0.01746851604666009, + "flos": 18617456217600.0, + "grad_norm": 3.1828637615162796, + "language_loss": 0.81730658, + "learning_rate": 3.688276733258688e-06, + "loss": 0.8408339, + "num_input_tokens_seen": 16417665, + "router_z_loss_clip": 1.83496094, + "router_z_loss_mlp": 0.30639648, + "step": 602, + "time_per_iteration": 2.43766450881958 + }, + { + "auxiliary_loss_clip": 0.01073518, + "auxiliary_loss_mlp": 0.01029119, + "balance_loss_clip": 1.01981568, + "balance_loss_mlp": 1.02539945, + "epoch": 0.017497533515176137, + "flos": 74083283637120.0, + "grad_norm": 0.7520457199776976, + "language_loss": 0.53976548, + "learning_rate": 3.689233198305862e-06, + "loss": 0.56079185, + "num_input_tokens_seen": 16483255, + "router_z_loss_clip": 0.5390625, + "router_z_loss_mlp": 0.03710938, + "step": 603, + "time_per_iteration": 3.2007808685302734 + }, + { + "auxiliary_loss_clip": 0.0107346, + "auxiliary_loss_mlp": 0.01019477, + "balance_loss_clip": 1.01996946, + "balance_loss_mlp": 1.01585317, + "epoch": 0.01752655098369218, + "flos": 53908238983680.0, + "grad_norm": 0.7439364910230636, + "language_loss": 0.51238602, + "learning_rate": 3.6901880784889333e-06, + "loss": 0.53331542, + "num_input_tokens_seen": 16536955, + "router_z_loss_clip": 0.53515625, + "router_z_loss_mlp": 0.03613281, + "step": 604, + "time_per_iteration": 3.032623052597046 + }, + { + "auxiliary_loss_clip": 0.01273485, + "auxiliary_loss_mlp": 0.01077764, + "balance_loss_clip": 1.08747363, + "balance_loss_mlp": 1.04474354, + "epoch": 0.01755556845220823, + "flos": 26753880147840.0, + "grad_norm": 3.341736413821845, + "language_loss": 1.10774112, + "learning_rate": 3.6911413790514606e-06, + "loss": 1.13125372, + "num_input_tokens_seen": 16550330, + "router_z_loss_clip": 1.86035156, + "router_z_loss_mlp": 0.33007812, + "step": 605, + "time_per_iteration": 2.5718863010406494 + }, + { + "auxiliary_loss_clip": 0.01272159, + "auxiliary_loss_mlp": 0.01077603, + "balance_loss_clip": 1.08981371, + "balance_loss_mlp": 1.04958868, + "epoch": 0.017584585920724274, + "flos": 12487289506560.0, + "grad_norm": 3.462855778260563, + "language_loss": 0.89826351, + "learning_rate": 3.6920931052110214e-06, + "loss": 0.92176116, + "num_input_tokens_seen": 16560930, + "router_z_loss_clip": 1.82226562, + "router_z_loss_mlp": 0.28027344, + "step": 606, + "time_per_iteration": 2.455230474472046 + }, + { + "auxiliary_loss_clip": 0.01070378, + "auxiliary_loss_mlp": 0.01008067, + "balance_loss_clip": 1.01793885, + "balance_loss_mlp": 1.00437188, + "epoch": 0.017613603389240323, + "flos": 74779914983040.0, + "grad_norm": 0.6471286144705196, + "language_loss": 0.52320492, + "learning_rate": 3.693043262159385e-06, + "loss": 0.5439893, + "num_input_tokens_seen": 16629165, + "router_z_loss_clip": 0.5234375, + "router_z_loss_mlp": 0.03686523, + "step": 607, + "time_per_iteration": 3.1992547512054443 + }, + { + "auxiliary_loss_clip": 0.01072094, + "auxiliary_loss_mlp": 0.01017691, + "balance_loss_clip": 1.01953602, + "balance_loss_mlp": 1.01397157, + "epoch": 0.01764262085775637, + "flos": 60796586995200.0, + "grad_norm": 0.7490610030176942, + "language_loss": 0.56958973, + "learning_rate": 3.6939918550626825e-06, + "loss": 0.5904876, + "num_input_tokens_seen": 16691410, + "router_z_loss_clip": 0.5234375, + "router_z_loss_mlp": 0.03710938, + "step": 608, + "time_per_iteration": 3.09908390045166 + }, + { + "auxiliary_loss_clip": 0.01248164, + "auxiliary_loss_mlp": 0.01062147, + "balance_loss_clip": 1.08222318, + "balance_loss_mlp": 1.03497958, + "epoch": 0.017671638326272415, + "flos": 19638432397440.0, + "grad_norm": 2.2129515149820818, + "language_loss": 0.8097266, + "learning_rate": 3.694938889061574e-06, + "loss": 0.83282971, + "num_input_tokens_seen": 16707460, + "router_z_loss_clip": 1.65917969, + "router_z_loss_mlp": 0.27185059, + "step": 609, + "time_per_iteration": 2.541170358657837 + }, + { + "auxiliary_loss_clip": 0.0127055, + "auxiliary_loss_mlp": 0.01072143, + "balance_loss_clip": 1.0894084, + "balance_loss_mlp": 1.04200661, + "epoch": 0.017700655794788463, + "flos": 25913749968000.0, + "grad_norm": 2.8213252240655327, + "language_loss": 1.00312757, + "learning_rate": 3.695884369271419e-06, + "loss": 1.02655447, + "num_input_tokens_seen": 16721175, + "router_z_loss_clip": 1.8125, + "router_z_loss_mlp": 0.30126953, + "step": 610, + "time_per_iteration": 2.5615479946136475 + }, + { + "auxiliary_loss_clip": 0.01265726, + "auxiliary_loss_mlp": 0.01077988, + "balance_loss_clip": 1.09280503, + "balance_loss_mlp": 1.04875827, + "epoch": 0.017729673263304508, + "flos": 12230420371200.0, + "grad_norm": 2.8182074433218554, + "language_loss": 0.88247967, + "learning_rate": 3.6968283007824383e-06, + "loss": 0.90591675, + "num_input_tokens_seen": 16733305, + "router_z_loss_clip": 1.72851562, + "router_z_loss_mlp": 0.29211426, + "step": 611, + "time_per_iteration": 2.45025634765625 + }, + { + "auxiliary_loss_clip": 0.01266081, + "auxiliary_loss_mlp": 0.01081744, + "balance_loss_clip": 1.08564281, + "balance_loss_mlp": 1.05247784, + "epoch": 0.017758690731820556, + "flos": 13217137954560.0, + "grad_norm": 3.0354158631776937, + "language_loss": 1.11749673, + "learning_rate": 3.697770688659881e-06, + "loss": 1.140975, + "num_input_tokens_seen": 16744370, + "router_z_loss_clip": 1.80566406, + "router_z_loss_mlp": 0.29284668, + "step": 612, + "time_per_iteration": 2.523357629776001 + }, + { + "auxiliary_loss_clip": 0.01264825, + "auxiliary_loss_mlp": 0.01076642, + "balance_loss_clip": 1.08708, + "balance_loss_mlp": 1.04784083, + "epoch": 0.017787708200336604, + "flos": 37012840600320.0, + "grad_norm": 2.4264106124876785, + "language_loss": 0.93848705, + "learning_rate": 3.6987115379441873e-06, + "loss": 0.96190161, + "num_input_tokens_seen": 16762245, + "router_z_loss_clip": 1.77636719, + "router_z_loss_mlp": 0.28796387, + "step": 613, + "time_per_iteration": 2.684645891189575 + }, + { + "auxiliary_loss_clip": 0.01079128, + "auxiliary_loss_mlp": 0.01026628, + "balance_loss_clip": 1.02627552, + "balance_loss_mlp": 1.02297974, + "epoch": 0.01781672566885265, + "flos": 60691404994560.0, + "grad_norm": 0.7299083624074766, + "language_loss": 0.59014058, + "learning_rate": 3.6996508536511475e-06, + "loss": 0.61119819, + "num_input_tokens_seen": 16823525, + "router_z_loss_clip": 0.52734375, + "router_z_loss_mlp": 0.03637695, + "step": 614, + "time_per_iteration": 3.0821847915649414 + }, + { + "auxiliary_loss_clip": 0.01268465, + "auxiliary_loss_mlp": 0.01071844, + "balance_loss_clip": 1.08778584, + "balance_loss_mlp": 1.04313838, + "epoch": 0.017845743137368697, + "flos": 36825925201920.0, + "grad_norm": 6.1781271286354595, + "language_loss": 0.89681822, + "learning_rate": 3.7005886407720676e-06, + "loss": 0.92022133, + "num_input_tokens_seen": 16839510, + "router_z_loss_clip": 1.80761719, + "router_z_loss_mlp": 0.28723145, + "step": 615, + "time_per_iteration": 2.690915584564209 + }, + { + "auxiliary_loss_clip": 0.01075029, + "auxiliary_loss_mlp": 0.01004938, + "balance_loss_clip": 1.02283168, + "balance_loss_mlp": 1.00140905, + "epoch": 0.01787476060588474, + "flos": 67070539848960.0, + "grad_norm": 0.6762180825232003, + "language_loss": 0.54564792, + "learning_rate": 3.7015249042739234e-06, + "loss": 0.56644762, + "num_input_tokens_seen": 16901575, + "router_z_loss_clip": 0.5234375, + "router_z_loss_mlp": 0.03540039, + "step": 616, + "time_per_iteration": 3.0788233280181885 + }, + { + "auxiliary_loss_clip": 0.01254919, + "auxiliary_loss_mlp": 0.01077796, + "balance_loss_clip": 1.08106351, + "balance_loss_mlp": 1.04968643, + "epoch": 0.01790377807440079, + "flos": 18985899974400.0, + "grad_norm": 6.24904571780512, + "language_loss": 0.93660015, + "learning_rate": 3.7024596490995227e-06, + "loss": 0.95992732, + "num_input_tokens_seen": 16915745, + "router_z_loss_clip": 1.73828125, + "router_z_loss_mlp": 0.28125, + "step": 617, + "time_per_iteration": 2.5328705310821533 + }, + { + "auxiliary_loss_clip": 0.01070706, + "auxiliary_loss_mlp": 0.0100971, + "balance_loss_clip": 1.0186913, + "balance_loss_mlp": 1.00594282, + "epoch": 0.017932795542916834, + "flos": 64955042392320.0, + "grad_norm": 0.7457093743412875, + "language_loss": 0.53625613, + "learning_rate": 3.7033928801676558e-06, + "loss": 0.5570603, + "num_input_tokens_seen": 16975495, + "router_z_loss_clip": 0.51953125, + "router_z_loss_mlp": 0.03759766, + "step": 618, + "time_per_iteration": 3.0333316326141357 + }, + { + "auxiliary_loss_clip": 0.01268255, + "auxiliary_loss_mlp": 0.01077803, + "balance_loss_clip": 1.08583355, + "balance_loss_mlp": 1.0465703, + "epoch": 0.017961813011432883, + "flos": 16939386587520.0, + "grad_norm": 3.396341770030467, + "language_loss": 0.85340536, + "learning_rate": 3.70432460237326e-06, + "loss": 0.87686586, + "num_input_tokens_seen": 16990735, + "router_z_loss_clip": 1.82226562, + "router_z_loss_mlp": 0.31225586, + "step": 619, + "time_per_iteration": 2.5222649574279785 + }, + { + "auxiliary_loss_clip": 0.01070957, + "auxiliary_loss_mlp": 0.01010876, + "balance_loss_clip": 1.01934385, + "balance_loss_mlp": 1.00725174, + "epoch": 0.01799083047994893, + "flos": 62114868046080.0, + "grad_norm": 0.7170313484582185, + "language_loss": 0.51120567, + "learning_rate": 3.705254820587563e-06, + "loss": 0.53202403, + "num_input_tokens_seen": 17051900, + "router_z_loss_clip": 0.515625, + "router_z_loss_mlp": 0.03613281, + "step": 620, + "time_per_iteration": 2.9794461727142334 + }, + { + "auxiliary_loss_clip": 0.01070781, + "auxiliary_loss_mlp": 0.01007857, + "balance_loss_clip": 1.01898134, + "balance_loss_mlp": 1.00408959, + "epoch": 0.018019847948464975, + "flos": 55723486654080.0, + "grad_norm": 0.6740836824301483, + "language_loss": 0.537835, + "learning_rate": 3.7061835396582444e-06, + "loss": 0.55862129, + "num_input_tokens_seen": 17105620, + "router_z_loss_clip": 0.51953125, + "router_z_loss_mlp": 0.03759766, + "step": 621, + "time_per_iteration": 2.899606943130493 + }, + { + "auxiliary_loss_clip": 0.01256837, + "auxiliary_loss_mlp": 0.01063712, + "balance_loss_clip": 1.08842444, + "balance_loss_mlp": 1.03562593, + "epoch": 0.018048865416981023, + "flos": 74731036573440.0, + "grad_norm": 3.4607322703153547, + "language_loss": 1.10437226, + "learning_rate": 3.707110764409583e-06, + "loss": 1.12757754, + "num_input_tokens_seen": 17128415, + "router_z_loss_clip": 1.68359375, + "router_z_loss_mlp": 0.28088379, + "step": 622, + "time_per_iteration": 2.8991286754608154 + }, + { + "auxiliary_loss_clip": 0.01266634, + "auxiliary_loss_mlp": 0.01061952, + "balance_loss_clip": 1.08932805, + "balance_loss_mlp": 1.03337717, + "epoch": 0.018077882885497068, + "flos": 11720991732480.0, + "grad_norm": 2.5228558831510695, + "language_loss": 0.83194721, + "learning_rate": 3.708036499642607e-06, + "loss": 0.85523307, + "num_input_tokens_seen": 17142540, + "router_z_loss_clip": 1.77246094, + "router_z_loss_mlp": 0.28564453, + "step": 623, + "time_per_iteration": 2.5488977432250977 + }, + { + "auxiliary_loss_clip": 0.0125336, + "auxiliary_loss_mlp": 0.01077054, + "balance_loss_clip": 1.08137381, + "balance_loss_mlp": 1.04895616, + "epoch": 0.018106900354013116, + "flos": 33722381358720.0, + "grad_norm": 2.8861094486800694, + "language_loss": 1.07267475, + "learning_rate": 3.708960750135246e-06, + "loss": 1.09597898, + "num_input_tokens_seen": 17158405, + "router_z_loss_clip": 1.72070312, + "router_z_loss_mlp": 0.28100586, + "step": 624, + "time_per_iteration": 2.6192963123321533 + }, + { + "auxiliary_loss_clip": 0.01262039, + "auxiliary_loss_mlp": 0.0106783, + "balance_loss_clip": 1.08339906, + "balance_loss_mlp": 1.03892207, + "epoch": 0.018135917822529164, + "flos": 74732257635840.0, + "grad_norm": 4.389886501535285, + "language_loss": 0.83051461, + "learning_rate": 3.7098835206424755e-06, + "loss": 0.85381323, + "num_input_tokens_seen": 17182745, + "router_z_loss_clip": 1.78710938, + "router_z_loss_mlp": 0.28918457, + "step": 625, + "time_per_iteration": 2.9425625801086426 + }, + { + "auxiliary_loss_clip": 0.01263656, + "auxiliary_loss_mlp": 0.01088366, + "balance_loss_clip": 1.08296299, + "balance_loss_mlp": 1.05610824, + "epoch": 0.01816493529104521, + "flos": 20842804442880.0, + "grad_norm": 2.7002801510447707, + "language_loss": 0.97596622, + "learning_rate": 3.7108048158964674e-06, + "loss": 0.99948639, + "num_input_tokens_seen": 17197765, + "router_z_loss_clip": 1.8046875, + "router_z_loss_mlp": 0.3223877, + "step": 626, + "time_per_iteration": 2.552898645401001 + }, + { + "auxiliary_loss_clip": 0.01274818, + "auxiliary_loss_mlp": 0.01089885, + "balance_loss_clip": 1.09466171, + "balance_loss_mlp": 1.05709076, + "epoch": 0.018193952759561257, + "flos": 11321054726400.0, + "grad_norm": 3.1402135009647916, + "language_loss": 0.77881169, + "learning_rate": 3.711724640606732e-06, + "loss": 0.80245876, + "num_input_tokens_seen": 17210110, + "router_z_loss_clip": 1.80078125, + "router_z_loss_mlp": 0.32763672, + "step": 627, + "time_per_iteration": 2.4882194995880127 + }, + { + "auxiliary_loss_clip": 0.01266744, + "auxiliary_loss_mlp": 0.01084784, + "balance_loss_clip": 1.08620501, + "balance_loss_mlp": 1.05396819, + "epoch": 0.0182229702280773, + "flos": 38757057125760.0, + "grad_norm": 2.634845165249222, + "language_loss": 0.89709342, + "learning_rate": 3.712642999460262e-06, + "loss": 0.92060876, + "num_input_tokens_seen": 17231000, + "router_z_loss_clip": 1.80664062, + "router_z_loss_mlp": 0.30810547, + "step": 628, + "time_per_iteration": 2.717193365097046 + }, + { + "auxiliary_loss_clip": 0.01255172, + "auxiliary_loss_mlp": 0.01080889, + "balance_loss_clip": 1.08349371, + "balance_loss_mlp": 1.05566406, + "epoch": 0.01825198769659335, + "flos": 39272160113280.0, + "grad_norm": 2.770696050981816, + "language_loss": 0.86501861, + "learning_rate": 3.713559897121683e-06, + "loss": 0.88837922, + "num_input_tokens_seen": 17248095, + "router_z_loss_clip": 1.71582031, + "router_z_loss_mlp": 0.25231934, + "step": 629, + "time_per_iteration": 2.6617650985717773 + }, + { + "auxiliary_loss_clip": 0.01265984, + "auxiliary_loss_mlp": 0.01079425, + "balance_loss_clip": 1.08980107, + "balance_loss_mlp": 1.05062413, + "epoch": 0.018281005165109394, + "flos": 22158715196160.0, + "grad_norm": 2.6882328570381864, + "language_loss": 0.94989949, + "learning_rate": 3.7144753382333854e-06, + "loss": 0.97335362, + "num_input_tokens_seen": 17263175, + "router_z_loss_clip": 1.76171875, + "router_z_loss_mlp": 0.28820801, + "step": 630, + "time_per_iteration": 2.5523245334625244 + }, + { + "auxiliary_loss_clip": 0.0109029, + "auxiliary_loss_mlp": 0.01069665, + "balance_loss_clip": 1.03420031, + "balance_loss_mlp": 1.0646342, + "epoch": 0.018310022633625442, + "flos": 69160469800320.0, + "grad_norm": 0.7597583731313579, + "language_loss": 0.50819445, + "learning_rate": 3.7153893274156738e-06, + "loss": 0.52979398, + "num_input_tokens_seen": 17327605, + "router_z_loss_clip": 0.5625, + "router_z_loss_mlp": 0.05029297, + "step": 631, + "time_per_iteration": 3.1645946502685547 + }, + { + "auxiliary_loss_clip": 0.01250595, + "auxiliary_loss_mlp": 0.01071776, + "balance_loss_clip": 1.08203804, + "balance_loss_mlp": 1.0420686, + "epoch": 0.01833904010214149, + "flos": 17139302709120.0, + "grad_norm": 4.156772739399987, + "language_loss": 0.90218329, + "learning_rate": 3.7163018692669016e-06, + "loss": 0.92540699, + "num_input_tokens_seen": 17342545, + "router_z_loss_clip": 1.68457031, + "router_z_loss_mlp": 0.29724121, + "step": 632, + "time_per_iteration": 2.668933153152466 + }, + { + "auxiliary_loss_clip": 0.01077811, + "auxiliary_loss_mlp": 0.0101066, + "balance_loss_clip": 1.02523363, + "balance_loss_mlp": 1.00624943, + "epoch": 0.018368057570657535, + "flos": 57011962394880.0, + "grad_norm": 0.6384660602655263, + "language_loss": 0.50159597, + "learning_rate": 3.717212968363613e-06, + "loss": 0.52248067, + "num_input_tokens_seen": 17405975, + "router_z_loss_clip": 0.5234375, + "router_z_loss_mlp": 0.04418945, + "step": 633, + "time_per_iteration": 3.177367687225342 + }, + { + "auxiliary_loss_clip": 0.01264864, + "auxiliary_loss_mlp": 0.01073482, + "balance_loss_clip": 1.08242154, + "balance_loss_mlp": 1.0421896, + "epoch": 0.018397075039173583, + "flos": 10151120845440.0, + "grad_norm": 3.553972551057223, + "language_loss": 0.92804182, + "learning_rate": 3.7181226292606785e-06, + "loss": 0.95142531, + "num_input_tokens_seen": 17415530, + "router_z_loss_clip": 1.82324219, + "router_z_loss_mlp": 0.31323242, + "step": 634, + "time_per_iteration": 2.520512819290161 + }, + { + "auxiliary_loss_clip": 0.01250439, + "auxiliary_loss_mlp": 0.01071878, + "balance_loss_clip": 1.08138561, + "balance_loss_mlp": 1.04497242, + "epoch": 0.018426092507689628, + "flos": 28251031950720.0, + "grad_norm": 2.9665049639969756, + "language_loss": 0.80699706, + "learning_rate": 3.7190308564914345e-06, + "loss": 0.83022022, + "num_input_tokens_seen": 17436895, + "router_z_loss_clip": 1.69042969, + "router_z_loss_mlp": 0.26916504, + "step": 635, + "time_per_iteration": 2.8216803073883057 + }, + { + "auxiliary_loss_clip": 0.01273973, + "auxiliary_loss_mlp": 0.0110259, + "balance_loss_clip": 1.08631396, + "balance_loss_mlp": 1.06820965, + "epoch": 0.018455109976205676, + "flos": 24169389788160.0, + "grad_norm": 2.679221665011376, + "language_loss": 0.93199956, + "learning_rate": 3.719937654567814e-06, + "loss": 0.95576519, + "num_input_tokens_seen": 17454505, + "router_z_loss_clip": 1.875, + "router_z_loss_mlp": 0.34375, + "step": 636, + "time_per_iteration": 2.5480055809020996 + }, + { + "auxiliary_loss_clip": 0.01077462, + "auxiliary_loss_mlp": 0.01038642, + "balance_loss_clip": 1.02385068, + "balance_loss_mlp": 1.03439856, + "epoch": 0.01848412744472172, + "flos": 63136885720320.0, + "grad_norm": 0.6672869392873008, + "language_loss": 0.55217379, + "learning_rate": 3.7208430279804867e-06, + "loss": 0.57333481, + "num_input_tokens_seen": 17519505, + "router_z_loss_clip": 0.53515625, + "router_z_loss_mlp": 0.04248047, + "step": 637, + "time_per_iteration": 3.0966734886169434 + }, + { + "auxiliary_loss_clip": 0.01077347, + "auxiliary_loss_mlp": 0.01037379, + "balance_loss_clip": 1.02407444, + "balance_loss_mlp": 1.03323007, + "epoch": 0.01851314491323777, + "flos": 64670666417280.0, + "grad_norm": 0.7100987655101599, + "language_loss": 0.55904686, + "learning_rate": 3.7217469811989875e-06, + "loss": 0.58019412, + "num_input_tokens_seen": 17580635, + "router_z_loss_clip": 0.53125, + "router_z_loss_mlp": 0.04150391, + "step": 638, + "time_per_iteration": 3.0167455673217773 + }, + { + "auxiliary_loss_clip": 0.01260745, + "auxiliary_loss_mlp": 0.01080072, + "balance_loss_clip": 1.08288193, + "balance_loss_mlp": 1.04986429, + "epoch": 0.018542162381753817, + "flos": 16793553369600.0, + "grad_norm": 3.72024426049118, + "language_loss": 0.88579851, + "learning_rate": 3.722649518671853e-06, + "loss": 0.90920669, + "num_input_tokens_seen": 17593785, + "router_z_loss_clip": 1.77539062, + "router_z_loss_mlp": 0.30249023, + "step": 639, + "time_per_iteration": 2.5984108448028564 + }, + { + "auxiliary_loss_clip": 0.01076397, + "auxiliary_loss_mlp": 0.0100627, + "balance_loss_clip": 1.02433157, + "balance_loss_mlp": 1.00240731, + "epoch": 0.01857117985026986, + "flos": 74774348375040.0, + "grad_norm": 0.7083056260752514, + "language_loss": 0.55128598, + "learning_rate": 3.7235506448267494e-06, + "loss": 0.57211262, + "num_input_tokens_seen": 17659045, + "router_z_loss_clip": 0.5234375, + "router_z_loss_mlp": 0.03857422, + "step": 640, + "time_per_iteration": 3.1563680171966553 + }, + { + "auxiliary_loss_clip": 0.01075866, + "auxiliary_loss_mlp": 0.01007816, + "balance_loss_clip": 1.02434945, + "balance_loss_mlp": 1.00392997, + "epoch": 0.01860019731878591, + "flos": 68642781033600.0, + "grad_norm": 0.7431630641719688, + "language_loss": 0.52907723, + "learning_rate": 3.724450364070606e-06, + "loss": 0.54991406, + "num_input_tokens_seen": 17713855, + "router_z_loss_clip": 0.515625, + "router_z_loss_mlp": 0.03881836, + "step": 641, + "time_per_iteration": 2.995194435119629 + }, + { + "auxiliary_loss_clip": 0.01078039, + "auxiliary_loss_mlp": 0.01015817, + "balance_loss_clip": 1.02543128, + "balance_loss_mlp": 1.01185882, + "epoch": 0.018629214787301954, + "flos": 65475029629440.0, + "grad_norm": 0.7037282208440876, + "language_loss": 0.52018124, + "learning_rate": 3.7253486807897415e-06, + "loss": 0.54111975, + "num_input_tokens_seen": 17775570, + "router_z_loss_clip": 0.52734375, + "router_z_loss_mlp": 0.03955078, + "step": 642, + "time_per_iteration": 3.0877366065979004 + }, + { + "auxiliary_loss_clip": 0.01264851, + "auxiliary_loss_mlp": 0.01071403, + "balance_loss_clip": 1.08248472, + "balance_loss_mlp": 1.04213703, + "epoch": 0.018658232255818002, + "flos": 37626373831680.0, + "grad_norm": 3.0837574021057987, + "language_loss": 0.98027509, + "learning_rate": 3.726245599349994e-06, + "loss": 1.00363755, + "num_input_tokens_seen": 17791295, + "router_z_loss_clip": 1.82128906, + "router_z_loss_mlp": 0.29296875, + "step": 643, + "time_per_iteration": 2.663940668106079 + }, + { + "auxiliary_loss_clip": 0.01079781, + "auxiliary_loss_mlp": 0.0102495, + "balance_loss_clip": 1.02706361, + "balance_loss_mlp": 1.021088, + "epoch": 0.01868724972433405, + "flos": 69620089253760.0, + "grad_norm": 0.7830185415768317, + "language_loss": 0.52921736, + "learning_rate": 3.7271411240968497e-06, + "loss": 0.55026466, + "num_input_tokens_seen": 17846505, + "router_z_loss_clip": 0.52734375, + "router_z_loss_mlp": 0.03857422, + "step": 644, + "time_per_iteration": 3.035417079925537 + }, + { + "auxiliary_loss_clip": 0.01261951, + "auxiliary_loss_mlp": 0.01069004, + "balance_loss_clip": 1.08350551, + "balance_loss_mlp": 1.04099023, + "epoch": 0.018716267192850095, + "flos": 27703897009920.0, + "grad_norm": 2.932574109330675, + "language_loss": 1.0785346, + "learning_rate": 3.728035259355564e-06, + "loss": 1.10184419, + "num_input_tokens_seen": 17864505, + "router_z_loss_clip": 1.78417969, + "router_z_loss_mlp": 0.28015137, + "step": 645, + "time_per_iteration": 4.907215118408203 + }, + { + "auxiliary_loss_clip": 0.01246097, + "auxiliary_loss_mlp": 0.01062308, + "balance_loss_clip": 1.08076286, + "balance_loss_mlp": 1.03539073, + "epoch": 0.018745284661366143, + "flos": 28432416654720.0, + "grad_norm": 2.67154511126872, + "language_loss": 0.69762766, + "learning_rate": 3.7289280094312938e-06, + "loss": 0.72071171, + "num_input_tokens_seen": 17884095, + "router_z_loss_clip": 1.65136719, + "router_z_loss_mlp": 0.26928711, + "step": 646, + "time_per_iteration": 4.922529220581055 + }, + { + "auxiliary_loss_clip": 0.01239697, + "auxiliary_loss_mlp": 0.01062808, + "balance_loss_clip": 1.07925463, + "balance_loss_mlp": 1.03890705, + "epoch": 0.018774302129882188, + "flos": 23543107228800.0, + "grad_norm": 2.770227359317249, + "language_loss": 0.83408028, + "learning_rate": 3.729819378609217e-06, + "loss": 0.85710526, + "num_input_tokens_seen": 17899715, + "router_z_loss_clip": 1.60351562, + "router_z_loss_mlp": 0.23925781, + "step": 647, + "time_per_iteration": 7.419363021850586 + }, + { + "auxiliary_loss_clip": 0.01262914, + "auxiliary_loss_mlp": 0.01069424, + "balance_loss_clip": 1.09018326, + "balance_loss_mlp": 1.0420773, + "epoch": 0.018803319598398236, + "flos": 11975275088640.0, + "grad_norm": 3.4329821900997715, + "language_loss": 0.98644292, + "learning_rate": 3.730709371154657e-06, + "loss": 1.00976634, + "num_input_tokens_seen": 17909490, + "router_z_loss_clip": 1.7265625, + "router_z_loss_mlp": 0.27319336, + "step": 648, + "time_per_iteration": 2.5989880561828613 + }, + { + "auxiliary_loss_clip": 0.01257876, + "auxiliary_loss_mlp": 0.01077486, + "balance_loss_clip": 1.08682275, + "balance_loss_mlp": 1.05009222, + "epoch": 0.01883233706691428, + "flos": 15991309059840.0, + "grad_norm": 3.0308281253942644, + "language_loss": 0.96505332, + "learning_rate": 3.731597991313208e-06, + "loss": 0.9884069, + "num_input_tokens_seen": 17924135, + "router_z_loss_clip": 1.70996094, + "router_z_loss_mlp": 0.27380371, + "step": 649, + "time_per_iteration": 2.5094821453094482 + }, + { + "auxiliary_loss_clip": 0.01075676, + "auxiliary_loss_mlp": 0.01002212, + "balance_loss_clip": 1.02411854, + "balance_loss_mlp": 0.99880296, + "epoch": 0.01886135453543033, + "flos": 63609685464960.0, + "grad_norm": 0.7080446277871814, + "language_loss": 0.53445375, + "learning_rate": 3.732485243310849e-06, + "loss": 0.55523264, + "num_input_tokens_seen": 17988495, + "router_z_loss_clip": 0.515625, + "router_z_loss_mlp": 0.03417969, + "step": 650, + "time_per_iteration": 3.1962881088256836 + }, + { + "auxiliary_loss_clip": 0.01250586, + "auxiliary_loss_mlp": 0.01058116, + "balance_loss_clip": 1.08377087, + "balance_loss_mlp": 1.03218818, + "epoch": 0.018890372003946377, + "flos": 30986563000320.0, + "grad_norm": 12.340719499728337, + "language_loss": 1.01377261, + "learning_rate": 3.733371131354075e-06, + "loss": 1.03685963, + "num_input_tokens_seen": 18003780, + "router_z_loss_clip": 1.66796875, + "router_z_loss_mlp": 0.25927734, + "step": 651, + "time_per_iteration": 2.6083269119262695 + }, + { + "auxiliary_loss_clip": 0.01245521, + "auxiliary_loss_mlp": 0.01069545, + "balance_loss_clip": 1.08302236, + "balance_loss_mlp": 1.04240072, + "epoch": 0.01891938947246242, + "flos": 36784771194240.0, + "grad_norm": 2.720650478669134, + "language_loss": 0.95490372, + "learning_rate": 3.734255659630009e-06, + "loss": 0.97805434, + "num_input_tokens_seen": 18019785, + "router_z_loss_clip": 1.62597656, + "router_z_loss_mlp": 0.27185059, + "step": 652, + "time_per_iteration": 2.713486671447754 + }, + { + "auxiliary_loss_clip": 0.01267398, + "auxiliary_loss_mlp": 0.0106227, + "balance_loss_clip": 1.09281778, + "balance_loss_mlp": 1.03404117, + "epoch": 0.01894840694097847, + "flos": 33215179363200.0, + "grad_norm": 2.7725866072424843, + "language_loss": 0.95710778, + "learning_rate": 3.7351388323065203e-06, + "loss": 0.98040444, + "num_input_tokens_seen": 18035145, + "router_z_loss_clip": 1.74609375, + "router_z_loss_mlp": 0.28222656, + "step": 653, + "time_per_iteration": 2.6868276596069336 + }, + { + "auxiliary_loss_clip": 0.01263855, + "auxiliary_loss_mlp": 0.01070864, + "balance_loss_clip": 1.09030962, + "balance_loss_mlp": 1.04258752, + "epoch": 0.018977424409494514, + "flos": 18762068373120.0, + "grad_norm": 7.054807444563945, + "language_loss": 0.99860704, + "learning_rate": 3.7360206535323494e-06, + "loss": 1.0219543, + "num_input_tokens_seen": 18048260, + "router_z_loss_clip": 1.73730469, + "router_z_loss_mlp": 0.28295898, + "step": 654, + "time_per_iteration": 2.536318063735962 + }, + { + "auxiliary_loss_clip": 0.01247345, + "auxiliary_loss_mlp": 0.0106921, + "balance_loss_clip": 1.08211374, + "balance_loss_mlp": 1.04192352, + "epoch": 0.019006441878010562, + "flos": 12159317399040.0, + "grad_norm": 2.892569135352073, + "language_loss": 1.01223969, + "learning_rate": 3.7369011274372165e-06, + "loss": 1.03540528, + "num_input_tokens_seen": 18060555, + "router_z_loss_clip": 1.65234375, + "router_z_loss_mlp": 0.27258301, + "step": 655, + "time_per_iteration": 2.4518179893493652 + }, + { + "auxiliary_loss_clip": 0.01249565, + "auxiliary_loss_mlp": 0.01064442, + "balance_loss_clip": 1.08295369, + "balance_loss_mlp": 1.03670144, + "epoch": 0.01903545934652661, + "flos": 15921858113280.0, + "grad_norm": 9.58127674974507, + "language_loss": 0.82017893, + "learning_rate": 3.737780258131944e-06, + "loss": 0.843319, + "num_input_tokens_seen": 18073535, + "router_z_loss_clip": 1.66601562, + "router_z_loss_mlp": 0.27746582, + "step": 656, + "time_per_iteration": 2.49375581741333 + }, + { + "auxiliary_loss_clip": 0.01245997, + "auxiliary_loss_mlp": 0.01061448, + "balance_loss_clip": 1.08510232, + "balance_loss_mlp": 1.03710604, + "epoch": 0.019064476815042655, + "flos": 27445663157760.0, + "grad_norm": 3.5159051558758874, + "language_loss": 0.98691332, + "learning_rate": 3.738658049708568e-06, + "loss": 1.00998783, + "num_input_tokens_seen": 18089265, + "router_z_loss_clip": 1.61035156, + "router_z_loss_mlp": 0.24353027, + "step": 657, + "time_per_iteration": 2.5683257579803467 + }, + { + "auxiliary_loss_clip": 0.01257102, + "auxiliary_loss_mlp": 0.0107128, + "balance_loss_clip": 1.08735061, + "balance_loss_mlp": 1.0441004, + "epoch": 0.019093494283558703, + "flos": 16501420056960.0, + "grad_norm": 2.672685330853211, + "language_loss": 0.82759893, + "learning_rate": 3.739534506240455e-06, + "loss": 0.85088277, + "num_input_tokens_seen": 18102775, + "router_z_loss_clip": 1.69921875, + "router_z_loss_mlp": 0.27197266, + "step": 658, + "time_per_iteration": 2.527101755142212 + }, + { + "auxiliary_loss_clip": 0.01070721, + "auxiliary_loss_mlp": 0.01003978, + "balance_loss_clip": 1.02128816, + "balance_loss_mlp": 1.00080705, + "epoch": 0.019122511752074748, + "flos": 74595441709440.0, + "grad_norm": 0.7019095508562159, + "language_loss": 0.53684229, + "learning_rate": 3.7404096317824104e-06, + "loss": 0.55758929, + "num_input_tokens_seen": 18168635, + "router_z_loss_clip": 0.49414062, + "router_z_loss_mlp": 0.03173828, + "step": 659, + "time_per_iteration": 3.1520724296569824 + }, + { + "auxiliary_loss_clip": 0.01261798, + "auxiliary_loss_mlp": 0.01071689, + "balance_loss_clip": 1.08621716, + "balance_loss_mlp": 1.04480696, + "epoch": 0.019151529220590796, + "flos": 45835301364480.0, + "grad_norm": 3.36500997945657, + "language_loss": 0.89931327, + "learning_rate": 3.741283430370799e-06, + "loss": 0.92264813, + "num_input_tokens_seen": 18185620, + "router_z_loss_clip": 1.75585938, + "router_z_loss_mlp": 0.26867676, + "step": 660, + "time_per_iteration": 2.711902618408203 + }, + { + "auxiliary_loss_clip": 0.01234872, + "auxiliary_loss_mlp": 0.01052341, + "balance_loss_clip": 1.07668447, + "balance_loss_mlp": 1.03024006, + "epoch": 0.01918054668910684, + "flos": 47951553006720.0, + "grad_norm": 9.524303104948498, + "language_loss": 0.97735727, + "learning_rate": 3.74215590602365e-06, + "loss": 1.00022936, + "num_input_tokens_seen": 18203070, + "router_z_loss_clip": 1.58300781, + "router_z_loss_mlp": 0.22106934, + "step": 661, + "time_per_iteration": 2.756683588027954 + }, + { + "auxiliary_loss_clip": 0.01255292, + "auxiliary_loss_mlp": 0.01064991, + "balance_loss_clip": 1.07957077, + "balance_loss_mlp": 1.03474784, + "epoch": 0.01920956415762289, + "flos": 74732724512640.0, + "grad_norm": 2.065216472555214, + "language_loss": 0.90356386, + "learning_rate": 3.743027062740771e-06, + "loss": 0.92676669, + "num_input_tokens_seen": 18236040, + "router_z_loss_clip": 1.75683594, + "router_z_loss_mlp": 0.30236816, + "step": 662, + "time_per_iteration": 2.9195499420166016 + }, + { + "auxiliary_loss_clip": 0.01248144, + "auxiliary_loss_mlp": 0.01068974, + "balance_loss_clip": 1.08182263, + "balance_loss_mlp": 1.04087675, + "epoch": 0.019238581626138937, + "flos": 27673588909440.0, + "grad_norm": 2.975738260489116, + "language_loss": 1.00803268, + "learning_rate": 3.743896904503857e-06, + "loss": 1.03120387, + "num_input_tokens_seen": 18251270, + "router_z_loss_clip": 1.6640625, + "router_z_loss_mlp": 0.28076172, + "step": 663, + "time_per_iteration": 2.5781471729278564 + }, + { + "auxiliary_loss_clip": 0.01070217, + "auxiliary_loss_mlp": 0.01013397, + "balance_loss_clip": 1.02121866, + "balance_loss_mlp": 1.01022613, + "epoch": 0.01926759909465498, + "flos": 61312839217920.0, + "grad_norm": 0.9520051075257574, + "language_loss": 0.5646255, + "learning_rate": 3.7447654352766005e-06, + "loss": 0.58546174, + "num_input_tokens_seen": 18310320, + "router_z_loss_clip": 0.49023438, + "router_z_loss_mlp": 0.03173828, + "step": 664, + "time_per_iteration": 2.9791722297668457 + }, + { + "auxiliary_loss_clip": 0.01255085, + "auxiliary_loss_mlp": 0.01077173, + "balance_loss_clip": 1.08480513, + "balance_loss_mlp": 1.05101824, + "epoch": 0.01929661656317103, + "flos": 46819756391040.0, + "grad_norm": 2.2705517721743487, + "language_loss": 0.75147188, + "learning_rate": 3.7456326590047978e-06, + "loss": 0.77479446, + "num_input_tokens_seen": 18331365, + "router_z_loss_clip": 1.70410156, + "router_z_loss_mlp": 0.26147461, + "step": 665, + "time_per_iteration": 2.7919726371765137 + }, + { + "auxiliary_loss_clip": 0.01253347, + "auxiliary_loss_mlp": 0.01068169, + "balance_loss_clip": 1.08334959, + "balance_loss_mlp": 1.04041672, + "epoch": 0.019325634031687074, + "flos": 22170961733760.0, + "grad_norm": 2.1935700410579413, + "language_loss": 0.89074475, + "learning_rate": 3.746498579616459e-06, + "loss": 0.91395992, + "num_input_tokens_seen": 18352115, + "router_z_loss_clip": 1.70019531, + "router_z_loss_mlp": 0.27746582, + "step": 666, + "time_per_iteration": 2.568605899810791 + }, + { + "auxiliary_loss_clip": 0.01259623, + "auxiliary_loss_mlp": 0.0106889, + "balance_loss_clip": 1.08579254, + "balance_loss_mlp": 1.04126906, + "epoch": 0.019354651500203122, + "flos": 29306338554240.0, + "grad_norm": 2.137673558224185, + "language_loss": 0.87209356, + "learning_rate": 3.747363201021913e-06, + "loss": 0.89537871, + "num_input_tokens_seen": 18370185, + "router_z_loss_clip": 1.73925781, + "router_z_loss_mlp": 0.27587891, + "step": 667, + "time_per_iteration": 2.5903451442718506 + }, + { + "auxiliary_loss_clip": 0.01245237, + "auxiliary_loss_mlp": 0.01065493, + "balance_loss_clip": 1.08253026, + "balance_loss_mlp": 1.03988695, + "epoch": 0.01938366896871917, + "flos": 28834795785600.0, + "grad_norm": 2.115644288007423, + "language_loss": 0.88804108, + "learning_rate": 3.7482265271139155e-06, + "loss": 0.91114837, + "num_input_tokens_seen": 18386270, + "router_z_loss_clip": 1.62695312, + "router_z_loss_mlp": 0.25622559, + "step": 668, + "time_per_iteration": 2.586818218231201 + }, + { + "auxiliary_loss_clip": 0.01068833, + "auxiliary_loss_mlp": 0.01010201, + "balance_loss_clip": 1.02101421, + "balance_loss_mlp": 1.00695825, + "epoch": 0.019412686437235215, + "flos": 74783254947840.0, + "grad_norm": 0.6979747759847875, + "language_loss": 0.54104602, + "learning_rate": 3.7490885617677517e-06, + "loss": 0.56183636, + "num_input_tokens_seen": 18454165, + "router_z_loss_clip": 0.47851562, + "router_z_loss_mlp": 0.0324707, + "step": 669, + "time_per_iteration": 3.179426908493042 + }, + { + "auxiliary_loss_clip": 0.01272136, + "auxiliary_loss_mlp": 0.01072075, + "balance_loss_clip": 1.08948219, + "balance_loss_mlp": 1.04167616, + "epoch": 0.019441703905751263, + "flos": 30366313925760.0, + "grad_norm": 2.610803726231014, + "language_loss": 1.14941978, + "learning_rate": 3.7499493088413417e-06, + "loss": 1.17286205, + "num_input_tokens_seen": 18471430, + "router_z_loss_clip": 1.82617188, + "router_z_loss_mlp": 0.30407715, + "step": 670, + "time_per_iteration": 2.61970591545105 + }, + { + "auxiliary_loss_clip": 0.01244975, + "auxiliary_loss_mlp": 0.01067073, + "balance_loss_clip": 1.08293581, + "balance_loss_mlp": 1.04295719, + "epoch": 0.019470721374267308, + "flos": 27631788456960.0, + "grad_norm": 2.835764020818319, + "language_loss": 0.99429023, + "learning_rate": 3.750808772175345e-06, + "loss": 1.01741076, + "num_input_tokens_seen": 18489680, + "router_z_loss_clip": 1.62109375, + "router_z_loss_mlp": 0.24108887, + "step": 671, + "time_per_iteration": 2.597099542617798 + }, + { + "auxiliary_loss_clip": 0.01245493, + "auxiliary_loss_mlp": 0.01066772, + "balance_loss_clip": 1.08082008, + "balance_loss_mlp": 1.03979528, + "epoch": 0.019499738842783356, + "flos": 32955113917440.0, + "grad_norm": 2.794793124730692, + "language_loss": 0.811234, + "learning_rate": 3.7516669555932624e-06, + "loss": 0.83435661, + "num_input_tokens_seen": 18504105, + "router_z_loss_clip": 1.64550781, + "router_z_loss_mlp": 0.26977539, + "step": 672, + "time_per_iteration": 2.5549168586730957 + }, + { + "auxiliary_loss_clip": 0.01067759, + "auxiliary_loss_mlp": 0.01007462, + "balance_loss_clip": 1.0202713, + "balance_loss_mlp": 1.00431526, + "epoch": 0.0195287563112994, + "flos": 65981082389760.0, + "grad_norm": 0.7105069849651359, + "language_loss": 0.58092737, + "learning_rate": 3.7525238629015374e-06, + "loss": 0.60167956, + "num_input_tokens_seen": 18562865, + "router_z_loss_clip": 0.47460938, + "router_z_loss_mlp": 0.03149414, + "step": 673, + "time_per_iteration": 3.010805368423462 + }, + { + "auxiliary_loss_clip": 0.01068314, + "auxiliary_loss_mlp": 0.01005483, + "balance_loss_clip": 1.02067733, + "balance_loss_mlp": 1.00244296, + "epoch": 0.01955777377981545, + "flos": 56650629530880.0, + "grad_norm": 0.6980202824728641, + "language_loss": 0.54093796, + "learning_rate": 3.7533794978896586e-06, + "loss": 0.56167591, + "num_input_tokens_seen": 18625005, + "router_z_loss_clip": 0.4765625, + "router_z_loss_mlp": 0.03039551, + "step": 674, + "time_per_iteration": 3.0357110500335693 + }, + { + "auxiliary_loss_clip": 0.01251784, + "auxiliary_loss_mlp": 0.0107603, + "balance_loss_clip": 1.08560395, + "balance_loss_mlp": 1.04644215, + "epoch": 0.019586791248331497, + "flos": 39596971824000.0, + "grad_norm": 3.073901969054375, + "language_loss": 1.055897, + "learning_rate": 3.7542338643302607e-06, + "loss": 1.07917523, + "num_input_tokens_seen": 18642340, + "router_z_loss_clip": 1.66113281, + "router_z_loss_mlp": 0.29577637, + "step": 675, + "time_per_iteration": 2.682215929031372 + }, + { + "auxiliary_loss_clip": 0.0106753, + "auxiliary_loss_mlp": 0.01001548, + "balance_loss_clip": 1.0204277, + "balance_loss_mlp": 0.99837703, + "epoch": 0.01961580871684754, + "flos": 53725956439680.0, + "grad_norm": 0.7735821984421519, + "language_loss": 0.55505741, + "learning_rate": 3.7550869659792225e-06, + "loss": 0.57574821, + "num_input_tokens_seen": 18699755, + "router_z_loss_clip": 0.47070312, + "router_z_loss_mlp": 0.03173828, + "step": 676, + "time_per_iteration": 3.0313844680786133 + }, + { + "auxiliary_loss_clip": 0.01249219, + "auxiliary_loss_mlp": 0.01068305, + "balance_loss_clip": 1.0803293, + "balance_loss_mlp": 1.04154289, + "epoch": 0.01964482618536359, + "flos": 15516929116800.0, + "grad_norm": 3.279058461987378, + "language_loss": 0.82892966, + "learning_rate": 3.755938806575768e-06, + "loss": 0.8521049, + "num_input_tokens_seen": 18711300, + "router_z_loss_clip": 1.68847656, + "router_z_loss_mlp": 0.2677002, + "step": 677, + "time_per_iteration": 2.5072927474975586 + }, + { + "auxiliary_loss_clip": 0.01067277, + "auxiliary_loss_mlp": 0.01004621, + "balance_loss_clip": 1.01991177, + "balance_loss_mlp": 1.00113988, + "epoch": 0.019673843653879634, + "flos": 74771008410240.0, + "grad_norm": 0.9436467762170717, + "language_loss": 0.54030591, + "learning_rate": 3.756789389842562e-06, + "loss": 0.5610249, + "num_input_tokens_seen": 18767475, + "router_z_loss_clip": 0.47265625, + "router_z_loss_mlp": 0.03491211, + "step": 678, + "time_per_iteration": 3.0358550548553467 + }, + { + "auxiliary_loss_clip": 0.01251318, + "auxiliary_loss_mlp": 0.01063584, + "balance_loss_clip": 1.08603621, + "balance_loss_mlp": 1.04000473, + "epoch": 0.019702861122395682, + "flos": 28249200357120.0, + "grad_norm": 3.3606409802404205, + "language_loss": 0.92886376, + "learning_rate": 3.7576387194858126e-06, + "loss": 0.95201284, + "num_input_tokens_seen": 18783635, + "router_z_loss_clip": 1.65332031, + "router_z_loss_mlp": 0.23571777, + "step": 679, + "time_per_iteration": 2.565511465072632 + }, + { + "auxiliary_loss_clip": 0.01241322, + "auxiliary_loss_mlp": 0.01063421, + "balance_loss_clip": 1.0826087, + "balance_loss_mlp": 1.03915048, + "epoch": 0.01973187859091173, + "flos": 50188752720000.0, + "grad_norm": 2.913283513124242, + "language_loss": 1.07730234, + "learning_rate": 3.7584867991953607e-06, + "loss": 1.10034966, + "num_input_tokens_seen": 18807275, + "router_z_loss_clip": 1.5859375, + "router_z_loss_mlp": 0.24255371, + "step": 680, + "time_per_iteration": 2.8140270709991455 + }, + { + "auxiliary_loss_clip": 0.01253673, + "auxiliary_loss_mlp": 0.01057995, + "balance_loss_clip": 1.08966315, + "balance_loss_mlp": 1.03241313, + "epoch": 0.019760896059427775, + "flos": 41277734974080.0, + "grad_norm": 2.5317860655482045, + "language_loss": 0.9757607, + "learning_rate": 3.7593336326447845e-06, + "loss": 0.99887735, + "num_input_tokens_seen": 18824035, + "router_z_loss_clip": 1.640625, + "router_z_loss_mlp": 0.25598145, + "step": 681, + "time_per_iteration": 2.741102695465088 + }, + { + "auxiliary_loss_clip": 0.0126028, + "auxiliary_loss_mlp": 0.01072363, + "balance_loss_clip": 1.08668804, + "balance_loss_mlp": 1.04483736, + "epoch": 0.019789913527943823, + "flos": 25842539255040.0, + "grad_norm": 3.106226745594547, + "language_loss": 0.97738004, + "learning_rate": 3.760179223491489e-06, + "loss": 1.00070643, + "num_input_tokens_seen": 18845295, + "router_z_loss_clip": 1.73535156, + "router_z_loss_mlp": 0.27514648, + "step": 682, + "time_per_iteration": 2.7879316806793213 + }, + { + "auxiliary_loss_clip": 0.0124925, + "auxiliary_loss_mlp": 0.01062446, + "balance_loss_clip": 1.08574867, + "balance_loss_mlp": 1.03664863, + "epoch": 0.019818930996459868, + "flos": 16468993054080.0, + "grad_norm": 3.490696560397538, + "language_loss": 0.81777751, + "learning_rate": 3.761023575376802e-06, + "loss": 0.84089452, + "num_input_tokens_seen": 18859190, + "router_z_loss_clip": 1.63183594, + "router_z_loss_mlp": 0.25793457, + "step": 683, + "time_per_iteration": 2.481989622116089 + }, + { + "auxiliary_loss_clip": 0.0126197, + "auxiliary_loss_mlp": 0.01075732, + "balance_loss_clip": 1.08635795, + "balance_loss_mlp": 1.04640627, + "epoch": 0.019847948464975916, + "flos": 25184763446400.0, + "grad_norm": 2.9679946601733898, + "language_loss": 1.05911851, + "learning_rate": 3.7618666919260695e-06, + "loss": 1.08249545, + "num_input_tokens_seen": 18873435, + "router_z_loss_clip": 1.75878906, + "router_z_loss_mlp": 0.29333496, + "step": 684, + "time_per_iteration": 2.5484795570373535 + }, + { + "auxiliary_loss_clip": 0.01244145, + "auxiliary_loss_mlp": 0.01066324, + "balance_loss_clip": 1.08343601, + "balance_loss_mlp": 1.04252958, + "epoch": 0.01987696593349196, + "flos": 12779350992000.0, + "grad_norm": 2.6796542887528845, + "language_loss": 0.76521409, + "learning_rate": 3.7627085767487498e-06, + "loss": 0.78831875, + "num_input_tokens_seen": 18886415, + "router_z_loss_clip": 1.60546875, + "router_z_loss_mlp": 0.23779297, + "step": 685, + "time_per_iteration": 2.4632444381713867 + }, + { + "auxiliary_loss_clip": 0.01253886, + "auxiliary_loss_mlp": 0.01070162, + "balance_loss_clip": 1.08384037, + "balance_loss_mlp": 1.04379344, + "epoch": 0.01990598340200801, + "flos": 25112116189440.0, + "grad_norm": 3.231097080774256, + "language_loss": 0.78758109, + "learning_rate": 3.7635492334385024e-06, + "loss": 0.81082159, + "num_input_tokens_seen": 18900680, + "router_z_loss_clip": 1.70214844, + "router_z_loss_mlp": 0.26403809, + "step": 686, + "time_per_iteration": 2.489377737045288 + }, + { + "auxiliary_loss_clip": 0.01071541, + "auxiliary_loss_mlp": 0.01008337, + "balance_loss_clip": 1.02459884, + "balance_loss_mlp": 1.00514209, + "epoch": 0.019935000870524057, + "flos": 56356844192640.0, + "grad_norm": 0.6815387214534854, + "language_loss": 0.52861929, + "learning_rate": 3.7643886655732852e-06, + "loss": 0.54941809, + "num_input_tokens_seen": 18959550, + "router_z_loss_clip": 0.46875, + "router_z_loss_mlp": 0.03198242, + "step": 687, + "time_per_iteration": 3.146796941757202 + }, + { + "auxiliary_loss_clip": 0.0124205, + "auxiliary_loss_mlp": 0.01054934, + "balance_loss_clip": 1.08325124, + "balance_loss_mlp": 1.03200984, + "epoch": 0.0199640183390401, + "flos": 41129495544960.0, + "grad_norm": 4.28635340236473, + "language_loss": 0.76392341, + "learning_rate": 3.76522687671544e-06, + "loss": 0.78689331, + "num_input_tokens_seen": 18975105, + "router_z_loss_clip": 1.58789062, + "router_z_loss_mlp": 0.22924805, + "step": 688, + "time_per_iteration": 2.6914756298065186 + }, + { + "auxiliary_loss_clip": 0.01252772, + "auxiliary_loss_mlp": 0.01068246, + "balance_loss_clip": 1.08368361, + "balance_loss_mlp": 1.04066098, + "epoch": 0.01999303580755615, + "flos": 12743224888320.0, + "grad_norm": 3.349043438395561, + "language_loss": 1.04548264, + "learning_rate": 3.7660638704117904e-06, + "loss": 1.0686928, + "num_input_tokens_seen": 18985985, + "router_z_loss_clip": 1.68847656, + "router_z_loss_mlp": 0.27624512, + "step": 689, + "time_per_iteration": 2.455453872680664 + }, + { + "auxiliary_loss_clip": 0.01262818, + "auxiliary_loss_mlp": 0.01073317, + "balance_loss_clip": 1.08947313, + "balance_loss_mlp": 1.04207253, + "epoch": 0.020022053276072194, + "flos": 29964329844480.0, + "grad_norm": 5.035830259690678, + "language_loss": 1.08459604, + "learning_rate": 3.766899650193724e-06, + "loss": 1.10795748, + "num_input_tokens_seen": 18999805, + "router_z_loss_clip": 1.73242188, + "router_z_loss_mlp": 0.31201172, + "step": 690, + "time_per_iteration": 2.58838152885437 + }, + { + "auxiliary_loss_clip": 0.01258049, + "auxiliary_loss_mlp": 0.01067935, + "balance_loss_clip": 1.0872376, + "balance_loss_mlp": 1.03949213, + "epoch": 0.020051070744588242, + "flos": 16209179003520.0, + "grad_norm": 3.287131741884498, + "language_loss": 0.99536932, + "learning_rate": 3.7677342195772886e-06, + "loss": 1.01862907, + "num_input_tokens_seen": 19013040, + "router_z_loss_clip": 1.70605469, + "router_z_loss_mlp": 0.28442383, + "step": 691, + "time_per_iteration": 2.515986442565918 + }, + { + "auxiliary_loss_clip": 0.01067749, + "auxiliary_loss_mlp": 0.01002581, + "balance_loss_clip": 1.02111793, + "balance_loss_mlp": 0.9992668, + "epoch": 0.020080088213104287, + "flos": 68279401094400.0, + "grad_norm": 0.7686316046552427, + "language_loss": 0.51263875, + "learning_rate": 3.7685675820632748e-06, + "loss": 0.533342, + "num_input_tokens_seen": 19076865, + "router_z_loss_clip": 0.46679688, + "router_z_loss_mlp": 0.03320312, + "step": 692, + "time_per_iteration": 3.131937265396118 + }, + { + "auxiliary_loss_clip": 0.01252262, + "auxiliary_loss_mlp": 0.01062426, + "balance_loss_clip": 1.07999432, + "balance_loss_mlp": 1.03360105, + "epoch": 0.020109105681620335, + "flos": 30220013831040.0, + "grad_norm": 2.904577235590966, + "language_loss": 0.89141232, + "learning_rate": 3.7693997411373113e-06, + "loss": 0.91455913, + "num_input_tokens_seen": 19091515, + "router_z_loss_clip": 1.72265625, + "router_z_loss_mlp": 0.28833008, + "step": 693, + "time_per_iteration": 2.5547542572021484 + }, + { + "auxiliary_loss_clip": 0.01068152, + "auxiliary_loss_mlp": 0.01002331, + "balance_loss_clip": 1.02133107, + "balance_loss_mlp": 0.99887353, + "epoch": 0.020138123150136383, + "flos": 67433812047360.0, + "grad_norm": 0.8548121885966526, + "language_loss": 0.53819323, + "learning_rate": 3.770230700269945e-06, + "loss": 0.55889803, + "num_input_tokens_seen": 19148115, + "router_z_loss_clip": 0.46875, + "router_z_loss_mlp": 0.03466797, + "step": 694, + "time_per_iteration": 3.0570461750030518 + }, + { + "auxiliary_loss_clip": 0.01234139, + "auxiliary_loss_mlp": 0.01065827, + "balance_loss_clip": 1.08153319, + "balance_loss_mlp": 1.04097152, + "epoch": 0.020167140618652428, + "flos": 32302761062400.0, + "grad_norm": 2.666960377651372, + "language_loss": 0.82245147, + "learning_rate": 3.7710604629167325e-06, + "loss": 0.84545112, + "num_input_tokens_seen": 19166560, + "router_z_loss_clip": 1.52441406, + "router_z_loss_mlp": 0.24890137, + "step": 695, + "time_per_iteration": 2.59849214553833 + }, + { + "auxiliary_loss_clip": 0.01249995, + "auxiliary_loss_mlp": 0.0107549, + "balance_loss_clip": 1.08450961, + "balance_loss_mlp": 1.04974103, + "epoch": 0.020196158087168476, + "flos": 35180677624320.0, + "grad_norm": 2.8496628149531293, + "language_loss": 1.00075233, + "learning_rate": 3.771889032518326e-06, + "loss": 1.0240072, + "num_input_tokens_seen": 19181395, + "router_z_loss_clip": 1.65625, + "router_z_loss_mlp": 0.25732422, + "step": 696, + "time_per_iteration": 2.710571050643921 + }, + { + "auxiliary_loss_clip": 0.01238742, + "auxiliary_loss_mlp": 0.01061785, + "balance_loss_clip": 1.07930636, + "balance_loss_mlp": 1.03725147, + "epoch": 0.02022517555568452, + "flos": 28506105406080.0, + "grad_norm": 2.553925370489592, + "language_loss": 0.75719923, + "learning_rate": 3.7727164125005555e-06, + "loss": 0.78020453, + "num_input_tokens_seen": 19197925, + "router_z_loss_clip": 1.59375, + "router_z_loss_mlp": 0.24536133, + "step": 697, + "time_per_iteration": 2.5776686668395996 + }, + { + "auxiliary_loss_clip": 0.01242996, + "auxiliary_loss_mlp": 0.01065369, + "balance_loss_clip": 1.08089232, + "balance_loss_mlp": 1.03983402, + "epoch": 0.02025419302420057, + "flos": 13542955246080.0, + "grad_norm": 5.586706352617531, + "language_loss": 1.09604514, + "learning_rate": 3.7735426062745193e-06, + "loss": 1.11912882, + "num_input_tokens_seen": 19207700, + "router_z_loss_clip": 1.62011719, + "router_z_loss_mlp": 0.25500488, + "step": 698, + "time_per_iteration": 2.4862637519836426 + }, + { + "auxiliary_loss_clip": 0.01256714, + "auxiliary_loss_mlp": 0.01068589, + "balance_loss_clip": 1.09017253, + "balance_loss_mlp": 1.04074121, + "epoch": 0.020283210492716617, + "flos": 27046444423680.0, + "grad_norm": 4.231335580201617, + "language_loss": 1.02955925, + "learning_rate": 3.7743676172366622e-06, + "loss": 1.05281234, + "num_input_tokens_seen": 19222820, + "router_z_loss_clip": 1.66503906, + "router_z_loss_mlp": 0.27856445, + "step": 699, + "time_per_iteration": 2.545625686645508 + }, + { + "auxiliary_loss_clip": 0.01259159, + "auxiliary_loss_mlp": 0.01063739, + "balance_loss_clip": 1.087291, + "balance_loss_mlp": 1.03733397, + "epoch": 0.02031222796123266, + "flos": 33620862545280.0, + "grad_norm": 2.1504903896205505, + "language_loss": 0.76564932, + "learning_rate": 3.775191448768865e-06, + "loss": 0.78887826, + "num_input_tokens_seen": 19239690, + "router_z_loss_clip": 1.72265625, + "router_z_loss_mlp": 0.26367188, + "step": 700, + "time_per_iteration": 2.556943655014038 + }, + { + "auxiliary_loss_clip": 0.01240917, + "auxiliary_loss_mlp": 0.01066284, + "balance_loss_clip": 1.08114314, + "balance_loss_mlp": 1.04150033, + "epoch": 0.02034124542974871, + "flos": 11430294963840.0, + "grad_norm": 7.288667532421857, + "language_loss": 1.06843221, + "learning_rate": 3.776014104238524e-06, + "loss": 1.09150422, + "num_input_tokens_seen": 19251150, + "router_z_loss_clip": 1.59765625, + "router_z_loss_mlp": 0.24804688, + "step": 701, + "time_per_iteration": 2.451050043106079 + }, + { + "auxiliary_loss_clip": 0.01257137, + "auxiliary_loss_mlp": 0.01067687, + "balance_loss_clip": 1.08176577, + "balance_loss_mlp": 1.03794408, + "epoch": 0.020370262898264754, + "flos": 10517984403840.0, + "grad_norm": 2.9839941233339076, + "language_loss": 0.84742284, + "learning_rate": 3.7768355869986333e-06, + "loss": 0.87067115, + "num_input_tokens_seen": 19261715, + "router_z_loss_clip": 1.75390625, + "router_z_loss_mlp": 0.29736328, + "step": 702, + "time_per_iteration": 2.4992480278015137 + }, + { + "auxiliary_loss_clip": 0.01072958, + "auxiliary_loss_mlp": 0.01022162, + "balance_loss_clip": 1.02352142, + "balance_loss_mlp": 1.01875293, + "epoch": 0.020399280366780802, + "flos": 57698322451200.0, + "grad_norm": 0.7041249922565237, + "language_loss": 0.51533866, + "learning_rate": 3.7776559003878716e-06, + "loss": 0.53628993, + "num_input_tokens_seen": 19322490, + "router_z_loss_clip": 0.49414062, + "router_z_loss_mlp": 0.03417969, + "step": 703, + "time_per_iteration": 3.082115888595581 + }, + { + "auxiliary_loss_clip": 0.01072162, + "auxiliary_loss_mlp": 0.01009296, + "balance_loss_clip": 1.02318597, + "balance_loss_mlp": 1.00591075, + "epoch": 0.020428297835296847, + "flos": 69305117869440.0, + "grad_norm": 0.7265692530247303, + "language_loss": 0.52407479, + "learning_rate": 3.7784750477306753e-06, + "loss": 0.54488933, + "num_input_tokens_seen": 19388305, + "router_z_loss_clip": 0.49023438, + "router_z_loss_mlp": 0.03393555, + "step": 704, + "time_per_iteration": 3.123145341873169 + }, + { + "auxiliary_loss_clip": 0.01249739, + "auxiliary_loss_mlp": 0.01076936, + "balance_loss_clip": 1.08165836, + "balance_loss_mlp": 1.05029321, + "epoch": 0.020457315303812895, + "flos": 52183445760000.0, + "grad_norm": 2.207274875965419, + "language_loss": 0.93495011, + "learning_rate": 3.7792930323373297e-06, + "loss": 0.95821691, + "num_input_tokens_seen": 19410815, + "router_z_loss_clip": 1.68066406, + "router_z_loss_mlp": 0.26635742, + "step": 705, + "time_per_iteration": 2.8110828399658203 + }, + { + "auxiliary_loss_clip": 0.01261903, + "auxiliary_loss_mlp": 0.01075183, + "balance_loss_clip": 1.08079648, + "balance_loss_mlp": 1.04381871, + "epoch": 0.020486332772328943, + "flos": 31938195974400.0, + "grad_norm": 3.454049450514723, + "language_loss": 1.0698657, + "learning_rate": 3.780109857504039e-06, + "loss": 1.09323657, + "num_input_tokens_seen": 19426225, + "router_z_loss_clip": 1.81054688, + "router_z_loss_mlp": 0.31384277, + "step": 706, + "time_per_iteration": 2.598447799682617 + }, + { + "auxiliary_loss_clip": 0.01249884, + "auxiliary_loss_mlp": 0.01074934, + "balance_loss_clip": 1.08456278, + "balance_loss_mlp": 1.04805183, + "epoch": 0.020515350240844988, + "flos": 11173138519680.0, + "grad_norm": 2.9815908673159273, + "language_loss": 0.83136547, + "learning_rate": 3.7809255265130137e-06, + "loss": 0.85461366, + "num_input_tokens_seen": 19436275, + "router_z_loss_clip": 1.65429688, + "router_z_loss_mlp": 0.2689209, + "step": 707, + "time_per_iteration": 2.4698917865753174 + }, + { + "auxiliary_loss_clip": 0.01257448, + "auxiliary_loss_mlp": 0.01075654, + "balance_loss_clip": 1.08623099, + "balance_loss_mlp": 1.04537499, + "epoch": 0.020544367709361036, + "flos": 16719720963840.0, + "grad_norm": 2.9429349886488327, + "language_loss": 0.6974048, + "learning_rate": 3.7817400426325455e-06, + "loss": 0.72073579, + "num_input_tokens_seen": 19448860, + "router_z_loss_clip": 1.71289062, + "router_z_loss_mlp": 0.30273438, + "step": 708, + "time_per_iteration": 2.4753336906433105 + }, + { + "auxiliary_loss_clip": 0.01072334, + "auxiliary_loss_mlp": 0.01014316, + "balance_loss_clip": 1.02419949, + "balance_loss_mlp": 1.01064479, + "epoch": 0.02057338517787708, + "flos": 64736633744640.0, + "grad_norm": 0.7579412421793997, + "language_loss": 0.52815115, + "learning_rate": 3.782553409117088e-06, + "loss": 0.54901767, + "num_input_tokens_seen": 19509640, + "router_z_loss_clip": 0.48046875, + "router_z_loss_mlp": 0.03662109, + "step": 709, + "time_per_iteration": 3.0479116439819336 + }, + { + "auxiliary_loss_clip": 0.0126183, + "auxiliary_loss_mlp": 0.01074917, + "balance_loss_clip": 1.08569717, + "balance_loss_mlp": 1.04559135, + "epoch": 0.02060240264639313, + "flos": 20301667073280.0, + "grad_norm": 3.128102967794606, + "language_loss": 0.95766425, + "learning_rate": 3.783365629207333e-06, + "loss": 0.98103178, + "num_input_tokens_seen": 19527220, + "router_z_loss_clip": 1.76074219, + "router_z_loss_mlp": 0.29333496, + "step": 710, + "time_per_iteration": 2.537468671798706 + }, + { + "auxiliary_loss_clip": 0.01254488, + "auxiliary_loss_mlp": 0.01074681, + "balance_loss_clip": 1.08801937, + "balance_loss_mlp": 1.04619002, + "epoch": 0.020631420114909177, + "flos": 29893693749120.0, + "grad_norm": 17.5300635160985, + "language_loss": 0.88201189, + "learning_rate": 3.7841767061302886e-06, + "loss": 0.9053036, + "num_input_tokens_seen": 19544115, + "router_z_loss_clip": 1.66308594, + "router_z_loss_mlp": 0.28503418, + "step": 711, + "time_per_iteration": 2.597478151321411 + }, + { + "auxiliary_loss_clip": 0.01254967, + "auxiliary_loss_mlp": 0.0106809, + "balance_loss_clip": 1.0844059, + "balance_loss_mlp": 1.03969419, + "epoch": 0.02066043758342522, + "flos": 31943295705600.0, + "grad_norm": 2.182512143636868, + "language_loss": 0.71702862, + "learning_rate": 3.7849866430993588e-06, + "loss": 0.74025917, + "num_input_tokens_seen": 19562355, + "router_z_loss_clip": 1.70410156, + "router_z_loss_mlp": 0.28393555, + "step": 712, + "time_per_iteration": 2.5750198364257812 + }, + { + "auxiliary_loss_clip": 0.01259408, + "auxiliary_loss_mlp": 0.0109409, + "balance_loss_clip": 1.08381522, + "balance_loss_mlp": 1.06376362, + "epoch": 0.02068945505194127, + "flos": 37700637200640.0, + "grad_norm": 2.193133966504159, + "language_loss": 0.96542799, + "learning_rate": 3.7857954433144147e-06, + "loss": 0.98896301, + "num_input_tokens_seen": 19581335, + "router_z_loss_clip": 1.75488281, + "router_z_loss_mlp": 0.30310059, + "step": 713, + "time_per_iteration": 2.6587538719177246 + }, + { + "auxiliary_loss_clip": 0.01069045, + "auxiliary_loss_mlp": 0.01012068, + "balance_loss_clip": 1.02109993, + "balance_loss_mlp": 1.00870657, + "epoch": 0.020718472520457314, + "flos": 74775533523840.0, + "grad_norm": 0.7107901629604971, + "language_loss": 0.56514561, + "learning_rate": 3.7866031099618737e-06, + "loss": 0.58595669, + "num_input_tokens_seen": 19642865, + "router_z_loss_clip": 0.48046875, + "router_z_loss_mlp": 0.03369141, + "step": 714, + "time_per_iteration": 3.1530051231384277 + }, + { + "auxiliary_loss_clip": 0.01269035, + "auxiliary_loss_mlp": 0.0107984, + "balance_loss_clip": 1.08982956, + "balance_loss_mlp": 1.04896438, + "epoch": 0.020747489988973362, + "flos": 20341097228160.0, + "grad_norm": 2.408143615268931, + "language_loss": 0.89014709, + "learning_rate": 3.787409646214775e-06, + "loss": 0.91363585, + "num_input_tokens_seen": 19664640, + "router_z_loss_clip": 1.79296875, + "router_z_loss_mlp": 0.30883789, + "step": 715, + "time_per_iteration": 2.651585340499878 + }, + { + "auxiliary_loss_clip": 0.01069078, + "auxiliary_loss_mlp": 0.01006117, + "balance_loss_clip": 1.02093005, + "balance_loss_mlp": 1.00280333, + "epoch": 0.020776507457489407, + "flos": 66002379154560.0, + "grad_norm": 0.6586669384880776, + "language_loss": 0.52861726, + "learning_rate": 3.788215055232854e-06, + "loss": 0.54936922, + "num_input_tokens_seen": 19735205, + "router_z_loss_clip": 0.48046875, + "router_z_loss_mlp": 0.03320312, + "step": 716, + "time_per_iteration": 5.557220697402954 + }, + { + "auxiliary_loss_clip": 0.01069617, + "auxiliary_loss_mlp": 0.01005694, + "balance_loss_clip": 1.0216794, + "balance_loss_mlp": 1.00226057, + "epoch": 0.020805524926005455, + "flos": 56277121956480.0, + "grad_norm": 0.7348961391610491, + "language_loss": 0.56194484, + "learning_rate": 3.789019340162615e-06, + "loss": 0.58269787, + "num_input_tokens_seen": 19792645, + "router_z_loss_clip": 0.48046875, + "router_z_loss_mlp": 0.03442383, + "step": 717, + "time_per_iteration": 5.322114944458008 + }, + { + "auxiliary_loss_clip": 0.01250486, + "auxiliary_loss_mlp": 0.01076702, + "balance_loss_clip": 1.0839057, + "balance_loss_mlp": 1.04787707, + "epoch": 0.020834542394521503, + "flos": 12051262310400.0, + "grad_norm": 5.008158926965863, + "language_loss": 0.8531993, + "learning_rate": 3.7898225041374074e-06, + "loss": 0.87647116, + "num_input_tokens_seen": 19803710, + "router_z_loss_clip": 1.66601562, + "router_z_loss_mlp": 0.28833008, + "step": 718, + "time_per_iteration": 4.9694788455963135 + }, + { + "auxiliary_loss_clip": 0.01068406, + "auxiliary_loss_mlp": 0.01002739, + "balance_loss_clip": 1.02067065, + "balance_loss_mlp": 0.99949616, + "epoch": 0.020863559863037548, + "flos": 68889953496960.0, + "grad_norm": 0.7480654949415988, + "language_loss": 0.53443158, + "learning_rate": 3.790624550277496e-06, + "loss": 0.55514306, + "num_input_tokens_seen": 19857780, + "router_z_loss_clip": 0.4765625, + "router_z_loss_mlp": 0.0324707, + "step": 719, + "time_per_iteration": 2.985157012939453 + }, + { + "auxiliary_loss_clip": 0.01250054, + "auxiliary_loss_mlp": 0.01064317, + "balance_loss_clip": 1.08414626, + "balance_loss_mlp": 1.03531349, + "epoch": 0.020892577331553596, + "flos": 18289735505280.0, + "grad_norm": 2.8736573560585423, + "language_loss": 0.77287209, + "learning_rate": 3.7914254816901373e-06, + "loss": 0.7960158, + "num_input_tokens_seen": 19870425, + "router_z_loss_clip": 1.65917969, + "router_z_loss_mlp": 0.28991699, + "step": 720, + "time_per_iteration": 2.416999101638794 + }, + { + "auxiliary_loss_clip": 0.01067032, + "auxiliary_loss_mlp": 0.01010538, + "balance_loss_clip": 1.01989055, + "balance_loss_mlp": 1.00724816, + "epoch": 0.02092159480006964, + "flos": 57372900209280.0, + "grad_norm": 0.6833172584387436, + "language_loss": 0.54751849, + "learning_rate": 3.792225301469649e-06, + "loss": 0.56829417, + "num_input_tokens_seen": 19931845, + "router_z_loss_clip": 0.47070312, + "router_z_loss_mlp": 0.03295898, + "step": 721, + "time_per_iteration": 3.083284616470337 + }, + { + "auxiliary_loss_clip": 0.01248672, + "auxiliary_loss_mlp": 0.01064536, + "balance_loss_clip": 1.07973635, + "balance_loss_mlp": 1.03531778, + "epoch": 0.02095061226858569, + "flos": 28652118192000.0, + "grad_norm": 2.9264723400968276, + "language_loss": 1.1234678, + "learning_rate": 3.793024012697482e-06, + "loss": 1.14659989, + "num_input_tokens_seen": 19953645, + "router_z_loss_clip": 1.69140625, + "router_z_loss_mlp": 0.29199219, + "step": 722, + "time_per_iteration": 2.6102864742279053 + }, + { + "auxiliary_loss_clip": 0.01247478, + "auxiliary_loss_mlp": 0.01071829, + "balance_loss_clip": 1.08023417, + "balance_loss_mlp": 1.04276621, + "epoch": 0.020979629737101737, + "flos": 16153662533760.0, + "grad_norm": 2.6933922521588713, + "language_loss": 0.91357893, + "learning_rate": 3.7938216184422938e-06, + "loss": 0.93677199, + "num_input_tokens_seen": 19969390, + "router_z_loss_clip": 1.67382812, + "router_z_loss_mlp": 0.29077148, + "step": 723, + "time_per_iteration": 2.5271570682525635 + }, + { + "auxiliary_loss_clip": 0.01068324, + "auxiliary_loss_mlp": 0.01014221, + "balance_loss_clip": 1.02102566, + "balance_loss_mlp": 1.01045382, + "epoch": 0.02100864720561778, + "flos": 70190172984960.0, + "grad_norm": 0.7056176279261286, + "language_loss": 0.53796011, + "learning_rate": 3.7946181217600164e-06, + "loss": 0.55878562, + "num_input_tokens_seen": 20034860, + "router_z_loss_clip": 0.47265625, + "router_z_loss_mlp": 0.03759766, + "step": 724, + "time_per_iteration": 3.1863796710968018 + }, + { + "auxiliary_loss_clip": 0.01265616, + "auxiliary_loss_mlp": 0.01068887, + "balance_loss_clip": 1.08593202, + "balance_loss_mlp": 1.03925192, + "epoch": 0.02103766467413383, + "flos": 17523904608000.0, + "grad_norm": 3.8333418608802274, + "language_loss": 0.80860138, + "learning_rate": 3.795413525693929e-06, + "loss": 0.83194643, + "num_input_tokens_seen": 20050690, + "router_z_loss_clip": 1.79589844, + "router_z_loss_mlp": 0.29614258, + "step": 725, + "time_per_iteration": 2.48465895652771 + }, + { + "auxiliary_loss_clip": 0.01260359, + "auxiliary_loss_mlp": 0.01068713, + "balance_loss_clip": 1.08301806, + "balance_loss_mlp": 1.03657448, + "epoch": 0.021066682142649874, + "flos": 34818482833920.0, + "grad_norm": 2.6452620939842575, + "language_loss": 0.99654549, + "learning_rate": 3.7962078332747247e-06, + "loss": 1.01983619, + "num_input_tokens_seen": 20067375, + "router_z_loss_clip": 1.77539062, + "router_z_loss_mlp": 0.32141113, + "step": 726, + "time_per_iteration": 2.651540756225586 + }, + { + "auxiliary_loss_clip": 0.01249913, + "auxiliary_loss_mlp": 0.01089768, + "balance_loss_clip": 1.07989824, + "balance_loss_mlp": 1.06087184, + "epoch": 0.021095699611165922, + "flos": 74733047735040.0, + "grad_norm": 2.831905109290866, + "language_loss": 1.04215956, + "learning_rate": 3.7970010475205834e-06, + "loss": 1.06555629, + "num_input_tokens_seen": 20089985, + "router_z_loss_clip": 1.69726562, + "router_z_loss_mlp": 0.28869629, + "step": 727, + "time_per_iteration": 2.89263653755188 + }, + { + "auxiliary_loss_clip": 0.0125041, + "auxiliary_loss_mlp": 0.01069429, + "balance_loss_clip": 1.07712197, + "balance_loss_mlp": 1.04035389, + "epoch": 0.021124717079681967, + "flos": 18545096269440.0, + "grad_norm": 3.1813605151045783, + "language_loss": 0.82384896, + "learning_rate": 3.7977931714372386e-06, + "loss": 0.84704733, + "num_input_tokens_seen": 20107145, + "router_z_loss_clip": 1.73242188, + "router_z_loss_mlp": 0.29101562, + "step": 728, + "time_per_iteration": 2.5796632766723633 + }, + { + "auxiliary_loss_clip": 0.01251267, + "auxiliary_loss_mlp": 0.01059695, + "balance_loss_clip": 1.08622837, + "balance_loss_mlp": 1.03305173, + "epoch": 0.021153734548198015, + "flos": 74731934413440.0, + "grad_norm": 2.828354376369737, + "language_loss": 0.6661461, + "learning_rate": 3.7985842080180446e-06, + "loss": 0.68925571, + "num_input_tokens_seen": 20128560, + "router_z_loss_clip": 1.65039062, + "router_z_loss_mlp": 0.26623535, + "step": 729, + "time_per_iteration": 2.929410934448242 + }, + { + "auxiliary_loss_clip": 0.01249554, + "auxiliary_loss_mlp": 0.01066886, + "balance_loss_clip": 1.07900894, + "balance_loss_mlp": 1.03750038, + "epoch": 0.021182752016714063, + "flos": 16831080103680.0, + "grad_norm": 3.923290013604154, + "language_loss": 0.777156, + "learning_rate": 3.7993741602440483e-06, + "loss": 0.80032051, + "num_input_tokens_seen": 20142450, + "router_z_loss_clip": 1.70703125, + "router_z_loss_mlp": 0.2935791, + "step": 730, + "time_per_iteration": 2.520411968231201 + }, + { + "auxiliary_loss_clip": 0.01262567, + "auxiliary_loss_mlp": 0.01075284, + "balance_loss_clip": 1.08553612, + "balance_loss_mlp": 1.04558873, + "epoch": 0.021211769485230108, + "flos": 22887234840960.0, + "grad_norm": 2.988540167693218, + "language_loss": 0.87203753, + "learning_rate": 3.8001630310840514e-06, + "loss": 0.89541602, + "num_input_tokens_seen": 20157885, + "router_z_loss_clip": 1.76953125, + "router_z_loss_mlp": 0.29699707, + "step": 731, + "time_per_iteration": 2.5068671703338623 + }, + { + "auxiliary_loss_clip": 0.01253619, + "auxiliary_loss_mlp": 0.01066604, + "balance_loss_clip": 1.08721733, + "balance_loss_mlp": 1.03932905, + "epoch": 0.021240786953746156, + "flos": 13364407716480.0, + "grad_norm": 2.4174539183877677, + "language_loss": 0.79657334, + "learning_rate": 3.800950823494683e-06, + "loss": 0.81977552, + "num_input_tokens_seen": 20172155, + "router_z_loss_clip": 1.66503906, + "router_z_loss_mlp": 0.27294922, + "step": 732, + "time_per_iteration": 2.515233039855957 + }, + { + "auxiliary_loss_clip": 0.01257832, + "auxiliary_loss_mlp": 0.01077328, + "balance_loss_clip": 1.08507061, + "balance_loss_mlp": 1.04716802, + "epoch": 0.0212698044222622, + "flos": 30347605918080.0, + "grad_norm": 2.709771227528617, + "language_loss": 0.95139802, + "learning_rate": 3.8017375404204606e-06, + "loss": 0.97474962, + "num_input_tokens_seen": 20194245, + "router_z_loss_clip": 1.72851562, + "router_z_loss_mlp": 0.30151367, + "step": 733, + "time_per_iteration": 2.6813642978668213 + }, + { + "auxiliary_loss_clip": 0.01239421, + "auxiliary_loss_mlp": 0.01074801, + "balance_loss_clip": 1.07676923, + "balance_loss_mlp": 1.04669106, + "epoch": 0.02129882189077825, + "flos": 27669279277440.0, + "grad_norm": 2.6091725140602433, + "language_loss": 1.11447573, + "learning_rate": 3.802523184793859e-06, + "loss": 1.13761795, + "num_input_tokens_seen": 20213170, + "router_z_loss_clip": 1.62597656, + "router_z_loss_mlp": 0.28100586, + "step": 734, + "time_per_iteration": 2.579730987548828 + }, + { + "auxiliary_loss_clip": 0.01077344, + "auxiliary_loss_mlp": 0.0100504, + "balance_loss_clip": 1.03004074, + "balance_loss_mlp": 1.00136876, + "epoch": 0.021327839359294293, + "flos": 68644073923200.0, + "grad_norm": 0.784979257360011, + "language_loss": 0.5287472, + "learning_rate": 3.8033077595353777e-06, + "loss": 0.54957104, + "num_input_tokens_seen": 20273980, + "router_z_loss_clip": 0.47265625, + "router_z_loss_mlp": 0.03662109, + "step": 735, + "time_per_iteration": 3.0912137031555176 + }, + { + "auxiliary_loss_clip": 0.01257454, + "auxiliary_loss_mlp": 0.01083949, + "balance_loss_clip": 1.08760941, + "balance_loss_mlp": 1.05599499, + "epoch": 0.02135685682781034, + "flos": 30767364253440.0, + "grad_norm": 4.648745813511679, + "language_loss": 0.81159842, + "learning_rate": 3.8040912675536016e-06, + "loss": 0.83501244, + "num_input_tokens_seen": 20288030, + "router_z_loss_clip": 1.69824219, + "router_z_loss_mlp": 0.27929688, + "step": 736, + "time_per_iteration": 2.531303644180298 + }, + { + "auxiliary_loss_clip": 0.01072565, + "auxiliary_loss_mlp": 0.01002656, + "balance_loss_clip": 1.02553749, + "balance_loss_mlp": 0.99891263, + "epoch": 0.02138587429632639, + "flos": 69807722924160.0, + "grad_norm": 0.7641405358084519, + "language_loss": 0.60171735, + "learning_rate": 3.8048737117452677e-06, + "loss": 0.62246954, + "num_input_tokens_seen": 20349820, + "router_z_loss_clip": 0.46875, + "router_z_loss_mlp": 0.03735352, + "step": 737, + "time_per_iteration": 3.056368112564087 + }, + { + "auxiliary_loss_clip": 0.01250683, + "auxiliary_loss_mlp": 0.01064073, + "balance_loss_clip": 1.08571887, + "balance_loss_mlp": 1.03766799, + "epoch": 0.021414891764842434, + "flos": 15807589971840.0, + "grad_norm": 2.7593644896297773, + "language_loss": 0.84012067, + "learning_rate": 3.8056550949953317e-06, + "loss": 0.86326826, + "num_input_tokens_seen": 20361700, + "router_z_loss_clip": 1.65039062, + "router_z_loss_mlp": 0.26403809, + "step": 738, + "time_per_iteration": 2.4680464267730713 + }, + { + "auxiliary_loss_clip": 0.01069222, + "auxiliary_loss_mlp": 0.01004609, + "balance_loss_clip": 1.02255702, + "balance_loss_mlp": 1.00108039, + "epoch": 0.021443909233358482, + "flos": 68509553489280.0, + "grad_norm": 1.0453453676119682, + "language_loss": 0.52872574, + "learning_rate": 3.806435420177029e-06, + "loss": 0.54946399, + "num_input_tokens_seen": 20422835, + "router_z_loss_clip": 0.46679688, + "router_z_loss_mlp": 0.03540039, + "step": 739, + "time_per_iteration": 3.152878522872925 + }, + { + "auxiliary_loss_clip": 0.01068213, + "auxiliary_loss_mlp": 0.01003422, + "balance_loss_clip": 1.02171409, + "balance_loss_mlp": 1.00017953, + "epoch": 0.021472926701874527, + "flos": 74777688339840.0, + "grad_norm": 0.6614298676070118, + "language_loss": 0.57411456, + "learning_rate": 3.8072146901519385e-06, + "loss": 0.59483087, + "num_input_tokens_seen": 20488955, + "router_z_loss_clip": 0.46484375, + "router_z_loss_mlp": 0.0324707, + "step": 740, + "time_per_iteration": 3.132631301879883 + }, + { + "auxiliary_loss_clip": 0.01257026, + "auxiliary_loss_mlp": 0.01072647, + "balance_loss_clip": 1.08428788, + "balance_loss_mlp": 1.04072261, + "epoch": 0.021501944170390575, + "flos": 36934914044160.0, + "grad_norm": 5.859827598735575, + "language_loss": 1.08238423, + "learning_rate": 3.8079929077700457e-06, + "loss": 1.10568106, + "num_input_tokens_seen": 20508400, + "router_z_loss_clip": 1.72753906, + "router_z_loss_mlp": 0.31958008, + "step": 741, + "time_per_iteration": 2.7152328491210938 + }, + { + "auxiliary_loss_clip": 0.01242545, + "auxiliary_loss_mlp": 0.01071401, + "balance_loss_clip": 1.08138013, + "balance_loss_mlp": 1.04329157, + "epoch": 0.021530961638906623, + "flos": 42596016024960.0, + "grad_norm": 2.448747358100891, + "language_loss": 1.1421926, + "learning_rate": 3.8087700758698065e-06, + "loss": 1.16533208, + "num_input_tokens_seen": 20531245, + "router_z_loss_clip": 1.61230469, + "router_z_loss_mlp": 0.28088379, + "step": 742, + "time_per_iteration": 2.7192225456237793 + }, + { + "auxiliary_loss_clip": 0.01066398, + "auxiliary_loss_mlp": 0.01006581, + "balance_loss_clip": 1.02106988, + "balance_loss_mlp": 1.00340974, + "epoch": 0.021559979107422668, + "flos": 65505696865920.0, + "grad_norm": 1.1359846652090246, + "language_loss": 0.46614367, + "learning_rate": 3.809546197278207e-06, + "loss": 0.48687342, + "num_input_tokens_seen": 20586370, + "router_z_loss_clip": 0.453125, + "router_z_loss_mlp": 0.03173828, + "step": 743, + "time_per_iteration": 3.01947021484375 + }, + { + "auxiliary_loss_clip": 0.01254545, + "auxiliary_loss_mlp": 0.01069445, + "balance_loss_clip": 1.08423054, + "balance_loss_mlp": 1.03666222, + "epoch": 0.021588996575938716, + "flos": 31751029180800.0, + "grad_norm": 2.901802131492519, + "language_loss": 0.96719366, + "learning_rate": 3.810321274810827e-06, + "loss": 0.99043357, + "num_input_tokens_seen": 20604280, + "router_z_loss_clip": 1.70410156, + "router_z_loss_mlp": 0.32775879, + "step": 744, + "time_per_iteration": 2.6355600357055664 + }, + { + "auxiliary_loss_clip": 0.01253384, + "auxiliary_loss_mlp": 0.0106725, + "balance_loss_clip": 1.08090091, + "balance_loss_mlp": 1.03790104, + "epoch": 0.02161801404445476, + "flos": 28505494874880.0, + "grad_norm": 3.2945784332646326, + "language_loss": 0.99970549, + "learning_rate": 3.8110953112719017e-06, + "loss": 1.02291179, + "num_input_tokens_seen": 20618850, + "router_z_loss_clip": 1.72558594, + "router_z_loss_mlp": 0.2935791, + "step": 745, + "time_per_iteration": 2.5354669094085693 + }, + { + "auxiliary_loss_clip": 0.01238528, + "auxiliary_loss_mlp": 0.01074583, + "balance_loss_clip": 1.07874727, + "balance_loss_mlp": 1.04661632, + "epoch": 0.02164703151297081, + "flos": 32443961425920.0, + "grad_norm": 20.160339066072375, + "language_loss": 1.10842073, + "learning_rate": 3.81186830945438e-06, + "loss": 1.13155186, + "num_input_tokens_seen": 20632240, + "router_z_loss_clip": 1.59570312, + "router_z_loss_mlp": 0.27954102, + "step": 746, + "time_per_iteration": 2.610490083694458 + }, + { + "auxiliary_loss_clip": 0.01238218, + "auxiliary_loss_mlp": 0.01073991, + "balance_loss_clip": 1.07925034, + "balance_loss_mlp": 1.04527354, + "epoch": 0.021676048981486853, + "flos": 38028896616960.0, + "grad_norm": 2.781046001986463, + "language_loss": 1.0826149, + "learning_rate": 3.812640272139988e-06, + "loss": 1.10573685, + "num_input_tokens_seen": 20648830, + "router_z_loss_clip": 1.58886719, + "router_z_loss_mlp": 0.28735352, + "step": 747, + "time_per_iteration": 2.639296770095825 + }, + { + "auxiliary_loss_clip": 0.01255601, + "auxiliary_loss_mlp": 0.01077736, + "balance_loss_clip": 1.08493209, + "balance_loss_mlp": 1.04733729, + "epoch": 0.0217050664500029, + "flos": 35258029562880.0, + "grad_norm": 2.4103588689469335, + "language_loss": 0.73075366, + "learning_rate": 3.813411202099287e-06, + "loss": 0.75408697, + "num_input_tokens_seen": 20668395, + "router_z_loss_clip": 1.70703125, + "router_z_loss_mlp": 0.30419922, + "step": 748, + "time_per_iteration": 2.6485164165496826 + }, + { + "auxiliary_loss_clip": 0.01244501, + "auxiliary_loss_mlp": 0.01070523, + "balance_loss_clip": 1.07752764, + "balance_loss_mlp": 1.0432713, + "epoch": 0.02173408391851895, + "flos": 29423874833280.0, + "grad_norm": 2.6753748146932237, + "language_loss": 1.10580468, + "learning_rate": 3.8141811020917338e-06, + "loss": 1.12895489, + "num_input_tokens_seen": 20688865, + "router_z_loss_clip": 1.66894531, + "router_z_loss_mlp": 0.27258301, + "step": 749, + "time_per_iteration": 2.5539801120758057 + }, + { + "auxiliary_loss_clip": 0.01064424, + "auxiliary_loss_mlp": 0.01018353, + "balance_loss_clip": 1.02012491, + "balance_loss_mlp": 1.0154444, + "epoch": 0.021763101387034994, + "flos": 62630042860800.0, + "grad_norm": 0.7903721935653789, + "language_loss": 0.56792188, + "learning_rate": 3.81494997486574e-06, + "loss": 0.58874965, + "num_input_tokens_seen": 20753735, + "router_z_loss_clip": 0.44335938, + "router_z_loss_mlp": 0.02905273, + "step": 750, + "time_per_iteration": 3.1086015701293945 + }, + { + "auxiliary_loss_clip": 0.01251166, + "auxiliary_loss_mlp": 0.01069755, + "balance_loss_clip": 1.08361483, + "balance_loss_mlp": 1.03752112, + "epoch": 0.021792118855551042, + "flos": 22047320142720.0, + "grad_norm": 2.310431268673291, + "language_loss": 0.82842219, + "learning_rate": 3.815717823158732e-06, + "loss": 0.8516314, + "num_input_tokens_seen": 20770205, + "router_z_loss_clip": 1.67675781, + "router_z_loss_mlp": 0.32226562, + "step": 751, + "time_per_iteration": 2.572899341583252 + }, + { + "auxiliary_loss_clip": 0.01246609, + "auxiliary_loss_mlp": 0.01076069, + "balance_loss_clip": 1.08977735, + "balance_loss_mlp": 1.04903197, + "epoch": 0.021821136324067087, + "flos": 11066053098240.0, + "grad_norm": 3.8162485266316732, + "language_loss": 0.8694675, + "learning_rate": 3.816484649697207e-06, + "loss": 0.89269423, + "num_input_tokens_seen": 20780935, + "router_z_loss_clip": 1.56933594, + "router_z_loss_mlp": 0.27038574, + "step": 752, + "time_per_iteration": 2.538648843765259 + }, + { + "auxiliary_loss_clip": 0.01065984, + "auxiliary_loss_mlp": 0.01016941, + "balance_loss_clip": 1.02115977, + "balance_loss_mlp": 1.01381826, + "epoch": 0.021850153792583135, + "flos": 63391635953280.0, + "grad_norm": 0.7561861786667957, + "language_loss": 0.53866696, + "learning_rate": 3.817250457196791e-06, + "loss": 0.55949628, + "num_input_tokens_seen": 20840120, + "router_z_loss_clip": 0.44921875, + "router_z_loss_mlp": 0.03125, + "step": 753, + "time_per_iteration": 3.0136756896972656 + }, + { + "auxiliary_loss_clip": 0.01253154, + "auxiliary_loss_mlp": 0.01069649, + "balance_loss_clip": 1.08243847, + "balance_loss_mlp": 1.04232609, + "epoch": 0.021879171261099183, + "flos": 42735241140480.0, + "grad_norm": 2.2126470238764995, + "language_loss": 0.95042998, + "learning_rate": 3.818015248362302e-06, + "loss": 0.97365797, + "num_input_tokens_seen": 20862485, + "router_z_loss_clip": 1.70605469, + "router_z_loss_mlp": 0.27319336, + "step": 754, + "time_per_iteration": 2.7609691619873047 + }, + { + "auxiliary_loss_clip": 0.01255221, + "auxiliary_loss_mlp": 0.01087375, + "balance_loss_clip": 1.0848484, + "balance_loss_mlp": 1.05735803, + "epoch": 0.021908188729615228, + "flos": 56057597009280.0, + "grad_norm": 1.8726133114636674, + "language_loss": 0.98227257, + "learning_rate": 3.818779025887801e-06, + "loss": 1.00569856, + "num_input_tokens_seen": 20891275, + "router_z_loss_clip": 1.70507812, + "router_z_loss_mlp": 0.3001709, + "step": 755, + "time_per_iteration": 2.875412940979004 + }, + { + "auxiliary_loss_clip": 0.01258384, + "auxiliary_loss_mlp": 0.01081154, + "balance_loss_clip": 1.08582449, + "balance_loss_mlp": 1.04610622, + "epoch": 0.021937206198131276, + "flos": 25551950227200.0, + "grad_norm": 2.42149095483029, + "language_loss": 1.05354679, + "learning_rate": 3.81954179245665e-06, + "loss": 1.07694209, + "num_input_tokens_seen": 20906840, + "router_z_loss_clip": 1.72558594, + "router_z_loss_mlp": 0.35058594, + "step": 756, + "time_per_iteration": 2.566013813018799 + }, + { + "auxiliary_loss_clip": 0.01247389, + "auxiliary_loss_mlp": 0.01082276, + "balance_loss_clip": 1.08320284, + "balance_loss_mlp": 1.05393958, + "epoch": 0.02196622366664732, + "flos": 29635352156160.0, + "grad_norm": 3.0873136344918866, + "language_loss": 0.77436954, + "learning_rate": 3.820303550741571e-06, + "loss": 0.79766619, + "num_input_tokens_seen": 20926260, + "router_z_loss_clip": 1.640625, + "router_z_loss_mlp": 0.2833252, + "step": 757, + "time_per_iteration": 2.7123377323150635 + }, + { + "auxiliary_loss_clip": 0.01252844, + "auxiliary_loss_mlp": 0.01069582, + "balance_loss_clip": 1.08717775, + "balance_loss_mlp": 1.04132962, + "epoch": 0.02199524113516337, + "flos": 17380082551680.0, + "grad_norm": 2.720306891482898, + "language_loss": 1.00778008, + "learning_rate": 3.8210643034047025e-06, + "loss": 1.03100443, + "num_input_tokens_seen": 20940335, + "router_z_loss_clip": 1.65625, + "router_z_loss_mlp": 0.28222656, + "step": 758, + "time_per_iteration": 2.479130268096924 + }, + { + "auxiliary_loss_clip": 0.01066931, + "auxiliary_loss_mlp": 0.01006163, + "balance_loss_clip": 1.02120399, + "balance_loss_mlp": 1.00306344, + "epoch": 0.022024258603679413, + "flos": 74770792928640.0, + "grad_norm": 0.6736375744890061, + "language_loss": 0.52309698, + "learning_rate": 3.8218240530976505e-06, + "loss": 0.54382789, + "num_input_tokens_seen": 21005590, + "router_z_loss_clip": 0.45703125, + "router_z_loss_mlp": 0.03100586, + "step": 759, + "time_per_iteration": 3.256314992904663 + }, + { + "auxiliary_loss_clip": 0.01244252, + "auxiliary_loss_mlp": 0.0106713, + "balance_loss_clip": 1.08626306, + "balance_loss_mlp": 1.04175043, + "epoch": 0.02205327607219546, + "flos": 29709076821120.0, + "grad_norm": 3.248926515190084, + "language_loss": 0.72669172, + "learning_rate": 3.82258280246155e-06, + "loss": 0.74980557, + "num_input_tokens_seen": 21021005, + "router_z_loss_clip": 1.58007812, + "router_z_loss_mlp": 0.25378418, + "step": 760, + "time_per_iteration": 2.5647099018096924 + }, + { + "auxiliary_loss_clip": 0.010671, + "auxiliary_loss_mlp": 0.01005298, + "balance_loss_clip": 1.02144945, + "balance_loss_mlp": 1.0022465, + "epoch": 0.02208229354071151, + "flos": 66423286725120.0, + "grad_norm": 0.6625653256473126, + "language_loss": 0.51102751, + "learning_rate": 3.823340554127116e-06, + "loss": 0.53175151, + "num_input_tokens_seen": 21082605, + "router_z_loss_clip": 0.45703125, + "router_z_loss_mlp": 0.03051758, + "step": 761, + "time_per_iteration": 3.099398612976074 + }, + { + "auxiliary_loss_clip": 0.01251374, + "auxiliary_loss_mlp": 0.01067261, + "balance_loss_clip": 1.08603287, + "balance_loss_mlp": 1.0392704, + "epoch": 0.022111311009227554, + "flos": 26134241604480.0, + "grad_norm": 2.3060606052174037, + "language_loss": 0.81708479, + "learning_rate": 3.824097310714699e-06, + "loss": 0.84027112, + "num_input_tokens_seen": 21099205, + "router_z_loss_clip": 1.65039062, + "router_z_loss_mlp": 0.27978516, + "step": 762, + "time_per_iteration": 2.523514747619629 + }, + { + "auxiliary_loss_clip": 0.01238074, + "auxiliary_loss_mlp": 0.01063514, + "balance_loss_clip": 1.08378029, + "balance_loss_mlp": 1.03964818, + "epoch": 0.022140328477743602, + "flos": 14531468509440.0, + "grad_norm": 2.2820766076372614, + "language_loss": 0.90253675, + "learning_rate": 3.824853074834342e-06, + "loss": 0.92555255, + "num_input_tokens_seen": 21112380, + "router_z_loss_clip": 1.54296875, + "router_z_loss_mlp": 0.23864746, + "step": 763, + "time_per_iteration": 2.55084490776062 + }, + { + "auxiliary_loss_clip": 0.01241954, + "auxiliary_loss_mlp": 0.01071211, + "balance_loss_clip": 1.08437681, + "balance_loss_mlp": 1.04478228, + "epoch": 0.022169345946259647, + "flos": 36166856503680.0, + "grad_norm": 2.6655168525336084, + "language_loss": 0.87902284, + "learning_rate": 3.82560784908583e-06, + "loss": 0.90215451, + "num_input_tokens_seen": 21130490, + "router_z_loss_clip": 1.57324219, + "router_z_loss_mlp": 0.26416016, + "step": 764, + "time_per_iteration": 2.6324265003204346 + }, + { + "auxiliary_loss_clip": 0.01240722, + "auxiliary_loss_mlp": 0.01073636, + "balance_loss_clip": 1.08418334, + "balance_loss_mlp": 1.04624128, + "epoch": 0.022198363414775695, + "flos": 27701131662720.0, + "grad_norm": 2.7434731449837737, + "language_loss": 0.890674, + "learning_rate": 3.826361636058748e-06, + "loss": 0.91381752, + "num_input_tokens_seen": 21148605, + "router_z_loss_clip": 1.56640625, + "router_z_loss_mlp": 0.27392578, + "step": 765, + "time_per_iteration": 2.578378677368164 + }, + { + "auxiliary_loss_clip": 0.01065091, + "auxiliary_loss_mlp": 0.01003388, + "balance_loss_clip": 1.02063704, + "balance_loss_mlp": 1.00045514, + "epoch": 0.022227380883291743, + "flos": 67334519877120.0, + "grad_norm": 0.7019149750690886, + "language_loss": 0.51905847, + "learning_rate": 3.827114438332532e-06, + "loss": 0.5397433, + "num_input_tokens_seen": 21207490, + "router_z_loss_clip": 0.4453125, + "router_z_loss_mlp": 0.02929688, + "step": 766, + "time_per_iteration": 3.0879392623901367 + }, + { + "auxiliary_loss_clip": 0.01251925, + "auxiliary_loss_mlp": 0.01064548, + "balance_loss_clip": 1.08674216, + "balance_loss_mlp": 1.03568769, + "epoch": 0.022256398351807788, + "flos": 28688854826880.0, + "grad_norm": 3.6849937836836113, + "language_loss": 0.9765591, + "learning_rate": 3.827866258476522e-06, + "loss": 0.99972379, + "num_input_tokens_seen": 21222515, + "router_z_loss_clip": 1.65234375, + "router_z_loss_mlp": 0.28894043, + "step": 767, + "time_per_iteration": 2.578927516937256 + }, + { + "auxiliary_loss_clip": 0.0106398, + "auxiliary_loss_mlp": 0.0100226, + "balance_loss_clip": 1.01982617, + "balance_loss_mlp": 0.99928015, + "epoch": 0.022285415820323836, + "flos": 61312372341120.0, + "grad_norm": 0.6937001401849658, + "language_loss": 0.50172746, + "learning_rate": 3.828617099050014e-06, + "loss": 0.52238989, + "num_input_tokens_seen": 21272320, + "router_z_loss_clip": 0.44140625, + "router_z_loss_mlp": 0.02978516, + "step": 768, + "time_per_iteration": 2.9888412952423096 + }, + { + "auxiliary_loss_clip": 0.01248366, + "auxiliary_loss_mlp": 0.01077124, + "balance_loss_clip": 1.08753705, + "balance_loss_mlp": 1.0488472, + "epoch": 0.02231443328883988, + "flos": 24017271690240.0, + "grad_norm": 2.317130818298539, + "language_loss": 0.84136623, + "learning_rate": 3.8293669626023145e-06, + "loss": 0.86462116, + "num_input_tokens_seen": 21286465, + "router_z_loss_clip": 1.60742188, + "router_z_loss_mlp": 0.28283691, + "step": 769, + "time_per_iteration": 2.5229814052581787 + }, + { + "auxiliary_loss_clip": 0.01248255, + "auxiliary_loss_mlp": 0.01066174, + "balance_loss_clip": 1.08174908, + "balance_loss_mlp": 1.03767097, + "epoch": 0.02234345075735593, + "flos": 20077979126400.0, + "grad_norm": 2.7808519240195726, + "language_loss": 1.03489125, + "learning_rate": 3.830115851672791e-06, + "loss": 1.05803549, + "num_input_tokens_seen": 21302395, + "router_z_loss_clip": 1.6640625, + "router_z_loss_mlp": 0.28515625, + "step": 770, + "time_per_iteration": 2.5279815196990967 + }, + { + "auxiliary_loss_clip": 0.01241017, + "auxiliary_loss_mlp": 0.01060494, + "balance_loss_clip": 1.07901096, + "balance_loss_mlp": 1.03411269, + "epoch": 0.022372468225871973, + "flos": 39491071551360.0, + "grad_norm": 3.7739156592365912, + "language_loss": 1.03286922, + "learning_rate": 3.830863768790924e-06, + "loss": 1.05588424, + "num_input_tokens_seen": 21319210, + "router_z_loss_clip": 1.62011719, + "router_z_loss_mlp": 0.26367188, + "step": 771, + "time_per_iteration": 2.647238254547119 + }, + { + "auxiliary_loss_clip": 0.01063182, + "auxiliary_loss_mlp": 0.01008183, + "balance_loss_clip": 1.01941466, + "balance_loss_mlp": 1.00526285, + "epoch": 0.02240148569438802, + "flos": 73242435185280.0, + "grad_norm": 0.6863233048589507, + "language_loss": 0.51115477, + "learning_rate": 3.831610716476358e-06, + "loss": 0.5318684, + "num_input_tokens_seen": 21379720, + "router_z_loss_clip": 0.4375, + "router_z_loss_mlp": 0.0291748, + "step": 772, + "time_per_iteration": 3.1192657947540283 + }, + { + "auxiliary_loss_clip": 0.01236534, + "auxiliary_loss_mlp": 0.01058797, + "balance_loss_clip": 1.08078766, + "balance_loss_mlp": 1.03276134, + "epoch": 0.02243050316290407, + "flos": 15257653770240.0, + "grad_norm": 4.363264667622451, + "language_loss": 1.03814292, + "learning_rate": 3.83235669723895e-06, + "loss": 1.06109619, + "num_input_tokens_seen": 21390245, + "router_z_loss_clip": 1.55957031, + "router_z_loss_mlp": 0.26037598, + "step": 773, + "time_per_iteration": 2.461189031600952 + }, + { + "auxiliary_loss_clip": 0.01062751, + "auxiliary_loss_mlp": 0.01007476, + "balance_loss_clip": 1.01935351, + "balance_loss_mlp": 1.00451934, + "epoch": 0.022459520631420114, + "flos": 65263803701760.0, + "grad_norm": 0.7291274340483628, + "language_loss": 0.52978998, + "learning_rate": 3.833101713578828e-06, + "loss": 0.55049223, + "num_input_tokens_seen": 21456720, + "router_z_loss_clip": 0.43359375, + "router_z_loss_mlp": 0.02954102, + "step": 774, + "time_per_iteration": 3.1744916439056396 + }, + { + "auxiliary_loss_clip": 0.01259312, + "auxiliary_loss_mlp": 0.01081011, + "balance_loss_clip": 1.08560801, + "balance_loss_mlp": 1.05141187, + "epoch": 0.022488538099936162, + "flos": 27008522640000.0, + "grad_norm": 2.7649777285142787, + "language_loss": 0.93903381, + "learning_rate": 3.83384576798643e-06, + "loss": 0.96243703, + "num_input_tokens_seen": 21471700, + "router_z_loss_clip": 1.73730469, + "router_z_loss_mlp": 0.29553223, + "step": 775, + "time_per_iteration": 2.5681474208831787 + }, + { + "auxiliary_loss_clip": 0.01243269, + "auxiliary_loss_mlp": 0.01080592, + "balance_loss_clip": 1.0803144, + "balance_loss_mlp": 1.05300736, + "epoch": 0.022517555568452207, + "flos": 26900252069760.0, + "grad_norm": 2.731880536564513, + "language_loss": 1.0274632, + "learning_rate": 3.834588862942565e-06, + "loss": 1.05070186, + "num_input_tokens_seen": 21491595, + "router_z_loss_clip": 1.62988281, + "router_z_loss_mlp": 0.27575684, + "step": 776, + "time_per_iteration": 2.619295120239258 + }, + { + "auxiliary_loss_clip": 0.01258479, + "auxiliary_loss_mlp": 0.01088019, + "balance_loss_clip": 1.09038901, + "balance_loss_mlp": 1.0565474, + "epoch": 0.022546573036968255, + "flos": 31393467244800.0, + "grad_norm": 2.474877083737226, + "language_loss": 1.04397106, + "learning_rate": 3.835331000918451e-06, + "loss": 1.06743598, + "num_input_tokens_seen": 21510460, + "router_z_loss_clip": 1.67871094, + "router_z_loss_mlp": 0.3145752, + "step": 777, + "time_per_iteration": 2.619532585144043 + }, + { + "auxiliary_loss_clip": 0.01062126, + "auxiliary_loss_mlp": 0.01008904, + "balance_loss_clip": 1.01890075, + "balance_loss_mlp": 1.00568569, + "epoch": 0.022575590505484303, + "flos": 74765369975040.0, + "grad_norm": 0.7649862837779966, + "language_loss": 0.55540991, + "learning_rate": 3.836072184375777e-06, + "loss": 0.57612014, + "num_input_tokens_seen": 21569265, + "router_z_loss_clip": 0.43359375, + "router_z_loss_mlp": 0.03222656, + "step": 778, + "time_per_iteration": 3.1295900344848633 + }, + { + "auxiliary_loss_clip": 0.01236842, + "auxiliary_loss_mlp": 0.01074353, + "balance_loss_clip": 1.07925427, + "balance_loss_mlp": 1.0497961, + "epoch": 0.022604607974000348, + "flos": 15703305811200.0, + "grad_norm": 2.9533678138049697, + "language_loss": 1.03679085, + "learning_rate": 3.8368124157667445e-06, + "loss": 1.05990267, + "num_input_tokens_seen": 21581555, + "router_z_loss_clip": 1.57519531, + "router_z_loss_mlp": 0.24536133, + "step": 779, + "time_per_iteration": 2.5084304809570312 + }, + { + "auxiliary_loss_clip": 0.01242753, + "auxiliary_loss_mlp": 0.01069938, + "balance_loss_clip": 1.08221388, + "balance_loss_mlp": 1.04385495, + "epoch": 0.022633625442516396, + "flos": 22887019359360.0, + "grad_norm": 2.294446611694585, + "language_loss": 1.05527425, + "learning_rate": 3.8375516975341135e-06, + "loss": 1.07840121, + "num_input_tokens_seen": 21598715, + "router_z_loss_clip": 1.60449219, + "router_z_loss_mlp": 0.26062012, + "step": 780, + "time_per_iteration": 2.6388564109802246 + }, + { + "auxiliary_loss_clip": 0.0106442, + "auxiliary_loss_mlp": 0.01007185, + "balance_loss_clip": 1.02125072, + "balance_loss_mlp": 1.00418127, + "epoch": 0.02266264291103244, + "flos": 60765201486720.0, + "grad_norm": 0.7041007824023117, + "language_loss": 0.57407808, + "learning_rate": 3.838290032111259e-06, + "loss": 0.59479415, + "num_input_tokens_seen": 21662190, + "router_z_loss_clip": 0.43164062, + "router_z_loss_mlp": 0.0300293, + "step": 781, + "time_per_iteration": 3.148550510406494 + }, + { + "auxiliary_loss_clip": 0.01248961, + "auxiliary_loss_mlp": 0.01088331, + "balance_loss_clip": 1.08223557, + "balance_loss_mlp": 1.05657327, + "epoch": 0.02269166037954849, + "flos": 39962111529600.0, + "grad_norm": 2.621845862302468, + "language_loss": 1.03776956, + "learning_rate": 3.8390274219222125e-06, + "loss": 1.06114244, + "num_input_tokens_seen": 21679115, + "router_z_loss_clip": 1.66699219, + "router_z_loss_mlp": 0.31762695, + "step": 782, + "time_per_iteration": 2.6608617305755615 + }, + { + "auxiliary_loss_clip": 0.01247851, + "auxiliary_loss_mlp": 0.01072898, + "balance_loss_clip": 1.08174539, + "balance_loss_mlp": 1.04406095, + "epoch": 0.022720677848064533, + "flos": 39086645345280.0, + "grad_norm": 2.943355332531316, + "language_loss": 0.82332951, + "learning_rate": 3.839763869381713e-06, + "loss": 0.84653705, + "num_input_tokens_seen": 21697935, + "router_z_loss_clip": 1.66210938, + "router_z_loss_mlp": 0.28869629, + "step": 783, + "time_per_iteration": 2.6599106788635254 + }, + { + "auxiliary_loss_clip": 0.01243305, + "auxiliary_loss_mlp": 0.01070896, + "balance_loss_clip": 1.07987225, + "balance_loss_mlp": 1.04159486, + "epoch": 0.02274969531658058, + "flos": 74746335767040.0, + "grad_norm": 2.7975677297066617, + "language_loss": 0.8672992, + "learning_rate": 3.840499376895254e-06, + "loss": 0.8904413, + "num_input_tokens_seen": 21721375, + "router_z_loss_clip": 1.63476562, + "router_z_loss_mlp": 0.29321289, + "step": 784, + "time_per_iteration": 2.9259445667266846 + }, + { + "auxiliary_loss_clip": 0.0124293, + "auxiliary_loss_mlp": 0.01073461, + "balance_loss_clip": 1.0832932, + "balance_loss_mlp": 1.0468061, + "epoch": 0.02277871278509663, + "flos": 14641211537280.0, + "grad_norm": 2.822132217598126, + "language_loss": 0.87304771, + "learning_rate": 3.841233946859129e-06, + "loss": 0.8962115, + "num_input_tokens_seen": 21733045, + "router_z_loss_clip": 1.59863281, + "router_z_loss_mlp": 0.26623535, + "step": 785, + "time_per_iteration": 2.543243885040283 + }, + { + "auxiliary_loss_clip": 0.01252575, + "auxiliary_loss_mlp": 0.0107531, + "balance_loss_clip": 1.08312774, + "balance_loss_mlp": 1.04610324, + "epoch": 0.022807730253612674, + "flos": 30518360196480.0, + "grad_norm": 2.5803735346086727, + "language_loss": 0.89575052, + "learning_rate": 3.8419675816604806e-06, + "loss": 0.91902936, + "num_input_tokens_seen": 21748950, + "router_z_loss_clip": 1.69433594, + "router_z_loss_mlp": 0.29174805, + "step": 786, + "time_per_iteration": 2.5510594844818115 + }, + { + "auxiliary_loss_clip": 0.0106292, + "auxiliary_loss_mlp": 0.01015214, + "balance_loss_clip": 1.01953411, + "balance_loss_mlp": 1.01262712, + "epoch": 0.022836747722128722, + "flos": 56302078930560.0, + "grad_norm": 0.7542817031061361, + "language_loss": 0.49618551, + "learning_rate": 3.842700283677345e-06, + "loss": 0.51696688, + "num_input_tokens_seen": 21803175, + "router_z_loss_clip": 0.43359375, + "router_z_loss_mlp": 0.02587891, + "step": 787, + "time_per_iteration": 2.9671757221221924 + }, + { + "auxiliary_loss_clip": 0.01249144, + "auxiliary_loss_mlp": 0.01079726, + "balance_loss_clip": 1.08271456, + "balance_loss_mlp": 1.04968536, + "epoch": 0.022865765190644767, + "flos": 26462321452800.0, + "grad_norm": 2.5203431280186317, + "language_loss": 0.90399587, + "learning_rate": 3.8434320552787e-06, + "loss": 0.9272846, + "num_input_tokens_seen": 21818255, + "router_z_loss_clip": 1.66503906, + "router_z_loss_mlp": 0.30029297, + "step": 788, + "time_per_iteration": 9.508500814437866 + }, + { + "auxiliary_loss_clip": 0.01237973, + "auxiliary_loss_mlp": 0.01081519, + "balance_loss_clip": 1.08144951, + "balance_loss_mlp": 1.05512643, + "epoch": 0.022894782659160815, + "flos": 47368399703040.0, + "grad_norm": 4.005230382944941, + "language_loss": 0.90344906, + "learning_rate": 3.844162898824509e-06, + "loss": 0.92664385, + "num_input_tokens_seen": 21835000, + "router_z_loss_clip": 1.56445312, + "router_z_loss_mlp": 0.26391602, + "step": 789, + "time_per_iteration": 2.772644519805908 + }, + { + "auxiliary_loss_clip": 0.01238359, + "auxiliary_loss_mlp": 0.01073745, + "balance_loss_clip": 1.07739139, + "balance_loss_mlp": 1.04617202, + "epoch": 0.02292380012767686, + "flos": 43355131079040.0, + "grad_norm": 3.8275622051896327, + "language_loss": 0.83264798, + "learning_rate": 3.844892816665769e-06, + "loss": 0.85576904, + "num_input_tokens_seen": 21851945, + "router_z_loss_clip": 1.61035156, + "router_z_loss_mlp": 0.27539062, + "step": 790, + "time_per_iteration": 5.104602813720703 + }, + { + "auxiliary_loss_clip": 0.01248658, + "auxiliary_loss_mlp": 0.01072774, + "balance_loss_clip": 1.08141923, + "balance_loss_mlp": 1.04299521, + "epoch": 0.022952817596192908, + "flos": 22813151040000.0, + "grad_norm": 2.9889650401517884, + "language_loss": 0.91918612, + "learning_rate": 3.845621811144555e-06, + "loss": 0.94240046, + "num_input_tokens_seen": 21865095, + "router_z_loss_clip": 1.671875, + "router_z_loss_mlp": 0.29797363, + "step": 791, + "time_per_iteration": 2.453662633895874 + }, + { + "auxiliary_loss_clip": 0.01235451, + "auxiliary_loss_mlp": 0.01068209, + "balance_loss_clip": 1.07936645, + "balance_loss_mlp": 1.04279315, + "epoch": 0.022981835064708956, + "flos": 31132072995840.0, + "grad_norm": 2.3385989873491826, + "language_loss": 0.76650238, + "learning_rate": 3.846349884594063e-06, + "loss": 0.78953898, + "num_input_tokens_seen": 21880565, + "router_z_loss_clip": 1.56054688, + "router_z_loss_mlp": 0.25390625, + "step": 792, + "time_per_iteration": 2.620572805404663 + }, + { + "auxiliary_loss_clip": 0.0125127, + "auxiliary_loss_mlp": 0.01071651, + "balance_loss_clip": 1.08191061, + "balance_loss_mlp": 1.04435205, + "epoch": 0.023010852533225, + "flos": 17744216676480.0, + "grad_norm": 2.732464610112242, + "language_loss": 0.88285339, + "learning_rate": 3.847077039338659e-06, + "loss": 0.90608257, + "num_input_tokens_seen": 21896470, + "router_z_loss_clip": 1.69433594, + "router_z_loss_mlp": 0.27307129, + "step": 793, + "time_per_iteration": 2.5586185455322266 + }, + { + "auxiliary_loss_clip": 0.01238837, + "auxiliary_loss_mlp": 0.01079483, + "balance_loss_clip": 1.08146763, + "balance_loss_mlp": 1.05452037, + "epoch": 0.02303987000174105, + "flos": 12268773118080.0, + "grad_norm": 4.314396856173407, + "language_loss": 1.29319859, + "learning_rate": 3.847803277693921e-06, + "loss": 1.31638169, + "num_input_tokens_seen": 21907555, + "router_z_loss_clip": 1.57324219, + "router_z_loss_mlp": 0.24975586, + "step": 794, + "time_per_iteration": 2.4872756004333496 + }, + { + "auxiliary_loss_clip": 0.01255182, + "auxiliary_loss_mlp": 0.01073576, + "balance_loss_clip": 1.08835363, + "balance_loss_mlp": 1.04651594, + "epoch": 0.023068887470257093, + "flos": 25627757880960.0, + "grad_norm": 2.7142719322723274, + "language_loss": 0.75479937, + "learning_rate": 3.848528601966682e-06, + "loss": 0.7780869, + "num_input_tokens_seen": 21930305, + "router_z_loss_clip": 1.66796875, + "router_z_loss_mlp": 0.27050781, + "step": 795, + "time_per_iteration": 2.658092498779297 + }, + { + "auxiliary_loss_clip": 0.01254353, + "auxiliary_loss_mlp": 0.01067515, + "balance_loss_clip": 1.0870893, + "balance_loss_mlp": 1.04072893, + "epoch": 0.02309790493877314, + "flos": 25075702776960.0, + "grad_norm": 2.67801865669879, + "language_loss": 0.93030334, + "learning_rate": 3.849253014455075e-06, + "loss": 0.95352203, + "num_input_tokens_seen": 21946535, + "router_z_loss_clip": 1.67285156, + "router_z_loss_mlp": 0.26782227, + "step": 796, + "time_per_iteration": 2.65147066116333 + }, + { + "auxiliary_loss_clip": 0.01246815, + "auxiliary_loss_mlp": 0.01078705, + "balance_loss_clip": 1.08064222, + "balance_loss_mlp": 1.04902172, + "epoch": 0.02312692240728919, + "flos": 18904561626240.0, + "grad_norm": 4.8828375865310605, + "language_loss": 1.17668951, + "learning_rate": 3.84997651744858e-06, + "loss": 1.19994473, + "num_input_tokens_seen": 21956120, + "router_z_loss_clip": 1.66308594, + "router_z_loss_mlp": 0.29699707, + "step": 797, + "time_per_iteration": 2.5236854553222656 + }, + { + "auxiliary_loss_clip": 0.01234971, + "auxiliary_loss_mlp": 0.01065527, + "balance_loss_clip": 1.0803721, + "balance_loss_mlp": 1.03732181, + "epoch": 0.023155939875805234, + "flos": 32884908785280.0, + "grad_norm": 1.995097276824581, + "language_loss": 0.82997525, + "learning_rate": 3.850699113228063e-06, + "loss": 0.85298026, + "num_input_tokens_seen": 21980765, + "router_z_loss_clip": 1.54589844, + "router_z_loss_mlp": 0.28210449, + "step": 798, + "time_per_iteration": 2.825413703918457 + }, + { + "auxiliary_loss_clip": 0.01263851, + "auxiliary_loss_mlp": 0.0107887, + "balance_loss_clip": 1.08688855, + "balance_loss_mlp": 1.04848337, + "epoch": 0.023184957344321282, + "flos": 36057867661440.0, + "grad_norm": 3.2837125332525527, + "language_loss": 0.95198941, + "learning_rate": 3.851420804065818e-06, + "loss": 0.97541666, + "num_input_tokens_seen": 21997965, + "router_z_loss_clip": 1.76953125, + "router_z_loss_mlp": 0.30395508, + "step": 799, + "time_per_iteration": 2.621530294418335 + }, + { + "auxiliary_loss_clip": 0.01249916, + "auxiliary_loss_mlp": 0.01061618, + "balance_loss_clip": 1.0819546, + "balance_loss_mlp": 1.0339731, + "epoch": 0.023213974812837327, + "flos": 26792376549120.0, + "grad_norm": 2.5101508226083626, + "language_loss": 0.98421955, + "learning_rate": 3.8521415922256166e-06, + "loss": 1.00733495, + "num_input_tokens_seen": 22017260, + "router_z_loss_clip": 1.67871094, + "router_z_loss_mlp": 0.27636719, + "step": 800, + "time_per_iteration": 2.5772271156311035 + }, + { + "auxiliary_loss_clip": 0.01254333, + "auxiliary_loss_mlp": 0.01075749, + "balance_loss_clip": 1.09164834, + "balance_loss_mlp": 1.04961836, + "epoch": 0.023242992281353375, + "flos": 34454779672320.0, + "grad_norm": 2.512568003584646, + "language_loss": 0.97800672, + "learning_rate": 3.852861479962747e-06, + "loss": 1.00130761, + "num_input_tokens_seen": 22034900, + "router_z_loss_clip": 1.62695312, + "router_z_loss_mlp": 0.26147461, + "step": 801, + "time_per_iteration": 2.5959880352020264 + }, + { + "auxiliary_loss_clip": 0.01235641, + "auxiliary_loss_mlp": 0.01064148, + "balance_loss_clip": 1.07906628, + "balance_loss_mlp": 1.03872085, + "epoch": 0.02327200974986942, + "flos": 12014992552320.0, + "grad_norm": 3.667759978019147, + "language_loss": 0.90583044, + "learning_rate": 3.853580469524051e-06, + "loss": 0.9288283, + "num_input_tokens_seen": 22047165, + "router_z_loss_clip": 1.56445312, + "router_z_loss_mlp": 0.25463867, + "step": 802, + "time_per_iteration": 2.4833991527557373 + }, + { + "auxiliary_loss_clip": 0.01234123, + "auxiliary_loss_mlp": 0.01066029, + "balance_loss_clip": 1.07847095, + "balance_loss_mlp": 1.03888488, + "epoch": 0.023301027218385468, + "flos": 16208963521920.0, + "grad_norm": 7.359186230737842, + "language_loss": 0.83537132, + "learning_rate": 3.854298563147975e-06, + "loss": 0.85837281, + "num_input_tokens_seen": 22060420, + "router_z_loss_clip": 1.55761719, + "router_z_loss_mlp": 0.27124023, + "step": 803, + "time_per_iteration": 2.496732473373413 + }, + { + "auxiliary_loss_clip": 0.01068927, + "auxiliary_loss_mlp": 0.01014946, + "balance_loss_clip": 1.02370214, + "balance_loss_mlp": 1.01216817, + "epoch": 0.023330044686901516, + "flos": 74773486448640.0, + "grad_norm": 0.7201788058314236, + "language_loss": 0.54997289, + "learning_rate": 3.855015763064606e-06, + "loss": 0.57081157, + "num_input_tokens_seen": 22125790, + "router_z_loss_clip": 0.453125, + "router_z_loss_mlp": 0.02783203, + "step": 804, + "time_per_iteration": 3.135089159011841 + }, + { + "auxiliary_loss_clip": 0.01067621, + "auxiliary_loss_mlp": 0.0100924, + "balance_loss_clip": 1.0230124, + "balance_loss_mlp": 1.00635493, + "epoch": 0.02335906215541756, + "flos": 56781881827200.0, + "grad_norm": 0.788883106283692, + "language_loss": 0.56249952, + "learning_rate": 3.855732071495717e-06, + "loss": 0.58326805, + "num_input_tokens_seen": 22182250, + "router_z_loss_clip": 0.4453125, + "router_z_loss_mlp": 0.02880859, + "step": 805, + "time_per_iteration": 2.986851930618286 + }, + { + "auxiliary_loss_clip": 0.01068346, + "auxiliary_loss_mlp": 0.01004476, + "balance_loss_clip": 1.02362239, + "balance_loss_mlp": 1.00159109, + "epoch": 0.02338807962393361, + "flos": 60798059452800.0, + "grad_norm": 0.6872948435671788, + "language_loss": 0.5364964, + "learning_rate": 3.856447490654803e-06, + "loss": 0.55722463, + "num_input_tokens_seen": 22243370, + "router_z_loss_clip": 0.44726562, + "router_z_loss_mlp": 0.02880859, + "step": 806, + "time_per_iteration": 2.9917502403259277 + }, + { + "auxiliary_loss_clip": 0.01244707, + "auxiliary_loss_mlp": 0.01070698, + "balance_loss_clip": 1.08463132, + "balance_loss_mlp": 1.04415059, + "epoch": 0.023417097092449653, + "flos": 36385660200960.0, + "grad_norm": 2.1533778969902073, + "language_loss": 0.87524152, + "learning_rate": 3.85716202274713e-06, + "loss": 0.8983956, + "num_input_tokens_seen": 22263715, + "router_z_loss_clip": 1.59960938, + "router_z_loss_mlp": 0.26538086, + "step": 807, + "time_per_iteration": 2.697164297103882 + }, + { + "auxiliary_loss_clip": 0.01246362, + "auxiliary_loss_mlp": 0.01070088, + "balance_loss_clip": 1.08473015, + "balance_loss_mlp": 1.04636526, + "epoch": 0.0234461145609657, + "flos": 30621135985920.0, + "grad_norm": 3.0340836340526742, + "language_loss": 0.91993535, + "learning_rate": 3.857875669969765e-06, + "loss": 0.9430998, + "num_input_tokens_seen": 22278910, + "router_z_loss_clip": 1.6171875, + "router_z_loss_mlp": 0.23718262, + "step": 808, + "time_per_iteration": 2.568380117416382 + }, + { + "auxiliary_loss_clip": 0.01228032, + "auxiliary_loss_mlp": 0.0107246, + "balance_loss_clip": 1.08323956, + "balance_loss_mlp": 1.04820085, + "epoch": 0.02347513202948175, + "flos": 13436013479040.0, + "grad_norm": 2.9915393137123996, + "language_loss": 0.8813144, + "learning_rate": 3.858588434511628e-06, + "loss": 0.90431929, + "num_input_tokens_seen": 22290310, + "router_z_loss_clip": 1.44628906, + "router_z_loss_mlp": 0.24279785, + "step": 809, + "time_per_iteration": 2.486053466796875 + }, + { + "auxiliary_loss_clip": 0.01248781, + "auxiliary_loss_mlp": 0.01079931, + "balance_loss_clip": 1.08319783, + "balance_loss_mlp": 1.04842448, + "epoch": 0.023504149497997794, + "flos": 52439632536960.0, + "grad_norm": 3.951855753786646, + "language_loss": 1.2788856, + "learning_rate": 3.859300318553524e-06, + "loss": 1.30217278, + "num_input_tokens_seen": 22311180, + "router_z_loss_clip": 1.65429688, + "router_z_loss_mlp": 0.31506348, + "step": 810, + "time_per_iteration": 2.7993898391723633 + }, + { + "auxiliary_loss_clip": 0.0125553, + "auxiliary_loss_mlp": 0.01078397, + "balance_loss_clip": 1.08785188, + "balance_loss_mlp": 1.05087209, + "epoch": 0.023533166966513842, + "flos": 11867148172800.0, + "grad_norm": 24.083928573857868, + "language_loss": 0.8979851, + "learning_rate": 3.860011324268188e-06, + "loss": 0.92132443, + "num_input_tokens_seen": 22323300, + "router_z_loss_clip": 1.67675781, + "router_z_loss_mlp": 0.27526855, + "step": 811, + "time_per_iteration": 2.4966161251068115 + }, + { + "auxiliary_loss_clip": 0.01247362, + "auxiliary_loss_mlp": 0.01077304, + "balance_loss_clip": 1.08234644, + "balance_loss_mlp": 1.05100644, + "epoch": 0.023562184435029887, + "flos": 22851467873280.0, + "grad_norm": 2.2319405317454124, + "language_loss": 0.83975685, + "learning_rate": 3.860721453820318e-06, + "loss": 0.86300349, + "num_input_tokens_seen": 22340530, + "router_z_loss_clip": 1.64941406, + "router_z_loss_mlp": 0.26293945, + "step": 812, + "time_per_iteration": 2.509347438812256 + }, + { + "auxiliary_loss_clip": 0.01260316, + "auxiliary_loss_mlp": 0.01085448, + "balance_loss_clip": 1.08670616, + "balance_loss_mlp": 1.05533612, + "epoch": 0.023591201903545935, + "flos": 32819408334720.0, + "grad_norm": 2.452320036640301, + "language_loss": 0.87660408, + "learning_rate": 3.861430709366625e-06, + "loss": 0.90006173, + "num_input_tokens_seen": 22365065, + "router_z_loss_clip": 1.73535156, + "router_z_loss_mlp": 0.30126953, + "step": 813, + "time_per_iteration": 2.739375591278076 + }, + { + "auxiliary_loss_clip": 0.01250948, + "auxiliary_loss_mlp": 0.01097889, + "balance_loss_clip": 1.08891749, + "balance_loss_mlp": 1.07168663, + "epoch": 0.02362021937206198, + "flos": 32738177727360.0, + "grad_norm": 2.062704000175503, + "language_loss": 0.86644018, + "learning_rate": 3.8621390930558644e-06, + "loss": 0.88992846, + "num_input_tokens_seen": 22381910, + "router_z_loss_clip": 1.6171875, + "router_z_loss_mlp": 0.26196289, + "step": 814, + "time_per_iteration": 2.6085174083709717 + }, + { + "auxiliary_loss_clip": 0.01078171, + "auxiliary_loss_mlp": 0.01107944, + "balance_loss_clip": 1.03163576, + "balance_loss_mlp": 1.10467815, + "epoch": 0.023649236840578028, + "flos": 59088065610240.0, + "grad_norm": 0.7280761544821661, + "language_loss": 0.56430256, + "learning_rate": 3.862846607028876e-06, + "loss": 0.5861637, + "num_input_tokens_seen": 22442230, + "router_z_loss_clip": 0.46484375, + "router_z_loss_mlp": 0.03271484, + "step": 815, + "time_per_iteration": 3.1509642601013184 + }, + { + "auxiliary_loss_clip": 0.01244303, + "auxiliary_loss_mlp": 0.01069317, + "balance_loss_clip": 1.08769071, + "balance_loss_mlp": 1.04249513, + "epoch": 0.023678254309094076, + "flos": 21099098960640.0, + "grad_norm": 2.745214600485115, + "language_loss": 1.00926685, + "learning_rate": 3.863553253418625e-06, + "loss": 1.03240299, + "num_input_tokens_seen": 22455850, + "router_z_loss_clip": 1.56445312, + "router_z_loss_mlp": 0.26855469, + "step": 816, + "time_per_iteration": 2.5530571937561035 + }, + { + "auxiliary_loss_clip": 0.01253593, + "auxiliary_loss_mlp": 0.01063235, + "balance_loss_clip": 1.08733439, + "balance_loss_mlp": 1.03383768, + "epoch": 0.02370727177761012, + "flos": 34855327209600.0, + "grad_norm": 2.6625302509787763, + "language_loss": 0.83353806, + "learning_rate": 3.86425903435024e-06, + "loss": 0.85670632, + "num_input_tokens_seen": 22473240, + "router_z_loss_clip": 1.6640625, + "router_z_loss_mlp": 0.29418945, + "step": 817, + "time_per_iteration": 2.648815393447876 + }, + { + "auxiliary_loss_clip": 0.01254404, + "auxiliary_loss_mlp": 0.01066639, + "balance_loss_clip": 1.0877068, + "balance_loss_mlp": 1.03899479, + "epoch": 0.02373628924612617, + "flos": 25659143389440.0, + "grad_norm": 2.9175166725788304, + "language_loss": 0.75920057, + "learning_rate": 3.864963951941051e-06, + "loss": 0.78241098, + "num_input_tokens_seen": 22490925, + "router_z_loss_clip": 1.66503906, + "router_z_loss_mlp": 0.27685547, + "step": 818, + "time_per_iteration": 2.493804454803467 + }, + { + "auxiliary_loss_clip": 0.01240663, + "auxiliary_loss_mlp": 0.01069725, + "balance_loss_clip": 1.08228564, + "balance_loss_mlp": 1.04302216, + "epoch": 0.023765306714642213, + "flos": 11905213610880.0, + "grad_norm": 5.897911003946948, + "language_loss": 0.7740798, + "learning_rate": 3.8656680083006265e-06, + "loss": 0.79718363, + "num_input_tokens_seen": 22502800, + "router_z_loss_clip": 1.58203125, + "router_z_loss_mlp": 0.26708984, + "step": 819, + "time_per_iteration": 2.5052380561828613 + }, + { + "auxiliary_loss_clip": 0.01068553, + "auxiliary_loss_mlp": 0.01009806, + "balance_loss_clip": 1.02441573, + "balance_loss_mlp": 1.00706446, + "epoch": 0.02379432418315826, + "flos": 74776503191040.0, + "grad_norm": 0.6487380010260556, + "language_loss": 0.5319711, + "learning_rate": 3.866371205530811e-06, + "loss": 0.55275464, + "num_input_tokens_seen": 22573000, + "router_z_loss_clip": 0.44140625, + "router_z_loss_mlp": 0.02746582, + "step": 820, + "time_per_iteration": 3.3893234729766846 + }, + { + "auxiliary_loss_clip": 0.01231238, + "auxiliary_loss_mlp": 0.01066102, + "balance_loss_clip": 1.07937932, + "balance_loss_mlp": 1.04193854, + "epoch": 0.02382334165167431, + "flos": 15077023251840.0, + "grad_norm": 3.301752688644287, + "language_loss": 0.99738324, + "learning_rate": 3.86707354572577e-06, + "loss": 1.02035666, + "num_input_tokens_seen": 22584610, + "router_z_loss_clip": 1.51953125, + "router_z_loss_mlp": 0.24169922, + "step": 821, + "time_per_iteration": 2.6187894344329834 + }, + { + "auxiliary_loss_clip": 0.0124535, + "auxiliary_loss_mlp": 0.01089051, + "balance_loss_clip": 1.08502042, + "balance_loss_mlp": 1.0598681, + "epoch": 0.023852359120190354, + "flos": 10185379441920.0, + "grad_norm": 3.904009668165683, + "language_loss": 0.95653236, + "learning_rate": 3.867775030972013e-06, + "loss": 0.9798764, + "num_input_tokens_seen": 22593075, + "router_z_loss_clip": 1.60351562, + "router_z_loss_mlp": 0.29223633, + "step": 822, + "time_per_iteration": 2.5212459564208984 + }, + { + "auxiliary_loss_clip": 0.01243544, + "auxiliary_loss_mlp": 0.0107547, + "balance_loss_clip": 1.08341599, + "balance_loss_mlp": 1.04502344, + "epoch": 0.023881376588706402, + "flos": 12195084366720.0, + "grad_norm": 5.961413106412533, + "language_loss": 0.94910973, + "learning_rate": 3.868475663348448e-06, + "loss": 0.97229987, + "num_input_tokens_seen": 22604130, + "router_z_loss_clip": 1.60058594, + "router_z_loss_mlp": 0.3046875, + "step": 823, + "time_per_iteration": 2.47965145111084 + }, + { + "auxiliary_loss_clip": 0.010669, + "auxiliary_loss_mlp": 0.01018279, + "balance_loss_clip": 1.02201617, + "balance_loss_mlp": 1.01578796, + "epoch": 0.023910394057222447, + "flos": 60431088153600.0, + "grad_norm": 0.7100289834821197, + "language_loss": 0.55353123, + "learning_rate": 3.8691754449264e-06, + "loss": 0.57438308, + "num_input_tokens_seen": 22662890, + "router_z_loss_clip": 0.44921875, + "router_z_loss_mlp": 0.02490234, + "step": 824, + "time_per_iteration": 3.0353453159332275 + }, + { + "auxiliary_loss_clip": 0.01241503, + "auxiliary_loss_mlp": 0.01080335, + "balance_loss_clip": 1.08080244, + "balance_loss_mlp": 1.04976964, + "epoch": 0.023939411525738495, + "flos": 24497038673280.0, + "grad_norm": 2.745292108103284, + "language_loss": 0.81234539, + "learning_rate": 3.869874377769666e-06, + "loss": 0.83556378, + "num_input_tokens_seen": 22678465, + "router_z_loss_clip": 1.60644531, + "router_z_loss_mlp": 0.30578613, + "step": 825, + "time_per_iteration": 2.5989158153533936 + }, + { + "auxiliary_loss_clip": 0.01240407, + "auxiliary_loss_mlp": 0.01073907, + "balance_loss_clip": 1.08180737, + "balance_loss_mlp": 1.04788399, + "epoch": 0.02396842899425454, + "flos": 16865159132160.0, + "grad_norm": 2.683283257507046, + "language_loss": 0.93949574, + "learning_rate": 3.870572463934538e-06, + "loss": 0.96263891, + "num_input_tokens_seen": 22691670, + "router_z_loss_clip": 1.5859375, + "router_z_loss_mlp": 0.26037598, + "step": 826, + "time_per_iteration": 2.5393598079681396 + }, + { + "auxiliary_loss_clip": 0.01249814, + "auxiliary_loss_mlp": 0.01065661, + "balance_loss_clip": 1.0862987, + "balance_loss_mlp": 1.035954, + "epoch": 0.023997446462770588, + "flos": 11466097845120.0, + "grad_norm": 3.300536118159373, + "language_loss": 1.03730464, + "learning_rate": 3.871269705469845e-06, + "loss": 1.06045938, + "num_input_tokens_seen": 22702735, + "router_z_loss_clip": 1.63183594, + "router_z_loss_mlp": 0.29724121, + "step": 827, + "time_per_iteration": 2.443286418914795 + }, + { + "auxiliary_loss_clip": 0.01227499, + "auxiliary_loss_mlp": 0.01065978, + "balance_loss_clip": 1.07978201, + "balance_loss_mlp": 1.0432446, + "epoch": 0.024026463931286636, + "flos": 33759225734400.0, + "grad_norm": 3.017078258939447, + "language_loss": 0.70780683, + "learning_rate": 3.871966104416989e-06, + "loss": 0.73074162, + "num_input_tokens_seen": 22717825, + "router_z_loss_clip": 1.4765625, + "router_z_loss_mlp": 0.22729492, + "step": 828, + "time_per_iteration": 2.6898019313812256 + }, + { + "auxiliary_loss_clip": 0.01236713, + "auxiliary_loss_mlp": 0.01080047, + "balance_loss_clip": 1.0784924, + "balance_loss_mlp": 1.05237818, + "epoch": 0.02405548139980268, + "flos": 35327085459840.0, + "grad_norm": 2.4709143758669225, + "language_loss": 0.92395043, + "learning_rate": 3.872661662809979e-06, + "loss": 0.94711804, + "num_input_tokens_seen": 22735085, + "router_z_loss_clip": 1.58300781, + "router_z_loss_mlp": 0.27648926, + "step": 829, + "time_per_iteration": 2.628223180770874 + }, + { + "auxiliary_loss_clip": 0.01236296, + "auxiliary_loss_mlp": 0.01073171, + "balance_loss_clip": 1.08521485, + "balance_loss_mlp": 1.04767251, + "epoch": 0.02408449886831873, + "flos": 64704275591040.0, + "grad_norm": 1.9250924057470824, + "language_loss": 0.54731339, + "learning_rate": 3.873356382675468e-06, + "loss": 0.57040811, + "num_input_tokens_seen": 22760180, + "router_z_loss_clip": 1.51269531, + "router_z_loss_mlp": 0.25488281, + "step": 830, + "time_per_iteration": 2.8522591590881348 + }, + { + "auxiliary_loss_clip": 0.01237975, + "auxiliary_loss_mlp": 0.01077951, + "balance_loss_clip": 1.08137882, + "balance_loss_mlp": 1.05177259, + "epoch": 0.024113516336834773, + "flos": 25331781813120.0, + "grad_norm": 2.3581194291659497, + "language_loss": 0.860888, + "learning_rate": 3.87405026603279e-06, + "loss": 0.88404727, + "num_input_tokens_seen": 22776350, + "router_z_loss_clip": 1.56835938, + "router_z_loss_mlp": 0.26196289, + "step": 831, + "time_per_iteration": 2.5503923892974854 + }, + { + "auxiliary_loss_clip": 0.01244239, + "auxiliary_loss_mlp": 0.01073627, + "balance_loss_clip": 1.08672547, + "balance_loss_mlp": 1.04791403, + "epoch": 0.02414253380535082, + "flos": 26244128286720.0, + "grad_norm": 1.8252384411410263, + "language_loss": 0.66717786, + "learning_rate": 3.8747433148939905e-06, + "loss": 0.69035649, + "num_input_tokens_seen": 22794025, + "router_z_loss_clip": 1.57421875, + "router_z_loss_mlp": 0.25683594, + "step": 832, + "time_per_iteration": 2.635237455368042 + }, + { + "auxiliary_loss_clip": 0.01238836, + "auxiliary_loss_mlp": 0.01083049, + "balance_loss_clip": 1.08241045, + "balance_loss_mlp": 1.05884957, + "epoch": 0.024171551273866866, + "flos": 32737064405760.0, + "grad_norm": 2.6600582011112284, + "language_loss": 0.83006889, + "learning_rate": 3.875435531263866e-06, + "loss": 0.85328776, + "num_input_tokens_seen": 22809030, + "router_z_loss_clip": 1.56640625, + "router_z_loss_mlp": 0.24194336, + "step": 833, + "time_per_iteration": 2.6421093940734863 + }, + { + "auxiliary_loss_clip": 0.01070974, + "auxiliary_loss_mlp": 0.01091563, + "balance_loss_clip": 1.02738047, + "balance_loss_mlp": 1.08877361, + "epoch": 0.024200568742382914, + "flos": 74772409040640.0, + "grad_norm": 0.7439113082522958, + "language_loss": 0.5330478, + "learning_rate": 3.876126917139997e-06, + "loss": 0.55467319, + "num_input_tokens_seen": 22866415, + "router_z_loss_clip": 0.43554688, + "router_z_loss_mlp": 0.0279541, + "step": 834, + "time_per_iteration": 3.1569159030914307 + }, + { + "auxiliary_loss_clip": 0.01240971, + "auxiliary_loss_mlp": 0.01062, + "balance_loss_clip": 1.08395028, + "balance_loss_mlp": 1.03755045, + "epoch": 0.024229586210898962, + "flos": 17888002819200.0, + "grad_norm": 2.102982930946918, + "language_loss": 0.66979373, + "learning_rate": 3.876817474512782e-06, + "loss": 0.69282341, + "num_input_tokens_seen": 22880520, + "router_z_loss_clip": 1.56835938, + "router_z_loss_mlp": 0.24462891, + "step": 835, + "time_per_iteration": 2.6251587867736816 + }, + { + "auxiliary_loss_clip": 0.01238547, + "auxiliary_loss_mlp": 0.01073026, + "balance_loss_clip": 1.08506036, + "balance_loss_mlp": 1.04609692, + "epoch": 0.024258603679415007, + "flos": 27668561005440.0, + "grad_norm": 2.354150590000842, + "language_loss": 0.90512919, + "learning_rate": 3.8775072053654756e-06, + "loss": 0.92824495, + "num_input_tokens_seen": 22898465, + "router_z_loss_clip": 1.53417969, + "router_z_loss_mlp": 0.26928711, + "step": 836, + "time_per_iteration": 2.5729899406433105 + }, + { + "auxiliary_loss_clip": 0.01264142, + "auxiliary_loss_mlp": 0.01079755, + "balance_loss_clip": 1.09173179, + "balance_loss_mlp": 1.05158615, + "epoch": 0.024287621147931055, + "flos": 22778353739520.0, + "grad_norm": 2.5559473462782547, + "language_loss": 1.05865121, + "learning_rate": 3.878196111674215e-06, + "loss": 1.08209014, + "num_input_tokens_seen": 22914685, + "router_z_loss_clip": 1.7265625, + "router_z_loss_mlp": 0.28161621, + "step": 837, + "time_per_iteration": 2.6041030883789062 + }, + { + "auxiliary_loss_clip": 0.01249661, + "auxiliary_loss_mlp": 0.01078787, + "balance_loss_clip": 1.08574605, + "balance_loss_mlp": 1.05008161, + "epoch": 0.0243166386164471, + "flos": 15405139013760.0, + "grad_norm": 2.8620255265676096, + "language_loss": 0.9637965, + "learning_rate": 3.878884195408061e-06, + "loss": 0.98708099, + "num_input_tokens_seen": 22927160, + "router_z_loss_clip": 1.63769531, + "router_z_loss_mlp": 0.28723145, + "step": 838, + "time_per_iteration": 2.548220634460449 + }, + { + "auxiliary_loss_clip": 0.01254998, + "auxiliary_loss_mlp": 0.0106862, + "balance_loss_clip": 1.08632731, + "balance_loss_mlp": 1.03970039, + "epoch": 0.024345656084963148, + "flos": 11358330065280.0, + "grad_norm": 3.0146643032456044, + "language_loss": 0.84281838, + "learning_rate": 3.879571458529031e-06, + "loss": 0.86605453, + "num_input_tokens_seen": 22941175, + "router_z_loss_clip": 1.68652344, + "router_z_loss_mlp": 0.28918457, + "step": 839, + "time_per_iteration": 2.5876309871673584 + }, + { + "auxiliary_loss_clip": 0.01070572, + "auxiliary_loss_mlp": 0.01017364, + "balance_loss_clip": 1.02638876, + "balance_loss_mlp": 1.01474094, + "epoch": 0.024374673553479196, + "flos": 58315087906560.0, + "grad_norm": 0.7000163524239059, + "language_loss": 0.52421224, + "learning_rate": 3.88025790299213e-06, + "loss": 0.54509151, + "num_input_tokens_seen": 22992970, + "router_z_loss_clip": 0.44140625, + "router_z_loss_mlp": 0.02624512, + "step": 840, + "time_per_iteration": 3.0064616203308105 + }, + { + "auxiliary_loss_clip": 0.01239158, + "auxiliary_loss_mlp": 0.01067319, + "balance_loss_clip": 1.08353019, + "balance_loss_mlp": 1.03917336, + "epoch": 0.02440369102199524, + "flos": 10443721034880.0, + "grad_norm": 3.748322137849079, + "language_loss": 0.98759449, + "learning_rate": 3.880943530745382e-06, + "loss": 1.01065922, + "num_input_tokens_seen": 23004650, + "router_z_loss_clip": 1.55859375, + "router_z_loss_mlp": 0.28149414, + "step": 841, + "time_per_iteration": 2.5720629692077637 + }, + { + "auxiliary_loss_clip": 0.01240684, + "auxiliary_loss_mlp": 0.01081909, + "balance_loss_clip": 1.08181071, + "balance_loss_mlp": 1.05396652, + "epoch": 0.02443270849051129, + "flos": 19493497019520.0, + "grad_norm": 3.1080390389075503, + "language_loss": 1.01920915, + "learning_rate": 3.881628343729871e-06, + "loss": 1.04243505, + "num_input_tokens_seen": 23017855, + "router_z_loss_clip": 1.58886719, + "router_z_loss_mlp": 0.27954102, + "step": 842, + "time_per_iteration": 2.5633978843688965 + }, + { + "auxiliary_loss_clip": 0.01248843, + "auxiliary_loss_mlp": 0.01090495, + "balance_loss_clip": 1.08560371, + "balance_loss_mlp": 1.06281495, + "epoch": 0.024461725959027333, + "flos": 18726445059840.0, + "grad_norm": 2.731930472292028, + "language_loss": 0.89197481, + "learning_rate": 3.882312343879765e-06, + "loss": 0.9153682, + "num_input_tokens_seen": 23030935, + "router_z_loss_clip": 1.6328125, + "router_z_loss_mlp": 0.2767334, + "step": 843, + "time_per_iteration": 2.6169281005859375 + }, + { + "auxiliary_loss_clip": 0.01241052, + "auxiliary_loss_mlp": 0.01071778, + "balance_loss_clip": 1.08301711, + "balance_loss_mlp": 1.04645765, + "epoch": 0.02449074342754338, + "flos": 32852481782400.0, + "grad_norm": 2.4133095456367673, + "language_loss": 0.97994977, + "learning_rate": 3.882995533122357e-06, + "loss": 1.0030781, + "num_input_tokens_seen": 23051470, + "router_z_loss_clip": 1.58007812, + "router_z_loss_mlp": 0.25354004, + "step": 844, + "time_per_iteration": 2.662419319152832 + }, + { + "auxiliary_loss_clip": 0.01245801, + "auxiliary_loss_mlp": 0.01069686, + "balance_loss_clip": 1.08254635, + "balance_loss_mlp": 1.04231596, + "epoch": 0.024519760896059426, + "flos": 16974686678400.0, + "grad_norm": 3.69653393582009, + "language_loss": 1.00761163, + "learning_rate": 3.88367791337809e-06, + "loss": 1.03076649, + "num_input_tokens_seen": 23062730, + "router_z_loss_clip": 1.6328125, + "router_z_loss_mlp": 0.27355957, + "step": 845, + "time_per_iteration": 2.528977155685425 + }, + { + "auxiliary_loss_clip": 0.01070389, + "auxiliary_loss_mlp": 0.0100661, + "balance_loss_clip": 1.02644777, + "balance_loss_mlp": 1.00401115, + "epoch": 0.024548778364575474, + "flos": 53322499900800.0, + "grad_norm": 0.7178789704942669, + "language_loss": 0.52525443, + "learning_rate": 3.884359486560594e-06, + "loss": 0.54602438, + "num_input_tokens_seen": 23118095, + "router_z_loss_clip": 0.43945312, + "router_z_loss_mlp": 0.02600098, + "step": 846, + "time_per_iteration": 3.0204224586486816 + }, + { + "auxiliary_loss_clip": 0.01246191, + "auxiliary_loss_mlp": 0.0106269, + "balance_loss_clip": 1.09235835, + "balance_loss_mlp": 1.03661907, + "epoch": 0.024577795833091522, + "flos": 13619768480640.0, + "grad_norm": 2.6915051625054307, + "language_loss": 0.83592725, + "learning_rate": 3.885040254576717e-06, + "loss": 0.85901606, + "num_input_tokens_seen": 23131945, + "router_z_loss_clip": 1.53808594, + "router_z_loss_mlp": 0.26098633, + "step": 847, + "time_per_iteration": 2.539546012878418 + }, + { + "auxiliary_loss_clip": 0.01245769, + "auxiliary_loss_mlp": 0.01078081, + "balance_loss_clip": 1.08574736, + "balance_loss_mlp": 1.05056763, + "epoch": 0.024606813301607567, + "flos": 32299995715200.0, + "grad_norm": 8.96711237361451, + "language_loss": 0.78855038, + "learning_rate": 3.885720219326559e-06, + "loss": 0.8117888, + "num_input_tokens_seen": 23146705, + "router_z_loss_clip": 1.60253906, + "router_z_loss_mlp": 0.27514648, + "step": 848, + "time_per_iteration": 2.6296589374542236 + }, + { + "auxiliary_loss_clip": 0.01071246, + "auxiliary_loss_mlp": 0.01001412, + "balance_loss_clip": 1.02862477, + "balance_loss_mlp": 0.99878979, + "epoch": 0.024635830770123615, + "flos": 59633440784640.0, + "grad_norm": 0.7457722455440348, + "language_loss": 0.5504899, + "learning_rate": 3.886399382703498e-06, + "loss": 0.57121646, + "num_input_tokens_seen": 23204840, + "router_z_loss_clip": 0.42578125, + "router_z_loss_mlp": 0.02624512, + "step": 849, + "time_per_iteration": 2.994229316711426 + }, + { + "auxiliary_loss_clip": 0.01069762, + "auxiliary_loss_mlp": 0.01001426, + "balance_loss_clip": 1.02751327, + "balance_loss_mlp": 0.9987914, + "epoch": 0.02466484823863966, + "flos": 68311540788480.0, + "grad_norm": 0.7539705107376002, + "language_loss": 0.54339939, + "learning_rate": 3.887077746594228e-06, + "loss": 0.56411129, + "num_input_tokens_seen": 23267380, + "router_z_loss_clip": 0.421875, + "router_z_loss_mlp": 0.02636719, + "step": 850, + "time_per_iteration": 3.050306558609009 + }, + { + "auxiliary_loss_clip": 0.01237501, + "auxiliary_loss_mlp": 0.01079764, + "balance_loss_clip": 1.08219433, + "balance_loss_mlp": 1.05073714, + "epoch": 0.024693865707155708, + "flos": 12268413982080.0, + "grad_norm": 3.2200301793642616, + "language_loss": 0.97411859, + "learning_rate": 3.88775531287879e-06, + "loss": 0.99729127, + "num_input_tokens_seen": 23277990, + "router_z_loss_clip": 1.55371094, + "router_z_loss_mlp": 0.2902832, + "step": 851, + "time_per_iteration": 2.473684549331665 + }, + { + "auxiliary_loss_clip": 0.01067606, + "auxiliary_loss_mlp": 0.01025787, + "balance_loss_clip": 1.02530813, + "balance_loss_mlp": 1.02315271, + "epoch": 0.024722883175671756, + "flos": 58501105464960.0, + "grad_norm": 0.8551415871998761, + "language_loss": 0.57438284, + "learning_rate": 3.888432083430597e-06, + "loss": 0.59531683, + "num_input_tokens_seen": 23335060, + "router_z_loss_clip": 0.421875, + "router_z_loss_mlp": 0.02636719, + "step": 852, + "time_per_iteration": 3.0955748558044434 + }, + { + "auxiliary_loss_clip": 0.01235958, + "auxiliary_loss_mlp": 0.0107367, + "balance_loss_clip": 1.0861249, + "balance_loss_mlp": 1.0491488, + "epoch": 0.0247519006441878, + "flos": 43317927567360.0, + "grad_norm": 3.0796550562804397, + "language_loss": 0.92183292, + "learning_rate": 3.889108060116473e-06, + "loss": 0.94492918, + "num_input_tokens_seen": 23351345, + "router_z_loss_clip": 1.49707031, + "router_z_loss_mlp": 0.24511719, + "step": 853, + "time_per_iteration": 2.730483293533325 + }, + { + "auxiliary_loss_clip": 0.01064905, + "auxiliary_loss_mlp": 0.01016619, + "balance_loss_clip": 1.02301049, + "balance_loss_mlp": 1.01403224, + "epoch": 0.02478091811270385, + "flos": 68928378071040.0, + "grad_norm": 0.7030418043285895, + "language_loss": 0.52304476, + "learning_rate": 3.889783244796675e-06, + "loss": 0.54385996, + "num_input_tokens_seen": 23408405, + "router_z_loss_clip": 0.41796875, + "router_z_loss_mlp": 0.02587891, + "step": 854, + "time_per_iteration": 3.0347251892089844 + }, + { + "auxiliary_loss_clip": 0.01249522, + "auxiliary_loss_mlp": 0.01079932, + "balance_loss_clip": 1.08880472, + "balance_loss_mlp": 1.05214465, + "epoch": 0.024809935581219893, + "flos": 21548055052800.0, + "grad_norm": 2.3519735067303755, + "language_loss": 0.83490682, + "learning_rate": 3.890457639324937e-06, + "loss": 0.85820138, + "num_input_tokens_seen": 23427245, + "router_z_loss_clip": 1.60742188, + "router_z_loss_mlp": 0.27758789, + "step": 855, + "time_per_iteration": 2.670048952102661 + }, + { + "auxiliary_loss_clip": 0.01251229, + "auxiliary_loss_mlp": 0.01066919, + "balance_loss_clip": 1.08858883, + "balance_loss_mlp": 1.03845191, + "epoch": 0.02483895304973594, + "flos": 16792152739200.0, + "grad_norm": 3.169886003789659, + "language_loss": 1.00161576, + "learning_rate": 3.891131245548486e-06, + "loss": 1.0247972, + "num_input_tokens_seen": 23440585, + "router_z_loss_clip": 1.62597656, + "router_z_loss_mlp": 0.28479004, + "step": 856, + "time_per_iteration": 2.551182985305786 + }, + { + "auxiliary_loss_clip": 0.01234465, + "auxiliary_loss_mlp": 0.01060098, + "balance_loss_clip": 1.08272326, + "balance_loss_mlp": 1.03477764, + "epoch": 0.024867970518251986, + "flos": 15881817427200.0, + "grad_norm": 5.786910573302485, + "language_loss": 0.86439848, + "learning_rate": 3.89180406530808e-06, + "loss": 0.88734412, + "num_input_tokens_seen": 23453110, + "router_z_loss_clip": 1.51660156, + "router_z_loss_mlp": 0.25305176, + "step": 857, + "time_per_iteration": 2.540032148361206 + }, + { + "auxiliary_loss_clip": 0.01242753, + "auxiliary_loss_mlp": 0.01067116, + "balance_loss_clip": 1.08028388, + "balance_loss_mlp": 1.03955483, + "epoch": 0.024896987986768034, + "flos": 11063180010240.0, + "grad_norm": 3.583359466099892, + "language_loss": 0.83883774, + "learning_rate": 3.892476100438039e-06, + "loss": 0.86193645, + "num_input_tokens_seen": 23466395, + "router_z_loss_clip": 1.62304688, + "router_z_loss_mlp": 0.2755127, + "step": 858, + "time_per_iteration": 4.889093637466431 + }, + { + "auxiliary_loss_clip": 0.01063868, + "auxiliary_loss_mlp": 0.01009902, + "balance_loss_clip": 1.02191806, + "balance_loss_mlp": 1.00726783, + "epoch": 0.024926005455284082, + "flos": 58359725533440.0, + "grad_norm": 0.7297670254627896, + "language_loss": 0.54202312, + "learning_rate": 3.8931473527662725e-06, + "loss": 0.56276077, + "num_input_tokens_seen": 23529420, + "router_z_loss_clip": 0.41796875, + "router_z_loss_mlp": 0.02636719, + "step": 859, + "time_per_iteration": 8.864942789077759 + }, + { + "auxiliary_loss_clip": 0.01246553, + "auxiliary_loss_mlp": 0.01063282, + "balance_loss_clip": 1.08363819, + "balance_loss_mlp": 1.03761613, + "epoch": 0.024955022923800127, + "flos": 25257985320960.0, + "grad_norm": 2.681982504144006, + "language_loss": 0.69391483, + "learning_rate": 3.893817824114308e-06, + "loss": 0.71701318, + "num_input_tokens_seen": 23544890, + "router_z_loss_clip": 1.62695312, + "router_z_loss_mlp": 0.25671387, + "step": 860, + "time_per_iteration": 2.7815465927124023 + }, + { + "auxiliary_loss_clip": 0.01061403, + "auxiliary_loss_mlp": 0.01009964, + "balance_loss_clip": 1.01950562, + "balance_loss_mlp": 1.00743675, + "epoch": 0.024984040392316175, + "flos": 73452332309760.0, + "grad_norm": 0.7377155722497727, + "language_loss": 0.48964834, + "learning_rate": 3.894487516297324e-06, + "loss": 0.51036203, + "num_input_tokens_seen": 23595790, + "router_z_loss_clip": 0.41796875, + "router_z_loss_mlp": 0.02526855, + "step": 861, + "time_per_iteration": 5.493698835372925 + }, + { + "auxiliary_loss_clip": 0.01230175, + "auxiliary_loss_mlp": 0.01069308, + "balance_loss_clip": 1.08105922, + "balance_loss_mlp": 1.04126978, + "epoch": 0.02501305786083222, + "flos": 12378587973120.0, + "grad_norm": 3.0469696755751645, + "language_loss": 0.91425896, + "learning_rate": 3.895156431124179e-06, + "loss": 0.93725377, + "num_input_tokens_seen": 23606470, + "router_z_loss_clip": 1.4921875, + "router_z_loss_mlp": 0.28063965, + "step": 862, + "time_per_iteration": 2.5400149822235107 + }, + { + "auxiliary_loss_clip": 0.01230347, + "auxiliary_loss_mlp": 0.01064952, + "balance_loss_clip": 1.08097863, + "balance_loss_mlp": 1.0401566, + "epoch": 0.025042075329348268, + "flos": 15222640988160.0, + "grad_norm": 2.666327634283882, + "language_loss": 0.76892781, + "learning_rate": 3.895824570397436e-06, + "loss": 0.79188085, + "num_input_tokens_seen": 23618450, + "router_z_loss_clip": 1.49316406, + "router_z_loss_mlp": 0.2479248, + "step": 863, + "time_per_iteration": 2.547414779663086 + }, + { + "auxiliary_loss_clip": 0.01239249, + "auxiliary_loss_mlp": 0.01094617, + "balance_loss_clip": 1.08654714, + "balance_loss_mlp": 1.06719923, + "epoch": 0.025071092797864316, + "flos": 10881400256640.0, + "grad_norm": 6.038272225704115, + "language_loss": 0.84653699, + "learning_rate": 3.896491935913401e-06, + "loss": 0.86987567, + "num_input_tokens_seen": 23629475, + "router_z_loss_clip": 1.5234375, + "router_z_loss_mlp": 0.27429199, + "step": 864, + "time_per_iteration": 2.540681838989258 + }, + { + "auxiliary_loss_clip": 0.01234154, + "auxiliary_loss_mlp": 0.01073566, + "balance_loss_clip": 1.0815587, + "balance_loss_mlp": 1.05003393, + "epoch": 0.02510011026638036, + "flos": 24418860721920.0, + "grad_norm": 3.3404515465487337, + "language_loss": 1.00210881, + "learning_rate": 3.897158529462142e-06, + "loss": 1.02518594, + "num_input_tokens_seen": 23646720, + "router_z_loss_clip": 1.52636719, + "router_z_loss_mlp": 0.23547363, + "step": 865, + "time_per_iteration": 2.6788129806518555 + }, + { + "auxiliary_loss_clip": 0.01228663, + "auxiliary_loss_mlp": 0.01061802, + "balance_loss_clip": 1.07904291, + "balance_loss_mlp": 1.03862762, + "epoch": 0.02512912773489641, + "flos": 29966233265280.0, + "grad_norm": 2.9374033420513976, + "language_loss": 0.79451859, + "learning_rate": 3.8978243528275245e-06, + "loss": 0.81742322, + "num_input_tokens_seen": 23665625, + "router_z_loss_clip": 1.49707031, + "router_z_loss_mlp": 0.23181152, + "step": 866, + "time_per_iteration": 2.690728187561035 + }, + { + "auxiliary_loss_clip": 0.0106825, + "auxiliary_loss_mlp": 0.01018532, + "balance_loss_clip": 1.0262959, + "balance_loss_mlp": 1.01582551, + "epoch": 0.025158145203412453, + "flos": 74777724253440.0, + "grad_norm": 0.6485707294660125, + "language_loss": 0.52879399, + "learning_rate": 3.898489407787237e-06, + "loss": 0.54966182, + "num_input_tokens_seen": 23736630, + "router_z_loss_clip": 0.41796875, + "router_z_loss_mlp": 0.02709961, + "step": 867, + "time_per_iteration": 3.4229581356048584 + }, + { + "auxiliary_loss_clip": 0.01258113, + "auxiliary_loss_mlp": 0.01083135, + "balance_loss_clip": 1.08365452, + "balance_loss_mlp": 1.05261731, + "epoch": 0.0251871626719285, + "flos": 32446798600320.0, + "grad_norm": 2.5547369259363486, + "language_loss": 0.83002949, + "learning_rate": 3.89915369611282e-06, + "loss": 0.85344195, + "num_input_tokens_seen": 23764165, + "router_z_loss_clip": 1.74511719, + "router_z_loss_mlp": 0.30517578, + "step": 868, + "time_per_iteration": 2.8863041400909424 + }, + { + "auxiliary_loss_clip": 0.01237883, + "auxiliary_loss_mlp": 0.01067258, + "balance_loss_clip": 1.0771327, + "balance_loss_mlp": 1.04117477, + "epoch": 0.025216180140444546, + "flos": 15992458295040.0, + "grad_norm": 10.975430528887971, + "language_loss": 0.85903275, + "learning_rate": 3.899817219569695e-06, + "loss": 0.88208425, + "num_input_tokens_seen": 23776725, + "router_z_loss_clip": 1.60839844, + "router_z_loss_mlp": 0.26074219, + "step": 869, + "time_per_iteration": 2.5519089698791504 + }, + { + "auxiliary_loss_clip": 0.01239068, + "auxiliary_loss_mlp": 0.01068688, + "balance_loss_clip": 1.08051443, + "balance_loss_mlp": 1.04246163, + "epoch": 0.025245197608960594, + "flos": 12560834603520.0, + "grad_norm": 3.518713845237544, + "language_loss": 0.97518897, + "learning_rate": 3.900479979917193e-06, + "loss": 0.99826658, + "num_input_tokens_seen": 23789535, + "router_z_loss_clip": 1.58691406, + "router_z_loss_mlp": 0.26220703, + "step": 870, + "time_per_iteration": 2.525245189666748 + }, + { + "auxiliary_loss_clip": 0.01240793, + "auxiliary_loss_mlp": 0.01078527, + "balance_loss_clip": 1.07928181, + "balance_loss_mlp": 1.05123973, + "epoch": 0.025274215077476642, + "flos": 47738998275840.0, + "grad_norm": 2.6611138057933075, + "language_loss": 0.90596378, + "learning_rate": 3.901141978908582e-06, + "loss": 0.92915696, + "num_input_tokens_seen": 23812190, + "router_z_loss_clip": 1.61425781, + "router_z_loss_mlp": 0.27282715, + "step": 871, + "time_per_iteration": 2.8097145557403564 + }, + { + "auxiliary_loss_clip": 0.01231394, + "auxiliary_loss_mlp": 0.01065805, + "balance_loss_clip": 1.07879436, + "balance_loss_mlp": 1.04019856, + "epoch": 0.025303232545992687, + "flos": 16405575592320.0, + "grad_norm": 2.80867963260008, + "language_loss": 0.9086926, + "learning_rate": 3.901803218291094e-06, + "loss": 0.93166459, + "num_input_tokens_seen": 23826655, + "router_z_loss_clip": 1.52734375, + "router_z_loss_mlp": 0.25610352, + "step": 872, + "time_per_iteration": 2.618051290512085 + }, + { + "auxiliary_loss_clip": 0.01236674, + "auxiliary_loss_mlp": 0.01067965, + "balance_loss_clip": 1.08487582, + "balance_loss_mlp": 1.04258537, + "epoch": 0.025332250014508735, + "flos": 47818145894400.0, + "grad_norm": 3.261677664279955, + "language_loss": 0.94279027, + "learning_rate": 3.902463699805952e-06, + "loss": 0.9658367, + "num_input_tokens_seen": 23849390, + "router_z_loss_clip": 1.51757812, + "router_z_loss_mlp": 0.25366211, + "step": 873, + "time_per_iteration": 2.8308966159820557 + }, + { + "auxiliary_loss_clip": 0.01238653, + "auxiliary_loss_mlp": 0.01068135, + "balance_loss_clip": 1.08499289, + "balance_loss_mlp": 1.04385209, + "epoch": 0.02536126748302478, + "flos": 20444052585600.0, + "grad_norm": 2.359520487274988, + "language_loss": 0.85696793, + "learning_rate": 3.903123425188401e-06, + "loss": 0.88003576, + "num_input_tokens_seen": 23865620, + "router_z_loss_clip": 1.53515625, + "router_z_loss_mlp": 0.24267578, + "step": 874, + "time_per_iteration": 2.563338279724121 + }, + { + "auxiliary_loss_clip": 0.01236052, + "auxiliary_loss_mlp": 0.01070963, + "balance_loss_clip": 1.07984424, + "balance_loss_mlp": 1.04372382, + "epoch": 0.025390284951540828, + "flos": 33941759673600.0, + "grad_norm": 2.5609415703140948, + "language_loss": 1.16369832, + "learning_rate": 3.903782396167732e-06, + "loss": 1.18676853, + "num_input_tokens_seen": 23882290, + "router_z_loss_clip": 1.56152344, + "router_z_loss_mlp": 0.27233887, + "step": 875, + "time_per_iteration": 2.6742823123931885 + }, + { + "auxiliary_loss_clip": 0.01061233, + "auxiliary_loss_mlp": 0.01006405, + "balance_loss_clip": 1.01989579, + "balance_loss_mlp": 1.00346041, + "epoch": 0.025419302420056876, + "flos": 59194863722880.0, + "grad_norm": 0.6583928335649651, + "language_loss": 0.54528022, + "learning_rate": 3.904440614467313e-06, + "loss": 0.56595659, + "num_input_tokens_seen": 23942490, + "router_z_loss_clip": 0.4140625, + "router_z_loss_mlp": 0.02941895, + "step": 876, + "time_per_iteration": 3.1518383026123047 + }, + { + "auxiliary_loss_clip": 0.01207995, + "auxiliary_loss_mlp": 0.0105056, + "balance_loss_clip": 1.0720365, + "balance_loss_mlp": 1.0287087, + "epoch": 0.02544831988857292, + "flos": 10771908624000.0, + "grad_norm": 3.1519276669926413, + "language_loss": 0.94345671, + "learning_rate": 3.905098081804608e-06, + "loss": 0.96604216, + "num_input_tokens_seen": 23954680, + "router_z_loss_clip": 1.36035156, + "router_z_loss_mlp": 0.21838379, + "step": 877, + "time_per_iteration": 2.5326766967773438 + }, + { + "auxiliary_loss_clip": 0.01241702, + "auxiliary_loss_mlp": 0.01063489, + "balance_loss_clip": 1.08840704, + "balance_loss_mlp": 1.03975451, + "epoch": 0.02547733735708897, + "flos": 10443864689280.0, + "grad_norm": 3.0651451133512753, + "language_loss": 0.98660898, + "learning_rate": 3.905754799891214e-06, + "loss": 1.00966084, + "num_input_tokens_seen": 23966875, + "router_z_loss_clip": 1.53027344, + "router_z_loss_mlp": 0.23742676, + "step": 878, + "time_per_iteration": 2.5206682682037354 + }, + { + "auxiliary_loss_clip": 0.01233175, + "auxiliary_loss_mlp": 0.01056915, + "balance_loss_clip": 1.08087182, + "balance_loss_mlp": 1.03110611, + "epoch": 0.025506354825605013, + "flos": 39231293414400.0, + "grad_norm": 2.8296792380713574, + "language_loss": 1.00205231, + "learning_rate": 3.9064107704328816e-06, + "loss": 1.02495337, + "num_input_tokens_seen": 23986835, + "router_z_loss_clip": 1.52246094, + "router_z_loss_mlp": 0.25817871, + "step": 879, + "time_per_iteration": 2.8013131618499756 + }, + { + "auxiliary_loss_clip": 0.01237035, + "auxiliary_loss_mlp": 0.01074238, + "balance_loss_clip": 1.08111787, + "balance_loss_mlp": 1.04446006, + "epoch": 0.02553537229412106, + "flos": 12561445134720.0, + "grad_norm": 3.255249055930344, + "language_loss": 1.07118106, + "learning_rate": 3.9070659951295425e-06, + "loss": 1.09429383, + "num_input_tokens_seen": 23998250, + "router_z_loss_clip": 1.55957031, + "router_z_loss_mlp": 0.29785156, + "step": 880, + "time_per_iteration": 2.6185708045959473 + }, + { + "auxiliary_loss_clip": 0.01238331, + "auxiliary_loss_mlp": 0.01079678, + "balance_loss_clip": 1.08119571, + "balance_loss_mlp": 1.05277288, + "epoch": 0.025564389762637106, + "flos": 16980432854400.0, + "grad_norm": 3.543989500902007, + "language_loss": 0.96733212, + "learning_rate": 3.907720475675338e-06, + "loss": 0.99051219, + "num_input_tokens_seen": 24014605, + "router_z_loss_clip": 1.57226562, + "router_z_loss_mlp": 0.26928711, + "step": 881, + "time_per_iteration": 2.564403533935547 + }, + { + "auxiliary_loss_clip": 0.01227083, + "auxiliary_loss_mlp": 0.01064494, + "balance_loss_clip": 1.08034146, + "balance_loss_mlp": 1.04024649, + "epoch": 0.025593407231153154, + "flos": 15842602753920.0, + "grad_norm": 3.196486848141009, + "language_loss": 0.74393451, + "learning_rate": 3.908374213758642e-06, + "loss": 0.76685023, + "num_input_tokens_seen": 24027300, + "router_z_loss_clip": 1.46777344, + "router_z_loss_mlp": 0.24243164, + "step": 882, + "time_per_iteration": 2.514450788497925 + }, + { + "auxiliary_loss_clip": 0.01062341, + "auxiliary_loss_mlp": 0.01004569, + "balance_loss_clip": 1.02139497, + "balance_loss_mlp": 1.00158882, + "epoch": 0.025622424699669202, + "flos": 59821720899840.0, + "grad_norm": 0.6952560989395623, + "language_loss": 0.55075771, + "learning_rate": 3.909027211062089e-06, + "loss": 0.57142681, + "num_input_tokens_seen": 24092200, + "router_z_loss_clip": 0.41015625, + "router_z_loss_mlp": 0.02978516, + "step": 883, + "time_per_iteration": 3.1586811542510986 + }, + { + "auxiliary_loss_clip": 0.01243532, + "auxiliary_loss_mlp": 0.01067793, + "balance_loss_clip": 1.08331382, + "balance_loss_mlp": 1.04126847, + "epoch": 0.025651442168185247, + "flos": 27265643170560.0, + "grad_norm": 2.490680743286625, + "language_loss": 0.91827309, + "learning_rate": 3.909679469262601e-06, + "loss": 0.9413864, + "num_input_tokens_seen": 24107905, + "router_z_loss_clip": 1.6015625, + "router_z_loss_mlp": 0.26538086, + "step": 884, + "time_per_iteration": 2.5592026710510254 + }, + { + "auxiliary_loss_clip": 0.01240232, + "auxiliary_loss_mlp": 0.01066416, + "balance_loss_clip": 1.0849787, + "balance_loss_mlp": 1.04177535, + "epoch": 0.025680459636701295, + "flos": 50074376837760.0, + "grad_norm": 2.174665261775655, + "language_loss": 0.91949034, + "learning_rate": 3.910330990031413e-06, + "loss": 0.94255686, + "num_input_tokens_seen": 24131570, + "router_z_loss_clip": 1.55078125, + "router_z_loss_mlp": 0.24621582, + "step": 885, + "time_per_iteration": 2.821596145629883 + }, + { + "auxiliary_loss_clip": 0.01243148, + "auxiliary_loss_mlp": 0.01071981, + "balance_loss_clip": 1.07908106, + "balance_loss_mlp": 1.04282212, + "epoch": 0.02570947710521734, + "flos": 17814206327040.0, + "grad_norm": 3.299617581821387, + "language_loss": 1.12964094, + "learning_rate": 3.910981775034096e-06, + "loss": 1.15279222, + "num_input_tokens_seen": 24143230, + "router_z_loss_clip": 1.640625, + "router_z_loss_mlp": 0.29162598, + "step": 886, + "time_per_iteration": 2.53300142288208 + }, + { + "auxiliary_loss_clip": 0.01225795, + "auxiliary_loss_mlp": 0.0106155, + "balance_loss_clip": 1.07793272, + "balance_loss_mlp": 1.03354764, + "epoch": 0.025738494573733388, + "flos": 16687976319360.0, + "grad_norm": 4.000583578984584, + "language_loss": 0.78568172, + "learning_rate": 3.911631825930584e-06, + "loss": 0.80855513, + "num_input_tokens_seen": 24158850, + "router_z_loss_clip": 1.47851562, + "router_z_loss_mlp": 0.2800293, + "step": 887, + "time_per_iteration": 2.5680854320526123 + }, + { + "auxiliary_loss_clip": 0.01226252, + "auxiliary_loss_mlp": 0.01075032, + "balance_loss_clip": 1.08340216, + "balance_loss_mlp": 1.04911637, + "epoch": 0.025767512042249432, + "flos": 27556411766400.0, + "grad_norm": 2.5172573020617763, + "language_loss": 0.95911264, + "learning_rate": 3.9122811443752026e-06, + "loss": 0.98212552, + "num_input_tokens_seen": 24173490, + "router_z_loss_clip": 1.42773438, + "router_z_loss_mlp": 0.25927734, + "step": 888, + "time_per_iteration": 2.615175724029541 + }, + { + "auxiliary_loss_clip": 0.01243881, + "auxiliary_loss_mlp": 0.01064797, + "balance_loss_clip": 1.08496952, + "balance_loss_mlp": 1.03953671, + "epoch": 0.02579652951076548, + "flos": 41094159540480.0, + "grad_norm": 2.806478179115039, + "language_loss": 0.91481316, + "learning_rate": 3.912929732016691e-06, + "loss": 0.93789995, + "num_input_tokens_seen": 24189735, + "router_z_loss_clip": 1.58886719, + "router_z_loss_mlp": 0.25305176, + "step": 889, + "time_per_iteration": 2.6314542293548584 + }, + { + "auxiliary_loss_clip": 0.01063084, + "auxiliary_loss_mlp": 0.0100381, + "balance_loss_clip": 1.02257156, + "balance_loss_mlp": 1.00071073, + "epoch": 0.02582554697928153, + "flos": 63967211487360.0, + "grad_norm": 0.6797189637747895, + "language_loss": 0.5088321, + "learning_rate": 3.913577590498226e-06, + "loss": 0.52950108, + "num_input_tokens_seen": 24249930, + "router_z_loss_clip": 0.40429688, + "router_z_loss_mlp": 0.03100586, + "step": 890, + "time_per_iteration": 3.077139139175415 + }, + { + "auxiliary_loss_clip": 0.01229482, + "auxiliary_loss_mlp": 0.01057346, + "balance_loss_clip": 1.07803047, + "balance_loss_mlp": 1.03175199, + "epoch": 0.025854564447797573, + "flos": 25991173733760.0, + "grad_norm": 4.686327240411136, + "language_loss": 1.03801775, + "learning_rate": 3.91422472145745e-06, + "loss": 1.06088614, + "num_input_tokens_seen": 24266130, + "router_z_loss_clip": 1.51464844, + "router_z_loss_mlp": 0.25598145, + "step": 891, + "time_per_iteration": 2.587041139602661 + }, + { + "auxiliary_loss_clip": 0.01224933, + "auxiliary_loss_mlp": 0.01062649, + "balance_loss_clip": 1.0823307, + "balance_loss_mlp": 1.0405004, + "epoch": 0.02588358191631362, + "flos": 16391892510720.0, + "grad_norm": 3.091597688174363, + "language_loss": 0.79414976, + "learning_rate": 3.914871126526495e-06, + "loss": 0.81702554, + "num_input_tokens_seen": 24278130, + "router_z_loss_clip": 1.42578125, + "router_z_loss_mlp": 0.22143555, + "step": 892, + "time_per_iteration": 2.524545907974243 + }, + { + "auxiliary_loss_clip": 0.01229576, + "auxiliary_loss_mlp": 0.01062969, + "balance_loss_clip": 1.0795455, + "balance_loss_mlp": 1.03530073, + "epoch": 0.025912599384829666, + "flos": 12924753246720.0, + "grad_norm": 3.505787286515889, + "language_loss": 1.01362371, + "learning_rate": 3.915516807332006e-06, + "loss": 1.03654921, + "num_input_tokens_seen": 24288865, + "router_z_loss_clip": 1.50097656, + "router_z_loss_mlp": 0.27661133, + "step": 893, + "time_per_iteration": 2.482672691345215 + }, + { + "auxiliary_loss_clip": 0.01242136, + "auxiliary_loss_mlp": 0.01065223, + "balance_loss_clip": 1.08198225, + "balance_loss_mlp": 1.03790069, + "epoch": 0.025941616853345714, + "flos": 22447364889600.0, + "grad_norm": 3.4488461782075786, + "language_loss": 0.98287499, + "learning_rate": 3.916161765495166e-06, + "loss": 1.00594854, + "num_input_tokens_seen": 24300935, + "router_z_loss_clip": 1.6015625, + "router_z_loss_mlp": 0.27307129, + "step": 894, + "time_per_iteration": 2.584247350692749 + }, + { + "auxiliary_loss_clip": 0.01061857, + "auxiliary_loss_mlp": 0.01002736, + "balance_loss_clip": 1.02147818, + "balance_loss_mlp": 0.9995653, + "epoch": 0.025970634321861762, + "flos": 72729810236160.0, + "grad_norm": 0.7854042349005231, + "language_loss": 0.55693114, + "learning_rate": 3.916806002631721e-06, + "loss": 0.57757705, + "num_input_tokens_seen": 24366515, + "router_z_loss_clip": 0.40234375, + "router_z_loss_mlp": 0.03173828, + "step": 895, + "time_per_iteration": 3.145401954650879 + }, + { + "auxiliary_loss_clip": 0.01061592, + "auxiliary_loss_mlp": 0.01004544, + "balance_loss_clip": 1.02141178, + "balance_loss_mlp": 1.00130105, + "epoch": 0.025999651790377807, + "flos": 74779986810240.0, + "grad_norm": 0.6647990289198724, + "language_loss": 0.53100622, + "learning_rate": 3.917449520352006e-06, + "loss": 0.55166757, + "num_input_tokens_seen": 24437660, + "router_z_loss_clip": 0.40234375, + "router_z_loss_mlp": 0.0324707, + "step": 896, + "time_per_iteration": 3.279995918273926 + }, + { + "auxiliary_loss_clip": 0.01061077, + "auxiliary_loss_mlp": 0.01001175, + "balance_loss_clip": 1.02110851, + "balance_loss_mlp": 0.99805176, + "epoch": 0.026028669258893855, + "flos": 56668511525760.0, + "grad_norm": 0.7323890157195496, + "language_loss": 0.51646924, + "learning_rate": 3.918092320260965e-06, + "loss": 0.53709173, + "num_input_tokens_seen": 24491695, + "router_z_loss_clip": 0.3984375, + "router_z_loss_mlp": 0.03112793, + "step": 897, + "time_per_iteration": 2.9143896102905273 + }, + { + "auxiliary_loss_clip": 0.0106142, + "auxiliary_loss_mlp": 0.01006491, + "balance_loss_clip": 1.02168632, + "balance_loss_mlp": 1.00327253, + "epoch": 0.0260576867274099, + "flos": 56355766784640.0, + "grad_norm": 0.696423215999549, + "language_loss": 0.54952973, + "learning_rate": 3.918734403958178e-06, + "loss": 0.57020885, + "num_input_tokens_seen": 24551640, + "router_z_loss_clip": 0.3984375, + "router_z_loss_mlp": 0.03222656, + "step": 898, + "time_per_iteration": 3.189929485321045 + }, + { + "auxiliary_loss_clip": 0.01237312, + "auxiliary_loss_mlp": 0.01056151, + "balance_loss_clip": 1.0794642, + "balance_loss_mlp": 1.03019893, + "epoch": 0.026086704195925948, + "flos": 29235702458880.0, + "grad_norm": 4.094161464740947, + "language_loss": 0.85058957, + "learning_rate": 3.919375773037884e-06, + "loss": 0.87352419, + "num_input_tokens_seen": 24573575, + "router_z_loss_clip": 1.578125, + "router_z_loss_mlp": 0.25927734, + "step": 899, + "time_per_iteration": 2.675370216369629 + }, + { + "auxiliary_loss_clip": 0.01227717, + "auxiliary_loss_mlp": 0.01064663, + "balance_loss_clip": 1.07954097, + "balance_loss_mlp": 1.04181099, + "epoch": 0.026115721664441992, + "flos": 14384629710720.0, + "grad_norm": 3.9782453602379313, + "language_loss": 0.90342224, + "learning_rate": 3.9200164290890045e-06, + "loss": 0.92634606, + "num_input_tokens_seen": 24584115, + "router_z_loss_clip": 1.48339844, + "router_z_loss_mlp": 0.22875977, + "step": 900, + "time_per_iteration": 2.5152814388275146 + }, + { + "auxiliary_loss_clip": 0.01060227, + "auxiliary_loss_mlp": 0.01007799, + "balance_loss_clip": 1.02040219, + "balance_loss_mlp": 1.00462842, + "epoch": 0.02614473913295804, + "flos": 61970830508160.0, + "grad_norm": 0.7882896547404445, + "language_loss": 0.54980022, + "learning_rate": 3.92065637369517e-06, + "loss": 0.57048047, + "num_input_tokens_seen": 24644865, + "router_z_loss_clip": 0.3984375, + "router_z_loss_mlp": 0.03173828, + "step": 901, + "time_per_iteration": 3.0208778381347656 + }, + { + "auxiliary_loss_clip": 0.01222672, + "auxiliary_loss_mlp": 0.01059072, + "balance_loss_clip": 1.07801104, + "balance_loss_mlp": 1.03538465, + "epoch": 0.02617375660147409, + "flos": 30512254884480.0, + "grad_norm": 3.2322870518636937, + "language_loss": 0.82809907, + "learning_rate": 3.921295608434737e-06, + "loss": 0.8509165, + "num_input_tokens_seen": 24661255, + "router_z_loss_clip": 1.4453125, + "router_z_loss_mlp": 0.23681641, + "step": 902, + "time_per_iteration": 2.659738779067993 + }, + { + "auxiliary_loss_clip": 0.01059521, + "auxiliary_loss_mlp": 0.01004738, + "balance_loss_clip": 1.02015209, + "balance_loss_mlp": 1.00163865, + "epoch": 0.026202774069990133, + "flos": 58320546773760.0, + "grad_norm": 0.6816112438849654, + "language_loss": 0.51371092, + "learning_rate": 3.92193413488082e-06, + "loss": 0.53435349, + "num_input_tokens_seen": 24722775, + "router_z_loss_clip": 0.39257812, + "router_z_loss_mlp": 0.03088379, + "step": 903, + "time_per_iteration": 3.2255117893218994 + }, + { + "auxiliary_loss_clip": 0.01243405, + "auxiliary_loss_mlp": 0.01065337, + "balance_loss_clip": 1.08735859, + "balance_loss_mlp": 1.03777623, + "epoch": 0.02623179153850618, + "flos": 17338713062400.0, + "grad_norm": 4.381495266295244, + "language_loss": 0.84802002, + "learning_rate": 3.922571954601306e-06, + "loss": 0.87110746, + "num_input_tokens_seen": 24733740, + "router_z_loss_clip": 1.55957031, + "router_z_loss_mlp": 0.27563477, + "step": 904, + "time_per_iteration": 2.517422914505005 + }, + { + "auxiliary_loss_clip": 0.01242393, + "auxiliary_loss_mlp": 0.01078533, + "balance_loss_clip": 1.08441532, + "balance_loss_mlp": 1.05234277, + "epoch": 0.026260809007022226, + "flos": 19821002250240.0, + "grad_norm": 6.849123332709156, + "language_loss": 0.95548242, + "learning_rate": 3.9232090691588845e-06, + "loss": 0.9786917, + "num_input_tokens_seen": 24745660, + "router_z_loss_clip": 1.58105469, + "router_z_loss_mlp": 0.26171875, + "step": 905, + "time_per_iteration": 2.509121894836426 + }, + { + "auxiliary_loss_clip": 0.01221056, + "auxiliary_loss_mlp": 0.01055352, + "balance_loss_clip": 1.07869339, + "balance_loss_mlp": 1.03315544, + "epoch": 0.026289826475538274, + "flos": 18253824883200.0, + "grad_norm": 2.6435251237958255, + "language_loss": 0.73645604, + "learning_rate": 3.923845480111065e-06, + "loss": 0.75922006, + "num_input_tokens_seen": 24758895, + "router_z_loss_clip": 1.42480469, + "router_z_loss_mlp": 0.22216797, + "step": 906, + "time_per_iteration": 2.5408263206481934 + }, + { + "auxiliary_loss_clip": 0.01231171, + "auxiliary_loss_mlp": 0.01073303, + "balance_loss_clip": 1.07667446, + "balance_loss_mlp": 1.04586148, + "epoch": 0.026318843944054322, + "flos": 16279958753280.0, + "grad_norm": 4.629239011680649, + "language_loss": 0.97383738, + "learning_rate": 3.924481189010205e-06, + "loss": 0.9968822, + "num_input_tokens_seen": 24769185, + "router_z_loss_clip": 1.54296875, + "router_z_loss_mlp": 0.27453613, + "step": 907, + "time_per_iteration": 2.527207612991333 + }, + { + "auxiliary_loss_clip": 0.01229361, + "auxiliary_loss_mlp": 0.01066969, + "balance_loss_clip": 1.08218467, + "balance_loss_mlp": 1.04502249, + "epoch": 0.026347861412570367, + "flos": 12160610288640.0, + "grad_norm": 2.74919132279394, + "language_loss": 0.87917602, + "learning_rate": 3.925116197403529e-06, + "loss": 0.90213931, + "num_input_tokens_seen": 24784365, + "router_z_loss_clip": 1.46972656, + "router_z_loss_mlp": 0.21948242, + "step": 908, + "time_per_iteration": 2.5788733959198 + }, + { + "auxiliary_loss_clip": 0.0122733, + "auxiliary_loss_mlp": 0.01059726, + "balance_loss_clip": 1.07858729, + "balance_loss_mlp": 1.03365505, + "epoch": 0.026376878881086415, + "flos": 36022423916160.0, + "grad_norm": 3.167581129220127, + "language_loss": 0.83848399, + "learning_rate": 3.925750506833153e-06, + "loss": 0.86135459, + "num_input_tokens_seen": 24801230, + "router_z_loss_clip": 1.48632812, + "router_z_loss_mlp": 0.26062012, + "step": 909, + "time_per_iteration": 2.6813018321990967 + }, + { + "auxiliary_loss_clip": 0.01226203, + "auxiliary_loss_mlp": 0.01053332, + "balance_loss_clip": 1.08123446, + "balance_loss_mlp": 1.02990746, + "epoch": 0.02640589634960246, + "flos": 32262612635520.0, + "grad_norm": 2.8714084129338446, + "language_loss": 1.0168941, + "learning_rate": 3.926384118836106e-06, + "loss": 1.03968942, + "num_input_tokens_seen": 24816915, + "router_z_loss_clip": 1.45019531, + "router_z_loss_mlp": 0.23413086, + "step": 910, + "time_per_iteration": 2.6616592407226562 + }, + { + "auxiliary_loss_clip": 0.01061064, + "auxiliary_loss_mlp": 0.0100916, + "balance_loss_clip": 1.02196527, + "balance_loss_mlp": 1.00639486, + "epoch": 0.026434913818118508, + "flos": 71438679866880.0, + "grad_norm": 0.6981551122434553, + "language_loss": 0.53284085, + "learning_rate": 3.9270170349443515e-06, + "loss": 0.55354309, + "num_input_tokens_seen": 24883480, + "router_z_loss_clip": 0.390625, + "router_z_loss_mlp": 0.02770996, + "step": 911, + "time_per_iteration": 3.321171522140503 + }, + { + "auxiliary_loss_clip": 0.01060731, + "auxiliary_loss_mlp": 0.01004537, + "balance_loss_clip": 1.02216387, + "balance_loss_mlp": 1.00168824, + "epoch": 0.026463931286634552, + "flos": 65796824597760.0, + "grad_norm": 0.7187111429907533, + "language_loss": 0.55692083, + "learning_rate": 3.927649256684814e-06, + "loss": 0.57757348, + "num_input_tokens_seen": 24941805, + "router_z_loss_clip": 0.38671875, + "router_z_loss_mlp": 0.02844238, + "step": 912, + "time_per_iteration": 3.025604724884033 + }, + { + "auxiliary_loss_clip": 0.01060914, + "auxiliary_loss_mlp": 0.01003794, + "balance_loss_clip": 1.02170515, + "balance_loss_mlp": 1.00096881, + "epoch": 0.0264929487551506, + "flos": 60325906152960.0, + "grad_norm": 0.673513328764273, + "language_loss": 0.53057349, + "learning_rate": 3.928280785579394e-06, + "loss": 0.55122054, + "num_input_tokens_seen": 25006025, + "router_z_loss_clip": 0.39257812, + "router_z_loss_mlp": 0.02819824, + "step": 913, + "time_per_iteration": 3.1619060039520264 + }, + { + "auxiliary_loss_clip": 0.0123532, + "auxiliary_loss_mlp": 0.01064721, + "balance_loss_clip": 1.08498406, + "balance_loss_mlp": 1.03860259, + "epoch": 0.02652196622366665, + "flos": 21063762956160.0, + "grad_norm": 2.5948052138703503, + "language_loss": 0.89402938, + "learning_rate": 3.928911623144997e-06, + "loss": 0.91702986, + "num_input_tokens_seen": 25019310, + "router_z_loss_clip": 1.50390625, + "router_z_loss_mlp": 0.2611084, + "step": 914, + "time_per_iteration": 2.6085731983184814 + }, + { + "auxiliary_loss_clip": 0.01230035, + "auxiliary_loss_mlp": 0.01070567, + "balance_loss_clip": 1.08144975, + "balance_loss_mlp": 1.04658175, + "epoch": 0.026550983692182693, + "flos": 12891141095040.0, + "grad_norm": 3.567768036805144, + "language_loss": 0.84956229, + "learning_rate": 3.92954177089355e-06, + "loss": 0.87256825, + "num_input_tokens_seen": 25031890, + "router_z_loss_clip": 1.484375, + "router_z_loss_mlp": 0.23986816, + "step": 915, + "time_per_iteration": 2.5578839778900146 + }, + { + "auxiliary_loss_clip": 0.01253145, + "auxiliary_loss_mlp": 0.01074152, + "balance_loss_clip": 1.08574224, + "balance_loss_mlp": 1.04517245, + "epoch": 0.02658000116069874, + "flos": 13218215362560.0, + "grad_norm": 2.544280858837013, + "language_loss": 0.76163214, + "learning_rate": 3.9301712303320286e-06, + "loss": 0.78490508, + "num_input_tokens_seen": 25045930, + "router_z_loss_clip": 1.67285156, + "router_z_loss_mlp": 0.28979492, + "step": 916, + "time_per_iteration": 2.516235589981079 + }, + { + "auxiliary_loss_clip": 0.01221179, + "auxiliary_loss_mlp": 0.01065164, + "balance_loss_clip": 1.07808864, + "balance_loss_mlp": 1.03824604, + "epoch": 0.026609018629214786, + "flos": 48024487572480.0, + "grad_norm": 2.4923382178214855, + "language_loss": 0.79752052, + "learning_rate": 3.930800002962473e-06, + "loss": 0.82038397, + "num_input_tokens_seen": 25062715, + "router_z_loss_clip": 1.42871094, + "router_z_loss_mlp": 0.26904297, + "step": 917, + "time_per_iteration": 2.912956476211548 + }, + { + "auxiliary_loss_clip": 0.01060857, + "auxiliary_loss_mlp": 0.01004748, + "balance_loss_clip": 1.02204585, + "balance_loss_mlp": 1.00216126, + "epoch": 0.026638036097730834, + "flos": 65328693621120.0, + "grad_norm": 0.713482028961135, + "language_loss": 0.5060581, + "learning_rate": 3.931428090282013e-06, + "loss": 0.52671421, + "num_input_tokens_seen": 25125725, + "router_z_loss_clip": 0.38671875, + "router_z_loss_mlp": 0.02587891, + "step": 918, + "time_per_iteration": 3.0897669792175293 + }, + { + "auxiliary_loss_clip": 0.01060899, + "auxiliary_loss_mlp": 0.01003509, + "balance_loss_clip": 1.02173138, + "balance_loss_mlp": 1.00089824, + "epoch": 0.026667053566246882, + "flos": 74787456839040.0, + "grad_norm": 0.692039764659511, + "language_loss": 0.55723393, + "learning_rate": 3.932055493782887e-06, + "loss": 0.577878, + "num_input_tokens_seen": 25196390, + "router_z_loss_clip": 0.390625, + "router_z_loss_mlp": 0.02612305, + "step": 919, + "time_per_iteration": 3.2855610847473145 + }, + { + "auxiliary_loss_clip": 0.01227645, + "auxiliary_loss_mlp": 0.01083959, + "balance_loss_clip": 1.08122528, + "balance_loss_mlp": 1.05930686, + "epoch": 0.026696071034762927, + "flos": 30187084037760.0, + "grad_norm": 2.677216172973356, + "language_loss": 0.95313966, + "learning_rate": 3.932682214952469e-06, + "loss": 0.97625566, + "num_input_tokens_seen": 25213175, + "router_z_loss_clip": 1.46289062, + "router_z_loss_mlp": 0.24658203, + "step": 920, + "time_per_iteration": 2.6303465366363525 + }, + { + "auxiliary_loss_clip": 0.01233617, + "auxiliary_loss_mlp": 0.01071967, + "balance_loss_clip": 1.07953835, + "balance_loss_mlp": 1.04584789, + "epoch": 0.026725088503278975, + "flos": 29050977790080.0, + "grad_norm": 3.1914690212212653, + "language_loss": 0.93289685, + "learning_rate": 3.933308255273279e-06, + "loss": 0.95595276, + "num_input_tokens_seen": 25225940, + "router_z_loss_clip": 1.54199219, + "router_z_loss_mlp": 0.2611084, + "step": 921, + "time_per_iteration": 2.6385815143585205 + }, + { + "auxiliary_loss_clip": 0.01058189, + "auxiliary_loss_mlp": 0.01006422, + "balance_loss_clip": 1.01962733, + "balance_loss_mlp": 1.00397873, + "epoch": 0.02675410597179502, + "flos": 60909849555840.0, + "grad_norm": 0.6766346103261375, + "language_loss": 0.50266892, + "learning_rate": 3.933933616223017e-06, + "loss": 0.52331501, + "num_input_tokens_seen": 25287290, + "router_z_loss_clip": 0.38476562, + "router_z_loss_mlp": 0.02441406, + "step": 922, + "time_per_iteration": 3.099605083465576 + }, + { + "auxiliary_loss_clip": 0.01230582, + "auxiliary_loss_mlp": 0.01060087, + "balance_loss_clip": 1.08008981, + "balance_loss_mlp": 1.03502941, + "epoch": 0.026783123440311068, + "flos": 24126871063680.0, + "grad_norm": 5.194410791432975, + "language_loss": 0.93522877, + "learning_rate": 3.934558299274573e-06, + "loss": 0.95813543, + "num_input_tokens_seen": 25300365, + "router_z_loss_clip": 1.50585938, + "router_z_loss_mlp": 0.25036621, + "step": 923, + "time_per_iteration": 2.7258808612823486 + }, + { + "auxiliary_loss_clip": 0.01230196, + "auxiliary_loss_mlp": 0.01065936, + "balance_loss_clip": 1.08031964, + "balance_loss_mlp": 1.04258299, + "epoch": 0.026812140908827112, + "flos": 36567835004160.0, + "grad_norm": 2.341700720874623, + "language_loss": 0.98114538, + "learning_rate": 3.9351823058960555e-06, + "loss": 1.00410664, + "num_input_tokens_seen": 25321065, + "router_z_loss_clip": 1.5, + "router_z_loss_mlp": 0.23364258, + "step": 924, + "time_per_iteration": 2.7170562744140625 + }, + { + "auxiliary_loss_clip": 0.01227756, + "auxiliary_loss_mlp": 0.01056997, + "balance_loss_clip": 1.07721686, + "balance_loss_mlp": 1.03326297, + "epoch": 0.02684115837734316, + "flos": 14566517205120.0, + "grad_norm": 3.7654847059547736, + "language_loss": 0.76543629, + "learning_rate": 3.935805637550806e-06, + "loss": 0.78828382, + "num_input_tokens_seen": 25333200, + "router_z_loss_clip": 1.50683594, + "router_z_loss_mlp": 0.23742676, + "step": 925, + "time_per_iteration": 2.575843095779419 + }, + { + "auxiliary_loss_clip": 0.01238263, + "auxiliary_loss_mlp": 0.01055629, + "balance_loss_clip": 1.08429635, + "balance_loss_mlp": 1.0308094, + "epoch": 0.02687017584585921, + "flos": 29782873313280.0, + "grad_norm": 2.9717526292404997, + "language_loss": 0.89331335, + "learning_rate": 3.936428295697425e-06, + "loss": 0.9162522, + "num_input_tokens_seen": 25350220, + "router_z_loss_clip": 1.5390625, + "router_z_loss_mlp": 0.24816895, + "step": 926, + "time_per_iteration": 2.6396710872650146 + }, + { + "auxiliary_loss_clip": 0.01240021, + "auxiliary_loss_mlp": 0.01072354, + "balance_loss_clip": 1.08488464, + "balance_loss_mlp": 1.04678392, + "epoch": 0.026899193314375253, + "flos": 46929786727680.0, + "grad_norm": 3.3301106935594293, + "language_loss": 0.99068034, + "learning_rate": 3.937050281789788e-06, + "loss": 1.01380408, + "num_input_tokens_seen": 25366365, + "router_z_loss_clip": 1.55078125, + "router_z_loss_mlp": 0.25585938, + "step": 927, + "time_per_iteration": 2.6683990955352783 + }, + { + "auxiliary_loss_clip": 0.01241207, + "auxiliary_loss_mlp": 0.01070304, + "balance_loss_clip": 1.0850668, + "balance_loss_mlp": 1.04385161, + "epoch": 0.0269282107828913, + "flos": 27337248933120.0, + "grad_norm": 3.1479642002726504, + "language_loss": 0.97802371, + "learning_rate": 3.93767159727707e-06, + "loss": 1.00113869, + "num_input_tokens_seen": 25384070, + "router_z_loss_clip": 1.56152344, + "router_z_loss_mlp": 0.2644043, + "step": 928, + "time_per_iteration": 2.746894359588623 + }, + { + "auxiliary_loss_clip": 0.01060077, + "auxiliary_loss_mlp": 0.01013631, + "balance_loss_clip": 1.02136946, + "balance_loss_mlp": 1.01119924, + "epoch": 0.026957228251407346, + "flos": 68715966994560.0, + "grad_norm": 0.7070603326820097, + "language_loss": 0.50683308, + "learning_rate": 3.938292243603762e-06, + "loss": 0.52757013, + "num_input_tokens_seen": 25446115, + "router_z_loss_clip": 0.38671875, + "router_z_loss_mlp": 0.02429199, + "step": 929, + "time_per_iteration": 5.483400583267212 + }, + { + "auxiliary_loss_clip": 0.01218234, + "auxiliary_loss_mlp": 0.01052765, + "balance_loss_clip": 1.07509124, + "balance_loss_mlp": 1.0310514, + "epoch": 0.026986245719923394, + "flos": 27552030307200.0, + "grad_norm": 4.5508572015926365, + "language_loss": 1.03275132, + "learning_rate": 3.938912222209695e-06, + "loss": 1.05546129, + "num_input_tokens_seen": 25457680, + "router_z_loss_clip": 1.43164062, + "router_z_loss_mlp": 0.21716309, + "step": 930, + "time_per_iteration": 7.497909784317017 + }, + { + "auxiliary_loss_clip": 0.01059959, + "auxiliary_loss_mlp": 0.01010995, + "balance_loss_clip": 1.02122641, + "balance_loss_mlp": 1.00882494, + "epoch": 0.02701526318843944, + "flos": 59405622773760.0, + "grad_norm": 0.6690654150821398, + "language_loss": 0.50515628, + "learning_rate": 3.939531534530054e-06, + "loss": 0.52586579, + "num_input_tokens_seen": 25518550, + "router_z_loss_clip": 0.38671875, + "router_z_loss_mlp": 0.02172852, + "step": 931, + "time_per_iteration": 3.0732717514038086 + }, + { + "auxiliary_loss_clip": 0.01223329, + "auxiliary_loss_mlp": 0.01056774, + "balance_loss_clip": 1.07993233, + "balance_loss_mlp": 1.03395748, + "epoch": 0.027044280656955487, + "flos": 27557812396800.0, + "grad_norm": 2.3273534949686554, + "language_loss": 0.75405931, + "learning_rate": 3.9401501819954064e-06, + "loss": 0.77686036, + "num_input_tokens_seen": 25540010, + "router_z_loss_clip": 1.43457031, + "router_z_loss_mlp": 0.22790527, + "step": 932, + "time_per_iteration": 5.140233755111694 + }, + { + "auxiliary_loss_clip": 0.01241477, + "auxiliary_loss_mlp": 0.01078282, + "balance_loss_clip": 1.08231807, + "balance_loss_mlp": 1.054726, + "epoch": 0.027073298125471535, + "flos": 31181020254720.0, + "grad_norm": 2.4146306286083927, + "language_loss": 0.89434439, + "learning_rate": 3.940768166031714e-06, + "loss": 0.91754198, + "num_input_tokens_seen": 25563520, + "router_z_loss_clip": 1.58984375, + "router_z_loss_mlp": 0.2355957, + "step": 933, + "time_per_iteration": 2.703184127807617 + }, + { + "auxiliary_loss_clip": 0.0122456, + "auxiliary_loss_mlp": 0.0106482, + "balance_loss_clip": 1.080603, + "balance_loss_mlp": 1.04382682, + "epoch": 0.02710231559398758, + "flos": 38320203916800.0, + "grad_norm": 2.1538060576526874, + "language_loss": 0.93836313, + "learning_rate": 3.941385488060358e-06, + "loss": 0.96125698, + "num_input_tokens_seen": 25583720, + "router_z_loss_clip": 1.44042969, + "router_z_loss_mlp": 0.21008301, + "step": 934, + "time_per_iteration": 2.718600273132324 + }, + { + "auxiliary_loss_clip": 0.01229051, + "auxiliary_loss_mlp": 0.0105642, + "balance_loss_clip": 1.08063149, + "balance_loss_mlp": 1.03176796, + "epoch": 0.027131333062503628, + "flos": 36652728798720.0, + "grad_norm": 2.1230381228101773, + "language_loss": 0.94272912, + "learning_rate": 3.942002149498154e-06, + "loss": 0.9655838, + "num_input_tokens_seen": 25606675, + "router_z_loss_clip": 1.48339844, + "router_z_loss_mlp": 0.24645996, + "step": 935, + "time_per_iteration": 2.740248680114746 + }, + { + "auxiliary_loss_clip": 0.01060626, + "auxiliary_loss_mlp": 0.01004002, + "balance_loss_clip": 1.02167892, + "balance_loss_mlp": 1.00194013, + "epoch": 0.027160350531019672, + "flos": 68557348535040.0, + "grad_norm": 0.7485804146026694, + "language_loss": 0.52418143, + "learning_rate": 3.9426181517573775e-06, + "loss": 0.5448277, + "num_input_tokens_seen": 25658530, + "router_z_loss_clip": 0.38867188, + "router_z_loss_mlp": 0.02062988, + "step": 936, + "time_per_iteration": 2.9781527519226074 + }, + { + "auxiliary_loss_clip": 0.0106103, + "auxiliary_loss_mlp": 0.01005493, + "balance_loss_clip": 1.02241814, + "balance_loss_mlp": 1.0033114, + "epoch": 0.02718936799953572, + "flos": 74783075379840.0, + "grad_norm": 0.6227072235616687, + "language_loss": 0.47316843, + "learning_rate": 3.943233496245778e-06, + "loss": 0.49383366, + "num_input_tokens_seen": 25724825, + "router_z_loss_clip": 0.38671875, + "router_z_loss_mlp": 0.02185059, + "step": 937, + "time_per_iteration": 3.2389538288116455 + }, + { + "auxiliary_loss_clip": 0.0123124, + "auxiliary_loss_mlp": 0.01058222, + "balance_loss_clip": 1.08108687, + "balance_loss_mlp": 1.03417778, + "epoch": 0.02721838546805177, + "flos": 29275491749760.0, + "grad_norm": 2.804192915806399, + "language_loss": 0.95050079, + "learning_rate": 3.943848184366598e-06, + "loss": 0.97339541, + "num_input_tokens_seen": 25741665, + "router_z_loss_clip": 1.50195312, + "router_z_loss_mlp": 0.24035645, + "step": 938, + "time_per_iteration": 2.6378042697906494 + }, + { + "auxiliary_loss_clip": 0.01246889, + "auxiliary_loss_mlp": 0.01085517, + "balance_loss_clip": 1.08750701, + "balance_loss_mlp": 1.05620384, + "epoch": 0.027247402936567813, + "flos": 36971219715840.0, + "grad_norm": 2.701513289575716, + "language_loss": 0.94911414, + "learning_rate": 3.9444622175186e-06, + "loss": 0.97243822, + "num_input_tokens_seen": 25764955, + "router_z_loss_clip": 1.59472656, + "router_z_loss_mlp": 0.29309082, + "step": 939, + "time_per_iteration": 2.7994613647460938 + }, + { + "auxiliary_loss_clip": 0.01229236, + "auxiliary_loss_mlp": 0.01067227, + "balance_loss_clip": 1.07911658, + "balance_loss_mlp": 1.0419904, + "epoch": 0.02727642040508386, + "flos": 15992781517440.0, + "grad_norm": 2.3057166102340725, + "language_loss": 0.80436969, + "learning_rate": 3.945075597096074e-06, + "loss": 0.8273344, + "num_input_tokens_seen": 25778015, + "router_z_loss_clip": 1.50195312, + "router_z_loss_mlp": 0.25256348, + "step": 940, + "time_per_iteration": 2.572314977645874 + }, + { + "auxiliary_loss_clip": 0.01228013, + "auxiliary_loss_mlp": 0.01061344, + "balance_loss_clip": 1.08177698, + "balance_loss_mlp": 1.03769314, + "epoch": 0.027305437873599906, + "flos": 29897033713920.0, + "grad_norm": 2.3182102457208504, + "language_loss": 0.93363148, + "learning_rate": 3.945688324488866e-06, + "loss": 0.95652509, + "num_input_tokens_seen": 25793635, + "router_z_loss_clip": 1.46289062, + "router_z_loss_mlp": 0.23632812, + "step": 941, + "time_per_iteration": 2.6026089191436768 + }, + { + "auxiliary_loss_clip": 0.0105909, + "auxiliary_loss_mlp": 0.01005123, + "balance_loss_clip": 1.02030373, + "balance_loss_mlp": 1.0030489, + "epoch": 0.027334455342115954, + "flos": 66057285093120.0, + "grad_norm": 0.7095230808293153, + "language_loss": 0.51256269, + "learning_rate": 3.946300401082393e-06, + "loss": 0.53320479, + "num_input_tokens_seen": 25852085, + "router_z_loss_clip": 0.38671875, + "router_z_loss_mlp": 0.02075195, + "step": 942, + "time_per_iteration": 3.226095676422119 + }, + { + "auxiliary_loss_clip": 0.01244708, + "auxiliary_loss_mlp": 0.01071248, + "balance_loss_clip": 1.08513522, + "balance_loss_mlp": 1.04219651, + "epoch": 0.027363472810632, + "flos": 30921349858560.0, + "grad_norm": 3.8403516428172177, + "language_loss": 1.11488104, + "learning_rate": 3.946911828257664e-06, + "loss": 1.13804078, + "num_input_tokens_seen": 25873450, + "router_z_loss_clip": 1.59472656, + "router_z_loss_mlp": 0.29040527, + "step": 943, + "time_per_iteration": 2.6821413040161133 + }, + { + "auxiliary_loss_clip": 0.01058422, + "auxiliary_loss_mlp": 0.01004202, + "balance_loss_clip": 1.01937413, + "balance_loss_mlp": 1.00204456, + "epoch": 0.027392490279148047, + "flos": 60278610919680.0, + "grad_norm": 0.734498344695878, + "language_loss": 0.50399029, + "learning_rate": 3.94752260739129e-06, + "loss": 0.52461648, + "num_input_tokens_seen": 25929255, + "router_z_loss_clip": 0.390625, + "router_z_loss_mlp": 0.02160645, + "step": 944, + "time_per_iteration": 3.0097875595092773 + }, + { + "auxiliary_loss_clip": 0.01231585, + "auxiliary_loss_mlp": 0.0105761, + "balance_loss_clip": 1.08250201, + "balance_loss_mlp": 1.03497219, + "epoch": 0.027421507747664095, + "flos": 16791398553600.0, + "grad_norm": 2.960265288012635, + "language_loss": 0.8063401, + "learning_rate": 3.9481327398555175e-06, + "loss": 0.8292321, + "num_input_tokens_seen": 25940965, + "router_z_loss_clip": 1.49023438, + "router_z_loss_mlp": 0.22644043, + "step": 945, + "time_per_iteration": 2.508424758911133 + }, + { + "auxiliary_loss_clip": 0.01057551, + "auxiliary_loss_mlp": 0.0100233, + "balance_loss_clip": 1.01870489, + "balance_loss_mlp": 1.00014877, + "epoch": 0.02745052521618014, + "flos": 61863242296320.0, + "grad_norm": 0.6572646521548177, + "language_loss": 0.48150319, + "learning_rate": 3.948742227018233e-06, + "loss": 0.50210202, + "num_input_tokens_seen": 26004465, + "router_z_loss_clip": 0.38867188, + "router_z_loss_mlp": 0.02185059, + "step": 946, + "time_per_iteration": 3.1177799701690674 + }, + { + "auxiliary_loss_clip": 0.01241221, + "auxiliary_loss_mlp": 0.01071273, + "balance_loss_clip": 1.08223212, + "balance_loss_mlp": 1.04374743, + "epoch": 0.027479542684696188, + "flos": 27924568214400.0, + "grad_norm": 2.6639773490275447, + "language_loss": 0.98489141, + "learning_rate": 3.949351070242994e-06, + "loss": 1.00801635, + "num_input_tokens_seen": 26025085, + "router_z_loss_clip": 1.58984375, + "router_z_loss_mlp": 0.27526855, + "step": 947, + "time_per_iteration": 2.666386604309082 + }, + { + "auxiliary_loss_clip": 0.01245631, + "auxiliary_loss_mlp": 0.01076463, + "balance_loss_clip": 1.08535028, + "balance_loss_mlp": 1.04821038, + "epoch": 0.027508560153212232, + "flos": 21940450202880.0, + "grad_norm": 2.605555715643518, + "language_loss": 0.99836171, + "learning_rate": 3.949959270889033e-06, + "loss": 1.0215826, + "num_input_tokens_seen": 26041380, + "router_z_loss_clip": 1.60351562, + "router_z_loss_mlp": 0.28271484, + "step": 948, + "time_per_iteration": 2.561279535293579 + }, + { + "auxiliary_loss_clip": 0.01237743, + "auxiliary_loss_mlp": 0.01070075, + "balance_loss_clip": 1.08087432, + "balance_loss_mlp": 1.04448628, + "epoch": 0.02753757762172828, + "flos": 57914752872960.0, + "grad_norm": 2.4097715737986736, + "language_loss": 0.80532187, + "learning_rate": 3.950566830311289e-06, + "loss": 0.82840008, + "num_input_tokens_seen": 26063680, + "router_z_loss_clip": 1.57128906, + "router_z_loss_mlp": 0.25592041, + "step": 949, + "time_per_iteration": 2.8978796005249023 + }, + { + "auxiliary_loss_clip": 0.01229174, + "auxiliary_loss_mlp": 0.01059887, + "balance_loss_clip": 1.07761168, + "balance_loss_mlp": 1.03416157, + "epoch": 0.02756659509024433, + "flos": 26609016597120.0, + "grad_norm": 2.3015357561371057, + "language_loss": 0.92435718, + "learning_rate": 3.951173749860417e-06, + "loss": 0.94724774, + "num_input_tokens_seen": 26080575, + "router_z_loss_clip": 1.51464844, + "router_z_loss_mlp": 0.25720215, + "step": 950, + "time_per_iteration": 2.6716766357421875 + }, + { + "auxiliary_loss_clip": 0.01058738, + "auxiliary_loss_mlp": 0.00999961, + "balance_loss_clip": 1.01971364, + "balance_loss_mlp": 0.99786252, + "epoch": 0.027595612558760373, + "flos": 54237719462400.0, + "grad_norm": 0.6852375731411249, + "language_loss": 0.50789261, + "learning_rate": 3.9517800308828105e-06, + "loss": 0.52847958, + "num_input_tokens_seen": 26141950, + "router_z_loss_clip": 0.390625, + "router_z_loss_mlp": 0.02099609, + "step": 951, + "time_per_iteration": 3.16227388381958 + }, + { + "auxiliary_loss_clip": 0.01058308, + "auxiliary_loss_mlp": 0.01002152, + "balance_loss_clip": 1.019696, + "balance_loss_mlp": 0.99988693, + "epoch": 0.02762463002727642, + "flos": 60064655558400.0, + "grad_norm": 0.7220972617709347, + "language_loss": 0.48960492, + "learning_rate": 3.9523856747206175e-06, + "loss": 0.51020956, + "num_input_tokens_seen": 26193555, + "router_z_loss_clip": 0.38671875, + "router_z_loss_mlp": 0.02270508, + "step": 952, + "time_per_iteration": 2.9106037616729736 + }, + { + "auxiliary_loss_clip": 0.01226027, + "auxiliary_loss_mlp": 0.01063517, + "balance_loss_clip": 1.07913923, + "balance_loss_mlp": 1.03918624, + "epoch": 0.027653647495792466, + "flos": 25631169672960.0, + "grad_norm": 2.495338767114376, + "language_loss": 1.00579119, + "learning_rate": 3.952990682711758e-06, + "loss": 1.02868664, + "num_input_tokens_seen": 26210310, + "router_z_loss_clip": 1.47070312, + "router_z_loss_mlp": 0.2434082, + "step": 953, + "time_per_iteration": 2.589776039123535 + }, + { + "auxiliary_loss_clip": 0.01056639, + "auxiliary_loss_mlp": 0.01003095, + "balance_loss_clip": 1.01807261, + "balance_loss_mlp": 1.00084233, + "epoch": 0.027682664964308514, + "flos": 60806750544000.0, + "grad_norm": 0.7416048053546256, + "language_loss": 0.51462322, + "learning_rate": 3.953595056189946e-06, + "loss": 0.53522056, + "num_input_tokens_seen": 26275435, + "router_z_loss_clip": 0.38671875, + "router_z_loss_mlp": 0.02258301, + "step": 954, + "time_per_iteration": 3.193633556365967 + }, + { + "auxiliary_loss_clip": 0.01229428, + "auxiliary_loss_mlp": 0.01063614, + "balance_loss_clip": 1.0800643, + "balance_loss_mlp": 1.03828239, + "epoch": 0.02771168243282456, + "flos": 28067312862720.0, + "grad_norm": 4.453018091213798, + "language_loss": 1.02448177, + "learning_rate": 3.954198796484698e-06, + "loss": 1.04741216, + "num_input_tokens_seen": 26290105, + "router_z_loss_clip": 1.49316406, + "router_z_loss_mlp": 0.25311279, + "step": 955, + "time_per_iteration": 2.540583848953247 + }, + { + "auxiliary_loss_clip": 0.0105612, + "auxiliary_loss_mlp": 0.0100521, + "balance_loss_clip": 1.01761031, + "balance_loss_mlp": 1.00298059, + "epoch": 0.027740699901340607, + "flos": 58576266673920.0, + "grad_norm": 0.7053633595278944, + "language_loss": 0.54771405, + "learning_rate": 3.954801904921359e-06, + "loss": 0.56832737, + "num_input_tokens_seen": 26348590, + "router_z_loss_clip": 0.38671875, + "router_z_loss_mlp": 0.02233887, + "step": 956, + "time_per_iteration": 3.0330193042755127 + }, + { + "auxiliary_loss_clip": 0.01056007, + "auxiliary_loss_mlp": 0.01006822, + "balance_loss_clip": 1.01746488, + "balance_loss_mlp": 1.00460458, + "epoch": 0.027769717369856655, + "flos": 60128255566080.0, + "grad_norm": 0.632509870297192, + "language_loss": 0.47226289, + "learning_rate": 3.955404382821119e-06, + "loss": 0.49289119, + "num_input_tokens_seen": 26411455, + "router_z_loss_clip": 0.38671875, + "router_z_loss_mlp": 0.0222168, + "step": 957, + "time_per_iteration": 3.218322515487671 + }, + { + "auxiliary_loss_clip": 0.01055338, + "auxiliary_loss_mlp": 0.01004094, + "balance_loss_clip": 1.01673841, + "balance_loss_mlp": 1.00192475, + "epoch": 0.0277987348383727, + "flos": 70028465955840.0, + "grad_norm": 0.6856190179161219, + "language_loss": 0.5036872, + "learning_rate": 3.956006231501026e-06, + "loss": 0.5242815, + "num_input_tokens_seen": 26472470, + "router_z_loss_clip": 0.38671875, + "router_z_loss_mlp": 0.02172852, + "step": 958, + "time_per_iteration": 3.068801164627075 + }, + { + "auxiliary_loss_clip": 0.01235506, + "auxiliary_loss_mlp": 0.01081112, + "balance_loss_clip": 1.07854223, + "balance_loss_mlp": 1.05224001, + "epoch": 0.027827752306888748, + "flos": 44522084131200.0, + "grad_norm": 2.929847871587081, + "language_loss": 1.05179346, + "learning_rate": 3.9566074522740066e-06, + "loss": 1.07495964, + "num_input_tokens_seen": 26493700, + "router_z_loss_clip": 1.56933594, + "router_z_loss_mlp": 0.28881836, + "step": 959, + "time_per_iteration": 2.7587554454803467 + }, + { + "auxiliary_loss_clip": 0.01054637, + "auxiliary_loss_mlp": 0.01003873, + "balance_loss_clip": 1.016289, + "balance_loss_mlp": 1.00181031, + "epoch": 0.027856769775404792, + "flos": 74771978077440.0, + "grad_norm": 0.7052640433174867, + "language_loss": 0.55463958, + "learning_rate": 3.9572080464488815e-06, + "loss": 0.57522476, + "num_input_tokens_seen": 26551680, + "router_z_loss_clip": 0.3828125, + "router_z_loss_mlp": 0.02062988, + "step": 960, + "time_per_iteration": 3.146177053451538 + }, + { + "auxiliary_loss_clip": 0.01242555, + "auxiliary_loss_mlp": 0.01069646, + "balance_loss_clip": 1.08359599, + "balance_loss_mlp": 1.04113078, + "epoch": 0.02788578724392084, + "flos": 12961849017600.0, + "grad_norm": 2.7284205148645015, + "language_loss": 0.81245029, + "learning_rate": 3.957808015330385e-06, + "loss": 0.8355723, + "num_input_tokens_seen": 26564550, + "router_z_loss_clip": 1.58886719, + "router_z_loss_mlp": 0.28515625, + "step": 961, + "time_per_iteration": 2.623457431793213 + }, + { + "auxiliary_loss_clip": 0.01235131, + "auxiliary_loss_mlp": 0.01060813, + "balance_loss_clip": 1.08246553, + "balance_loss_mlp": 1.03385937, + "epoch": 0.02791480471243689, + "flos": 43354879683840.0, + "grad_norm": 2.6158701601933347, + "language_loss": 0.81669182, + "learning_rate": 3.958407360219179e-06, + "loss": 0.83965123, + "num_input_tokens_seen": 26580135, + "router_z_loss_clip": 1.52441406, + "router_z_loss_mlp": 0.26940918, + "step": 962, + "time_per_iteration": 2.6853837966918945 + }, + { + "auxiliary_loss_clip": 0.01248345, + "auxiliary_loss_mlp": 0.01076923, + "balance_loss_clip": 1.08355772, + "balance_loss_mlp": 1.04853964, + "epoch": 0.027943822180952933, + "flos": 21135045496320.0, + "grad_norm": 2.893102125791652, + "language_loss": 1.04910278, + "learning_rate": 3.9590060824118735e-06, + "loss": 1.07235551, + "num_input_tokens_seen": 26592885, + "router_z_loss_clip": 1.64941406, + "router_z_loss_mlp": 0.28393555, + "step": 963, + "time_per_iteration": 2.5191562175750732 + }, + { + "auxiliary_loss_clip": 0.01055027, + "auxiliary_loss_mlp": 0.01000988, + "balance_loss_clip": 1.01670134, + "balance_loss_mlp": 0.99884254, + "epoch": 0.02797283964946898, + "flos": 66933864599040.0, + "grad_norm": 0.6527660984502832, + "language_loss": 0.54597789, + "learning_rate": 3.959604183201038e-06, + "loss": 0.56653804, + "num_input_tokens_seen": 26656860, + "router_z_loss_clip": 0.3828125, + "router_z_loss_mlp": 0.02148438, + "step": 964, + "time_per_iteration": 3.121638774871826 + }, + { + "auxiliary_loss_clip": 0.0123416, + "auxiliary_loss_mlp": 0.01068503, + "balance_loss_clip": 1.08083534, + "balance_loss_mlp": 1.04040515, + "epoch": 0.028001857117985026, + "flos": 29489267543040.0, + "grad_norm": 3.2484418550788976, + "language_loss": 0.8543871, + "learning_rate": 3.960201663875225e-06, + "loss": 0.87741369, + "num_input_tokens_seen": 26671265, + "router_z_loss_clip": 1.53222656, + "router_z_loss_mlp": 0.28100586, + "step": 965, + "time_per_iteration": 2.58693265914917 + }, + { + "auxiliary_loss_clip": 0.01237394, + "auxiliary_loss_mlp": 0.0106304, + "balance_loss_clip": 1.08432245, + "balance_loss_mlp": 1.03617024, + "epoch": 0.028030874586501074, + "flos": 10661698719360.0, + "grad_norm": 3.2548946653178983, + "language_loss": 0.838763, + "learning_rate": 3.960798525718981e-06, + "loss": 0.86176741, + "num_input_tokens_seen": 26681225, + "router_z_loss_clip": 1.53125, + "router_z_loss_mlp": 0.26855469, + "step": 966, + "time_per_iteration": 2.4577770233154297 + }, + { + "auxiliary_loss_clip": 0.01226799, + "auxiliary_loss_mlp": 0.01054199, + "balance_loss_clip": 1.07970762, + "balance_loss_mlp": 1.0298326, + "epoch": 0.02805989205501712, + "flos": 12342138647040.0, + "grad_norm": 2.7142647633035897, + "language_loss": 0.90645099, + "learning_rate": 3.961394770012866e-06, + "loss": 0.92926097, + "num_input_tokens_seen": 26693995, + "router_z_loss_clip": 1.47167969, + "router_z_loss_mlp": 0.24377441, + "step": 967, + "time_per_iteration": 2.54106068611145 + }, + { + "auxiliary_loss_clip": 0.01227084, + "auxiliary_loss_mlp": 0.01056821, + "balance_loss_clip": 1.07820106, + "balance_loss_mlp": 1.03009427, + "epoch": 0.028088909523533167, + "flos": 25732508918400.0, + "grad_norm": 2.317173724203715, + "language_loss": 0.95854086, + "learning_rate": 3.9619903980334684e-06, + "loss": 0.98137993, + "num_input_tokens_seen": 26707370, + "router_z_loss_clip": 1.48828125, + "router_z_loss_mlp": 0.26721191, + "step": 968, + "time_per_iteration": 2.5648951530456543 + }, + { + "auxiliary_loss_clip": 0.01228731, + "auxiliary_loss_mlp": 0.01065137, + "balance_loss_clip": 1.0798192, + "balance_loss_mlp": 1.03920913, + "epoch": 0.028117926992049215, + "flos": 25841749155840.0, + "grad_norm": 2.2281932691785915, + "language_loss": 0.85538983, + "learning_rate": 3.9625854110534254e-06, + "loss": 0.87832856, + "num_input_tokens_seen": 26722675, + "router_z_loss_clip": 1.48828125, + "router_z_loss_mlp": 0.25939941, + "step": 969, + "time_per_iteration": 2.75724196434021 + }, + { + "auxiliary_loss_clip": 0.01057246, + "auxiliary_loss_mlp": 0.01006219, + "balance_loss_clip": 1.01860929, + "balance_loss_mlp": 1.00404918, + "epoch": 0.02814694446056526, + "flos": 68558318202240.0, + "grad_norm": 0.7193778431199317, + "language_loss": 0.56998682, + "learning_rate": 3.963179810341432e-06, + "loss": 0.59062147, + "num_input_tokens_seen": 26781100, + "router_z_loss_clip": 0.38671875, + "router_z_loss_mlp": 0.02172852, + "step": 970, + "time_per_iteration": 3.060539484024048 + }, + { + "auxiliary_loss_clip": 0.01247897, + "auxiliary_loss_mlp": 0.01075398, + "balance_loss_clip": 1.08808768, + "balance_loss_mlp": 1.04699028, + "epoch": 0.028175961929081308, + "flos": 35437115796480.0, + "grad_norm": 3.7012801299541582, + "language_loss": 0.87824315, + "learning_rate": 3.9637735971622635e-06, + "loss": 0.90147614, + "num_input_tokens_seen": 26796080, + "router_z_loss_clip": 1.59863281, + "router_z_loss_mlp": 0.28405762, + "step": 971, + "time_per_iteration": 2.6348683834075928 + }, + { + "auxiliary_loss_clip": 0.01220167, + "auxiliary_loss_mlp": 0.01058218, + "balance_loss_clip": 1.07909727, + "balance_loss_mlp": 1.03568149, + "epoch": 0.028204979397597352, + "flos": 30656938867200.0, + "grad_norm": 5.0175014693623865, + "language_loss": 0.94185555, + "learning_rate": 3.964366772776789e-06, + "loss": 0.96463943, + "num_input_tokens_seen": 26812295, + "router_z_loss_clip": 1.41015625, + "router_z_loss_mlp": 0.22528076, + "step": 972, + "time_per_iteration": 2.6071956157684326 + }, + { + "auxiliary_loss_clip": 0.01233535, + "auxiliary_loss_mlp": 0.01063678, + "balance_loss_clip": 1.07719088, + "balance_loss_mlp": 1.03862023, + "epoch": 0.0282339968661134, + "flos": 23291122343040.0, + "grad_norm": 2.819906593169006, + "language_loss": 0.86186671, + "learning_rate": 3.964959338441989e-06, + "loss": 0.88483894, + "num_input_tokens_seen": 26828600, + "router_z_loss_clip": 1.56347656, + "router_z_loss_mlp": 0.25061035, + "step": 973, + "time_per_iteration": 2.528775215148926 + }, + { + "auxiliary_loss_clip": 0.01057442, + "auxiliary_loss_mlp": 0.01000402, + "balance_loss_clip": 1.01896477, + "balance_loss_mlp": 0.99819678, + "epoch": 0.02826301433462945, + "flos": 68287266172800.0, + "grad_norm": 0.6330223208170824, + "language_loss": 0.52064753, + "learning_rate": 3.96555129541097e-06, + "loss": 0.54122591, + "num_input_tokens_seen": 26896545, + "router_z_loss_clip": 0.38476562, + "router_z_loss_mlp": 0.02209473, + "step": 974, + "time_per_iteration": 3.22257137298584 + }, + { + "auxiliary_loss_clip": 0.01242706, + "auxiliary_loss_mlp": 0.01066394, + "balance_loss_clip": 1.08344924, + "balance_loss_mlp": 1.03909481, + "epoch": 0.028292031803145493, + "flos": 12487504988160.0, + "grad_norm": 2.8238578089226354, + "language_loss": 0.83277392, + "learning_rate": 3.9661426449329815e-06, + "loss": 0.85586494, + "num_input_tokens_seen": 26908390, + "router_z_loss_clip": 1.59375, + "router_z_loss_mlp": 0.27307129, + "step": 975, + "time_per_iteration": 2.492258071899414 + }, + { + "auxiliary_loss_clip": 0.01233551, + "auxiliary_loss_mlp": 0.01060964, + "balance_loss_clip": 1.0812676, + "balance_loss_mlp": 1.03442836, + "epoch": 0.02832104927166154, + "flos": 40363197770880.0, + "grad_norm": 2.5902511072427195, + "language_loss": 1.10049391, + "learning_rate": 3.966733388253427e-06, + "loss": 1.12343907, + "num_input_tokens_seen": 26926780, + "router_z_loss_clip": 1.52441406, + "router_z_loss_mlp": 0.26513672, + "step": 976, + "time_per_iteration": 2.7040040493011475 + }, + { + "auxiliary_loss_clip": 0.01233758, + "auxiliary_loss_mlp": 0.01056178, + "balance_loss_clip": 1.0870533, + "balance_loss_mlp": 1.03086948, + "epoch": 0.028350066740177586, + "flos": 16499444808960.0, + "grad_norm": 3.547457057490907, + "language_loss": 0.93984443, + "learning_rate": 3.967323526613891e-06, + "loss": 0.96274388, + "num_input_tokens_seen": 26937800, + "router_z_loss_clip": 1.46582031, + "router_z_loss_mlp": 0.25292969, + "step": 977, + "time_per_iteration": 2.4762227535247803 + }, + { + "auxiliary_loss_clip": 0.01057195, + "auxiliary_loss_mlp": 0.01003286, + "balance_loss_clip": 1.01901531, + "balance_loss_mlp": 1.00118756, + "epoch": 0.028379084208693634, + "flos": 63293241623040.0, + "grad_norm": 0.6568449680568399, + "language_loss": 0.49581924, + "learning_rate": 3.967913061252141e-06, + "loss": 0.516424, + "num_input_tokens_seen": 27003420, + "router_z_loss_clip": 0.38085938, + "router_z_loss_mlp": 0.02099609, + "step": 978, + "time_per_iteration": 3.250293731689453 + }, + { + "auxiliary_loss_clip": 0.0122431, + "auxiliary_loss_mlp": 0.01054004, + "balance_loss_clip": 1.07945502, + "balance_loss_mlp": 1.02968526, + "epoch": 0.02840810167720968, + "flos": 20114248884480.0, + "grad_norm": 2.964999004881547, + "language_loss": 0.77891612, + "learning_rate": 3.968501993402152e-06, + "loss": 0.80169928, + "num_input_tokens_seen": 27017875, + "router_z_loss_clip": 1.44824219, + "router_z_loss_mlp": 0.24316406, + "step": 979, + "time_per_iteration": 2.5605199337005615 + }, + { + "auxiliary_loss_clip": 0.01058469, + "auxiliary_loss_mlp": 0.01005466, + "balance_loss_clip": 1.02011037, + "balance_loss_mlp": 1.00341606, + "epoch": 0.028437119145725727, + "flos": 74780597341440.0, + "grad_norm": 0.7182320640133315, + "language_loss": 0.59977692, + "learning_rate": 3.969090324294122e-06, + "loss": 0.62041628, + "num_input_tokens_seen": 27079345, + "router_z_loss_clip": 0.3828125, + "router_z_loss_mlp": 0.02050781, + "step": 980, + "time_per_iteration": 3.2260260581970215 + }, + { + "auxiliary_loss_clip": 0.01242891, + "auxiliary_loss_mlp": 0.01086222, + "balance_loss_clip": 1.0866003, + "balance_loss_mlp": 1.05948305, + "epoch": 0.028466136614241775, + "flos": 26066586337920.0, + "grad_norm": 2.9432136347797657, + "language_loss": 0.69438589, + "learning_rate": 3.969678055154481e-06, + "loss": 0.717677, + "num_input_tokens_seen": 27093595, + "router_z_loss_clip": 1.56152344, + "router_z_loss_mlp": 0.26745605, + "step": 981, + "time_per_iteration": 2.6744308471679688 + }, + { + "auxiliary_loss_clip": 0.01057641, + "auxiliary_loss_mlp": 0.01002564, + "balance_loss_clip": 1.01964688, + "balance_loss_mlp": 1.00035906, + "epoch": 0.02849515408275782, + "flos": 72037991312640.0, + "grad_norm": 0.6786947718493213, + "language_loss": 0.54649043, + "learning_rate": 3.970265187205913e-06, + "loss": 0.56709254, + "num_input_tokens_seen": 27154075, + "router_z_loss_clip": 0.37890625, + "router_z_loss_mlp": 0.02209473, + "step": 982, + "time_per_iteration": 3.1387081146240234 + }, + { + "auxiliary_loss_clip": 0.01056724, + "auxiliary_loss_mlp": 0.01001331, + "balance_loss_clip": 1.01851165, + "balance_loss_mlp": 0.99922127, + "epoch": 0.028524171551273868, + "flos": 56899238538240.0, + "grad_norm": 0.7786847224313558, + "language_loss": 0.52196884, + "learning_rate": 3.970851721667367e-06, + "loss": 0.54254937, + "num_input_tokens_seen": 27206905, + "router_z_loss_clip": 0.3828125, + "router_z_loss_mlp": 0.02111816, + "step": 983, + "time_per_iteration": 3.0462443828582764 + }, + { + "auxiliary_loss_clip": 0.01055479, + "auxiliary_loss_mlp": 0.01001275, + "balance_loss_clip": 1.01778817, + "balance_loss_mlp": 0.99884284, + "epoch": 0.028553189019789912, + "flos": 68611072302720.0, + "grad_norm": 0.6376931922391331, + "language_loss": 0.56093854, + "learning_rate": 3.971437659754076e-06, + "loss": 0.58150607, + "num_input_tokens_seen": 27270340, + "router_z_loss_clip": 0.37695312, + "router_z_loss_mlp": 0.02429199, + "step": 984, + "time_per_iteration": 3.1827709674835205 + }, + { + "auxiliary_loss_clip": 0.01227466, + "auxiliary_loss_mlp": 0.01061107, + "balance_loss_clip": 1.08270884, + "balance_loss_mlp": 1.03939915, + "epoch": 0.02858220648830596, + "flos": 45506718725760.0, + "grad_norm": 2.2390206171901257, + "language_loss": 0.81849718, + "learning_rate": 3.9720230026775675e-06, + "loss": 0.84138298, + "num_input_tokens_seen": 27287480, + "router_z_loss_clip": 1.44824219, + "router_z_loss_mlp": 0.21716309, + "step": 985, + "time_per_iteration": 2.717686176300049 + }, + { + "auxiliary_loss_clip": 0.01241186, + "auxiliary_loss_mlp": 0.01071151, + "balance_loss_clip": 1.08450758, + "balance_loss_mlp": 1.04443622, + "epoch": 0.028611223956822005, + "flos": 25805766706560.0, + "grad_norm": 2.938606734490259, + "language_loss": 1.05267167, + "learning_rate": 3.972607751645682e-06, + "loss": 1.07579517, + "num_input_tokens_seen": 27302950, + "router_z_loss_clip": 1.56640625, + "router_z_loss_mlp": 0.26733398, + "step": 986, + "time_per_iteration": 2.638833999633789 + }, + { + "auxiliary_loss_clip": 0.01222092, + "auxiliary_loss_mlp": 0.01059172, + "balance_loss_clip": 1.08186579, + "balance_loss_mlp": 1.03689158, + "epoch": 0.028640241425338053, + "flos": 11319941404800.0, + "grad_norm": 3.2366119413175563, + "language_loss": 1.03597367, + "learning_rate": 3.973191907862586e-06, + "loss": 1.05878639, + "num_input_tokens_seen": 27314795, + "router_z_loss_clip": 1.40234375, + "router_z_loss_mlp": 0.22277832, + "step": 987, + "time_per_iteration": 2.5382885932922363 + }, + { + "auxiliary_loss_clip": 0.01239386, + "auxiliary_loss_mlp": 0.0107544, + "balance_loss_clip": 1.08644843, + "balance_loss_mlp": 1.05082273, + "epoch": 0.0286692588938541, + "flos": 27666406189440.0, + "grad_norm": 2.5121811597487405, + "language_loss": 0.85813659, + "learning_rate": 3.973775472528791e-06, + "loss": 0.88128483, + "num_input_tokens_seen": 27331450, + "router_z_loss_clip": 1.52832031, + "router_z_loss_mlp": 0.24584961, + "step": 988, + "time_per_iteration": 2.6620535850524902 + }, + { + "auxiliary_loss_clip": 0.0105399, + "auxiliary_loss_mlp": 0.0100487, + "balance_loss_clip": 1.01676786, + "balance_loss_mlp": 1.00250995, + "epoch": 0.028698276362370146, + "flos": 61237462527360.0, + "grad_norm": 0.7270476678504487, + "language_loss": 0.52422643, + "learning_rate": 3.9743584468411595e-06, + "loss": 0.544815, + "num_input_tokens_seen": 27393880, + "router_z_loss_clip": 0.37109375, + "router_z_loss_mlp": 0.02355957, + "step": 989, + "time_per_iteration": 3.083547353744507 + }, + { + "auxiliary_loss_clip": 0.01234745, + "auxiliary_loss_mlp": 0.01060311, + "balance_loss_clip": 1.08279896, + "balance_loss_mlp": 1.03321517, + "epoch": 0.028727293830886194, + "flos": 32702554414080.0, + "grad_norm": 2.484248160009939, + "language_loss": 0.82558298, + "learning_rate": 3.97494083199293e-06, + "loss": 0.84853357, + "num_input_tokens_seen": 27413805, + "router_z_loss_clip": 1.51855469, + "router_z_loss_mlp": 0.27124023, + "step": 990, + "time_per_iteration": 2.6589345932006836 + }, + { + "auxiliary_loss_clip": 0.01053372, + "auxiliary_loss_mlp": 0.01003607, + "balance_loss_clip": 1.01613116, + "balance_loss_mlp": 1.00141358, + "epoch": 0.02875631129940224, + "flos": 62407360494720.0, + "grad_norm": 0.6736643660035991, + "language_loss": 0.5198741, + "learning_rate": 3.975522629173727e-06, + "loss": 0.5404439, + "num_input_tokens_seen": 27478830, + "router_z_loss_clip": 0.37109375, + "router_z_loss_mlp": 0.02197266, + "step": 991, + "time_per_iteration": 3.0999112129211426 + }, + { + "auxiliary_loss_clip": 0.01232426, + "auxiliary_loss_mlp": 0.01059253, + "balance_loss_clip": 1.08161187, + "balance_loss_mlp": 1.03573263, + "epoch": 0.028785328767918287, + "flos": 61632943269120.0, + "grad_norm": 3.3219358413983215, + "language_loss": 0.76610422, + "learning_rate": 3.976103839569571e-06, + "loss": 0.78902102, + "num_input_tokens_seen": 27499010, + "router_z_loss_clip": 1.5078125, + "router_z_loss_mlp": 0.23522949, + "step": 992, + "time_per_iteration": 2.853160858154297 + }, + { + "auxiliary_loss_clip": 0.01221777, + "auxiliary_loss_mlp": 0.01058631, + "balance_loss_clip": 1.07989454, + "balance_loss_mlp": 1.03482521, + "epoch": 0.028814346236434335, + "flos": 18842113831680.0, + "grad_norm": 2.98381225649815, + "language_loss": 0.76039588, + "learning_rate": 3.976684464362904e-06, + "loss": 0.78319997, + "num_input_tokens_seen": 27516385, + "router_z_loss_clip": 1.41796875, + "router_z_loss_mlp": 0.23815918, + "step": 993, + "time_per_iteration": 2.543914318084717 + }, + { + "auxiliary_loss_clip": 0.01229862, + "auxiliary_loss_mlp": 0.01083062, + "balance_loss_clip": 1.08253801, + "balance_loss_mlp": 1.05612063, + "epoch": 0.02884336370495038, + "flos": 41971205923200.0, + "grad_norm": 2.3072558339566944, + "language_loss": 0.91456091, + "learning_rate": 3.9772645047325895e-06, + "loss": 0.9376902, + "num_input_tokens_seen": 27534935, + "router_z_loss_clip": 1.47460938, + "router_z_loss_mlp": 0.26940918, + "step": 994, + "time_per_iteration": 2.7315492630004883 + }, + { + "auxiliary_loss_clip": 0.01219528, + "auxiliary_loss_mlp": 0.01060593, + "balance_loss_clip": 1.07712066, + "balance_loss_mlp": 1.03709662, + "epoch": 0.028872381173466428, + "flos": 11318612601600.0, + "grad_norm": 4.664509219451033, + "language_loss": 0.88644171, + "learning_rate": 3.977843961853942e-06, + "loss": 0.90924299, + "num_input_tokens_seen": 27546160, + "router_z_loss_clip": 1.42382812, + "router_z_loss_mlp": 0.23486328, + "step": 995, + "time_per_iteration": 2.552112579345703 + }, + { + "auxiliary_loss_clip": 0.0123439, + "auxiliary_loss_mlp": 0.0106267, + "balance_loss_clip": 1.07996178, + "balance_loss_mlp": 1.03419113, + "epoch": 0.028901398641982472, + "flos": 16830218177280.0, + "grad_norm": 3.062282001077452, + "language_loss": 0.95720547, + "learning_rate": 3.978422836898733e-06, + "loss": 0.98017603, + "num_input_tokens_seen": 27560210, + "router_z_loss_clip": 1.54394531, + "router_z_loss_mlp": 0.28491211, + "step": 996, + "time_per_iteration": 2.5056700706481934 + }, + { + "auxiliary_loss_clip": 0.01059366, + "auxiliary_loss_mlp": 0.01023041, + "balance_loss_clip": 1.0220046, + "balance_loss_mlp": 1.02093148, + "epoch": 0.02893041611049852, + "flos": 74771978077440.0, + "grad_norm": 0.7070725692513952, + "language_loss": 0.5060792, + "learning_rate": 3.979001131035201e-06, + "loss": 0.52690327, + "num_input_tokens_seen": 27622735, + "router_z_loss_clip": 0.375, + "router_z_loss_mlp": 0.02111816, + "step": 997, + "time_per_iteration": 3.2436470985412598 + }, + { + "auxiliary_loss_clip": 0.01231845, + "auxiliary_loss_mlp": 0.01071029, + "balance_loss_clip": 1.0829407, + "balance_loss_mlp": 1.04598308, + "epoch": 0.028959433579014565, + "flos": 12197741973120.0, + "grad_norm": 3.0786561871334848, + "language_loss": 0.77195537, + "learning_rate": 3.979578845428077e-06, + "loss": 0.79498404, + "num_input_tokens_seen": 27635605, + "router_z_loss_clip": 1.48828125, + "router_z_loss_mlp": 0.25073242, + "step": 998, + "time_per_iteration": 2.5521960258483887 + }, + { + "auxiliary_loss_clip": 0.01230311, + "auxiliary_loss_mlp": 0.01076125, + "balance_loss_clip": 1.08300877, + "balance_loss_mlp": 1.05006599, + "epoch": 0.028988451047530613, + "flos": 14859153308160.0, + "grad_norm": 3.614969456951015, + "language_loss": 0.9920845, + "learning_rate": 3.9801559812385905e-06, + "loss": 1.01514888, + "num_input_tokens_seen": 27648280, + "router_z_loss_clip": 1.47265625, + "router_z_loss_mlp": 0.26025391, + "step": 999, + "time_per_iteration": 2.568708896636963 + }, + { + "auxiliary_loss_clip": 0.01056344, + "auxiliary_loss_mlp": 0.01006965, + "balance_loss_clip": 1.01886106, + "balance_loss_mlp": 1.00492609, + "epoch": 0.02901746851604666, + "flos": 63062766005760.0, + "grad_norm": 0.7080378749141798, + "language_loss": 0.53184664, + "learning_rate": 3.980732539624484e-06, + "loss": 0.55247974, + "num_input_tokens_seen": 27710000, + "router_z_loss_clip": 0.375, + "router_z_loss_mlp": 0.02038574, + "step": 1000, + "time_per_iteration": 3.1852424144744873 + }, + { + "auxiliary_loss_clip": 0.01233712, + "auxiliary_loss_mlp": 0.01052122, + "balance_loss_clip": 1.08259845, + "balance_loss_mlp": 1.0252049, + "epoch": 0.029046485984562706, + "flos": 29928562876800.0, + "grad_norm": 2.6137724883940585, + "language_loss": 1.0661391, + "learning_rate": 3.981308521740032e-06, + "loss": 1.0889976, + "num_input_tokens_seen": 27727140, + "router_z_loss_clip": 1.51074219, + "router_z_loss_mlp": 0.26916504, + "step": 1001, + "time_per_iteration": 10.931960105895996 + }, + { + "auxiliary_loss_clip": 0.01232553, + "auxiliary_loss_mlp": 0.01063757, + "balance_loss_clip": 1.08041835, + "balance_loss_mlp": 1.03875232, + "epoch": 0.029075503453078754, + "flos": 10810117716480.0, + "grad_norm": 6.3924884072939205, + "language_loss": 1.08086681, + "learning_rate": 3.981883928736047e-06, + "loss": 1.10382998, + "num_input_tokens_seen": 27739160, + "router_z_loss_clip": 1.52148438, + "router_z_loss_mlp": 0.25018311, + "step": 1002, + "time_per_iteration": 2.5015430450439453 + }, + { + "auxiliary_loss_clip": 0.01055573, + "auxiliary_loss_mlp": 0.01000202, + "balance_loss_clip": 1.01746678, + "balance_loss_mlp": 0.99818707, + "epoch": 0.0291045209215948, + "flos": 74770541533440.0, + "grad_norm": 0.7035554462781407, + "language_loss": 0.54466623, + "learning_rate": 3.982458761759901e-06, + "loss": 0.56522399, + "num_input_tokens_seen": 27796095, + "router_z_loss_clip": 0.38085938, + "router_z_loss_mlp": 0.0201416, + "step": 1003, + "time_per_iteration": 5.537184476852417 + }, + { + "auxiliary_loss_clip": 0.01225021, + "auxiliary_loss_mlp": 0.01064439, + "balance_loss_clip": 1.07656038, + "balance_loss_mlp": 1.03831995, + "epoch": 0.029133538390110847, + "flos": 26169577608960.0, + "grad_norm": 6.019685788527147, + "language_loss": 0.84099805, + "learning_rate": 3.983033021955535e-06, + "loss": 0.86389267, + "num_input_tokens_seen": 27809520, + "router_z_loss_clip": 1.48291016, + "router_z_loss_mlp": 0.2611084, + "step": 1004, + "time_per_iteration": 2.581404685974121 + }, + { + "auxiliary_loss_clip": 0.01235837, + "auxiliary_loss_mlp": 0.01063094, + "balance_loss_clip": 1.08358157, + "balance_loss_mlp": 1.03637874, + "epoch": 0.029162555858626895, + "flos": 21612657663360.0, + "grad_norm": 2.3478790884061165, + "language_loss": 0.94363165, + "learning_rate": 3.983606710463473e-06, + "loss": 0.96662092, + "num_input_tokens_seen": 27823675, + "router_z_loss_clip": 1.52148438, + "router_z_loss_mlp": 0.26708984, + "step": 1005, + "time_per_iteration": 2.5803563594818115 + }, + { + "auxiliary_loss_clip": 0.01238486, + "auxiliary_loss_mlp": 0.010703, + "balance_loss_clip": 1.07909632, + "balance_loss_mlp": 1.0430249, + "epoch": 0.02919157332714294, + "flos": 33757896931200.0, + "grad_norm": 2.642171052546041, + "language_loss": 0.97147375, + "learning_rate": 3.9841798284208365e-06, + "loss": 0.99456161, + "num_input_tokens_seen": 27839020, + "router_z_loss_clip": 1.59179688, + "router_z_loss_mlp": 0.27282715, + "step": 1006, + "time_per_iteration": 2.695396661758423 + }, + { + "auxiliary_loss_clip": 0.01210881, + "auxiliary_loss_mlp": 0.01057304, + "balance_loss_clip": 1.07384229, + "balance_loss_mlp": 1.03396332, + "epoch": 0.029220590795658988, + "flos": 18581976558720.0, + "grad_norm": 2.8879563416528553, + "language_loss": 0.84762776, + "learning_rate": 3.984752376961359e-06, + "loss": 0.87030965, + "num_input_tokens_seen": 27852880, + "router_z_loss_clip": 1.37109375, + "router_z_loss_mlp": 0.23352051, + "step": 1007, + "time_per_iteration": 2.587082624435425 + }, + { + "auxiliary_loss_clip": 0.01224734, + "auxiliary_loss_mlp": 0.01061076, + "balance_loss_clip": 1.0775249, + "balance_loss_mlp": 1.03650653, + "epoch": 0.029249608264175032, + "flos": 13328928057600.0, + "grad_norm": 2.991572297337879, + "language_loss": 0.87018991, + "learning_rate": 3.985324357215394e-06, + "loss": 0.89304793, + "num_input_tokens_seen": 27864250, + "router_z_loss_clip": 1.47070312, + "router_z_loss_mlp": 0.24560547, + "step": 1008, + "time_per_iteration": 2.550772190093994 + }, + { + "auxiliary_loss_clip": 0.0123281, + "auxiliary_loss_mlp": 0.01063325, + "balance_loss_clip": 1.08301425, + "balance_loss_mlp": 1.04015112, + "epoch": 0.02927862573269108, + "flos": 27665903399040.0, + "grad_norm": 2.556920662399157, + "language_loss": 0.79763806, + "learning_rate": 3.985895770309937e-06, + "loss": 0.82059944, + "num_input_tokens_seen": 27878130, + "router_z_loss_clip": 1.5, + "router_z_loss_mlp": 0.23181152, + "step": 1009, + "time_per_iteration": 2.6369876861572266 + }, + { + "auxiliary_loss_clip": 0.01219704, + "auxiliary_loss_mlp": 0.01066608, + "balance_loss_clip": 1.07651663, + "balance_loss_mlp": 1.04269445, + "epoch": 0.029307643201207125, + "flos": 25372073894400.0, + "grad_norm": 2.176500332940658, + "language_loss": 0.84760374, + "learning_rate": 3.986466617368632e-06, + "loss": 0.87046683, + "num_input_tokens_seen": 27897760, + "router_z_loss_clip": 1.43066406, + "router_z_loss_mlp": 0.23925781, + "step": 1010, + "time_per_iteration": 2.709099531173706 + }, + { + "auxiliary_loss_clip": 0.01060876, + "auxiliary_loss_mlp": 0.01021782, + "balance_loss_clip": 1.02074695, + "balance_loss_mlp": 1.01973152, + "epoch": 0.029336660669723173, + "flos": 60800393836800.0, + "grad_norm": 0.8686358323247027, + "language_loss": 0.49364164, + "learning_rate": 3.98703689951179e-06, + "loss": 0.51446819, + "num_input_tokens_seen": 27954750, + "router_z_loss_clip": 0.40234375, + "router_z_loss_mlp": 0.02050781, + "step": 1011, + "time_per_iteration": 3.069089412689209 + }, + { + "auxiliary_loss_clip": 0.01236754, + "auxiliary_loss_mlp": 0.01063228, + "balance_loss_clip": 1.08057237, + "balance_loss_mlp": 1.03684676, + "epoch": 0.02936567813823922, + "flos": 30292517433600.0, + "grad_norm": 2.446520891677654, + "language_loss": 0.96496302, + "learning_rate": 3.987606617856395e-06, + "loss": 0.98796284, + "num_input_tokens_seen": 27969840, + "router_z_loss_clip": 1.56152344, + "router_z_loss_mlp": 0.26391602, + "step": 1012, + "time_per_iteration": 2.732404947280884 + }, + { + "auxiliary_loss_clip": 0.01231622, + "auxiliary_loss_mlp": 0.01057052, + "balance_loss_clip": 1.08025289, + "balance_loss_mlp": 1.03112423, + "epoch": 0.029394695606755266, + "flos": 37559652318720.0, + "grad_norm": 2.5270741540910415, + "language_loss": 0.84265822, + "learning_rate": 3.988175773516123e-06, + "loss": 0.86554492, + "num_input_tokens_seen": 27987035, + "router_z_loss_clip": 1.51269531, + "router_z_loss_mlp": 0.25939941, + "step": 1013, + "time_per_iteration": 2.7099318504333496 + }, + { + "auxiliary_loss_clip": 0.01228371, + "auxiliary_loss_mlp": 0.01069952, + "balance_loss_clip": 1.08414459, + "balance_loss_mlp": 1.04761195, + "epoch": 0.029423713075271314, + "flos": 13582098092160.0, + "grad_norm": 2.8479740265009204, + "language_loss": 1.10120296, + "learning_rate": 3.988744367601354e-06, + "loss": 1.12418616, + "num_input_tokens_seen": 27999625, + "router_z_loss_clip": 1.44335938, + "router_z_loss_mlp": 0.22351074, + "step": 1014, + "time_per_iteration": 2.526477575302124 + }, + { + "auxiliary_loss_clip": 0.01238238, + "auxiliary_loss_mlp": 0.01064746, + "balance_loss_clip": 1.08475924, + "balance_loss_mlp": 1.03655314, + "epoch": 0.02945273054378736, + "flos": 34745045477760.0, + "grad_norm": 3.352165347359742, + "language_loss": 1.05597484, + "learning_rate": 3.9893124012191855e-06, + "loss": 1.07900453, + "num_input_tokens_seen": 28013200, + "router_z_loss_clip": 1.53515625, + "router_z_loss_mlp": 0.28186035, + "step": 1015, + "time_per_iteration": 2.6995327472686768 + }, + { + "auxiliary_loss_clip": 0.01226402, + "auxiliary_loss_mlp": 0.01056564, + "balance_loss_clip": 1.08062625, + "balance_loss_mlp": 1.03346169, + "epoch": 0.029481748012303407, + "flos": 33358570456320.0, + "grad_norm": 2.953552351707432, + "language_loss": 0.88841236, + "learning_rate": 3.989879875473443e-06, + "loss": 0.91124207, + "num_input_tokens_seen": 28032205, + "router_z_loss_clip": 1.45703125, + "router_z_loss_mlp": 0.23071289, + "step": 1016, + "time_per_iteration": 2.7236688137054443 + }, + { + "auxiliary_loss_clip": 0.01234894, + "auxiliary_loss_mlp": 0.01085893, + "balance_loss_clip": 1.08380079, + "balance_loss_mlp": 1.05810571, + "epoch": 0.029510765480819455, + "flos": 44668456053120.0, + "grad_norm": 2.047505506032142, + "language_loss": 0.93266815, + "learning_rate": 3.990446791464694e-06, + "loss": 0.95587599, + "num_input_tokens_seen": 28051985, + "router_z_loss_clip": 1.51171875, + "router_z_loss_mlp": 0.27807617, + "step": 1017, + "time_per_iteration": 2.8040997982025146 + }, + { + "auxiliary_loss_clip": 0.01235074, + "auxiliary_loss_mlp": 0.01078447, + "balance_loss_clip": 1.08445978, + "balance_loss_mlp": 1.05342531, + "epoch": 0.0295397829493355, + "flos": 36277892421120.0, + "grad_norm": 5.83866913457491, + "language_loss": 0.88260317, + "learning_rate": 3.991013150290262e-06, + "loss": 0.90573835, + "num_input_tokens_seen": 28066965, + "router_z_loss_clip": 1.50683594, + "router_z_loss_mlp": 0.25036621, + "step": 1018, + "time_per_iteration": 2.679429292678833 + }, + { + "auxiliary_loss_clip": 0.01064154, + "auxiliary_loss_mlp": 0.0102393, + "balance_loss_clip": 1.0241158, + "balance_loss_mlp": 1.02166545, + "epoch": 0.029568800417851548, + "flos": 74778370698240.0, + "grad_norm": 0.6629864130077536, + "language_loss": 0.50193036, + "learning_rate": 3.991578953044237e-06, + "loss": 0.52281117, + "num_input_tokens_seen": 28135700, + "router_z_loss_clip": 0.40039062, + "router_z_loss_mlp": 0.02270508, + "step": 1019, + "time_per_iteration": 3.3339948654174805 + }, + { + "auxiliary_loss_clip": 0.01226372, + "auxiliary_loss_mlp": 0.01055403, + "balance_loss_clip": 1.08288908, + "balance_loss_mlp": 1.03444624, + "epoch": 0.029597817886367592, + "flos": 9827135147520.0, + "grad_norm": 2.661362573751419, + "language_loss": 0.84671688, + "learning_rate": 3.992144200817493e-06, + "loss": 0.86953467, + "num_input_tokens_seen": 28147150, + "router_z_loss_clip": 1.43457031, + "router_z_loss_mlp": 0.20947266, + "step": 1020, + "time_per_iteration": 2.505955696105957 + }, + { + "auxiliary_loss_clip": 0.01230552, + "auxiliary_loss_mlp": 0.01064918, + "balance_loss_clip": 1.08384228, + "balance_loss_mlp": 1.04158831, + "epoch": 0.02962683535488364, + "flos": 66121992466560.0, + "grad_norm": 2.2089988797647604, + "language_loss": 0.76816994, + "learning_rate": 3.992708894697692e-06, + "loss": 0.7911247, + "num_input_tokens_seen": 28170010, + "router_z_loss_clip": 1.46679688, + "router_z_loss_mlp": 0.23327637, + "step": 1021, + "time_per_iteration": 2.973862648010254 + }, + { + "auxiliary_loss_clip": 0.01228755, + "auxiliary_loss_mlp": 0.0106588, + "balance_loss_clip": 1.08597493, + "balance_loss_mlp": 1.04356432, + "epoch": 0.029655852823399685, + "flos": 23542353043200.0, + "grad_norm": 2.345692026146459, + "language_loss": 0.9162159, + "learning_rate": 3.993273035769305e-06, + "loss": 0.93916231, + "num_input_tokens_seen": 28187690, + "router_z_loss_clip": 1.42675781, + "router_z_loss_mlp": 0.2232666, + "step": 1022, + "time_per_iteration": 2.6561391353607178 + }, + { + "auxiliary_loss_clip": 0.01222599, + "auxiliary_loss_mlp": 0.01062209, + "balance_loss_clip": 1.07851088, + "balance_loss_mlp": 1.03830767, + "epoch": 0.029684870291915733, + "flos": 13765278476160.0, + "grad_norm": 4.357571634823409, + "language_loss": 0.86721355, + "learning_rate": 3.99383662511362e-06, + "loss": 0.89006162, + "num_input_tokens_seen": 28200130, + "router_z_loss_clip": 1.44140625, + "router_z_loss_mlp": 0.23913574, + "step": 1023, + "time_per_iteration": 2.527026891708374 + }, + { + "auxiliary_loss_clip": 0.01060768, + "auxiliary_loss_mlp": 0.01003388, + "balance_loss_clip": 1.02080953, + "balance_loss_mlp": 1.00164759, + "epoch": 0.02971388776043178, + "flos": 74780345946240.0, + "grad_norm": 0.6672350115422275, + "language_loss": 0.55342001, + "learning_rate": 3.994399663808758e-06, + "loss": 0.57406157, + "num_input_tokens_seen": 28266870, + "router_z_loss_clip": 0.3984375, + "router_z_loss_mlp": 0.01745605, + "step": 1024, + "time_per_iteration": 3.2031638622283936 + }, + { + "auxiliary_loss_clip": 0.01060729, + "auxiliary_loss_mlp": 0.01003172, + "balance_loss_clip": 1.02090335, + "balance_loss_mlp": 1.00122857, + "epoch": 0.029742905228947826, + "flos": 58658035985280.0, + "grad_norm": 0.6919378193041104, + "language_loss": 0.49390993, + "learning_rate": 3.9949621529296794e-06, + "loss": 0.5145489, + "num_input_tokens_seen": 28329815, + "router_z_loss_clip": 0.3984375, + "router_z_loss_mlp": 0.01940918, + "step": 1025, + "time_per_iteration": 3.0748167037963867 + }, + { + "auxiliary_loss_clip": 0.01060461, + "auxiliary_loss_mlp": 0.01003544, + "balance_loss_clip": 1.02061808, + "balance_loss_mlp": 1.00167263, + "epoch": 0.029771922697463874, + "flos": 71278301640960.0, + "grad_norm": 1.1873560038820956, + "language_loss": 0.54756105, + "learning_rate": 3.995524093548202e-06, + "loss": 0.56820107, + "num_input_tokens_seen": 28395090, + "router_z_loss_clip": 0.3984375, + "router_z_loss_mlp": 0.01867676, + "step": 1026, + "time_per_iteration": 3.160339832305908 + }, + { + "auxiliary_loss_clip": 0.0121997, + "auxiliary_loss_mlp": 0.01059756, + "balance_loss_clip": 1.08144534, + "balance_loss_mlp": 1.03735685, + "epoch": 0.02980094016597992, + "flos": 12487864124160.0, + "grad_norm": 3.1689133970750576, + "language_loss": 0.87565899, + "learning_rate": 3.996085486733009e-06, + "loss": 0.89845622, + "num_input_tokens_seen": 28406805, + "router_z_loss_clip": 1.38574219, + "router_z_loss_mlp": 0.22399902, + "step": 1027, + "time_per_iteration": 2.49824595451355 + }, + { + "auxiliary_loss_clip": 0.01232519, + "auxiliary_loss_mlp": 0.01082986, + "balance_loss_clip": 1.07899714, + "balance_loss_mlp": 1.05927527, + "epoch": 0.029829957634495967, + "flos": 33440591162880.0, + "grad_norm": 2.07663904113132, + "language_loss": 0.982324, + "learning_rate": 3.996646333549668e-06, + "loss": 1.0054791, + "num_input_tokens_seen": 28428370, + "router_z_loss_clip": 1.53320312, + "router_z_loss_mlp": 0.23718262, + "step": 1028, + "time_per_iteration": 2.6786234378814697 + }, + { + "auxiliary_loss_clip": 0.0121584, + "auxiliary_loss_mlp": 0.01057494, + "balance_loss_clip": 1.07887733, + "balance_loss_mlp": 1.03749037, + "epoch": 0.02985897510301201, + "flos": 34198485154560.0, + "grad_norm": 2.5715418239061165, + "language_loss": 0.82330376, + "learning_rate": 3.997206635060634e-06, + "loss": 0.84603709, + "num_input_tokens_seen": 28444815, + "router_z_loss_clip": 1.36914062, + "router_z_loss_mlp": 0.19995117, + "step": 1029, + "time_per_iteration": 2.640687942504883 + }, + { + "auxiliary_loss_clip": 0.01239116, + "auxiliary_loss_mlp": 0.01078059, + "balance_loss_clip": 1.08414006, + "balance_loss_mlp": 1.05037892, + "epoch": 0.02988799257152806, + "flos": 30182415269760.0, + "grad_norm": 2.9519907213595267, + "language_loss": 0.90899396, + "learning_rate": 3.997766392325268e-06, + "loss": 0.93216574, + "num_input_tokens_seen": 28460255, + "router_z_loss_clip": 1.54980469, + "router_z_loss_mlp": 0.27685547, + "step": 1030, + "time_per_iteration": 2.609495162963867 + }, + { + "auxiliary_loss_clip": 0.01061413, + "auxiliary_loss_mlp": 0.01006907, + "balance_loss_clip": 1.02153635, + "balance_loss_mlp": 1.00502348, + "epoch": 0.029917010040044108, + "flos": 58205129397120.0, + "grad_norm": 0.7168076077501694, + "language_loss": 0.49556959, + "learning_rate": 3.998325606399846e-06, + "loss": 0.51625282, + "num_input_tokens_seen": 28519810, + "router_z_loss_clip": 0.3984375, + "router_z_loss_mlp": 0.01879883, + "step": 1031, + "time_per_iteration": 2.972410202026367 + }, + { + "auxiliary_loss_clip": 0.01233451, + "auxiliary_loss_mlp": 0.01060905, + "balance_loss_clip": 1.08498478, + "balance_loss_mlp": 1.03722978, + "epoch": 0.029946027508560152, + "flos": 18398041989120.0, + "grad_norm": 2.7879658852021776, + "language_loss": 0.79640102, + "learning_rate": 3.998884278337572e-06, + "loss": 0.81934458, + "num_input_tokens_seen": 28533395, + "router_z_loss_clip": 1.48535156, + "router_z_loss_mlp": 0.23693848, + "step": 1032, + "time_per_iteration": 2.535501480102539 + }, + { + "auxiliary_loss_clip": 0.01232545, + "auxiliary_loss_mlp": 0.01062521, + "balance_loss_clip": 1.08378518, + "balance_loss_mlp": 1.03797579, + "epoch": 0.0299750449770762, + "flos": 20844312814080.0, + "grad_norm": 2.8930080733914463, + "language_loss": 1.09355426, + "learning_rate": 3.999442409188591e-06, + "loss": 1.11650491, + "num_input_tokens_seen": 28549595, + "router_z_loss_clip": 1.48535156, + "router_z_loss_mlp": 0.24572754, + "step": 1033, + "time_per_iteration": 2.6170339584350586 + }, + { + "auxiliary_loss_clip": 0.01229682, + "auxiliary_loss_mlp": 0.01060059, + "balance_loss_clip": 1.08334899, + "balance_loss_mlp": 1.03584719, + "epoch": 0.030004062445592245, + "flos": 14823709562880.0, + "grad_norm": 6.232285293451581, + "language_loss": 0.87814641, + "learning_rate": 4e-06, + "loss": 0.90104389, + "num_input_tokens_seen": 28562100, + "router_z_loss_clip": 1.46386719, + "router_z_loss_mlp": 0.2421875, + "step": 1034, + "time_per_iteration": 2.525529623031616 + }, + { + "auxiliary_loss_clip": 0.01066659, + "auxiliary_loss_mlp": 0.01002985, + "balance_loss_clip": 1.0265193, + "balance_loss_mlp": 1.00107753, + "epoch": 0.030033079914108293, + "flos": 59924281207680.0, + "grad_norm": 0.7476942779719953, + "language_loss": 0.50852394, + "learning_rate": 3.999999991167595e-06, + "loss": 0.52922046, + "num_input_tokens_seen": 28622455, + "router_z_loss_clip": 0.40234375, + "router_z_loss_mlp": 0.01904297, + "step": 1035, + "time_per_iteration": 3.187279224395752 + }, + { + "auxiliary_loss_clip": 0.01233882, + "auxiliary_loss_mlp": 0.01069523, + "balance_loss_clip": 1.08178568, + "balance_loss_mlp": 1.04463232, + "epoch": 0.03006209738262434, + "flos": 62607991910400.0, + "grad_norm": 1.959341501337283, + "language_loss": 1.06701732, + "learning_rate": 3.999999964670382e-06, + "loss": 1.09005129, + "num_input_tokens_seen": 28652645, + "router_z_loss_clip": 1.52246094, + "router_z_loss_mlp": 0.2487793, + "step": 1036, + "time_per_iteration": 2.896721363067627 + }, + { + "auxiliary_loss_clip": 0.01236522, + "auxiliary_loss_mlp": 0.01077016, + "balance_loss_clip": 1.08480227, + "balance_loss_mlp": 1.05050397, + "epoch": 0.030091114851140386, + "flos": 36209339314560.0, + "grad_norm": 2.205982906495751, + "language_loss": 0.84439987, + "learning_rate": 3.999999920508358e-06, + "loss": 0.86753523, + "num_input_tokens_seen": 28677435, + "router_z_loss_clip": 1.51660156, + "router_z_loss_mlp": 0.26489258, + "step": 1037, + "time_per_iteration": 2.6796834468841553 + }, + { + "auxiliary_loss_clip": 0.01217031, + "auxiliary_loss_mlp": 0.01055361, + "balance_loss_clip": 1.08192849, + "balance_loss_mlp": 1.03613257, + "epoch": 0.030120132319656434, + "flos": 18799487366400.0, + "grad_norm": 3.127120887818971, + "language_loss": 0.8295157, + "learning_rate": 3.999999858681527e-06, + "loss": 0.85223961, + "num_input_tokens_seen": 28689935, + "router_z_loss_clip": 1.3515625, + "router_z_loss_mlp": 0.19238281, + "step": 1038, + "time_per_iteration": 2.5751113891601562 + }, + { + "auxiliary_loss_clip": 0.01229149, + "auxiliary_loss_mlp": 0.01052212, + "balance_loss_clip": 1.08458376, + "balance_loss_mlp": 1.03007507, + "epoch": 0.03014914978817248, + "flos": 25988192904960.0, + "grad_norm": 10.300888204923002, + "language_loss": 0.93328488, + "learning_rate": 3.999999779189888e-06, + "loss": 0.9560985, + "num_input_tokens_seen": 28708305, + "router_z_loss_clip": 1.44433594, + "router_z_loss_mlp": 0.22143555, + "step": 1039, + "time_per_iteration": 2.61574649810791 + }, + { + "auxiliary_loss_clip": 0.0106975, + "auxiliary_loss_mlp": 0.01007683, + "balance_loss_clip": 1.02913868, + "balance_loss_mlp": 1.00579917, + "epoch": 0.030178167256688527, + "flos": 61205215092480.0, + "grad_norm": 0.6628548819748673, + "language_loss": 0.51282489, + "learning_rate": 3.99999968203344e-06, + "loss": 0.53359914, + "num_input_tokens_seen": 28770205, + "router_z_loss_clip": 0.40625, + "router_z_loss_mlp": 0.01879883, + "step": 1040, + "time_per_iteration": 3.1224381923675537 + }, + { + "auxiliary_loss_clip": 0.01066389, + "auxiliary_loss_mlp": 0.01005094, + "balance_loss_clip": 1.02604103, + "balance_loss_mlp": 1.0030551, + "epoch": 0.03020718472520457, + "flos": 69188982220800.0, + "grad_norm": 0.8064127081612432, + "language_loss": 0.6074841, + "learning_rate": 3.999999567212187e-06, + "loss": 0.62819898, + "num_input_tokens_seen": 28830940, + "router_z_loss_clip": 0.40234375, + "router_z_loss_mlp": 0.02038574, + "step": 1041, + "time_per_iteration": 3.1521213054656982 + }, + { + "auxiliary_loss_clip": 0.01232871, + "auxiliary_loss_mlp": 0.01060763, + "balance_loss_clip": 1.08491337, + "balance_loss_mlp": 1.0348829, + "epoch": 0.03023620219372062, + "flos": 12487684556160.0, + "grad_norm": 3.0744095189329896, + "language_loss": 0.70643669, + "learning_rate": 3.9999994347261276e-06, + "loss": 0.72937304, + "num_input_tokens_seen": 28843830, + "router_z_loss_clip": 1.48046875, + "router_z_loss_mlp": 0.25878906, + "step": 1042, + "time_per_iteration": 2.591715097427368 + }, + { + "auxiliary_loss_clip": 0.01064363, + "auxiliary_loss_mlp": 0.01001806, + "balance_loss_clip": 1.02412355, + "balance_loss_mlp": 0.99970788, + "epoch": 0.030265219662236668, + "flos": 70976543483520.0, + "grad_norm": 0.7196566710068083, + "language_loss": 0.5669831, + "learning_rate": 3.999999284575265e-06, + "loss": 0.58764476, + "num_input_tokens_seen": 28900160, + "router_z_loss_clip": 0.40234375, + "router_z_loss_mlp": 0.02099609, + "step": 1043, + "time_per_iteration": 3.035817861557007 + }, + { + "auxiliary_loss_clip": 0.01063714, + "auxiliary_loss_mlp": 0.01003469, + "balance_loss_clip": 1.02378297, + "balance_loss_mlp": 1.0015254, + "epoch": 0.030294237130752712, + "flos": 63311195445120.0, + "grad_norm": 0.665278877638287, + "language_loss": 0.52829897, + "learning_rate": 3.999999116759598e-06, + "loss": 0.54897082, + "num_input_tokens_seen": 28962035, + "router_z_loss_clip": 0.3984375, + "router_z_loss_mlp": 0.01940918, + "step": 1044, + "time_per_iteration": 3.042367458343506 + }, + { + "auxiliary_loss_clip": 0.01228232, + "auxiliary_loss_mlp": 0.01059102, + "balance_loss_clip": 1.08325958, + "balance_loss_mlp": 1.03567719, + "epoch": 0.03032325459926876, + "flos": 20517058978560.0, + "grad_norm": 2.641932599812672, + "language_loss": 1.07879758, + "learning_rate": 3.999998931279131e-06, + "loss": 1.10167098, + "num_input_tokens_seen": 28975725, + "router_z_loss_clip": 1.44824219, + "router_z_loss_mlp": 0.23449707, + "step": 1045, + "time_per_iteration": 2.4968836307525635 + }, + { + "auxiliary_loss_clip": 0.01237333, + "auxiliary_loss_mlp": 0.01071046, + "balance_loss_clip": 1.08561981, + "balance_loss_mlp": 1.04496288, + "epoch": 0.030352272067784805, + "flos": 32088410651520.0, + "grad_norm": 1.915398046444334, + "language_loss": 1.04825103, + "learning_rate": 3.999998728133863e-06, + "loss": 1.07133484, + "num_input_tokens_seen": 28997230, + "router_z_loss_clip": 1.515625, + "router_z_loss_mlp": 0.2611084, + "step": 1046, + "time_per_iteration": 2.686091423034668 + }, + { + "auxiliary_loss_clip": 0.01234681, + "auxiliary_loss_mlp": 0.01079306, + "balance_loss_clip": 1.08621287, + "balance_loss_mlp": 1.05551159, + "epoch": 0.030381289536300853, + "flos": 32042767443840.0, + "grad_norm": 2.8933801684051152, + "language_loss": 0.93820357, + "learning_rate": 3.999998507323797e-06, + "loss": 0.96134341, + "num_input_tokens_seen": 29009920, + "router_z_loss_clip": 1.484375, + "router_z_loss_mlp": 0.23791504, + "step": 1047, + "time_per_iteration": 2.6263442039489746 + }, + { + "auxiliary_loss_clip": 0.01232179, + "auxiliary_loss_mlp": 0.01073992, + "balance_loss_clip": 1.07841659, + "balance_loss_mlp": 1.04740798, + "epoch": 0.0304103070048169, + "flos": 29451956290560.0, + "grad_norm": 3.789477093292238, + "language_loss": 0.99189186, + "learning_rate": 3.999998268848935e-06, + "loss": 1.01495361, + "num_input_tokens_seen": 29023615, + "router_z_loss_clip": 1.53808594, + "router_z_loss_mlp": 0.26574707, + "step": 1048, + "time_per_iteration": 2.5435233116149902 + }, + { + "auxiliary_loss_clip": 0.01243472, + "auxiliary_loss_mlp": 0.01077932, + "balance_loss_clip": 1.08407736, + "balance_loss_mlp": 1.05043101, + "epoch": 0.030439324473332946, + "flos": 18525418594560.0, + "grad_norm": 1.983815745189612, + "language_loss": 0.74613243, + "learning_rate": 3.99999801270928e-06, + "loss": 0.76934648, + "num_input_tokens_seen": 29042650, + "router_z_loss_clip": 1.59375, + "router_z_loss_mlp": 0.27514648, + "step": 1049, + "time_per_iteration": 2.6054294109344482 + }, + { + "auxiliary_loss_clip": 0.01221322, + "auxiliary_loss_mlp": 0.01050445, + "balance_loss_clip": 1.08008051, + "balance_loss_mlp": 1.02822483, + "epoch": 0.030468341941848994, + "flos": 20048425211520.0, + "grad_norm": 2.0215117116123813, + "language_loss": 0.77913713, + "learning_rate": 3.999997738904832e-06, + "loss": 0.80185479, + "num_input_tokens_seen": 29059945, + "router_z_loss_clip": 1.41308594, + "router_z_loss_mlp": 0.2220459, + "step": 1050, + "time_per_iteration": 2.5557713508605957 + }, + { + "auxiliary_loss_clip": 0.01063651, + "auxiliary_loss_mlp": 0.01019596, + "balance_loss_clip": 1.02374196, + "balance_loss_mlp": 1.01754522, + "epoch": 0.03049735941036504, + "flos": 67434278924160.0, + "grad_norm": 0.7552980342967687, + "language_loss": 0.54130816, + "learning_rate": 3.999997447435595e-06, + "loss": 0.5621407, + "num_input_tokens_seen": 29121730, + "router_z_loss_clip": 0.3984375, + "router_z_loss_mlp": 0.02050781, + "step": 1051, + "time_per_iteration": 3.0992860794067383 + }, + { + "auxiliary_loss_clip": 0.01232513, + "auxiliary_loss_mlp": 0.01076734, + "balance_loss_clip": 1.08165669, + "balance_loss_mlp": 1.05109191, + "epoch": 0.030526376878881087, + "flos": 37700421719040.0, + "grad_norm": 2.586672237483075, + "language_loss": 0.96063477, + "learning_rate": 3.999997138301571e-06, + "loss": 0.98372722, + "num_input_tokens_seen": 29138175, + "router_z_loss_clip": 1.50976562, + "router_z_loss_mlp": 0.25634766, + "step": 1052, + "time_per_iteration": 2.6018567085266113 + }, + { + "auxiliary_loss_clip": 0.01222274, + "auxiliary_loss_mlp": 0.01062098, + "balance_loss_clip": 1.08138371, + "balance_loss_mlp": 1.03853023, + "epoch": 0.03055539434739713, + "flos": 12120928738560.0, + "grad_norm": 2.574564680714619, + "language_loss": 0.80211186, + "learning_rate": 3.999996811502763e-06, + "loss": 0.82495558, + "num_input_tokens_seen": 29149015, + "router_z_loss_clip": 1.40820312, + "router_z_loss_mlp": 0.23583984, + "step": 1053, + "time_per_iteration": 2.5112738609313965 + }, + { + "auxiliary_loss_clip": 0.0123009, + "auxiliary_loss_mlp": 0.0106011, + "balance_loss_clip": 1.08144343, + "balance_loss_mlp": 1.03759134, + "epoch": 0.03058441181591318, + "flos": 14975791747200.0, + "grad_norm": 2.263623867334211, + "language_loss": 0.64118272, + "learning_rate": 3.999996467039174e-06, + "loss": 0.66408473, + "num_input_tokens_seen": 29164455, + "router_z_loss_clip": 1.48730469, + "router_z_loss_mlp": 0.2253418, + "step": 1054, + "time_per_iteration": 2.5092036724090576 + }, + { + "auxiliary_loss_clip": 0.01226805, + "auxiliary_loss_mlp": 0.01064464, + "balance_loss_clip": 1.08036697, + "balance_loss_mlp": 1.03978777, + "epoch": 0.030613429284429228, + "flos": 30548991519360.0, + "grad_norm": 2.425012444819279, + "language_loss": 0.95808238, + "learning_rate": 3.999996104910807e-06, + "loss": 0.98099506, + "num_input_tokens_seen": 29180150, + "router_z_loss_clip": 1.46484375, + "router_z_loss_mlp": 0.24694824, + "step": 1055, + "time_per_iteration": 2.636032819747925 + }, + { + "auxiliary_loss_clip": 0.01244627, + "auxiliary_loss_mlp": 0.01066377, + "balance_loss_clip": 1.0871551, + "balance_loss_mlp": 1.04024661, + "epoch": 0.030642446752945272, + "flos": 33722525013120.0, + "grad_norm": 2.253623952149246, + "language_loss": 0.92668307, + "learning_rate": 3.999995725117666e-06, + "loss": 0.9497931, + "num_input_tokens_seen": 29199245, + "router_z_loss_clip": 1.57519531, + "router_z_loss_mlp": 0.26135254, + "step": 1056, + "time_per_iteration": 2.6187427043914795 + }, + { + "auxiliary_loss_clip": 0.01228769, + "auxiliary_loss_mlp": 0.01061581, + "balance_loss_clip": 1.08189368, + "balance_loss_mlp": 1.03744161, + "epoch": 0.03067146422146132, + "flos": 30732171903360.0, + "grad_norm": 3.22158422076946, + "language_loss": 1.03675556, + "learning_rate": 3.999995327659752e-06, + "loss": 1.059659, + "num_input_tokens_seen": 29215115, + "router_z_loss_clip": 1.46777344, + "router_z_loss_mlp": 0.24145508, + "step": 1057, + "time_per_iteration": 2.6264894008636475 + }, + { + "auxiliary_loss_clip": 0.01063202, + "auxiliary_loss_mlp": 0.01004862, + "balance_loss_clip": 1.02419078, + "balance_loss_mlp": 1.00279999, + "epoch": 0.030700481689977365, + "flos": 61939265431680.0, + "grad_norm": 0.6641202261623632, + "language_loss": 0.58905745, + "learning_rate": 3.9999949125370706e-06, + "loss": 0.60973811, + "num_input_tokens_seen": 29285835, + "router_z_loss_clip": 0.390625, + "router_z_loss_mlp": 0.02062988, + "step": 1058, + "time_per_iteration": 3.301386833190918 + }, + { + "auxiliary_loss_clip": 0.0122988, + "auxiliary_loss_mlp": 0.01061265, + "balance_loss_clip": 1.08304739, + "balance_loss_mlp": 1.03829348, + "epoch": 0.030729499158493413, + "flos": 29854622730240.0, + "grad_norm": 3.5429590931226898, + "language_loss": 0.73670816, + "learning_rate": 3.999994479749624e-06, + "loss": 0.75961965, + "num_input_tokens_seen": 29306070, + "router_z_loss_clip": 1.46679688, + "router_z_loss_mlp": 0.22961426, + "step": 1059, + "time_per_iteration": 2.570394992828369 + }, + { + "auxiliary_loss_clip": 0.01229983, + "auxiliary_loss_mlp": 0.01068523, + "balance_loss_clip": 1.08260775, + "balance_loss_mlp": 1.0432508, + "epoch": 0.03075851662700946, + "flos": 20185315943040.0, + "grad_norm": 2.5599025834064246, + "language_loss": 1.03479922, + "learning_rate": 3.999994029297418e-06, + "loss": 1.05778432, + "num_input_tokens_seen": 29319195, + "router_z_loss_clip": 1.47167969, + "router_z_loss_mlp": 0.25305176, + "step": 1060, + "time_per_iteration": 2.5335445404052734 + }, + { + "auxiliary_loss_clip": 0.01241597, + "auxiliary_loss_mlp": 0.01070979, + "balance_loss_clip": 1.0873661, + "balance_loss_mlp": 1.04537344, + "epoch": 0.030787534095525506, + "flos": 19860791541120.0, + "grad_norm": 2.647706351986577, + "language_loss": 0.87684584, + "learning_rate": 3.999993561180455e-06, + "loss": 0.8999716, + "num_input_tokens_seen": 29334270, + "router_z_loss_clip": 1.54199219, + "router_z_loss_mlp": 0.25598145, + "step": 1061, + "time_per_iteration": 2.555541515350342 + }, + { + "auxiliary_loss_clip": 0.01063439, + "auxiliary_loss_mlp": 0.01002071, + "balance_loss_clip": 1.02436459, + "balance_loss_mlp": 1.00009191, + "epoch": 0.030816551564041554, + "flos": 74051395338240.0, + "grad_norm": 0.7571083409740194, + "language_loss": 0.55423284, + "learning_rate": 3.999993075398739e-06, + "loss": 0.57488793, + "num_input_tokens_seen": 29403110, + "router_z_loss_clip": 0.390625, + "router_z_loss_mlp": 0.01977539, + "step": 1062, + "time_per_iteration": 3.2175352573394775 + }, + { + "auxiliary_loss_clip": 0.01064059, + "auxiliary_loss_mlp": 0.0100318, + "balance_loss_clip": 1.0247581, + "balance_loss_mlp": 1.00116527, + "epoch": 0.0308455690325576, + "flos": 71240379857280.0, + "grad_norm": 0.8254548925924411, + "language_loss": 0.56330192, + "learning_rate": 3.999992571952275e-06, + "loss": 0.58397436, + "num_input_tokens_seen": 29464635, + "router_z_loss_clip": 0.39257812, + "router_z_loss_mlp": 0.0201416, + "step": 1063, + "time_per_iteration": 3.119818687438965 + }, + { + "auxiliary_loss_clip": 0.0122108, + "auxiliary_loss_mlp": 0.01050916, + "balance_loss_clip": 1.08306718, + "balance_loss_mlp": 1.0321281, + "epoch": 0.030874586501073647, + "flos": 14678235480960.0, + "grad_norm": 3.1684278141988824, + "language_loss": 0.85839689, + "learning_rate": 3.999992050841068e-06, + "loss": 0.88111687, + "num_input_tokens_seen": 29477410, + "router_z_loss_clip": 1.3828125, + "router_z_loss_mlp": 0.18798828, + "step": 1064, + "time_per_iteration": 2.525688409805298 + }, + { + "auxiliary_loss_clip": 0.01062592, + "auxiliary_loss_mlp": 0.01005328, + "balance_loss_clip": 1.02350163, + "balance_loss_mlp": 1.00340831, + "epoch": 0.03090360396958969, + "flos": 74764041171840.0, + "grad_norm": 0.6833361723731552, + "language_loss": 0.53048897, + "learning_rate": 3.999991512065121e-06, + "loss": 0.5511682, + "num_input_tokens_seen": 29538140, + "router_z_loss_clip": 0.390625, + "router_z_loss_mlp": 0.01916504, + "step": 1065, + "time_per_iteration": 3.0402162075042725 + }, + { + "auxiliary_loss_clip": 0.01217647, + "auxiliary_loss_mlp": 0.01050043, + "balance_loss_clip": 1.07890081, + "balance_loss_mlp": 1.02642715, + "epoch": 0.03093262143810574, + "flos": 20041529800320.0, + "grad_norm": 2.7550197439591817, + "language_loss": 0.80578256, + "learning_rate": 3.9999909556244405e-06, + "loss": 0.8284595, + "num_input_tokens_seen": 29552520, + "router_z_loss_clip": 1.38769531, + "router_z_loss_mlp": 0.23632812, + "step": 1066, + "time_per_iteration": 2.520481824874878 + }, + { + "auxiliary_loss_clip": 0.01224023, + "auxiliary_loss_mlp": 0.01079124, + "balance_loss_clip": 1.07622766, + "balance_loss_mlp": 1.05465031, + "epoch": 0.030961638906621788, + "flos": 16759366600320.0, + "grad_norm": 3.0546872339177416, + "language_loss": 0.91251671, + "learning_rate": 3.999990381519031e-06, + "loss": 0.93554813, + "num_input_tokens_seen": 29570035, + "router_z_loss_clip": 1.47949219, + "router_z_loss_mlp": 0.24487305, + "step": 1067, + "time_per_iteration": 2.4987175464630127 + }, + { + "auxiliary_loss_clip": 0.01237018, + "auxiliary_loss_mlp": 0.01072915, + "balance_loss_clip": 1.08222795, + "balance_loss_mlp": 1.04633164, + "epoch": 0.030990656375137832, + "flos": 20333519458560.0, + "grad_norm": 3.6324296266894702, + "language_loss": 0.89731151, + "learning_rate": 3.999989789748896e-06, + "loss": 0.92041081, + "num_input_tokens_seen": 29583255, + "router_z_loss_clip": 1.54882812, + "router_z_loss_mlp": 0.26586914, + "step": 1068, + "time_per_iteration": 2.532560110092163 + }, + { + "auxiliary_loss_clip": 0.01238075, + "auxiliary_loss_mlp": 0.01074146, + "balance_loss_clip": 1.08576798, + "balance_loss_mlp": 1.04757404, + "epoch": 0.03101967384365388, + "flos": 20442041424000.0, + "grad_norm": 3.9093288421535575, + "language_loss": 0.91026568, + "learning_rate": 3.999989180314042e-06, + "loss": 0.93338788, + "num_input_tokens_seen": 29596400, + "router_z_loss_clip": 1.5234375, + "router_z_loss_mlp": 0.26599121, + "step": 1069, + "time_per_iteration": 2.496823310852051 + }, + { + "auxiliary_loss_clip": 0.01063818, + "auxiliary_loss_mlp": 0.0101781, + "balance_loss_clip": 1.02490187, + "balance_loss_mlp": 1.01572347, + "epoch": 0.031048691312169925, + "flos": 74778191130240.0, + "grad_norm": 0.6221186456053501, + "language_loss": 0.50004959, + "learning_rate": 3.999988553214475e-06, + "loss": 0.5208658, + "num_input_tokens_seen": 29666475, + "router_z_loss_clip": 0.38867188, + "router_z_loss_mlp": 0.02087402, + "step": 1070, + "time_per_iteration": 3.2192530632019043 + }, + { + "auxiliary_loss_clip": 0.01062763, + "auxiliary_loss_mlp": 0.01011781, + "balance_loss_clip": 1.0241555, + "balance_loss_mlp": 1.00959957, + "epoch": 0.031077708780685973, + "flos": 57084501911040.0, + "grad_norm": 0.6965791458818434, + "language_loss": 0.54958463, + "learning_rate": 3.9999879084501984e-06, + "loss": 0.57033008, + "num_input_tokens_seen": 29728695, + "router_z_loss_clip": 0.38671875, + "router_z_loss_mlp": 0.02185059, + "step": 1071, + "time_per_iteration": 5.453380584716797 + }, + { + "auxiliary_loss_clip": 0.01237897, + "auxiliary_loss_mlp": 0.01074794, + "balance_loss_clip": 1.08484674, + "balance_loss_mlp": 1.04931879, + "epoch": 0.03110672624920202, + "flos": 20220508293120.0, + "grad_norm": 3.6960128693024714, + "language_loss": 1.15094733, + "learning_rate": 3.99998724602122e-06, + "loss": 1.17407429, + "num_input_tokens_seen": 29740860, + "router_z_loss_clip": 1.53125, + "router_z_loss_mlp": 0.25476074, + "step": 1072, + "time_per_iteration": 5.297034978866577 + }, + { + "auxiliary_loss_clip": 0.01233898, + "auxiliary_loss_mlp": 0.01073034, + "balance_loss_clip": 1.08249617, + "balance_loss_mlp": 1.04515052, + "epoch": 0.031135743717718066, + "flos": 27629597727360.0, + "grad_norm": 2.511414876721486, + "language_loss": 0.99642587, + "learning_rate": 3.999986565927545e-06, + "loss": 1.01949525, + "num_input_tokens_seen": 29757160, + "router_z_loss_clip": 1.51269531, + "router_z_loss_mlp": 0.27893066, + "step": 1073, + "time_per_iteration": 2.604795455932617 + }, + { + "auxiliary_loss_clip": 0.01233715, + "auxiliary_loss_mlp": 0.0106594, + "balance_loss_clip": 1.08531141, + "balance_loss_mlp": 1.04120386, + "epoch": 0.031164761186234114, + "flos": 30950760119040.0, + "grad_norm": 3.7315750562288965, + "language_loss": 1.05517113, + "learning_rate": 3.99998586816918e-06, + "loss": 1.07816768, + "num_input_tokens_seen": 29772100, + "router_z_loss_clip": 1.484375, + "router_z_loss_mlp": 0.24755859, + "step": 1074, + "time_per_iteration": 5.000601291656494 + }, + { + "auxiliary_loss_clip": 0.01231277, + "auxiliary_loss_mlp": 0.01073649, + "balance_loss_clip": 1.08378005, + "balance_loss_mlp": 1.04840076, + "epoch": 0.03119377865475016, + "flos": 33870010256640.0, + "grad_norm": 3.227614208599955, + "language_loss": 0.92709643, + "learning_rate": 3.99998515274613e-06, + "loss": 0.95014572, + "num_input_tokens_seen": 29791055, + "router_z_loss_clip": 1.47460938, + "router_z_loss_mlp": 0.25244141, + "step": 1075, + "time_per_iteration": 2.7255587577819824 + }, + { + "auxiliary_loss_clip": 0.01063274, + "auxiliary_loss_mlp": 0.01043032, + "balance_loss_clip": 1.02456474, + "balance_loss_mlp": 1.04077935, + "epoch": 0.031222796123266207, + "flos": 67511379467520.0, + "grad_norm": 0.7022586712655734, + "language_loss": 0.50306726, + "learning_rate": 3.999984419658401e-06, + "loss": 0.52413034, + "num_input_tokens_seen": 29847120, + "router_z_loss_clip": 0.38671875, + "router_z_loss_mlp": 0.02258301, + "step": 1076, + "time_per_iteration": 3.074472427368164 + }, + { + "auxiliary_loss_clip": 0.01061892, + "auxiliary_loss_mlp": 0.01016011, + "balance_loss_clip": 1.02321362, + "balance_loss_mlp": 1.01396108, + "epoch": 0.03125181359178225, + "flos": 74775820832640.0, + "grad_norm": 0.7365817724449816, + "language_loss": 0.56350195, + "learning_rate": 3.999983668906002e-06, + "loss": 0.58428097, + "num_input_tokens_seen": 29913725, + "router_z_loss_clip": 0.38671875, + "router_z_loss_mlp": 0.02050781, + "step": 1077, + "time_per_iteration": 3.2041497230529785 + }, + { + "auxiliary_loss_clip": 0.01060134, + "auxiliary_loss_mlp": 0.0100219, + "balance_loss_clip": 1.02171862, + "balance_loss_mlp": 1.00013971, + "epoch": 0.031280831060298296, + "flos": 65250407928960.0, + "grad_norm": 0.7238311235098921, + "language_loss": 0.46187818, + "learning_rate": 3.999982900488937e-06, + "loss": 0.48250148, + "num_input_tokens_seen": 29972275, + "router_z_loss_clip": 0.3828125, + "router_z_loss_mlp": 0.02050781, + "step": 1078, + "time_per_iteration": 3.2364559173583984 + }, + { + "auxiliary_loss_clip": 0.01231702, + "auxiliary_loss_mlp": 0.01071414, + "balance_loss_clip": 1.08259034, + "balance_loss_mlp": 1.04608178, + "epoch": 0.03130984852881435, + "flos": 31972921447680.0, + "grad_norm": 2.278708855828445, + "language_loss": 0.79338157, + "learning_rate": 3.999982114407214e-06, + "loss": 0.81641269, + "num_input_tokens_seen": 29987945, + "router_z_loss_clip": 1.4921875, + "router_z_loss_mlp": 0.25354004, + "step": 1079, + "time_per_iteration": 2.6204259395599365 + }, + { + "auxiliary_loss_clip": 0.01061925, + "auxiliary_loss_mlp": 0.0103947, + "balance_loss_clip": 1.02390838, + "balance_loss_mlp": 1.03727674, + "epoch": 0.03133886599733039, + "flos": 65413766983680.0, + "grad_norm": 0.7175472001984587, + "language_loss": 0.53230482, + "learning_rate": 3.999981310660839e-06, + "loss": 0.5533188, + "num_input_tokens_seen": 30051910, + "router_z_loss_clip": 0.37890625, + "router_z_loss_mlp": 0.02197266, + "step": 1080, + "time_per_iteration": 3.1768722534179688 + }, + { + "auxiliary_loss_clip": 0.01063345, + "auxiliary_loss_mlp": 0.0105059, + "balance_loss_clip": 1.02517414, + "balance_loss_mlp": 1.04844403, + "epoch": 0.03136788346584644, + "flos": 63309830728320.0, + "grad_norm": 0.7308925863111236, + "language_loss": 0.53040278, + "learning_rate": 3.99998048924982e-06, + "loss": 0.55154216, + "num_input_tokens_seen": 30110715, + "router_z_loss_clip": 0.3828125, + "router_z_loss_mlp": 0.02148438, + "step": 1081, + "time_per_iteration": 2.9746954441070557 + }, + { + "auxiliary_loss_clip": 0.01227663, + "auxiliary_loss_mlp": 0.01079551, + "balance_loss_clip": 1.08498573, + "balance_loss_mlp": 1.05603099, + "epoch": 0.03139690093436249, + "flos": 74731287968640.0, + "grad_norm": 2.6981827294092695, + "language_loss": 0.88158095, + "learning_rate": 3.999979650174164e-06, + "loss": 0.90465313, + "num_input_tokens_seen": 30136620, + "router_z_loss_clip": 1.42675781, + "router_z_loss_mlp": 0.23547363, + "step": 1082, + "time_per_iteration": 2.899092197418213 + }, + { + "auxiliary_loss_clip": 0.01243722, + "auxiliary_loss_mlp": 0.01066748, + "balance_loss_clip": 1.08242559, + "balance_loss_mlp": 1.04204798, + "epoch": 0.03142591840287853, + "flos": 36101715189120.0, + "grad_norm": 2.797024233049826, + "language_loss": 1.04863358, + "learning_rate": 3.9999787934338785e-06, + "loss": 1.07173836, + "num_input_tokens_seen": 30157695, + "router_z_loss_clip": 1.61425781, + "router_z_loss_mlp": 0.24694824, + "step": 1083, + "time_per_iteration": 2.668700933456421 + }, + { + "auxiliary_loss_clip": 0.01232908, + "auxiliary_loss_mlp": 0.0106936, + "balance_loss_clip": 1.08139753, + "balance_loss_mlp": 1.04516029, + "epoch": 0.03145493587139458, + "flos": 15659853333120.0, + "grad_norm": 3.3231835825817937, + "language_loss": 0.97453934, + "learning_rate": 3.999977919028971e-06, + "loss": 0.99756205, + "num_input_tokens_seen": 30168260, + "router_z_loss_clip": 1.51367188, + "router_z_loss_mlp": 0.24194336, + "step": 1084, + "time_per_iteration": 2.488548755645752 + }, + { + "auxiliary_loss_clip": 0.01230898, + "auxiliary_loss_mlp": 0.0106432, + "balance_loss_clip": 1.08557272, + "balance_loss_mlp": 1.03874934, + "epoch": 0.03148395333991063, + "flos": 39487408364160.0, + "grad_norm": 2.1667179366168163, + "language_loss": 0.99880552, + "learning_rate": 3.999977026959449e-06, + "loss": 1.02175772, + "num_input_tokens_seen": 30185235, + "router_z_loss_clip": 1.45214844, + "router_z_loss_mlp": 0.2557373, + "step": 1085, + "time_per_iteration": 2.682046413421631 + }, + { + "auxiliary_loss_clip": 0.01231377, + "auxiliary_loss_mlp": 0.01054383, + "balance_loss_clip": 1.08523655, + "balance_loss_mlp": 1.03272295, + "epoch": 0.031512970808426674, + "flos": 33686362995840.0, + "grad_norm": 2.972631280487473, + "language_loss": 0.84550667, + "learning_rate": 3.999976117225321e-06, + "loss": 0.86836421, + "num_input_tokens_seen": 30201055, + "router_z_loss_clip": 1.46191406, + "router_z_loss_mlp": 0.21655273, + "step": 1086, + "time_per_iteration": 2.546081304550171 + }, + { + "auxiliary_loss_clip": 0.01229273, + "auxiliary_loss_mlp": 0.01069172, + "balance_loss_clip": 1.08178294, + "balance_loss_mlp": 1.04338694, + "epoch": 0.03154198827694272, + "flos": 21463304912640.0, + "grad_norm": 3.048447186047043, + "language_loss": 0.80738688, + "learning_rate": 3.999975189826594e-06, + "loss": 0.83037132, + "num_input_tokens_seen": 30214355, + "router_z_loss_clip": 1.47558594, + "router_z_loss_mlp": 0.25793457, + "step": 1087, + "time_per_iteration": 2.5467417240142822 + }, + { + "auxiliary_loss_clip": 0.01233364, + "auxiliary_loss_mlp": 0.01058884, + "balance_loss_clip": 1.08314943, + "balance_loss_mlp": 1.03250265, + "epoch": 0.03157100574545876, + "flos": 29642283480960.0, + "grad_norm": 5.087958767265608, + "language_loss": 0.95039749, + "learning_rate": 3.9999742447632775e-06, + "loss": 0.97332001, + "num_input_tokens_seen": 30230100, + "router_z_loss_clip": 1.50097656, + "router_z_loss_mlp": 0.26379395, + "step": 1088, + "time_per_iteration": 2.592064142227173 + }, + { + "auxiliary_loss_clip": 0.01231114, + "auxiliary_loss_mlp": 0.01059576, + "balance_loss_clip": 1.08401465, + "balance_loss_mlp": 1.03351712, + "epoch": 0.031600023213974815, + "flos": 32811507342720.0, + "grad_norm": 2.2957679244303524, + "language_loss": 0.98609155, + "learning_rate": 3.99997328203538e-06, + "loss": 1.00899839, + "num_input_tokens_seen": 30249465, + "router_z_loss_clip": 1.47070312, + "router_z_loss_mlp": 0.26043701, + "step": 1089, + "time_per_iteration": 2.6177549362182617 + }, + { + "auxiliary_loss_clip": 0.01233391, + "auxiliary_loss_mlp": 0.01058109, + "balance_loss_clip": 1.08020556, + "balance_loss_mlp": 1.03189516, + "epoch": 0.03162904068249086, + "flos": 23548458355200.0, + "grad_norm": 2.851266265394564, + "language_loss": 0.95338869, + "learning_rate": 3.999972301642907e-06, + "loss": 0.97630376, + "num_input_tokens_seen": 30265955, + "router_z_loss_clip": 1.53320312, + "router_z_loss_mlp": 0.26208496, + "step": 1090, + "time_per_iteration": 2.5667266845703125 + }, + { + "auxiliary_loss_clip": 0.01234231, + "auxiliary_loss_mlp": 0.01067252, + "balance_loss_clip": 1.08452642, + "balance_loss_mlp": 1.04369652, + "epoch": 0.031658058151006904, + "flos": 30697985134080.0, + "grad_norm": 2.24460596316747, + "language_loss": 0.63312435, + "learning_rate": 3.999971303585871e-06, + "loss": 0.65613919, + "num_input_tokens_seen": 30282960, + "router_z_loss_clip": 1.49707031, + "router_z_loss_mlp": 0.23547363, + "step": 1091, + "time_per_iteration": 2.6313421726226807 + }, + { + "auxiliary_loss_clip": 0.01064101, + "auxiliary_loss_mlp": 0.01007308, + "balance_loss_clip": 1.02534258, + "balance_loss_mlp": 1.00499523, + "epoch": 0.031687075619522956, + "flos": 74593861511040.0, + "grad_norm": 0.7753155624635018, + "language_loss": 0.52627766, + "learning_rate": 3.999970287864279e-06, + "loss": 0.54699171, + "num_input_tokens_seen": 30344585, + "router_z_loss_clip": 0.38671875, + "router_z_loss_mlp": 0.02307129, + "step": 1092, + "time_per_iteration": 3.2099595069885254 + }, + { + "auxiliary_loss_clip": 0.01226473, + "auxiliary_loss_mlp": 0.01075533, + "balance_loss_clip": 1.07928944, + "balance_loss_mlp": 1.04931855, + "epoch": 0.031716093088039, + "flos": 33430643095680.0, + "grad_norm": 2.5711624996517832, + "language_loss": 0.87632513, + "learning_rate": 3.9999692544781385e-06, + "loss": 0.89934516, + "num_input_tokens_seen": 30360275, + "router_z_loss_clip": 1.47070312, + "router_z_loss_mlp": 0.2623291, + "step": 1093, + "time_per_iteration": 2.6609976291656494 + }, + { + "auxiliary_loss_clip": 0.0122415, + "auxiliary_loss_mlp": 0.01055193, + "balance_loss_clip": 1.0843401, + "balance_loss_mlp": 1.03465343, + "epoch": 0.031745110556555045, + "flos": 26207319824640.0, + "grad_norm": 2.3275538983583206, + "language_loss": 0.93900049, + "learning_rate": 3.999968203427463e-06, + "loss": 0.9617939, + "num_input_tokens_seen": 30377165, + "router_z_loss_clip": 1.39648438, + "router_z_loss_mlp": 0.20556641, + "step": 1094, + "time_per_iteration": 2.6152541637420654 + }, + { + "auxiliary_loss_clip": 0.01221219, + "auxiliary_loss_mlp": 0.01062943, + "balance_loss_clip": 1.07858014, + "balance_loss_mlp": 1.03914881, + "epoch": 0.03177412802507109, + "flos": 16688910072960.0, + "grad_norm": 2.582852211564546, + "language_loss": 0.83961964, + "learning_rate": 3.999967134712257e-06, + "loss": 0.86246127, + "num_input_tokens_seen": 30391495, + "router_z_loss_clip": 1.42578125, + "router_z_loss_mlp": 0.23803711, + "step": 1095, + "time_per_iteration": 2.4896352291107178 + }, + { + "auxiliary_loss_clip": 0.01232993, + "auxiliary_loss_mlp": 0.01068847, + "balance_loss_clip": 1.08414483, + "balance_loss_mlp": 1.0433718, + "epoch": 0.03180314549358714, + "flos": 35548726331520.0, + "grad_norm": 2.384895701487018, + "language_loss": 0.97635162, + "learning_rate": 3.999966048332532e-06, + "loss": 0.99937004, + "num_input_tokens_seen": 30408975, + "router_z_loss_clip": 1.48925781, + "router_z_loss_mlp": 0.25476074, + "step": 1096, + "time_per_iteration": 2.6419715881347656 + }, + { + "auxiliary_loss_clip": 0.01220024, + "auxiliary_loss_mlp": 0.01073738, + "balance_loss_clip": 1.07832313, + "balance_loss_mlp": 1.05019391, + "epoch": 0.031832162962103186, + "flos": 20074926470400.0, + "grad_norm": 4.652569939470712, + "language_loss": 0.92031187, + "learning_rate": 3.999964944288298e-06, + "loss": 0.94324946, + "num_input_tokens_seen": 30421450, + "router_z_loss_clip": 1.41894531, + "router_z_loss_mlp": 0.2355957, + "step": 1097, + "time_per_iteration": 2.4887685775756836 + }, + { + "auxiliary_loss_clip": 0.01218228, + "auxiliary_loss_mlp": 0.0105711, + "balance_loss_clip": 1.08343625, + "balance_loss_mlp": 1.03712511, + "epoch": 0.03186118043061923, + "flos": 29418667361280.0, + "grad_norm": 2.1355579513403713, + "language_loss": 0.75498056, + "learning_rate": 3.999963822579565e-06, + "loss": 0.77773392, + "num_input_tokens_seen": 30437610, + "router_z_loss_clip": 1.34765625, + "router_z_loss_mlp": 0.20001221, + "step": 1098, + "time_per_iteration": 2.579054355621338 + }, + { + "auxiliary_loss_clip": 0.01061439, + "auxiliary_loss_mlp": 0.01006409, + "balance_loss_clip": 1.02394176, + "balance_loss_mlp": 1.00437021, + "epoch": 0.03189019789913528, + "flos": 58102712743680.0, + "grad_norm": 0.8250729674939018, + "language_loss": 0.53373182, + "learning_rate": 3.999962683206341e-06, + "loss": 0.55441034, + "num_input_tokens_seen": 30495365, + "router_z_loss_clip": 0.375, + "router_z_loss_mlp": 0.02038574, + "step": 1099, + "time_per_iteration": 3.0120956897735596 + }, + { + "auxiliary_loss_clip": 0.01218966, + "auxiliary_loss_mlp": 0.01048315, + "balance_loss_clip": 1.08005619, + "balance_loss_mlp": 1.02565312, + "epoch": 0.03191921536765133, + "flos": 10626721850880.0, + "grad_norm": 3.252847270324612, + "language_loss": 0.86783141, + "learning_rate": 3.999961526168638e-06, + "loss": 0.89050424, + "num_input_tokens_seen": 30505960, + "router_z_loss_clip": 1.390625, + "router_z_loss_mlp": 0.2265625, + "step": 1100, + "time_per_iteration": 2.4873218536376953 + }, + { + "auxiliary_loss_clip": 0.01060378, + "auxiliary_loss_mlp": 0.01012302, + "balance_loss_clip": 1.02326989, + "balance_loss_mlp": 1.01034665, + "epoch": 0.03194823283616737, + "flos": 74788318765440.0, + "grad_norm": 0.6910897473141238, + "language_loss": 0.49452409, + "learning_rate": 3.999960351466465e-06, + "loss": 0.51525092, + "num_input_tokens_seen": 30574655, + "router_z_loss_clip": 0.37109375, + "router_z_loss_mlp": 0.01953125, + "step": 1101, + "time_per_iteration": 3.2392845153808594 + }, + { + "auxiliary_loss_clip": 0.01226319, + "auxiliary_loss_mlp": 0.01077222, + "balance_loss_clip": 1.08366489, + "balance_loss_mlp": 1.05416656, + "epoch": 0.031977250304683416, + "flos": 33332105111040.0, + "grad_norm": 2.6300092332462186, + "language_loss": 1.020226, + "learning_rate": 3.9999591590998334e-06, + "loss": 1.04326153, + "num_input_tokens_seen": 30592020, + "router_z_loss_clip": 1.42773438, + "router_z_loss_mlp": 0.23046875, + "step": 1102, + "time_per_iteration": 2.681151866912842 + }, + { + "auxiliary_loss_clip": 0.0121491, + "auxiliary_loss_mlp": 0.01059573, + "balance_loss_clip": 1.07865977, + "balance_loss_mlp": 1.03681588, + "epoch": 0.03200626777319947, + "flos": 16574462363520.0, + "grad_norm": 2.3905994700253506, + "language_loss": 0.80587608, + "learning_rate": 3.9999579490687525e-06, + "loss": 0.82862091, + "num_input_tokens_seen": 30604605, + "router_z_loss_clip": 1.36425781, + "router_z_loss_mlp": 0.22741699, + "step": 1103, + "time_per_iteration": 2.4884464740753174 + }, + { + "auxiliary_loss_clip": 0.01243619, + "auxiliary_loss_mlp": 0.01056037, + "balance_loss_clip": 1.08746457, + "balance_loss_mlp": 1.02998948, + "epoch": 0.03203528524171551, + "flos": 74738075639040.0, + "grad_norm": 1.9065588278433054, + "language_loss": 1.04129314, + "learning_rate": 3.999956721373235e-06, + "loss": 1.06428969, + "num_input_tokens_seen": 30634080, + "router_z_loss_clip": 1.56152344, + "router_z_loss_mlp": 0.26049805, + "step": 1104, + "time_per_iteration": 2.92010498046875 + }, + { + "auxiliary_loss_clip": 0.0123497, + "auxiliary_loss_mlp": 0.01083176, + "balance_loss_clip": 1.08622873, + "balance_loss_mlp": 1.05970407, + "epoch": 0.03206430271023156, + "flos": 74741343776640.0, + "grad_norm": 2.1246916260532247, + "language_loss": 0.76771891, + "learning_rate": 3.999955476013289e-06, + "loss": 0.79090029, + "num_input_tokens_seen": 30658905, + "router_z_loss_clip": 1.48730469, + "router_z_loss_mlp": 0.23461914, + "step": 1105, + "time_per_iteration": 2.948430299758911 + }, + { + "auxiliary_loss_clip": 0.01060076, + "auxiliary_loss_mlp": 0.01001586, + "balance_loss_clip": 1.02241552, + "balance_loss_mlp": 0.99970227, + "epoch": 0.03209332017874761, + "flos": 70433359038720.0, + "grad_norm": 0.7074593165504102, + "language_loss": 0.53126299, + "learning_rate": 3.999954212988927e-06, + "loss": 0.55187958, + "num_input_tokens_seen": 30719175, + "router_z_loss_clip": 0.37695312, + "router_z_loss_mlp": 0.01879883, + "step": 1106, + "time_per_iteration": 3.0710864067077637 + }, + { + "auxiliary_loss_clip": 0.01225128, + "auxiliary_loss_mlp": 0.01067708, + "balance_loss_clip": 1.08326435, + "balance_loss_mlp": 1.04199481, + "epoch": 0.03212233764726365, + "flos": 42335340048000.0, + "grad_norm": 2.236066602833272, + "language_loss": 1.1210146, + "learning_rate": 3.999952932300161e-06, + "loss": 1.14394295, + "num_input_tokens_seen": 30746765, + "router_z_loss_clip": 1.41992188, + "router_z_loss_mlp": 0.25695801, + "step": 1107, + "time_per_iteration": 2.862548589706421 + }, + { + "auxiliary_loss_clip": 0.01226737, + "auxiliary_loss_mlp": 0.01075872, + "balance_loss_clip": 1.07961619, + "balance_loss_mlp": 1.05009878, + "epoch": 0.0321513551157797, + "flos": 12779889696000.0, + "grad_norm": 2.5561340039322835, + "language_loss": 0.90430588, + "learning_rate": 3.9999516339470015e-06, + "loss": 0.92733198, + "num_input_tokens_seen": 30760580, + "router_z_loss_clip": 1.47167969, + "router_z_loss_mlp": 0.25769043, + "step": 1108, + "time_per_iteration": 2.5923333168029785 + }, + { + "auxiliary_loss_clip": 0.01237578, + "auxiliary_loss_mlp": 0.01066104, + "balance_loss_clip": 1.08378363, + "balance_loss_mlp": 1.03894854, + "epoch": 0.03218037258429575, + "flos": 18907722023040.0, + "grad_norm": 2.9772351601237883, + "language_loss": 0.9130637, + "learning_rate": 3.999950317929459e-06, + "loss": 0.9361006, + "num_input_tokens_seen": 30773240, + "router_z_loss_clip": 1.53710938, + "router_z_loss_mlp": 0.2713623, + "step": 1109, + "time_per_iteration": 2.5494978427886963 + }, + { + "auxiliary_loss_clip": 0.01233122, + "auxiliary_loss_mlp": 0.01061422, + "balance_loss_clip": 1.08067036, + "balance_loss_mlp": 1.03938019, + "epoch": 0.032209390052811794, + "flos": 17925385898880.0, + "grad_norm": 2.8087428970490764, + "language_loss": 0.86856925, + "learning_rate": 3.999948984247547e-06, + "loss": 0.89151472, + "num_input_tokens_seen": 30790030, + "router_z_loss_clip": 1.5234375, + "router_z_loss_mlp": 0.22045898, + "step": 1110, + "time_per_iteration": 2.575000762939453 + }, + { + "auxiliary_loss_clip": 0.01245684, + "auxiliary_loss_mlp": 0.01070684, + "balance_loss_clip": 1.09010005, + "balance_loss_mlp": 1.04260993, + "epoch": 0.03223840752132784, + "flos": 44338903747200.0, + "grad_norm": 2.413540055828144, + "language_loss": 0.96148646, + "learning_rate": 3.999947632901276e-06, + "loss": 0.98465014, + "num_input_tokens_seen": 30807085, + "router_z_loss_clip": 1.55566406, + "router_z_loss_mlp": 0.28051758, + "step": 1111, + "time_per_iteration": 2.7204151153564453 + }, + { + "auxiliary_loss_clip": 0.01058622, + "auxiliary_loss_mlp": 0.01003287, + "balance_loss_clip": 1.02086687, + "balance_loss_mlp": 1.00118887, + "epoch": 0.03226742498984388, + "flos": 70629037355520.0, + "grad_norm": 0.6808473043713754, + "language_loss": 0.55731559, + "learning_rate": 3.999946263890658e-06, + "loss": 0.57793468, + "num_input_tokens_seen": 30873065, + "router_z_loss_clip": 0.37695312, + "router_z_loss_mlp": 0.02099609, + "step": 1112, + "time_per_iteration": 3.3047616481781006 + }, + { + "auxiliary_loss_clip": 0.01222838, + "auxiliary_loss_mlp": 0.01069803, + "balance_loss_clip": 1.07698071, + "balance_loss_mlp": 1.04583025, + "epoch": 0.032296442458359935, + "flos": 20812316774400.0, + "grad_norm": 2.239795179132523, + "language_loss": 0.90865886, + "learning_rate": 3.999944877215704e-06, + "loss": 0.93158531, + "num_input_tokens_seen": 30892150, + "router_z_loss_clip": 1.45898438, + "router_z_loss_mlp": 0.23950195, + "step": 1113, + "time_per_iteration": 2.612121105194092 + }, + { + "auxiliary_loss_clip": 0.01230609, + "auxiliary_loss_mlp": 0.01077306, + "balance_loss_clip": 1.08427143, + "balance_loss_mlp": 1.05241561, + "epoch": 0.03232545992687598, + "flos": 25260319704960.0, + "grad_norm": 11.38504926847814, + "language_loss": 0.70394635, + "learning_rate": 3.99994347287643e-06, + "loss": 0.72702551, + "num_input_tokens_seen": 30907795, + "router_z_loss_clip": 1.46386719, + "router_z_loss_mlp": 0.24865723, + "step": 1114, + "time_per_iteration": 2.637134313583374 + }, + { + "auxiliary_loss_clip": 0.01057742, + "auxiliary_loss_mlp": 0.01000659, + "balance_loss_clip": 1.02073061, + "balance_loss_mlp": 0.99852484, + "epoch": 0.032354477395392024, + "flos": 60425629286400.0, + "grad_norm": 0.7144584729841114, + "language_loss": 0.47966886, + "learning_rate": 3.999942050872844e-06, + "loss": 0.5002529, + "num_input_tokens_seen": 30961055, + "router_z_loss_clip": 0.36914062, + "router_z_loss_mlp": 0.0213623, + "step": 1115, + "time_per_iteration": 2.9435558319091797 + }, + { + "auxiliary_loss_clip": 0.01239723, + "auxiliary_loss_mlp": 0.0107252, + "balance_loss_clip": 1.08662176, + "balance_loss_mlp": 1.04562676, + "epoch": 0.032383494863908076, + "flos": 26134169777280.0, + "grad_norm": 3.8269181539622394, + "language_loss": 0.96465123, + "learning_rate": 3.999940611204961e-06, + "loss": 0.98777372, + "num_input_tokens_seen": 30976655, + "router_z_loss_clip": 1.53027344, + "router_z_loss_mlp": 0.2689209, + "step": 1116, + "time_per_iteration": 2.63725209236145 + }, + { + "auxiliary_loss_clip": 0.01222676, + "auxiliary_loss_mlp": 0.0106275, + "balance_loss_clip": 1.07656944, + "balance_loss_mlp": 1.03802645, + "epoch": 0.03241251233242412, + "flos": 16426079280000.0, + "grad_norm": 3.74706440041978, + "language_loss": 0.91103935, + "learning_rate": 3.999939153872793e-06, + "loss": 0.93389368, + "num_input_tokens_seen": 30986590, + "router_z_loss_clip": 1.45996094, + "router_z_loss_mlp": 0.24743652, + "step": 1117, + "time_per_iteration": 2.5050852298736572 + }, + { + "auxiliary_loss_clip": 0.01233446, + "auxiliary_loss_mlp": 0.01076316, + "balance_loss_clip": 1.08092201, + "balance_loss_mlp": 1.05148435, + "epoch": 0.032441529800940165, + "flos": 33396348585600.0, + "grad_norm": 4.475242399263445, + "language_loss": 1.28954172, + "learning_rate": 3.999937678876355e-06, + "loss": 1.31263936, + "num_input_tokens_seen": 31007615, + "router_z_loss_clip": 1.52636719, + "router_z_loss_mlp": 0.24829102, + "step": 1118, + "time_per_iteration": 2.7111172676086426 + }, + { + "auxiliary_loss_clip": 0.01235002, + "auxiliary_loss_mlp": 0.01072351, + "balance_loss_clip": 1.08247828, + "balance_loss_mlp": 1.04501581, + "epoch": 0.03247054726945621, + "flos": 33139084400640.0, + "grad_norm": 6.542970542841416, + "language_loss": 0.88515031, + "learning_rate": 3.9999361862156565e-06, + "loss": 0.90822387, + "num_input_tokens_seen": 31024700, + "router_z_loss_clip": 1.52539062, + "router_z_loss_mlp": 0.27319336, + "step": 1119, + "time_per_iteration": 2.6211633682250977 + }, + { + "auxiliary_loss_clip": 0.0123028, + "auxiliary_loss_mlp": 0.01064523, + "balance_loss_clip": 1.0795691, + "balance_loss_mlp": 1.03807056, + "epoch": 0.03249956473797226, + "flos": 11940657356160.0, + "grad_norm": 2.865399129170839, + "language_loss": 0.87275505, + "learning_rate": 3.999934675890713e-06, + "loss": 0.89570314, + "num_input_tokens_seen": 31037760, + "router_z_loss_clip": 1.5078125, + "router_z_loss_mlp": 0.26452637, + "step": 1120, + "time_per_iteration": 2.496983051300049 + }, + { + "auxiliary_loss_clip": 0.01217466, + "auxiliary_loss_mlp": 0.01082359, + "balance_loss_clip": 1.0795157, + "balance_loss_mlp": 1.0577538, + "epoch": 0.032528582206488306, + "flos": 26099587958400.0, + "grad_norm": 2.3216486879831493, + "language_loss": 0.87411916, + "learning_rate": 3.999933147901536e-06, + "loss": 0.89711738, + "num_input_tokens_seen": 31052295, + "router_z_loss_clip": 1.37988281, + "router_z_loss_mlp": 0.24621582, + "step": 1121, + "time_per_iteration": 2.6082687377929688 + }, + { + "auxiliary_loss_clip": 0.01060452, + "auxiliary_loss_mlp": 0.01023538, + "balance_loss_clip": 1.02295017, + "balance_loss_mlp": 1.02120113, + "epoch": 0.03255759967500435, + "flos": 65717210102400.0, + "grad_norm": 0.7845979667276256, + "language_loss": 0.55222452, + "learning_rate": 3.999931602248141e-06, + "loss": 0.57306433, + "num_input_tokens_seen": 31107880, + "router_z_loss_clip": 0.375, + "router_z_loss_mlp": 0.02331543, + "step": 1122, + "time_per_iteration": 3.0305066108703613 + }, + { + "auxiliary_loss_clip": 0.01219177, + "auxiliary_loss_mlp": 0.01063307, + "balance_loss_clip": 1.0780549, + "balance_loss_mlp": 1.03978682, + "epoch": 0.0325866171435204, + "flos": 33648836261760.0, + "grad_norm": 3.8889732176380205, + "language_loss": 0.99877334, + "learning_rate": 3.999930038930541e-06, + "loss": 1.0215981, + "num_input_tokens_seen": 31122910, + "router_z_loss_clip": 1.40917969, + "router_z_loss_mlp": 0.23510742, + "step": 1123, + "time_per_iteration": 2.707817316055298 + }, + { + "auxiliary_loss_clip": 0.01059333, + "auxiliary_loss_mlp": 0.01009816, + "balance_loss_clip": 1.02201295, + "balance_loss_mlp": 1.00757492, + "epoch": 0.03261563461203645, + "flos": 69702792318720.0, + "grad_norm": 1.1606068944352677, + "language_loss": 0.52310622, + "learning_rate": 3.999928457948749e-06, + "loss": 0.54379773, + "num_input_tokens_seen": 31187025, + "router_z_loss_clip": 0.37109375, + "router_z_loss_mlp": 0.02246094, + "step": 1124, + "time_per_iteration": 3.0803310871124268 + }, + { + "auxiliary_loss_clip": 0.01058327, + "auxiliary_loss_mlp": 0.01004997, + "balance_loss_clip": 1.02109182, + "balance_loss_mlp": 1.00277925, + "epoch": 0.03264465208055249, + "flos": 67916021155200.0, + "grad_norm": 0.6926593895743555, + "language_loss": 0.5136562, + "learning_rate": 3.99992685930278e-06, + "loss": 0.53428942, + "num_input_tokens_seen": 31252435, + "router_z_loss_clip": 0.37109375, + "router_z_loss_mlp": 0.0222168, + "step": 1125, + "time_per_iteration": 3.140471935272217 + }, + { + "auxiliary_loss_clip": 0.01223908, + "auxiliary_loss_mlp": 0.0106455, + "balance_loss_clip": 1.0819695, + "balance_loss_mlp": 1.04061306, + "epoch": 0.032673669549068536, + "flos": 25842611082240.0, + "grad_norm": 2.3373306562127594, + "language_loss": 0.93891084, + "learning_rate": 3.9999252429926475e-06, + "loss": 0.96179545, + "num_input_tokens_seen": 31267990, + "router_z_loss_clip": 1.41894531, + "router_z_loss_mlp": 0.23937988, + "step": 1126, + "time_per_iteration": 2.559913158416748 + }, + { + "auxiliary_loss_clip": 0.01057885, + "auxiliary_loss_mlp": 0.01000422, + "balance_loss_clip": 1.02068758, + "balance_loss_mlp": 0.9982878, + "epoch": 0.03270268701758459, + "flos": 62050480917120.0, + "grad_norm": 0.7146712434746619, + "language_loss": 0.54885727, + "learning_rate": 3.999923609018365e-06, + "loss": 0.56944036, + "num_input_tokens_seen": 31332135, + "router_z_loss_clip": 0.37109375, + "router_z_loss_mlp": 0.0213623, + "step": 1127, + "time_per_iteration": 3.1930150985717773 + }, + { + "auxiliary_loss_clip": 0.01225193, + "auxiliary_loss_mlp": 0.01060865, + "balance_loss_clip": 1.08450115, + "balance_loss_mlp": 1.03797626, + "epoch": 0.03273170448610063, + "flos": 25331997294720.0, + "grad_norm": 2.4057512828464005, + "language_loss": 0.78553981, + "learning_rate": 3.99992195737995e-06, + "loss": 0.80840039, + "num_input_tokens_seen": 31346340, + "router_z_loss_clip": 1.40820312, + "router_z_loss_mlp": 0.22875977, + "step": 1128, + "time_per_iteration": 2.594557046890259 + }, + { + "auxiliary_loss_clip": 0.01230287, + "auxiliary_loss_mlp": 0.01073933, + "balance_loss_clip": 1.08364987, + "balance_loss_mlp": 1.04809999, + "epoch": 0.03276072195461668, + "flos": 45253476864000.0, + "grad_norm": 2.46355201524172, + "language_loss": 0.97108501, + "learning_rate": 3.999920288077414e-06, + "loss": 0.99412715, + "num_input_tokens_seen": 31366845, + "router_z_loss_clip": 1.46679688, + "router_z_loss_mlp": 0.25854492, + "step": 1129, + "time_per_iteration": 2.71125864982605 + }, + { + "auxiliary_loss_clip": 0.01215823, + "auxiliary_loss_mlp": 0.01071873, + "balance_loss_clip": 1.08104205, + "balance_loss_mlp": 1.05109501, + "epoch": 0.03278973942313273, + "flos": 35620511662080.0, + "grad_norm": 2.3937843094577405, + "language_loss": 0.83226275, + "learning_rate": 3.999918601110772e-06, + "loss": 0.85513979, + "num_input_tokens_seen": 31385380, + "router_z_loss_clip": 1.34863281, + "router_z_loss_mlp": 0.20776367, + "step": 1130, + "time_per_iteration": 2.684358596801758 + }, + { + "auxiliary_loss_clip": 0.01239811, + "auxiliary_loss_mlp": 0.01074947, + "balance_loss_clip": 1.08966279, + "balance_loss_mlp": 1.0492456, + "epoch": 0.03281875689164877, + "flos": 14350263373440.0, + "grad_norm": 2.574809130512908, + "language_loss": 0.71666348, + "learning_rate": 3.99991689648004e-06, + "loss": 0.73981106, + "num_input_tokens_seen": 31400620, + "router_z_loss_clip": 1.50097656, + "router_z_loss_mlp": 0.25720215, + "step": 1131, + "time_per_iteration": 2.5118298530578613 + }, + { + "auxiliary_loss_clip": 0.01217315, + "auxiliary_loss_mlp": 0.01058131, + "balance_loss_clip": 1.07882178, + "balance_loss_mlp": 1.03738856, + "epoch": 0.03284777436016482, + "flos": 29606085550080.0, + "grad_norm": 2.481360571808465, + "language_loss": 1.05181611, + "learning_rate": 3.999915174185233e-06, + "loss": 1.07457066, + "num_input_tokens_seen": 31419070, + "router_z_loss_clip": 1.38378906, + "router_z_loss_mlp": 0.20727539, + "step": 1132, + "time_per_iteration": 2.629671812057495 + }, + { + "auxiliary_loss_clip": 0.01231146, + "auxiliary_loss_mlp": 0.01074347, + "balance_loss_clip": 1.0849911, + "balance_loss_mlp": 1.04895592, + "epoch": 0.03287679182868086, + "flos": 20770300840320.0, + "grad_norm": 2.4433998241228903, + "language_loss": 0.62584209, + "learning_rate": 3.999913434226366e-06, + "loss": 0.64889705, + "num_input_tokens_seen": 31433095, + "router_z_loss_clip": 1.4609375, + "router_z_loss_mlp": 0.25378418, + "step": 1133, + "time_per_iteration": 2.64482045173645 + }, + { + "auxiliary_loss_clip": 0.01238977, + "auxiliary_loss_mlp": 0.01075974, + "balance_loss_clip": 1.08737528, + "balance_loss_mlp": 1.04892516, + "epoch": 0.032905809297196914, + "flos": 18763935880320.0, + "grad_norm": 2.318525433891448, + "language_loss": 1.04333591, + "learning_rate": 3.999911676603454e-06, + "loss": 1.06648529, + "num_input_tokens_seen": 31450930, + "router_z_loss_clip": 1.51757812, + "router_z_loss_mlp": 0.27087402, + "step": 1134, + "time_per_iteration": 2.6287786960601807 + }, + { + "auxiliary_loss_clip": 0.01227681, + "auxiliary_loss_mlp": 0.0106678, + "balance_loss_clip": 1.08140969, + "balance_loss_mlp": 1.04318881, + "epoch": 0.03293482676571296, + "flos": 17414233407360.0, + "grad_norm": 3.063550177591903, + "language_loss": 0.90338933, + "learning_rate": 3.999909901316513e-06, + "loss": 0.9263339, + "num_input_tokens_seen": 31464330, + "router_z_loss_clip": 1.46386719, + "router_z_loss_mlp": 0.23608398, + "step": 1135, + "time_per_iteration": 2.488687038421631 + }, + { + "auxiliary_loss_clip": 0.01239775, + "auxiliary_loss_mlp": 0.01072757, + "balance_loss_clip": 1.08792114, + "balance_loss_mlp": 1.04760361, + "epoch": 0.032963844234229, + "flos": 22628210889600.0, + "grad_norm": 2.428890426478567, + "language_loss": 0.90224826, + "learning_rate": 3.999908108365559e-06, + "loss": 0.92537355, + "num_input_tokens_seen": 31478310, + "router_z_loss_clip": 1.51757812, + "router_z_loss_mlp": 0.25170898, + "step": 1136, + "time_per_iteration": 2.6556832790374756 + }, + { + "auxiliary_loss_clip": 0.01060795, + "auxiliary_loss_mlp": 0.01016578, + "balance_loss_clip": 1.02355456, + "balance_loss_mlp": 1.01438427, + "epoch": 0.032992861702745055, + "flos": 74779124883840.0, + "grad_norm": 0.7465399967436406, + "language_loss": 0.56382847, + "learning_rate": 3.999906297750607e-06, + "loss": 0.58460218, + "num_input_tokens_seen": 31545475, + "router_z_loss_clip": 0.37109375, + "router_z_loss_mlp": 0.02197266, + "step": 1137, + "time_per_iteration": 3.2048585414886475 + }, + { + "auxiliary_loss_clip": 0.01058817, + "auxiliary_loss_mlp": 0.01002991, + "balance_loss_clip": 1.02202284, + "balance_loss_mlp": 1.00082147, + "epoch": 0.0330218791712611, + "flos": 67622559039360.0, + "grad_norm": 0.6756538950615467, + "language_loss": 0.52250957, + "learning_rate": 3.999904469471672e-06, + "loss": 0.54312766, + "num_input_tokens_seen": 31606580, + "router_z_loss_clip": 0.3671875, + "router_z_loss_mlp": 0.02172852, + "step": 1138, + "time_per_iteration": 3.0636179447174072 + }, + { + "auxiliary_loss_clip": 0.01213591, + "auxiliary_loss_mlp": 0.01052808, + "balance_loss_clip": 1.08063459, + "balance_loss_mlp": 1.03202963, + "epoch": 0.033050896639777144, + "flos": 34889154842880.0, + "grad_norm": 3.1676363284793374, + "language_loss": 0.79356384, + "learning_rate": 3.9999026235287725e-06, + "loss": 0.81622785, + "num_input_tokens_seen": 31621655, + "router_z_loss_clip": 1.33007812, + "router_z_loss_mlp": 0.2074585, + "step": 1139, + "time_per_iteration": 2.702899932861328 + }, + { + "auxiliary_loss_clip": 0.01227109, + "auxiliary_loss_mlp": 0.01066844, + "balance_loss_clip": 1.08370614, + "balance_loss_mlp": 1.04324067, + "epoch": 0.033079914108293196, + "flos": 27410075758080.0, + "grad_norm": 1.985925806021197, + "language_loss": 0.86822474, + "learning_rate": 3.999900759921924e-06, + "loss": 0.8911643, + "num_input_tokens_seen": 31635030, + "router_z_loss_clip": 1.43261719, + "router_z_loss_mlp": 0.23608398, + "step": 1140, + "time_per_iteration": 2.601228713989258 + }, + { + "auxiliary_loss_clip": 0.01232099, + "auxiliary_loss_mlp": 0.01080774, + "balance_loss_clip": 1.08784103, + "balance_loss_mlp": 1.05544233, + "epoch": 0.03310893157680924, + "flos": 10991646074880.0, + "grad_norm": 2.5504302095121174, + "language_loss": 0.89059895, + "learning_rate": 3.999898878651142e-06, + "loss": 0.91372764, + "num_input_tokens_seen": 31647265, + "router_z_loss_clip": 1.44238281, + "router_z_loss_mlp": 0.25317383, + "step": 1141, + "time_per_iteration": 2.5328569412231445 + }, + { + "auxiliary_loss_clip": 0.01227042, + "auxiliary_loss_mlp": 0.01078457, + "balance_loss_clip": 1.08515251, + "balance_loss_mlp": 1.054353, + "epoch": 0.033137949045325285, + "flos": 15881566032000.0, + "grad_norm": 3.6117781608941595, + "language_loss": 0.59699786, + "learning_rate": 3.999896979716444e-06, + "loss": 0.62005287, + "num_input_tokens_seen": 31661660, + "router_z_loss_clip": 1.41992188, + "router_z_loss_mlp": 0.2409668, + "step": 1142, + "time_per_iteration": 4.864759922027588 + }, + { + "auxiliary_loss_clip": 0.01062397, + "auxiliary_loss_mlp": 0.01027211, + "balance_loss_clip": 1.026191, + "balance_loss_mlp": 1.0250653, + "epoch": 0.03316696651384133, + "flos": 74775749005440.0, + "grad_norm": 0.6666313014548364, + "language_loss": 0.50154084, + "learning_rate": 3.999895063117847e-06, + "loss": 0.52243692, + "num_input_tokens_seen": 31727900, + "router_z_loss_clip": 0.36132812, + "router_z_loss_mlp": 0.02148438, + "step": 1143, + "time_per_iteration": 5.786648273468018 + }, + { + "auxiliary_loss_clip": 0.01222822, + "auxiliary_loss_mlp": 0.01082773, + "balance_loss_clip": 1.08219707, + "balance_loss_mlp": 1.0589664, + "epoch": 0.03319598398235738, + "flos": 16247639491200.0, + "grad_norm": 3.214915276803464, + "language_loss": 0.82130933, + "learning_rate": 3.999893128855368e-06, + "loss": 0.8443653, + "num_input_tokens_seen": 31742720, + "router_z_loss_clip": 1.40722656, + "router_z_loss_mlp": 0.23815918, + "step": 1144, + "time_per_iteration": 4.883395195007324 + }, + { + "auxiliary_loss_clip": 0.0122654, + "auxiliary_loss_mlp": 0.010618, + "balance_loss_clip": 1.08425033, + "balance_loss_mlp": 1.0382328, + "epoch": 0.033225001450873426, + "flos": 33830580101760.0, + "grad_norm": 2.499239124552077, + "language_loss": 0.89878631, + "learning_rate": 3.999891176929023e-06, + "loss": 0.92166972, + "num_input_tokens_seen": 31759635, + "router_z_loss_clip": 1.421875, + "router_z_loss_mlp": 0.23571777, + "step": 1145, + "time_per_iteration": 5.087206840515137 + }, + { + "auxiliary_loss_clip": 0.01224015, + "auxiliary_loss_mlp": 0.0106546, + "balance_loss_clip": 1.07998586, + "balance_loss_mlp": 1.04120088, + "epoch": 0.03325401891938947, + "flos": 45508119356160.0, + "grad_norm": 2.931094878156256, + "language_loss": 0.78167391, + "learning_rate": 3.999889207338829e-06, + "loss": 0.80456871, + "num_input_tokens_seen": 31777360, + "router_z_loss_clip": 1.43945312, + "router_z_loss_mlp": 0.24243164, + "step": 1146, + "time_per_iteration": 2.7147104740142822 + }, + { + "auxiliary_loss_clip": 0.01241452, + "auxiliary_loss_mlp": 0.01067876, + "balance_loss_clip": 1.08664715, + "balance_loss_mlp": 1.04095817, + "epoch": 0.03328303638790552, + "flos": 31022186313600.0, + "grad_norm": 2.4201597740483636, + "language_loss": 0.93351793, + "learning_rate": 3.999887220084805e-06, + "loss": 0.95661128, + "num_input_tokens_seen": 31792970, + "router_z_loss_clip": 1.54589844, + "router_z_loss_mlp": 0.26904297, + "step": 1147, + "time_per_iteration": 2.6112146377563477 + }, + { + "auxiliary_loss_clip": 0.01234841, + "auxiliary_loss_mlp": 0.01062468, + "balance_loss_clip": 1.08482027, + "balance_loss_mlp": 1.0348115, + "epoch": 0.03331205385642157, + "flos": 29672340186240.0, + "grad_norm": 2.5931447901411593, + "language_loss": 0.84360945, + "learning_rate": 3.999885215166969e-06, + "loss": 0.86658251, + "num_input_tokens_seen": 31808890, + "router_z_loss_clip": 1.5, + "router_z_loss_mlp": 0.27661133, + "step": 1148, + "time_per_iteration": 2.5942270755767822 + }, + { + "auxiliary_loss_clip": 0.01231341, + "auxiliary_loss_mlp": 0.01066403, + "balance_loss_clip": 1.08370495, + "balance_loss_mlp": 1.04166746, + "epoch": 0.03334107132493761, + "flos": 33588112320000.0, + "grad_norm": 1.8976749113269058, + "language_loss": 0.89843112, + "learning_rate": 3.999883192585336e-06, + "loss": 0.92140859, + "num_input_tokens_seen": 31835620, + "router_z_loss_clip": 1.4765625, + "router_z_loss_mlp": 0.24743652, + "step": 1149, + "time_per_iteration": 2.822004556655884 + }, + { + "auxiliary_loss_clip": 0.01231527, + "auxiliary_loss_mlp": 0.01064359, + "balance_loss_clip": 1.08077574, + "balance_loss_mlp": 1.03893209, + "epoch": 0.033370088793453656, + "flos": 30655825545600.0, + "grad_norm": 2.5892251908131065, + "language_loss": 0.87688941, + "learning_rate": 3.999881152339926e-06, + "loss": 0.89984828, + "num_input_tokens_seen": 31849535, + "router_z_loss_clip": 1.50488281, + "router_z_loss_mlp": 0.25445557, + "step": 1150, + "time_per_iteration": 2.589871644973755 + }, + { + "auxiliary_loss_clip": 0.01060868, + "auxiliary_loss_mlp": 0.01077589, + "balance_loss_clip": 1.02409911, + "balance_loss_mlp": 1.0756216, + "epoch": 0.03339910626196971, + "flos": 64479800522880.0, + "grad_norm": 0.7779758017067301, + "language_loss": 0.47962844, + "learning_rate": 3.999879094430756e-06, + "loss": 0.50101298, + "num_input_tokens_seen": 31906495, + "router_z_loss_clip": 0.3671875, + "router_z_loss_mlp": 0.01965332, + "step": 1151, + "time_per_iteration": 3.097970962524414 + }, + { + "auxiliary_loss_clip": 0.01211895, + "auxiliary_loss_mlp": 0.01067755, + "balance_loss_clip": 1.07893586, + "balance_loss_mlp": 1.04514074, + "epoch": 0.03342812373048575, + "flos": 29527907598720.0, + "grad_norm": 3.4228175617656027, + "language_loss": 0.91826427, + "learning_rate": 3.999877018857844e-06, + "loss": 0.94106078, + "num_input_tokens_seen": 31921320, + "router_z_loss_clip": 1.32910156, + "router_z_loss_mlp": 0.22631836, + "step": 1152, + "time_per_iteration": 2.6190896034240723 + }, + { + "auxiliary_loss_clip": 0.01217729, + "auxiliary_loss_mlp": 0.01067789, + "balance_loss_clip": 1.08073235, + "balance_loss_mlp": 1.04496598, + "epoch": 0.0334571411990018, + "flos": 32627752341120.0, + "grad_norm": 3.4945115179342072, + "language_loss": 0.92909855, + "learning_rate": 3.99987492562121e-06, + "loss": 0.95195377, + "num_input_tokens_seen": 31936155, + "router_z_loss_clip": 1.37011719, + "router_z_loss_mlp": 0.22808838, + "step": 1153, + "time_per_iteration": 2.626631736755371 + }, + { + "auxiliary_loss_clip": 0.01058136, + "auxiliary_loss_mlp": 0.01018716, + "balance_loss_clip": 1.02193558, + "balance_loss_mlp": 1.01679647, + "epoch": 0.03348615866751785, + "flos": 64417819605120.0, + "grad_norm": 0.7046714419395033, + "language_loss": 0.48023993, + "learning_rate": 3.999872814720871e-06, + "loss": 0.50100845, + "num_input_tokens_seen": 31996195, + "router_z_loss_clip": 0.36328125, + "router_z_loss_mlp": 0.01916504, + "step": 1154, + "time_per_iteration": 3.0608718395233154 + }, + { + "auxiliary_loss_clip": 0.01218503, + "auxiliary_loss_mlp": 0.01065535, + "balance_loss_clip": 1.08119917, + "balance_loss_mlp": 1.04166913, + "epoch": 0.03351517613603389, + "flos": 25988336559360.0, + "grad_norm": 6.682891515034318, + "language_loss": 0.76058567, + "learning_rate": 3.999870686156846e-06, + "loss": 0.78342605, + "num_input_tokens_seen": 32012965, + "router_z_loss_clip": 1.37207031, + "router_z_loss_mlp": 0.2388916, + "step": 1155, + "time_per_iteration": 2.6687278747558594 + }, + { + "auxiliary_loss_clip": 0.01223939, + "auxiliary_loss_mlp": 0.0105641, + "balance_loss_clip": 1.08223534, + "balance_loss_mlp": 1.03308105, + "epoch": 0.03354419360454994, + "flos": 19893936816000.0, + "grad_norm": 2.3851093856779917, + "language_loss": 0.88543904, + "learning_rate": 3.999868539929154e-06, + "loss": 0.90824258, + "num_input_tokens_seen": 32033140, + "router_z_loss_clip": 1.41503906, + "router_z_loss_mlp": 0.23339844, + "step": 1156, + "time_per_iteration": 2.6252331733703613 + }, + { + "auxiliary_loss_clip": 0.01060111, + "auxiliary_loss_mlp": 0.01012521, + "balance_loss_clip": 1.02259779, + "balance_loss_mlp": 1.01038694, + "epoch": 0.03357321107306598, + "flos": 56013070101120.0, + "grad_norm": 0.6854663727669815, + "language_loss": 0.55215442, + "learning_rate": 3.999866376037814e-06, + "loss": 0.57288074, + "num_input_tokens_seen": 32088540, + "router_z_loss_clip": 0.375, + "router_z_loss_mlp": 0.0213623, + "step": 1157, + "time_per_iteration": 2.905571460723877 + }, + { + "auxiliary_loss_clip": 0.01230094, + "auxiliary_loss_mlp": 0.01064858, + "balance_loss_clip": 1.08025336, + "balance_loss_mlp": 1.0373565, + "epoch": 0.033602228541582034, + "flos": 18690139388160.0, + "grad_norm": 2.8677414787926256, + "language_loss": 0.94240892, + "learning_rate": 3.999864194482844e-06, + "loss": 0.96535844, + "num_input_tokens_seen": 32101410, + "router_z_loss_clip": 1.49707031, + "router_z_loss_mlp": 0.27490234, + "step": 1158, + "time_per_iteration": 2.503831624984741 + }, + { + "auxiliary_loss_clip": 0.01208561, + "auxiliary_loss_mlp": 0.01057796, + "balance_loss_clip": 1.07547021, + "balance_loss_mlp": 1.03564644, + "epoch": 0.03363124601009808, + "flos": 15408335324160.0, + "grad_norm": 2.495605427837059, + "language_loss": 0.81834358, + "learning_rate": 3.999861995264266e-06, + "loss": 0.84100717, + "num_input_tokens_seen": 32116305, + "router_z_loss_clip": 1.33105469, + "router_z_loss_mlp": 0.22143555, + "step": 1159, + "time_per_iteration": 2.5187368392944336 + }, + { + "auxiliary_loss_clip": 0.01230681, + "auxiliary_loss_mlp": 0.01067238, + "balance_loss_clip": 1.08930612, + "balance_loss_mlp": 1.04511237, + "epoch": 0.03366026347861412, + "flos": 19750761204480.0, + "grad_norm": 2.421370532759707, + "language_loss": 0.8359769, + "learning_rate": 3.999859778382096e-06, + "loss": 0.8589561, + "num_input_tokens_seen": 32133150, + "router_z_loss_clip": 1.41503906, + "router_z_loss_mlp": 0.22131348, + "step": 1160, + "time_per_iteration": 2.5286636352539062 + }, + { + "auxiliary_loss_clip": 0.01208019, + "auxiliary_loss_mlp": 0.01051782, + "balance_loss_clip": 1.07378244, + "balance_loss_mlp": 1.03133726, + "epoch": 0.033689280947130175, + "flos": 35473888344960.0, + "grad_norm": 2.423755105188174, + "language_loss": 1.01893377, + "learning_rate": 3.9998575438363555e-06, + "loss": 1.0415318, + "num_input_tokens_seen": 32149280, + "router_z_loss_clip": 1.34277344, + "router_z_loss_mlp": 0.20446777, + "step": 1161, + "time_per_iteration": 2.6266887187957764 + }, + { + "auxiliary_loss_clip": 0.01061324, + "auxiliary_loss_mlp": 0.01006059, + "balance_loss_clip": 1.02430749, + "balance_loss_mlp": 1.00390136, + "epoch": 0.03371829841564622, + "flos": 54085314055680.0, + "grad_norm": 0.7428280954456852, + "language_loss": 0.51329839, + "learning_rate": 3.999855291627064e-06, + "loss": 0.53397226, + "num_input_tokens_seen": 32203435, + "router_z_loss_clip": 0.37109375, + "router_z_loss_mlp": 0.02160645, + "step": 1162, + "time_per_iteration": 2.986936330795288 + }, + { + "auxiliary_loss_clip": 0.01239397, + "auxiliary_loss_mlp": 0.0108401, + "balance_loss_clip": 1.08289003, + "balance_loss_mlp": 1.05747354, + "epoch": 0.033747315884162264, + "flos": 12817667825280.0, + "grad_norm": 3.117539263588283, + "language_loss": 0.77765, + "learning_rate": 3.999853021754241e-06, + "loss": 0.80088401, + "num_input_tokens_seen": 32218330, + "router_z_loss_clip": 1.56542969, + "router_z_loss_mlp": 0.26538086, + "step": 1163, + "time_per_iteration": 2.5042243003845215 + }, + { + "auxiliary_loss_clip": 0.01060853, + "auxiliary_loss_mlp": 0.00999971, + "balance_loss_clip": 1.02499628, + "balance_loss_mlp": 0.9980036, + "epoch": 0.033776333352678316, + "flos": 55283473048320.0, + "grad_norm": 0.7385106996844835, + "language_loss": 0.57599378, + "learning_rate": 3.999850734217907e-06, + "loss": 0.59660208, + "num_input_tokens_seen": 32274570, + "router_z_loss_clip": 0.359375, + "router_z_loss_mlp": 0.01965332, + "step": 1164, + "time_per_iteration": 2.961941957473755 + }, + { + "auxiliary_loss_clip": 0.01060908, + "auxiliary_loss_mlp": 0.01007868, + "balance_loss_clip": 1.025105, + "balance_loss_mlp": 1.00584114, + "epoch": 0.03380535082119436, + "flos": 62006202426240.0, + "grad_norm": 0.6988616169665612, + "language_loss": 0.50282896, + "learning_rate": 3.999848429018082e-06, + "loss": 0.52351671, + "num_input_tokens_seen": 32335785, + "router_z_loss_clip": 0.359375, + "router_z_loss_mlp": 0.02026367, + "step": 1165, + "time_per_iteration": 3.003934383392334 + }, + { + "auxiliary_loss_clip": 0.01221371, + "auxiliary_loss_mlp": 0.01061196, + "balance_loss_clip": 1.08112538, + "balance_loss_mlp": 1.03911805, + "epoch": 0.033834368289710405, + "flos": 67434599168640.0, + "grad_norm": 2.8489526121324826, + "language_loss": 0.81925201, + "learning_rate": 3.999846106154787e-06, + "loss": 0.84207767, + "num_input_tokens_seen": 32357945, + "router_z_loss_clip": 1.40234375, + "router_z_loss_mlp": 0.2208252, + "step": 1166, + "time_per_iteration": 2.8283851146698 + }, + { + "auxiliary_loss_clip": 0.0121488, + "auxiliary_loss_mlp": 0.01067017, + "balance_loss_clip": 1.07648897, + "balance_loss_mlp": 1.04410517, + "epoch": 0.03386338575822645, + "flos": 29126318567040.0, + "grad_norm": 3.272624552679031, + "language_loss": 0.85812235, + "learning_rate": 3.9998437656280415e-06, + "loss": 0.88094139, + "num_input_tokens_seen": 32374245, + "router_z_loss_clip": 1.38476562, + "router_z_loss_mlp": 0.22949219, + "step": 1167, + "time_per_iteration": 2.5970308780670166 + }, + { + "auxiliary_loss_clip": 0.0121816, + "auxiliary_loss_mlp": 0.01063862, + "balance_loss_clip": 1.07825983, + "balance_loss_mlp": 1.04148626, + "epoch": 0.0338924032267425, + "flos": 15589432719360.0, + "grad_norm": 2.6596376067927316, + "language_loss": 0.75562328, + "learning_rate": 3.999841407437867e-06, + "loss": 0.77844352, + "num_input_tokens_seen": 32387125, + "router_z_loss_clip": 1.3984375, + "router_z_loss_mlp": 0.22387695, + "step": 1168, + "time_per_iteration": 2.4466891288757324 + }, + { + "auxiliary_loss_clip": 0.01216335, + "auxiliary_loss_mlp": 0.01058506, + "balance_loss_clip": 1.07612371, + "balance_loss_mlp": 1.03598738, + "epoch": 0.033921420695258546, + "flos": 10841970101760.0, + "grad_norm": 3.9541371967968115, + "language_loss": 1.04082704, + "learning_rate": 3.999839031584283e-06, + "loss": 1.06357551, + "num_input_tokens_seen": 32396360, + "router_z_loss_clip": 1.40332031, + "router_z_loss_mlp": 0.2253418, + "step": 1169, + "time_per_iteration": 2.4808523654937744 + }, + { + "auxiliary_loss_clip": 0.01057094, + "auxiliary_loss_mlp": 0.0101872, + "balance_loss_clip": 1.02125502, + "balance_loss_mlp": 1.01663363, + "epoch": 0.03395043816377459, + "flos": 74773809671040.0, + "grad_norm": 0.6729706796291514, + "language_loss": 0.52168298, + "learning_rate": 3.999836638067312e-06, + "loss": 0.54244113, + "num_input_tokens_seen": 32458720, + "router_z_loss_clip": 0.359375, + "router_z_loss_mlp": 0.02087402, + "step": 1170, + "time_per_iteration": 3.082120895385742 + }, + { + "auxiliary_loss_clip": 0.01213397, + "auxiliary_loss_mlp": 0.01073902, + "balance_loss_clip": 1.07679653, + "balance_loss_mlp": 1.05059671, + "epoch": 0.03397945563229064, + "flos": 24490574225280.0, + "grad_norm": 2.536405201098417, + "language_loss": 1.06301522, + "learning_rate": 3.999834226886975e-06, + "loss": 1.08588815, + "num_input_tokens_seen": 32474510, + "router_z_loss_clip": 1.36621094, + "router_z_loss_mlp": 0.23303223, + "step": 1171, + "time_per_iteration": 2.5717625617980957 + }, + { + "auxiliary_loss_clip": 0.01056563, + "auxiliary_loss_mlp": 0.01004182, + "balance_loss_clip": 1.02103424, + "balance_loss_mlp": 1.00202394, + "epoch": 0.03400847310080669, + "flos": 65589297770880.0, + "grad_norm": 0.7551215765417711, + "language_loss": 0.55941212, + "learning_rate": 3.9998317980432924e-06, + "loss": 0.58001959, + "num_input_tokens_seen": 32535245, + "router_z_loss_clip": 0.35546875, + "router_z_loss_mlp": 0.02160645, + "step": 1172, + "time_per_iteration": 3.1233291625976562 + }, + { + "auxiliary_loss_clip": 0.01055481, + "auxiliary_loss_mlp": 0.01002173, + "balance_loss_clip": 1.02032113, + "balance_loss_mlp": 0.99999201, + "epoch": 0.03403749056932273, + "flos": 69676183319040.0, + "grad_norm": 0.627878256109273, + "language_loss": 0.48525026, + "learning_rate": 3.999829351536286e-06, + "loss": 0.50582683, + "num_input_tokens_seen": 32603280, + "router_z_loss_clip": 0.3515625, + "router_z_loss_mlp": 0.02185059, + "step": 1173, + "time_per_iteration": 3.2207024097442627 + }, + { + "auxiliary_loss_clip": 0.01055147, + "auxiliary_loss_mlp": 0.0100102, + "balance_loss_clip": 1.02027762, + "balance_loss_mlp": 0.99879116, + "epoch": 0.034066508037838776, + "flos": 70987425304320.0, + "grad_norm": 0.674763836953827, + "language_loss": 0.50108552, + "learning_rate": 3.999826887365978e-06, + "loss": 0.52164721, + "num_input_tokens_seen": 32671835, + "router_z_loss_clip": 0.34765625, + "router_z_loss_mlp": 0.02233887, + "step": 1174, + "time_per_iteration": 3.1457104682922363 + }, + { + "auxiliary_loss_clip": 0.01217139, + "auxiliary_loss_mlp": 0.01069432, + "balance_loss_clip": 1.08077896, + "balance_loss_mlp": 1.04872489, + "epoch": 0.03409552550635483, + "flos": 35583020841600.0, + "grad_norm": 2.7027764757655723, + "language_loss": 0.99005318, + "learning_rate": 3.9998244055323896e-06, + "loss": 1.01291895, + "num_input_tokens_seen": 32688635, + "router_z_loss_clip": 1.36328125, + "router_z_loss_mlp": 0.20703125, + "step": 1175, + "time_per_iteration": 2.6234207153320312 + }, + { + "auxiliary_loss_clip": 0.01218396, + "auxiliary_loss_mlp": 0.01055939, + "balance_loss_clip": 1.08062828, + "balance_loss_mlp": 1.03533387, + "epoch": 0.03412454297487087, + "flos": 24418142449920.0, + "grad_norm": 2.4705492439154813, + "language_loss": 0.76209259, + "learning_rate": 3.999821906035542e-06, + "loss": 0.78483593, + "num_input_tokens_seen": 32702555, + "router_z_loss_clip": 1.37695312, + "router_z_loss_mlp": 0.20611572, + "step": 1176, + "time_per_iteration": 2.57022762298584 + }, + { + "auxiliary_loss_clip": 0.01055203, + "auxiliary_loss_mlp": 0.01006479, + "balance_loss_clip": 1.02083647, + "balance_loss_mlp": 1.00452352, + "epoch": 0.03415356044338692, + "flos": 66168967455360.0, + "grad_norm": 0.5932215984189748, + "language_loss": 0.50244403, + "learning_rate": 3.999819388875459e-06, + "loss": 0.5230608, + "num_input_tokens_seen": 32768800, + "router_z_loss_clip": 0.34375, + "router_z_loss_mlp": 0.01953125, + "step": 1177, + "time_per_iteration": 3.196361541748047 + }, + { + "auxiliary_loss_clip": 0.01055178, + "auxiliary_loss_mlp": 0.01001856, + "balance_loss_clip": 1.0209887, + "balance_loss_mlp": 0.99996036, + "epoch": 0.03418257791190297, + "flos": 65899923609600.0, + "grad_norm": 0.6656221565629926, + "language_loss": 0.50398535, + "learning_rate": 3.999816854052162e-06, + "loss": 0.52455574, + "num_input_tokens_seen": 32825500, + "router_z_loss_clip": 0.34179688, + "router_z_loss_mlp": 0.0189209, + "step": 1178, + "time_per_iteration": 3.0706677436828613 + }, + { + "auxiliary_loss_clip": 0.01214208, + "auxiliary_loss_mlp": 0.01072631, + "balance_loss_clip": 1.08067715, + "balance_loss_mlp": 1.05070877, + "epoch": 0.03421159538041901, + "flos": 25445403509760.0, + "grad_norm": 2.8544277082061345, + "language_loss": 0.81489539, + "learning_rate": 3.999814301565673e-06, + "loss": 0.83776379, + "num_input_tokens_seen": 32844245, + "router_z_loss_clip": 1.33496094, + "router_z_loss_mlp": 0.21923828, + "step": 1179, + "time_per_iteration": 2.5878984928131104 + }, + { + "auxiliary_loss_clip": 0.01220948, + "auxiliary_loss_mlp": 0.01076333, + "balance_loss_clip": 1.07660747, + "balance_loss_mlp": 1.05065501, + "epoch": 0.03424061284893506, + "flos": 28109292883200.0, + "grad_norm": 2.9099959053128686, + "language_loss": 1.13510489, + "learning_rate": 3.999811731416015e-06, + "loss": 1.1580776, + "num_input_tokens_seen": 32866005, + "router_z_loss_clip": 1.44238281, + "router_z_loss_mlp": 0.25695801, + "step": 1180, + "time_per_iteration": 2.639751434326172 + }, + { + "auxiliary_loss_clip": 0.01224716, + "auxiliary_loss_mlp": 0.01061586, + "balance_loss_clip": 1.07479095, + "balance_loss_mlp": 1.03717196, + "epoch": 0.0342696303174511, + "flos": 70535018528640.0, + "grad_norm": 2.6859812042462616, + "language_loss": 0.74050486, + "learning_rate": 3.99980914360321e-06, + "loss": 0.76336789, + "num_input_tokens_seen": 32886890, + "router_z_loss_clip": 1.5, + "router_z_loss_mlp": 0.24414062, + "step": 1181, + "time_per_iteration": 2.920766830444336 + }, + { + "auxiliary_loss_clip": 0.01212506, + "auxiliary_loss_mlp": 0.01058645, + "balance_loss_clip": 1.07877398, + "balance_loss_mlp": 1.03610253, + "epoch": 0.034298647785967154, + "flos": 24929869559040.0, + "grad_norm": 2.605017159978999, + "language_loss": 0.93478554, + "learning_rate": 3.999806538127282e-06, + "loss": 0.957497, + "num_input_tokens_seen": 32900750, + "router_z_loss_clip": 1.33886719, + "router_z_loss_mlp": 0.22546387, + "step": 1182, + "time_per_iteration": 2.52909779548645 + }, + { + "auxiliary_loss_clip": 0.01053684, + "auxiliary_loss_mlp": 0.01007697, + "balance_loss_clip": 1.01914549, + "balance_loss_mlp": 1.00575399, + "epoch": 0.0343276652544832, + "flos": 60905216701440.0, + "grad_norm": 0.7057511494666638, + "language_loss": 0.50038934, + "learning_rate": 3.999803914988253e-06, + "loss": 0.52100319, + "num_input_tokens_seen": 32956575, + "router_z_loss_clip": 0.34570312, + "router_z_loss_mlp": 0.01940918, + "step": 1183, + "time_per_iteration": 3.0372440814971924 + }, + { + "auxiliary_loss_clip": 0.01214427, + "auxiliary_loss_mlp": 0.01057508, + "balance_loss_clip": 1.07536864, + "balance_loss_mlp": 1.03618157, + "epoch": 0.03435668272299924, + "flos": 23112574813440.0, + "grad_norm": 3.8647435793295397, + "language_loss": 0.91942531, + "learning_rate": 3.999801274186146e-06, + "loss": 0.94214463, + "num_input_tokens_seen": 32975570, + "router_z_loss_clip": 1.38964844, + "router_z_loss_mlp": 0.21325684, + "step": 1184, + "time_per_iteration": 2.5148963928222656 + }, + { + "auxiliary_loss_clip": 0.01227592, + "auxiliary_loss_mlp": 0.01068836, + "balance_loss_clip": 1.07964587, + "balance_loss_mlp": 1.04436195, + "epoch": 0.034385700191515295, + "flos": 74737465107840.0, + "grad_norm": 2.21632573124527, + "language_loss": 0.83771199, + "learning_rate": 3.999798615720986e-06, + "loss": 0.86067623, + "num_input_tokens_seen": 32999770, + "router_z_loss_clip": 1.47949219, + "router_z_loss_mlp": 0.24462891, + "step": 1185, + "time_per_iteration": 2.8951520919799805 + }, + { + "auxiliary_loss_clip": 0.01221408, + "auxiliary_loss_mlp": 0.01068125, + "balance_loss_clip": 1.07874215, + "balance_loss_mlp": 1.04473591, + "epoch": 0.03441471766003134, + "flos": 32666967014400.0, + "grad_norm": 3.0818542624099448, + "language_loss": 0.87647605, + "learning_rate": 3.999795939592795e-06, + "loss": 0.89937139, + "num_input_tokens_seen": 33014670, + "router_z_loss_clip": 1.42578125, + "router_z_loss_mlp": 0.23425293, + "step": 1186, + "time_per_iteration": 2.5703165531158447 + }, + { + "auxiliary_loss_clip": 0.01213858, + "auxiliary_loss_mlp": 0.01061733, + "balance_loss_clip": 1.0824877, + "balance_loss_mlp": 1.04224253, + "epoch": 0.034443735128547384, + "flos": 30586302771840.0, + "grad_norm": 2.7600135258739957, + "language_loss": 0.84404808, + "learning_rate": 3.9997932458015974e-06, + "loss": 0.866804, + "num_input_tokens_seen": 33028470, + "router_z_loss_clip": 1.3125, + "router_z_loss_mlp": 0.19512939, + "step": 1187, + "time_per_iteration": 2.5994300842285156 + }, + { + "auxiliary_loss_clip": 0.01054254, + "auxiliary_loss_mlp": 0.01000937, + "balance_loss_clip": 1.01950991, + "balance_loss_mlp": 0.99922079, + "epoch": 0.03447275259706343, + "flos": 54660458626560.0, + "grad_norm": 0.7161484842243012, + "language_loss": 0.48305309, + "learning_rate": 3.999790534347416e-06, + "loss": 0.50360501, + "num_input_tokens_seen": 33083380, + "router_z_loss_clip": 0.34765625, + "router_z_loss_mlp": 0.01721191, + "step": 1188, + "time_per_iteration": 2.923525094985962 + }, + { + "auxiliary_loss_clip": 0.01215521, + "auxiliary_loss_mlp": 0.01069068, + "balance_loss_clip": 1.07896876, + "balance_loss_mlp": 1.04554784, + "epoch": 0.03450177006557948, + "flos": 31426037902080.0, + "grad_norm": 2.110486647018455, + "language_loss": 0.93373287, + "learning_rate": 3.999787805230276e-06, + "loss": 0.95657873, + "num_input_tokens_seen": 33100870, + "router_z_loss_clip": 1.36328125, + "router_z_loss_mlp": 0.23498535, + "step": 1189, + "time_per_iteration": 2.6055307388305664 + }, + { + "auxiliary_loss_clip": 0.01213581, + "auxiliary_loss_mlp": 0.01061605, + "balance_loss_clip": 1.07737303, + "balance_loss_mlp": 1.03787088, + "epoch": 0.034530787534095525, + "flos": 30073749649920.0, + "grad_norm": 11.99703168211979, + "language_loss": 0.8191489, + "learning_rate": 3.9997850584502006e-06, + "loss": 0.84190077, + "num_input_tokens_seen": 33115540, + "router_z_loss_clip": 1.36328125, + "router_z_loss_mlp": 0.23730469, + "step": 1190, + "time_per_iteration": 2.5234763622283936 + }, + { + "auxiliary_loss_clip": 0.01055119, + "auxiliary_loss_mlp": 0.0100161, + "balance_loss_clip": 1.0205338, + "balance_loss_mlp": 0.99975073, + "epoch": 0.03455980500261157, + "flos": 63908319139200.0, + "grad_norm": 0.6722752644382142, + "language_loss": 0.52143639, + "learning_rate": 3.999782294007214e-06, + "loss": 0.54200369, + "num_input_tokens_seen": 33180150, + "router_z_loss_clip": 0.34570312, + "router_z_loss_mlp": 0.01855469, + "step": 1191, + "time_per_iteration": 3.105626344680786 + }, + { + "auxiliary_loss_clip": 0.01211583, + "auxiliary_loss_mlp": 0.01064597, + "balance_loss_clip": 1.07635999, + "balance_loss_mlp": 1.04391396, + "epoch": 0.03458882247112762, + "flos": 11867794617600.0, + "grad_norm": 4.322268383936197, + "language_loss": 0.83792746, + "learning_rate": 3.999779511901341e-06, + "loss": 0.86068928, + "num_input_tokens_seen": 33190350, + "router_z_loss_clip": 1.3515625, + "router_z_loss_mlp": 0.20690918, + "step": 1192, + "time_per_iteration": 2.513882637023926 + }, + { + "auxiliary_loss_clip": 0.01207059, + "auxiliary_loss_mlp": 0.01070835, + "balance_loss_clip": 1.07467306, + "balance_loss_mlp": 1.04987812, + "epoch": 0.034617839939643666, + "flos": 26789000670720.0, + "grad_norm": 2.7711533214195905, + "language_loss": 0.85719001, + "learning_rate": 3.999776712132606e-06, + "loss": 0.879969, + "num_input_tokens_seen": 33207540, + "router_z_loss_clip": 1.32324219, + "router_z_loss_mlp": 0.20941162, + "step": 1193, + "time_per_iteration": 2.5740151405334473 + }, + { + "auxiliary_loss_clip": 0.01227323, + "auxiliary_loss_mlp": 0.01062, + "balance_loss_clip": 1.0837723, + "balance_loss_mlp": 1.03756189, + "epoch": 0.03464685740815971, + "flos": 30986347518720.0, + "grad_norm": 3.2924109165837496, + "language_loss": 0.72413874, + "learning_rate": 3.999773894701034e-06, + "loss": 0.74703205, + "num_input_tokens_seen": 33222395, + "router_z_loss_clip": 1.43652344, + "router_z_loss_mlp": 0.24438477, + "step": 1194, + "time_per_iteration": 2.630434036254883 + }, + { + "auxiliary_loss_clip": 0.01054952, + "auxiliary_loss_mlp": 0.01000806, + "balance_loss_clip": 1.02065253, + "balance_loss_mlp": 0.9988991, + "epoch": 0.03467587487667576, + "flos": 57357098225280.0, + "grad_norm": 0.7395126047594536, + "language_loss": 0.53326529, + "learning_rate": 3.9997710596066505e-06, + "loss": 0.55382288, + "num_input_tokens_seen": 33272795, + "router_z_loss_clip": 0.34375, + "router_z_loss_mlp": 0.01904297, + "step": 1195, + "time_per_iteration": 2.8373007774353027 + }, + { + "auxiliary_loss_clip": 0.01208975, + "auxiliary_loss_mlp": 0.01046155, + "balance_loss_clip": 1.07691872, + "balance_loss_mlp": 1.02511501, + "epoch": 0.03470489234519181, + "flos": 44484198261120.0, + "grad_norm": 4.568338476291288, + "language_loss": 0.9507978, + "learning_rate": 3.9997682068494795e-06, + "loss": 0.97334909, + "num_input_tokens_seen": 33289355, + "router_z_loss_clip": 1.3203125, + "router_z_loss_mlp": 0.21057129, + "step": 1196, + "time_per_iteration": 2.7031447887420654 + }, + { + "auxiliary_loss_clip": 0.01201966, + "auxiliary_loss_mlp": 0.01048894, + "balance_loss_clip": 1.07427549, + "balance_loss_mlp": 1.02824688, + "epoch": 0.03473390981370785, + "flos": 15117028024320.0, + "grad_norm": 2.981561904685097, + "language_loss": 0.82731479, + "learning_rate": 3.9997653364295455e-06, + "loss": 0.84982336, + "num_input_tokens_seen": 33301815, + "router_z_loss_clip": 1.27636719, + "router_z_loss_mlp": 0.20678711, + "step": 1197, + "time_per_iteration": 2.516662836074829 + }, + { + "auxiliary_loss_clip": 0.01055223, + "auxiliary_loss_mlp": 0.01001839, + "balance_loss_clip": 1.02108431, + "balance_loss_mlp": 0.99994367, + "epoch": 0.034762927282223896, + "flos": 74779412192640.0, + "grad_norm": 0.6556309617605478, + "language_loss": 0.54054099, + "learning_rate": 3.999762448346876e-06, + "loss": 0.56111163, + "num_input_tokens_seen": 33371520, + "router_z_loss_clip": 0.34179688, + "router_z_loss_mlp": 0.0189209, + "step": 1198, + "time_per_iteration": 3.2444660663604736 + }, + { + "auxiliary_loss_clip": 0.01224544, + "auxiliary_loss_mlp": 0.01053429, + "balance_loss_clip": 1.08051741, + "balance_loss_mlp": 1.03091097, + "epoch": 0.03479194475073995, + "flos": 22237683246720.0, + "grad_norm": 2.994743954714972, + "language_loss": 1.06794608, + "learning_rate": 3.999759542601494e-06, + "loss": 1.09072566, + "num_input_tokens_seen": 33386930, + "router_z_loss_clip": 1.43847656, + "router_z_loss_mlp": 0.2253418, + "step": 1199, + "time_per_iteration": 2.468977451324463 + }, + { + "auxiliary_loss_clip": 0.01055577, + "auxiliary_loss_mlp": 0.01003621, + "balance_loss_clip": 1.02137709, + "balance_loss_mlp": 1.00167835, + "epoch": 0.03482096221925599, + "flos": 61604254258560.0, + "grad_norm": 0.8082469054975226, + "language_loss": 0.49845123, + "learning_rate": 3.999756619193427e-06, + "loss": 0.51904321, + "num_input_tokens_seen": 33446410, + "router_z_loss_clip": 0.34179688, + "router_z_loss_mlp": 0.01940918, + "step": 1200, + "time_per_iteration": 2.9808483123779297 + }, + { + "auxiliary_loss_clip": 0.01206869, + "auxiliary_loss_mlp": 0.010473, + "balance_loss_clip": 1.07206178, + "balance_loss_mlp": 1.02532935, + "epoch": 0.03484997968777204, + "flos": 24965277390720.0, + "grad_norm": 2.622888138145464, + "language_loss": 1.03637075, + "learning_rate": 3.999753678122701e-06, + "loss": 1.05891252, + "num_input_tokens_seen": 33460205, + "router_z_loss_clip": 1.34667969, + "router_z_loss_mlp": 0.21984863, + "step": 1201, + "time_per_iteration": 2.581292152404785 + }, + { + "auxiliary_loss_clip": 0.01218757, + "auxiliary_loss_mlp": 0.01054844, + "balance_loss_clip": 1.07771981, + "balance_loss_mlp": 1.03215826, + "epoch": 0.03487899715628809, + "flos": 22083410332800.0, + "grad_norm": 3.088376475458536, + "language_loss": 0.9249323, + "learning_rate": 3.999750719389341e-06, + "loss": 0.94766831, + "num_input_tokens_seen": 33474020, + "router_z_loss_clip": 1.41015625, + "router_z_loss_mlp": 0.22705078, + "step": 1202, + "time_per_iteration": 2.528491973876953 + }, + { + "auxiliary_loss_clip": 0.01212943, + "auxiliary_loss_mlp": 0.01057889, + "balance_loss_clip": 1.07977295, + "balance_loss_mlp": 1.03699195, + "epoch": 0.03490801462480413, + "flos": 16503000255360.0, + "grad_norm": 2.6835294304859305, + "language_loss": 0.81700408, + "learning_rate": 3.999747742993374e-06, + "loss": 0.8397125, + "num_input_tokens_seen": 33487345, + "router_z_loss_clip": 1.33007812, + "router_z_loss_mlp": 0.2088623, + "step": 1203, + "time_per_iteration": 2.4767868518829346 + }, + { + "auxiliary_loss_clip": 0.01217623, + "auxiliary_loss_mlp": 0.01064098, + "balance_loss_clip": 1.07920253, + "balance_loss_mlp": 1.04198456, + "epoch": 0.03493703209332018, + "flos": 32047831261440.0, + "grad_norm": 2.717439232332911, + "language_loss": 0.92908692, + "learning_rate": 3.999744748934825e-06, + "loss": 0.95190418, + "num_input_tokens_seen": 33502735, + "router_z_loss_clip": 1.3828125, + "router_z_loss_mlp": 0.22131348, + "step": 1204, + "time_per_iteration": 2.602555990219116 + }, + { + "auxiliary_loss_clip": 0.01056265, + "auxiliary_loss_mlp": 0.01014525, + "balance_loss_clip": 1.02191496, + "balance_loss_mlp": 1.01264179, + "epoch": 0.03496604956183622, + "flos": 74773450535040.0, + "grad_norm": 0.6910926194420121, + "language_loss": 0.54168272, + "learning_rate": 3.999741737213721e-06, + "loss": 0.56239057, + "num_input_tokens_seen": 33567165, + "router_z_loss_clip": 0.34375, + "router_z_loss_mlp": 0.01879883, + "step": 1205, + "time_per_iteration": 3.1253180503845215 + }, + { + "auxiliary_loss_clip": 0.01210923, + "auxiliary_loss_mlp": 0.01050375, + "balance_loss_clip": 1.07682419, + "balance_loss_mlp": 1.03128934, + "epoch": 0.034995067030352274, + "flos": 44341705008000.0, + "grad_norm": 3.4804526139422918, + "language_loss": 0.9448899, + "learning_rate": 3.99973870783009e-06, + "loss": 0.96750295, + "num_input_tokens_seen": 33588305, + "router_z_loss_clip": 1.34082031, + "router_z_loss_mlp": 0.19067383, + "step": 1206, + "time_per_iteration": 2.7044899463653564 + }, + { + "auxiliary_loss_clip": 0.01056226, + "auxiliary_loss_mlp": 0.01004806, + "balance_loss_clip": 1.0222857, + "balance_loss_mlp": 1.00310123, + "epoch": 0.03502408449886832, + "flos": 74762137751040.0, + "grad_norm": 0.7334097245685609, + "language_loss": 0.55433178, + "learning_rate": 3.9997356607839554e-06, + "loss": 0.57494211, + "num_input_tokens_seen": 33644095, + "router_z_loss_clip": 0.33984375, + "router_z_loss_mlp": 0.01708984, + "step": 1207, + "time_per_iteration": 2.9830455780029297 + }, + { + "auxiliary_loss_clip": 0.01211862, + "auxiliary_loss_mlp": 0.01055907, + "balance_loss_clip": 1.07931888, + "balance_loss_mlp": 1.03392529, + "epoch": 0.03505310196738436, + "flos": 13071268823040.0, + "grad_norm": 2.5568833281626566, + "language_loss": 0.88310128, + "learning_rate": 3.999732596075348e-06, + "loss": 0.90577894, + "num_input_tokens_seen": 33655200, + "router_z_loss_clip": 1.32617188, + "router_z_loss_mlp": 0.2199707, + "step": 1208, + "time_per_iteration": 2.490835428237915 + }, + { + "auxiliary_loss_clip": 0.0121502, + "auxiliary_loss_mlp": 0.01060528, + "balance_loss_clip": 1.07832932, + "balance_loss_mlp": 1.03780723, + "epoch": 0.035082119435900415, + "flos": 35362385550720.0, + "grad_norm": 2.658591631631523, + "language_loss": 1.01284671, + "learning_rate": 3.9997295137042925e-06, + "loss": 1.03560209, + "num_input_tokens_seen": 33669745, + "router_z_loss_clip": 1.36523438, + "router_z_loss_mlp": 0.22717285, + "step": 1209, + "time_per_iteration": 2.6865198612213135 + }, + { + "auxiliary_loss_clip": 0.01210105, + "auxiliary_loss_mlp": 0.01047889, + "balance_loss_clip": 1.07780337, + "balance_loss_mlp": 1.02817178, + "epoch": 0.03511113690441646, + "flos": 35547864405120.0, + "grad_norm": 2.5303647761922705, + "language_loss": 1.00659752, + "learning_rate": 3.999726413670816e-06, + "loss": 1.02917755, + "num_input_tokens_seen": 33688565, + "router_z_loss_clip": 1.32421875, + "router_z_loss_mlp": 0.19726562, + "step": 1210, + "time_per_iteration": 2.6384880542755127 + }, + { + "auxiliary_loss_clip": 0.0121245, + "auxiliary_loss_mlp": 0.01060436, + "balance_loss_clip": 1.07936764, + "balance_loss_mlp": 1.04101717, + "epoch": 0.035140154372932504, + "flos": 25294506474240.0, + "grad_norm": 3.039053768914526, + "language_loss": 0.93977058, + "learning_rate": 3.999723295974948e-06, + "loss": 0.96249938, + "num_input_tokens_seen": 33707555, + "router_z_loss_clip": 1.33105469, + "router_z_loss_mlp": 0.19390869, + "step": 1211, + "time_per_iteration": 2.694281578063965 + }, + { + "auxiliary_loss_clip": 0.01207249, + "auxiliary_loss_mlp": 0.01052933, + "balance_loss_clip": 1.07727885, + "balance_loss_mlp": 1.03149951, + "epoch": 0.03516917184144855, + "flos": 16903404138240.0, + "grad_norm": 2.7364309853147386, + "language_loss": 0.91720241, + "learning_rate": 3.999720160616714e-06, + "loss": 0.9398042, + "num_input_tokens_seen": 33721680, + "router_z_loss_clip": 1.29980469, + "router_z_loss_mlp": 0.2142334, + "step": 1212, + "time_per_iteration": 2.471935749053955 + }, + { + "auxiliary_loss_clip": 0.01057238, + "auxiliary_loss_mlp": 0.0100125, + "balance_loss_clip": 1.02349544, + "balance_loss_mlp": 0.99943841, + "epoch": 0.0351981893099646, + "flos": 62438315040000.0, + "grad_norm": 0.6704547296655576, + "language_loss": 0.51498961, + "learning_rate": 3.999717007596143e-06, + "loss": 0.5355745, + "num_input_tokens_seen": 33785430, + "router_z_loss_clip": 0.3359375, + "router_z_loss_mlp": 0.01806641, + "step": 1213, + "time_per_iteration": 3.175139904022217 + }, + { + "auxiliary_loss_clip": 0.01220194, + "auxiliary_loss_mlp": 0.0106782, + "balance_loss_clip": 1.08598554, + "balance_loss_mlp": 1.04590905, + "epoch": 0.035227206778480645, + "flos": 70862167601280.0, + "grad_norm": 3.09907777825493, + "language_loss": 0.83125901, + "learning_rate": 3.999713836913261e-06, + "loss": 0.85413915, + "num_input_tokens_seen": 33822670, + "router_z_loss_clip": 1.34179688, + "router_z_loss_mlp": 0.21887207, + "step": 1214, + "time_per_iteration": 7.922073602676392 + }, + { + "auxiliary_loss_clip": 0.01214827, + "auxiliary_loss_mlp": 0.01067098, + "balance_loss_clip": 1.07767308, + "balance_loss_mlp": 1.04192114, + "epoch": 0.03525622424699669, + "flos": 21282387085440.0, + "grad_norm": 2.3701187201518383, + "language_loss": 1.01710367, + "learning_rate": 3.999710648568098e-06, + "loss": 1.03992295, + "num_input_tokens_seen": 33836880, + "router_z_loss_clip": 1.37109375, + "router_z_loss_mlp": 0.25170898, + "step": 1215, + "time_per_iteration": 4.842917442321777 + }, + { + "auxiliary_loss_clip": 0.01213481, + "auxiliary_loss_mlp": 0.01061364, + "balance_loss_clip": 1.07701528, + "balance_loss_mlp": 1.04051447, + "epoch": 0.03528524171551274, + "flos": 32593565571840.0, + "grad_norm": 2.1344367215749864, + "language_loss": 0.82922715, + "learning_rate": 3.9997074425606804e-06, + "loss": 0.85197556, + "num_input_tokens_seen": 33855420, + "router_z_loss_clip": 1.36523438, + "router_z_loss_mlp": 0.20849609, + "step": 1216, + "time_per_iteration": 5.086705446243286 + }, + { + "auxiliary_loss_clip": 0.01218871, + "auxiliary_loss_mlp": 0.01067644, + "balance_loss_clip": 1.08518481, + "balance_loss_mlp": 1.0447911, + "epoch": 0.035314259184028786, + "flos": 26717646303360.0, + "grad_norm": 3.4361389712547172, + "language_loss": 0.86507237, + "learning_rate": 3.999704218891039e-06, + "loss": 0.88793755, + "num_input_tokens_seen": 33868655, + "router_z_loss_clip": 1.33691406, + "router_z_loss_mlp": 0.22851562, + "step": 1217, + "time_per_iteration": 2.546710252761841 + }, + { + "auxiliary_loss_clip": 0.01214327, + "auxiliary_loss_mlp": 0.01059263, + "balance_loss_clip": 1.07985473, + "balance_loss_mlp": 1.03710198, + "epoch": 0.03534327665254483, + "flos": 46571829742080.0, + "grad_norm": 3.767212553848738, + "language_loss": 0.70130515, + "learning_rate": 3.9997009775592e-06, + "loss": 0.7240411, + "num_input_tokens_seen": 33889470, + "router_z_loss_clip": 1.34472656, + "router_z_loss_mlp": 0.22180176, + "step": 1218, + "time_per_iteration": 2.9046289920806885 + }, + { + "auxiliary_loss_clip": 0.01060415, + "auxiliary_loss_mlp": 0.01005659, + "balance_loss_clip": 1.02640486, + "balance_loss_mlp": 1.0037632, + "epoch": 0.03537229412106088, + "flos": 61786357234560.0, + "grad_norm": 0.6557339969346951, + "language_loss": 0.48799711, + "learning_rate": 3.9996977185651925e-06, + "loss": 0.50865781, + "num_input_tokens_seen": 33947250, + "router_z_loss_clip": 0.33984375, + "router_z_loss_mlp": 0.0189209, + "step": 1219, + "time_per_iteration": 3.0476465225219727 + }, + { + "auxiliary_loss_clip": 0.0120614, + "auxiliary_loss_mlp": 0.01054883, + "balance_loss_clip": 1.0791651, + "balance_loss_mlp": 1.03521323, + "epoch": 0.03540131158957693, + "flos": 12413062051200.0, + "grad_norm": 16.657434786338406, + "language_loss": 1.00859249, + "learning_rate": 3.999694441909045e-06, + "loss": 1.03120267, + "num_input_tokens_seen": 33958700, + "router_z_loss_clip": 1.27050781, + "router_z_loss_mlp": 0.19689941, + "step": 1220, + "time_per_iteration": 2.5593202114105225 + }, + { + "auxiliary_loss_clip": 0.01059802, + "auxiliary_loss_mlp": 0.01004388, + "balance_loss_clip": 1.02637303, + "balance_loss_mlp": 1.00263584, + "epoch": 0.03543032905809297, + "flos": 69309607069440.0, + "grad_norm": 0.6689761452001839, + "language_loss": 0.48274389, + "learning_rate": 3.999691147590788e-06, + "loss": 0.50338578, + "num_input_tokens_seen": 34024825, + "router_z_loss_clip": 0.3359375, + "router_z_loss_mlp": 0.01757812, + "step": 1221, + "time_per_iteration": 3.2509331703186035 + }, + { + "auxiliary_loss_clip": 0.01213053, + "auxiliary_loss_mlp": 0.01056037, + "balance_loss_clip": 1.08311975, + "balance_loss_mlp": 1.03393602, + "epoch": 0.035459346526609016, + "flos": 24967001243520.0, + "grad_norm": 2.359649750912829, + "language_loss": 0.95754647, + "learning_rate": 3.999687835610449e-06, + "loss": 0.98023742, + "num_input_tokens_seen": 34039735, + "router_z_loss_clip": 1.29882812, + "router_z_loss_mlp": 0.22106934, + "step": 1222, + "time_per_iteration": 2.580012083053589 + }, + { + "auxiliary_loss_clip": 0.01218967, + "auxiliary_loss_mlp": 0.01058893, + "balance_loss_clip": 1.08334613, + "balance_loss_mlp": 1.03540909, + "epoch": 0.03548836399512507, + "flos": 25368087484800.0, + "grad_norm": 2.930415192749777, + "language_loss": 0.94516963, + "learning_rate": 3.999684505968059e-06, + "loss": 0.9679482, + "num_input_tokens_seen": 34057955, + "router_z_loss_clip": 1.35546875, + "router_z_loss_mlp": 0.23486328, + "step": 1223, + "time_per_iteration": 2.6188533306121826 + }, + { + "auxiliary_loss_clip": 0.01225342, + "auxiliary_loss_mlp": 0.01065765, + "balance_loss_clip": 1.08394289, + "balance_loss_mlp": 1.04120791, + "epoch": 0.03551738146364111, + "flos": 29162660152320.0, + "grad_norm": 7.949937242430619, + "language_loss": 0.9883976, + "learning_rate": 3.999681158663645e-06, + "loss": 1.01130867, + "num_input_tokens_seen": 34071810, + "router_z_loss_clip": 1.41357422, + "router_z_loss_mlp": 0.2454834, + "step": 1224, + "time_per_iteration": 2.559485912322998 + }, + { + "auxiliary_loss_clip": 0.01058206, + "auxiliary_loss_mlp": 0.01010741, + "balance_loss_clip": 1.0243957, + "balance_loss_mlp": 1.00897658, + "epoch": 0.03554639893215716, + "flos": 69804095650560.0, + "grad_norm": 0.7171389167519969, + "language_loss": 0.50836802, + "learning_rate": 3.999677793697238e-06, + "loss": 0.5290575, + "num_input_tokens_seen": 34126210, + "router_z_loss_clip": 0.33789062, + "router_z_loss_mlp": 0.0177002, + "step": 1225, + "time_per_iteration": 2.971177339553833 + }, + { + "auxiliary_loss_clip": 0.01057691, + "auxiliary_loss_mlp": 0.01010639, + "balance_loss_clip": 1.02396941, + "balance_loss_mlp": 1.00881529, + "epoch": 0.03557541640067321, + "flos": 66567360176640.0, + "grad_norm": 0.6565893754080889, + "language_loss": 0.51774311, + "learning_rate": 3.9996744110688685e-06, + "loss": 0.5384264, + "num_input_tokens_seen": 34188660, + "router_z_loss_clip": 0.33789062, + "router_z_loss_mlp": 0.01818848, + "step": 1226, + "time_per_iteration": 3.0623390674591064 + }, + { + "auxiliary_loss_clip": 0.01202972, + "auxiliary_loss_mlp": 0.01058578, + "balance_loss_clip": 1.07648802, + "balance_loss_mlp": 1.03876591, + "epoch": 0.03560443386918925, + "flos": 19092446691840.0, + "grad_norm": 2.482636172146491, + "language_loss": 0.8797645, + "learning_rate": 3.999671010778564e-06, + "loss": 0.90238005, + "num_input_tokens_seen": 34202710, + "router_z_loss_clip": 1.265625, + "router_z_loss_mlp": 0.19812012, + "step": 1227, + "time_per_iteration": 2.507634401321411 + }, + { + "auxiliary_loss_clip": 0.01223788, + "auxiliary_loss_mlp": 0.01076941, + "balance_loss_clip": 1.08302259, + "balance_loss_mlp": 1.05278969, + "epoch": 0.0356334513377053, + "flos": 16135993042560.0, + "grad_norm": 4.8278469773677575, + "language_loss": 0.81058973, + "learning_rate": 3.999667592826357e-06, + "loss": 0.83359706, + "num_input_tokens_seen": 34216430, + "router_z_loss_clip": 1.40820312, + "router_z_loss_mlp": 0.24157715, + "step": 1228, + "time_per_iteration": 2.5040652751922607 + }, + { + "auxiliary_loss_clip": 0.0105654, + "auxiliary_loss_mlp": 0.01008626, + "balance_loss_clip": 1.02315784, + "balance_loss_mlp": 1.00683784, + "epoch": 0.03566246880622134, + "flos": 67658074611840.0, + "grad_norm": 0.7191128893334651, + "language_loss": 0.4995203, + "learning_rate": 3.999664157212276e-06, + "loss": 0.52017194, + "num_input_tokens_seen": 34276440, + "router_z_loss_clip": 0.33398438, + "router_z_loss_mlp": 0.01782227, + "step": 1229, + "time_per_iteration": 3.079622983932495 + }, + { + "auxiliary_loss_clip": 0.01057704, + "auxiliary_loss_mlp": 0.01013486, + "balance_loss_clip": 1.02411962, + "balance_loss_mlp": 1.01167357, + "epoch": 0.035691486274737394, + "flos": 68193896768640.0, + "grad_norm": 0.7618059713025103, + "language_loss": 0.52678931, + "learning_rate": 3.999660703936352e-06, + "loss": 0.54750121, + "num_input_tokens_seen": 34324765, + "router_z_loss_clip": 0.3359375, + "router_z_loss_mlp": 0.01806641, + "step": 1230, + "time_per_iteration": 2.9247493743896484 + }, + { + "auxiliary_loss_clip": 0.01227644, + "auxiliary_loss_mlp": 0.01068155, + "balance_loss_clip": 1.0771842, + "balance_loss_mlp": 1.04008126, + "epoch": 0.03572050374325344, + "flos": 21757628954880.0, + "grad_norm": 4.013781506219665, + "language_loss": 1.17757857, + "learning_rate": 3.999657232998616e-06, + "loss": 1.20053661, + "num_input_tokens_seen": 34337470, + "router_z_loss_clip": 1.50390625, + "router_z_loss_mlp": 0.28076172, + "step": 1231, + "time_per_iteration": 2.559490442276001 + }, + { + "auxiliary_loss_clip": 0.01211607, + "auxiliary_loss_mlp": 0.01059941, + "balance_loss_clip": 1.0773406, + "balance_loss_mlp": 1.03742266, + "epoch": 0.03574952121176948, + "flos": 60908912824320.0, + "grad_norm": 2.38804668764699, + "language_loss": 0.81193298, + "learning_rate": 3.999653744399098e-06, + "loss": 0.83464849, + "num_input_tokens_seen": 34366815, + "router_z_loss_clip": 1.34179688, + "router_z_loss_mlp": 0.2253418, + "step": 1232, + "time_per_iteration": 2.929070472717285 + }, + { + "auxiliary_loss_clip": 0.01204932, + "auxiliary_loss_mlp": 0.01049126, + "balance_loss_clip": 1.07615685, + "balance_loss_mlp": 1.02828836, + "epoch": 0.035778538680285535, + "flos": 24365032191360.0, + "grad_norm": 1.9876641199277834, + "language_loss": 0.86729276, + "learning_rate": 3.999650238137829e-06, + "loss": 0.88983333, + "num_input_tokens_seen": 34390530, + "router_z_loss_clip": 1.28710938, + "router_z_loss_mlp": 0.20812988, + "step": 1233, + "time_per_iteration": 2.788933753967285 + }, + { + "auxiliary_loss_clip": 0.01219352, + "auxiliary_loss_mlp": 0.01077032, + "balance_loss_clip": 1.08107615, + "balance_loss_mlp": 1.05232036, + "epoch": 0.03580755614880158, + "flos": 47855205751680.0, + "grad_norm": 4.865282398126755, + "language_loss": 1.06737494, + "learning_rate": 3.999646714214839e-06, + "loss": 1.09033883, + "num_input_tokens_seen": 34409270, + "router_z_loss_clip": 1.3828125, + "router_z_loss_mlp": 0.24707031, + "step": 1234, + "time_per_iteration": 2.848583459854126 + }, + { + "auxiliary_loss_clip": 0.01058086, + "auxiliary_loss_mlp": 0.01001805, + "balance_loss_clip": 1.02383971, + "balance_loss_mlp": 1.00011253, + "epoch": 0.035836573617317624, + "flos": 71646170780160.0, + "grad_norm": 0.6971872033138632, + "language_loss": 0.52209282, + "learning_rate": 3.999643172630161e-06, + "loss": 0.54269171, + "num_input_tokens_seen": 34479295, + "router_z_loss_clip": 0.34375, + "router_z_loss_mlp": 0.01696777, + "step": 1235, + "time_per_iteration": 3.202822685241699 + }, + { + "auxiliary_loss_clip": 0.01223666, + "auxiliary_loss_mlp": 0.01062596, + "balance_loss_clip": 1.07795417, + "balance_loss_mlp": 1.03763318, + "epoch": 0.03586559108583367, + "flos": 30512506279680.0, + "grad_norm": 2.9506828864269994, + "language_loss": 1.02098322, + "learning_rate": 3.999639613383826e-06, + "loss": 1.04384589, + "num_input_tokens_seen": 34499115, + "router_z_loss_clip": 1.45703125, + "router_z_loss_mlp": 0.24951172, + "step": 1236, + "time_per_iteration": 2.6209616661071777 + }, + { + "auxiliary_loss_clip": 0.01210705, + "auxiliary_loss_mlp": 0.01066415, + "balance_loss_clip": 1.07579124, + "balance_loss_mlp": 1.04418206, + "epoch": 0.03589460855434972, + "flos": 20004110807040.0, + "grad_norm": 2.530502480134313, + "language_loss": 0.85596132, + "learning_rate": 3.999636036475864e-06, + "loss": 0.87873256, + "num_input_tokens_seen": 34514575, + "router_z_loss_clip": 1.34863281, + "router_z_loss_mlp": 0.22229004, + "step": 1237, + "time_per_iteration": 2.52374005317688 + }, + { + "auxiliary_loss_clip": 0.01223274, + "auxiliary_loss_mlp": 0.01056662, + "balance_loss_clip": 1.0831424, + "balance_loss_mlp": 1.03554988, + "epoch": 0.035923626022865765, + "flos": 39193874421120.0, + "grad_norm": 2.1829162526816623, + "language_loss": 0.93613309, + "learning_rate": 3.999632441906307e-06, + "loss": 0.95893252, + "num_input_tokens_seen": 34531345, + "router_z_loss_clip": 1.40234375, + "router_z_loss_mlp": 0.21118164, + "step": 1238, + "time_per_iteration": 2.6138668060302734 + }, + { + "auxiliary_loss_clip": 0.01196567, + "auxiliary_loss_mlp": 0.01051981, + "balance_loss_clip": 1.07402635, + "balance_loss_mlp": 1.03234744, + "epoch": 0.03595264349138181, + "flos": 23032098391680.0, + "grad_norm": 2.894880726584411, + "language_loss": 0.89456707, + "learning_rate": 3.999628829675188e-06, + "loss": 0.91705251, + "num_input_tokens_seen": 34545745, + "router_z_loss_clip": 1.22558594, + "router_z_loss_mlp": 0.1963501, + "step": 1239, + "time_per_iteration": 2.542646884918213 + }, + { + "auxiliary_loss_clip": 0.01224456, + "auxiliary_loss_mlp": 0.01070673, + "balance_loss_clip": 1.08226633, + "balance_loss_mlp": 1.04772484, + "epoch": 0.03598166095989786, + "flos": 42040872351360.0, + "grad_norm": 2.655877097860837, + "language_loss": 1.00523806, + "learning_rate": 3.999625199782537e-06, + "loss": 1.02818942, + "num_input_tokens_seen": 34561930, + "router_z_loss_clip": 1.42089844, + "router_z_loss_mlp": 0.22949219, + "step": 1240, + "time_per_iteration": 2.8636016845703125 + }, + { + "auxiliary_loss_clip": 0.01203296, + "auxiliary_loss_mlp": 0.01057479, + "balance_loss_clip": 1.07562649, + "balance_loss_mlp": 1.03436399, + "epoch": 0.036010678428413906, + "flos": 29090982562560.0, + "grad_norm": 1.9330785816983418, + "language_loss": 0.86923146, + "learning_rate": 3.999621552228387e-06, + "loss": 0.89183915, + "num_input_tokens_seen": 34579055, + "router_z_loss_clip": 1.27636719, + "router_z_loss_mlp": 0.2310791, + "step": 1241, + "time_per_iteration": 2.613783121109009 + }, + { + "auxiliary_loss_clip": 0.01217635, + "auxiliary_loss_mlp": 0.01070489, + "balance_loss_clip": 1.08086073, + "balance_loss_mlp": 1.04590762, + "epoch": 0.03603969589692995, + "flos": 34015125202560.0, + "grad_norm": 3.0122884368781757, + "language_loss": 0.91387308, + "learning_rate": 3.9996178870127715e-06, + "loss": 0.93675435, + "num_input_tokens_seen": 34595190, + "router_z_loss_clip": 1.36621094, + "router_z_loss_mlp": 0.24572754, + "step": 1242, + "time_per_iteration": 2.7284746170043945 + }, + { + "auxiliary_loss_clip": 0.01195736, + "auxiliary_loss_mlp": 0.010366, + "balance_loss_clip": 1.07416153, + "balance_loss_mlp": 1.01786041, + "epoch": 0.036068713365445995, + "flos": 35440348020480.0, + "grad_norm": 2.30959383674934, + "language_loss": 0.77869678, + "learning_rate": 3.99961420413572e-06, + "loss": 0.80102015, + "num_input_tokens_seen": 34617685, + "router_z_loss_clip": 1.21582031, + "router_z_loss_mlp": 0.18737793, + "step": 1243, + "time_per_iteration": 2.7362191677093506 + }, + { + "auxiliary_loss_clip": 0.01219504, + "auxiliary_loss_mlp": 0.01057836, + "balance_loss_clip": 1.08415556, + "balance_loss_mlp": 1.03669977, + "epoch": 0.03609773083396205, + "flos": 22776306664320.0, + "grad_norm": 2.1608737395956057, + "language_loss": 0.90768397, + "learning_rate": 3.999610503597269e-06, + "loss": 0.93045741, + "num_input_tokens_seen": 34632625, + "router_z_loss_clip": 1.35253906, + "router_z_loss_mlp": 0.21154785, + "step": 1244, + "time_per_iteration": 2.542292594909668 + }, + { + "auxiliary_loss_clip": 0.01208439, + "auxiliary_loss_mlp": 0.01052245, + "balance_loss_clip": 1.07736015, + "balance_loss_mlp": 1.02983391, + "epoch": 0.03612674830247809, + "flos": 20916349539840.0, + "grad_norm": 2.4354716412336823, + "language_loss": 0.84371978, + "learning_rate": 3.999606785397447e-06, + "loss": 0.86632663, + "num_input_tokens_seen": 34647585, + "router_z_loss_clip": 1.31152344, + "router_z_loss_mlp": 0.22412109, + "step": 1245, + "time_per_iteration": 2.6081392765045166 + }, + { + "auxiliary_loss_clip": 0.01220251, + "auxiliary_loss_mlp": 0.01070824, + "balance_loss_clip": 1.08185828, + "balance_loss_mlp": 1.04713702, + "epoch": 0.036155765770994136, + "flos": 14602212345600.0, + "grad_norm": 2.7034918386230116, + "language_loss": 0.88667846, + "learning_rate": 3.999603049536289e-06, + "loss": 0.90958917, + "num_input_tokens_seen": 34660910, + "router_z_loss_clip": 1.38378906, + "router_z_loss_mlp": 0.23706055, + "step": 1246, + "time_per_iteration": 2.523719310760498 + }, + { + "auxiliary_loss_clip": 0.01214902, + "auxiliary_loss_mlp": 0.01051123, + "balance_loss_clip": 1.0811795, + "balance_loss_mlp": 1.02936769, + "epoch": 0.03618478323951019, + "flos": 28154145991680.0, + "grad_norm": 2.040441177947115, + "language_loss": 0.73505437, + "learning_rate": 3.999599296013828e-06, + "loss": 0.75771463, + "num_input_tokens_seen": 34682705, + "router_z_loss_clip": 1.33691406, + "router_z_loss_mlp": 0.21740723, + "step": 1247, + "time_per_iteration": 2.7444496154785156 + }, + { + "auxiliary_loss_clip": 0.0121545, + "auxiliary_loss_mlp": 0.01078631, + "balance_loss_clip": 1.07827401, + "balance_loss_mlp": 1.05344248, + "epoch": 0.03621380070802623, + "flos": 33174743627520.0, + "grad_norm": 2.7574886944580514, + "language_loss": 1.05248117, + "learning_rate": 3.999595524830097e-06, + "loss": 1.07542205, + "num_input_tokens_seen": 34696720, + "router_z_loss_clip": 1.37207031, + "router_z_loss_mlp": 0.25195312, + "step": 1248, + "time_per_iteration": 2.7899322509765625 + }, + { + "auxiliary_loss_clip": 0.01219507, + "auxiliary_loss_mlp": 0.01067902, + "balance_loss_clip": 1.07833338, + "balance_loss_mlp": 1.04559195, + "epoch": 0.03624281817654228, + "flos": 34379833944960.0, + "grad_norm": 2.5013463164986587, + "language_loss": 1.10454381, + "learning_rate": 3.999591735985128e-06, + "loss": 1.1274178, + "num_input_tokens_seen": 34715475, + "router_z_loss_clip": 1.41210938, + "router_z_loss_mlp": 0.22320557, + "step": 1249, + "time_per_iteration": 2.741356611251831 + }, + { + "auxiliary_loss_clip": 0.01227275, + "auxiliary_loss_mlp": 0.01065159, + "balance_loss_clip": 1.08409691, + "balance_loss_mlp": 1.04174578, + "epoch": 0.03627183564505833, + "flos": 18069387523200.0, + "grad_norm": 2.3187941654252353, + "language_loss": 0.90464628, + "learning_rate": 3.999587929478957e-06, + "loss": 0.92757064, + "num_input_tokens_seen": 34729995, + "router_z_loss_clip": 1.42871094, + "router_z_loss_mlp": 0.234375, + "step": 1250, + "time_per_iteration": 2.503556489944458 + }, + { + "auxiliary_loss_clip": 0.0106448, + "auxiliary_loss_mlp": 0.01006128, + "balance_loss_clip": 1.02977657, + "balance_loss_mlp": 1.00426853, + "epoch": 0.03630085311357437, + "flos": 69553403654400.0, + "grad_norm": 0.7408057792476781, + "language_loss": 0.53098035, + "learning_rate": 3.999584105311616e-06, + "loss": 0.55168641, + "num_input_tokens_seen": 34794175, + "router_z_loss_clip": 0.34765625, + "router_z_loss_mlp": 0.01855469, + "step": 1251, + "time_per_iteration": 3.1541669368743896 + }, + { + "auxiliary_loss_clip": 0.0122116, + "auxiliary_loss_mlp": 0.01063184, + "balance_loss_clip": 1.08385205, + "balance_loss_mlp": 1.0403434, + "epoch": 0.03632987058209042, + "flos": 27991253813760.0, + "grad_norm": 4.843061233400709, + "language_loss": 1.06595564, + "learning_rate": 3.999580263483139e-06, + "loss": 1.088799, + "num_input_tokens_seen": 34805505, + "router_z_loss_clip": 1.37207031, + "router_z_loss_mlp": 0.22827148, + "step": 1252, + "time_per_iteration": 2.589967727661133 + }, + { + "auxiliary_loss_clip": 0.01210624, + "auxiliary_loss_mlp": 0.01058058, + "balance_loss_clip": 1.08158541, + "balance_loss_mlp": 1.03744102, + "epoch": 0.03635888805060646, + "flos": 36531924382080.0, + "grad_norm": 7.537327857182704, + "language_loss": 1.01864743, + "learning_rate": 3.999576403993559e-06, + "loss": 1.04133415, + "num_input_tokens_seen": 34821955, + "router_z_loss_clip": 1.29101562, + "router_z_loss_mlp": 0.20605469, + "step": 1253, + "time_per_iteration": 2.6589224338531494 + }, + { + "auxiliary_loss_clip": 0.01217881, + "auxiliary_loss_mlp": 0.01065137, + "balance_loss_clip": 1.08334243, + "balance_loss_mlp": 1.04359555, + "epoch": 0.036387905519122514, + "flos": 12123586344960.0, + "grad_norm": 3.0873878358383537, + "language_loss": 0.94907874, + "learning_rate": 3.999572526842912e-06, + "loss": 0.97190893, + "num_input_tokens_seen": 34833335, + "router_z_loss_clip": 1.34570312, + "router_z_loss_mlp": 0.21533203, + "step": 1254, + "time_per_iteration": 2.491394519805908 + }, + { + "auxiliary_loss_clip": 0.01223785, + "auxiliary_loss_mlp": 0.01061192, + "balance_loss_clip": 1.08664286, + "balance_loss_mlp": 1.03795767, + "epoch": 0.03641692298763856, + "flos": 20004218547840.0, + "grad_norm": 2.603811684696783, + "language_loss": 1.00983262, + "learning_rate": 3.999568632031231e-06, + "loss": 1.03268242, + "num_input_tokens_seen": 34846095, + "router_z_loss_clip": 1.37011719, + "router_z_loss_mlp": 0.2322998, + "step": 1255, + "time_per_iteration": 2.5845892429351807 + }, + { + "auxiliary_loss_clip": 0.0121503, + "auxiliary_loss_mlp": 0.01069416, + "balance_loss_clip": 1.08236635, + "balance_loss_mlp": 1.04717183, + "epoch": 0.0364459404561546, + "flos": 36387563621760.0, + "grad_norm": 2.296764966735242, + "language_loss": 0.8407805, + "learning_rate": 3.9995647195585516e-06, + "loss": 0.86362499, + "num_input_tokens_seen": 34863535, + "router_z_loss_clip": 1.328125, + "router_z_loss_mlp": 0.22241211, + "step": 1256, + "time_per_iteration": 2.5931851863861084 + }, + { + "auxiliary_loss_clip": 0.01211598, + "auxiliary_loss_mlp": 0.01064233, + "balance_loss_clip": 1.08073473, + "balance_loss_mlp": 1.04383683, + "epoch": 0.036474957924670655, + "flos": 30731381804160.0, + "grad_norm": 2.9826052891898622, + "language_loss": 0.83734632, + "learning_rate": 3.999560789424907e-06, + "loss": 0.86010468, + "num_input_tokens_seen": 34879955, + "router_z_loss_clip": 1.30957031, + "router_z_loss_mlp": 0.20397949, + "step": 1257, + "time_per_iteration": 2.614107370376587 + }, + { + "auxiliary_loss_clip": 0.01062791, + "auxiliary_loss_mlp": 0.01001934, + "balance_loss_clip": 1.02819037, + "balance_loss_mlp": 1.00006223, + "epoch": 0.0365039753931867, + "flos": 65676882107520.0, + "grad_norm": 0.7378449629463377, + "language_loss": 0.52128541, + "learning_rate": 3.999556841630332e-06, + "loss": 0.5419327, + "num_input_tokens_seen": 34934395, + "router_z_loss_clip": 0.34570312, + "router_z_loss_mlp": 0.01867676, + "step": 1258, + "time_per_iteration": 3.0304410457611084 + }, + { + "auxiliary_loss_clip": 0.01061597, + "auxiliary_loss_mlp": 0.01003485, + "balance_loss_clip": 1.02695584, + "balance_loss_mlp": 1.00154233, + "epoch": 0.036532992861702744, + "flos": 74768027581440.0, + "grad_norm": 0.6928294633722821, + "language_loss": 0.45179531, + "learning_rate": 3.999552876174862e-06, + "loss": 0.47244614, + "num_input_tokens_seen": 34990630, + "router_z_loss_clip": 0.34765625, + "router_z_loss_mlp": 0.01940918, + "step": 1259, + "time_per_iteration": 3.0335214138031006 + }, + { + "auxiliary_loss_clip": 0.0106054, + "auxiliary_loss_mlp": 0.01001026, + "balance_loss_clip": 1.02608812, + "balance_loss_mlp": 0.99921399, + "epoch": 0.03656201033021879, + "flos": 52926513390720.0, + "grad_norm": 0.6452242119764395, + "language_loss": 0.50301433, + "learning_rate": 3.9995488930585315e-06, + "loss": 0.52363002, + "num_input_tokens_seen": 35054365, + "router_z_loss_clip": 0.34375, + "router_z_loss_mlp": 0.01806641, + "step": 1260, + "time_per_iteration": 3.165480613708496 + }, + { + "auxiliary_loss_clip": 0.01213955, + "auxiliary_loss_mlp": 0.01058178, + "balance_loss_clip": 1.08095181, + "balance_loss_mlp": 1.03624344, + "epoch": 0.03659102779873484, + "flos": 12779279164800.0, + "grad_norm": 2.670614979403875, + "language_loss": 0.96972036, + "learning_rate": 3.999544892281377e-06, + "loss": 0.99244165, + "num_input_tokens_seen": 35067450, + "router_z_loss_clip": 1.33007812, + "router_z_loss_mlp": 0.21936035, + "step": 1261, + "time_per_iteration": 2.4855053424835205 + }, + { + "auxiliary_loss_clip": 0.01213257, + "auxiliary_loss_mlp": 0.01067901, + "balance_loss_clip": 1.07531929, + "balance_loss_mlp": 1.04512048, + "epoch": 0.036620045267250885, + "flos": 34998143685120.0, + "grad_norm": 2.5047488527889534, + "language_loss": 1.00848114, + "learning_rate": 3.999540873843432e-06, + "loss": 1.0312928, + "num_input_tokens_seen": 35088020, + "router_z_loss_clip": 1.37988281, + "router_z_loss_mlp": 0.22753906, + "step": 1262, + "time_per_iteration": 2.6741769313812256 + }, + { + "auxiliary_loss_clip": 0.01207935, + "auxiliary_loss_mlp": 0.01062753, + "balance_loss_clip": 1.0747081, + "balance_loss_mlp": 1.03887522, + "epoch": 0.03664906273576693, + "flos": 15483999323520.0, + "grad_norm": 3.1448963646996746, + "language_loss": 0.86657882, + "learning_rate": 3.9995368377447335e-06, + "loss": 0.88928568, + "num_input_tokens_seen": 35100805, + "router_z_loss_clip": 1.33203125, + "router_z_loss_mlp": 0.23876953, + "step": 1263, + "time_per_iteration": 2.5485339164733887 + }, + { + "auxiliary_loss_clip": 0.01058587, + "auxiliary_loss_mlp": 0.01003572, + "balance_loss_clip": 1.02445674, + "balance_loss_mlp": 1.0016886, + "epoch": 0.03667808020428298, + "flos": 57512376720000.0, + "grad_norm": 0.651783044674324, + "language_loss": 0.44365391, + "learning_rate": 3.999532783985316e-06, + "loss": 0.46427551, + "num_input_tokens_seen": 35158335, + "router_z_loss_clip": 0.33984375, + "router_z_loss_mlp": 0.01879883, + "step": 1264, + "time_per_iteration": 3.0156991481781006 + }, + { + "auxiliary_loss_clip": 0.01057469, + "auxiliary_loss_mlp": 0.01003079, + "balance_loss_clip": 1.0231967, + "balance_loss_mlp": 1.00119579, + "epoch": 0.036707097672799026, + "flos": 74775246215040.0, + "grad_norm": 0.6871704494179123, + "language_loss": 0.54796684, + "learning_rate": 3.999528712565216e-06, + "loss": 0.5685724, + "num_input_tokens_seen": 35224360, + "router_z_loss_clip": 0.34375, + "router_z_loss_mlp": 0.01879883, + "step": 1265, + "time_per_iteration": 3.114373207092285 + }, + { + "auxiliary_loss_clip": 0.01218883, + "auxiliary_loss_mlp": 0.01058063, + "balance_loss_clip": 1.08355474, + "balance_loss_mlp": 1.03648651, + "epoch": 0.03673611514131507, + "flos": 29345194091520.0, + "grad_norm": 2.7142475839713947, + "language_loss": 1.00416183, + "learning_rate": 3.9995246234844694e-06, + "loss": 1.02693129, + "num_input_tokens_seen": 35242600, + "router_z_loss_clip": 1.35253906, + "router_z_loss_mlp": 0.21582031, + "step": 1266, + "time_per_iteration": 2.5852749347686768 + }, + { + "auxiliary_loss_clip": 0.01214367, + "auxiliary_loss_mlp": 0.01062351, + "balance_loss_clip": 1.08336318, + "balance_loss_mlp": 1.0409286, + "epoch": 0.036765132609831115, + "flos": 74731323882240.0, + "grad_norm": 2.2092940073346328, + "language_loss": 1.10313296, + "learning_rate": 3.999520516743112e-06, + "loss": 1.12590003, + "num_input_tokens_seen": 35266790, + "router_z_loss_clip": 1.30810547, + "router_z_loss_mlp": 0.21429443, + "step": 1267, + "time_per_iteration": 2.971526622772217 + }, + { + "auxiliary_loss_clip": 0.01215744, + "auxiliary_loss_mlp": 0.01053494, + "balance_loss_clip": 1.08190167, + "balance_loss_mlp": 1.03191674, + "epoch": 0.03679415007834717, + "flos": 14278693524480.0, + "grad_norm": 2.5459663473682252, + "language_loss": 0.74392319, + "learning_rate": 3.99951639234118e-06, + "loss": 0.76661551, + "num_input_tokens_seen": 35279500, + "router_z_loss_clip": 1.33691406, + "router_z_loss_mlp": 0.21594238, + "step": 1268, + "time_per_iteration": 2.488034725189209 + }, + { + "auxiliary_loss_clip": 0.01056572, + "auxiliary_loss_mlp": 0.01003655, + "balance_loss_clip": 1.02271104, + "balance_loss_mlp": 1.00201035, + "epoch": 0.03682316754686321, + "flos": 61525178467200.0, + "grad_norm": 0.7347308769767372, + "language_loss": 0.5171479, + "learning_rate": 3.999512250278711e-06, + "loss": 0.53775012, + "num_input_tokens_seen": 35338470, + "router_z_loss_clip": 0.33789062, + "router_z_loss_mlp": 0.01647949, + "step": 1269, + "time_per_iteration": 3.028096914291382 + }, + { + "auxiliary_loss_clip": 0.01217496, + "auxiliary_loss_mlp": 0.01055086, + "balance_loss_clip": 1.0860281, + "balance_loss_mlp": 1.03290105, + "epoch": 0.036852185015379256, + "flos": 12744158641920.0, + "grad_norm": 2.732271088051641, + "language_loss": 0.85902393, + "learning_rate": 3.99950809055574e-06, + "loss": 0.88174975, + "num_input_tokens_seen": 35357605, + "router_z_loss_clip": 1.31347656, + "router_z_loss_mlp": 0.2220459, + "step": 1270, + "time_per_iteration": 2.6521143913269043 + }, + { + "auxiliary_loss_clip": 0.01227427, + "auxiliary_loss_mlp": 0.01073467, + "balance_loss_clip": 1.08401227, + "balance_loss_mlp": 1.04576278, + "epoch": 0.03688120248389531, + "flos": 14166005581440.0, + "grad_norm": 3.463476043583455, + "language_loss": 0.85884923, + "learning_rate": 3.999503913172305e-06, + "loss": 0.88185823, + "num_input_tokens_seen": 35372910, + "router_z_loss_clip": 1.43359375, + "router_z_loss_mlp": 0.27709961, + "step": 1271, + "time_per_iteration": 2.5342209339141846 + }, + { + "auxiliary_loss_clip": 0.0105528, + "auxiliary_loss_mlp": 0.01001969, + "balance_loss_clip": 1.02154195, + "balance_loss_mlp": 1.00025225, + "epoch": 0.03691021995241135, + "flos": 58678252364160.0, + "grad_norm": 0.7241910126800506, + "language_loss": 0.48589915, + "learning_rate": 3.999499718128441e-06, + "loss": 0.50647163, + "num_input_tokens_seen": 35426990, + "router_z_loss_clip": 0.33789062, + "router_z_loss_mlp": 0.01721191, + "step": 1272, + "time_per_iteration": 3.0646305084228516 + }, + { + "auxiliary_loss_clip": 0.01214465, + "auxiliary_loss_mlp": 0.0105959, + "balance_loss_clip": 1.07813668, + "balance_loss_mlp": 1.03745329, + "epoch": 0.0369392374209274, + "flos": 56818830965760.0, + "grad_norm": 2.177416766973467, + "language_loss": 0.64343059, + "learning_rate": 3.999495505424188e-06, + "loss": 0.66617119, + "num_input_tokens_seen": 35449765, + "router_z_loss_clip": 1.36328125, + "router_z_loss_mlp": 0.22149658, + "step": 1273, + "time_per_iteration": 2.833768844604492 + }, + { + "auxiliary_loss_clip": 0.01211139, + "auxiliary_loss_mlp": 0.01051053, + "balance_loss_clip": 1.08116949, + "balance_loss_mlp": 1.02983356, + "epoch": 0.03696825488944344, + "flos": 12311435496960.0, + "grad_norm": 2.3692232925154633, + "language_loss": 0.83166206, + "learning_rate": 3.9994912750595805e-06, + "loss": 0.85428393, + "num_input_tokens_seen": 35461620, + "router_z_loss_clip": 1.29785156, + "router_z_loss_mlp": 0.2121582, + "step": 1274, + "time_per_iteration": 2.576624870300293 + }, + { + "auxiliary_loss_clip": 0.0122702, + "auxiliary_loss_mlp": 0.01083604, + "balance_loss_clip": 1.08467436, + "balance_loss_mlp": 1.06037009, + "epoch": 0.03699727235795949, + "flos": 21071987170560.0, + "grad_norm": 2.9642964209889975, + "language_loss": 0.80812252, + "learning_rate": 3.999487027034657e-06, + "loss": 0.83122873, + "num_input_tokens_seen": 35476800, + "router_z_loss_clip": 1.421875, + "router_z_loss_mlp": 0.23205566, + "step": 1275, + "time_per_iteration": 2.5265214443206787 + }, + { + "auxiliary_loss_clip": 0.01199969, + "auxiliary_loss_mlp": 0.01062062, + "balance_loss_clip": 1.0750246, + "balance_loss_mlp": 1.04204035, + "epoch": 0.03702628982647554, + "flos": 17667116133120.0, + "grad_norm": 3.110990117693486, + "language_loss": 0.83825195, + "learning_rate": 3.999482761349455e-06, + "loss": 0.86087221, + "num_input_tokens_seen": 35489345, + "router_z_loss_clip": 1.25097656, + "router_z_loss_mlp": 0.20037842, + "step": 1276, + "time_per_iteration": 2.5173494815826416 + }, + { + "auxiliary_loss_clip": 0.01220988, + "auxiliary_loss_mlp": 0.0105957, + "balance_loss_clip": 1.07889128, + "balance_loss_mlp": 1.03552532, + "epoch": 0.03705530729499158, + "flos": 25991963832960.0, + "grad_norm": 3.263666305039841, + "language_loss": 0.81784719, + "learning_rate": 3.999478478004013e-06, + "loss": 0.84065282, + "num_input_tokens_seen": 35508105, + "router_z_loss_clip": 1.41894531, + "router_z_loss_mlp": 0.24072266, + "step": 1277, + "time_per_iteration": 2.5628867149353027 + }, + { + "auxiliary_loss_clip": 0.01206765, + "auxiliary_loss_mlp": 0.01063636, + "balance_loss_clip": 1.07425749, + "balance_loss_mlp": 1.04078388, + "epoch": 0.037084324763507634, + "flos": 34234036640640.0, + "grad_norm": 2.0864938750079878, + "language_loss": 0.75432116, + "learning_rate": 3.999474176998368e-06, + "loss": 0.7770251, + "num_input_tokens_seen": 35524875, + "router_z_loss_clip": 1.32519531, + "router_z_loss_mlp": 0.2286377, + "step": 1278, + "time_per_iteration": 2.752269983291626 + }, + { + "auxiliary_loss_clip": 0.01222642, + "auxiliary_loss_mlp": 0.0105884, + "balance_loss_clip": 1.08510089, + "balance_loss_mlp": 1.03649402, + "epoch": 0.03711334223202368, + "flos": 14530534755840.0, + "grad_norm": 2.978213076881666, + "language_loss": 0.81577647, + "learning_rate": 3.999469858332557e-06, + "loss": 0.83859128, + "num_input_tokens_seen": 35536425, + "router_z_loss_clip": 1.37695312, + "router_z_loss_mlp": 0.22332764, + "step": 1279, + "time_per_iteration": 2.544481039047241 + }, + { + "auxiliary_loss_clip": 0.0105675, + "auxiliary_loss_mlp": 0.01009691, + "balance_loss_clip": 1.02275491, + "balance_loss_mlp": 1.00787914, + "epoch": 0.03714235970053972, + "flos": 65690888411520.0, + "grad_norm": 0.6309809401966839, + "language_loss": 0.46713299, + "learning_rate": 3.99946552200662e-06, + "loss": 0.48779741, + "num_input_tokens_seen": 35598590, + "router_z_loss_clip": 0.33984375, + "router_z_loss_mlp": 0.01806641, + "step": 1280, + "time_per_iteration": 3.2179243564605713 + }, + { + "auxiliary_loss_clip": 0.01231247, + "auxiliary_loss_mlp": 0.01082062, + "balance_loss_clip": 1.08247101, + "balance_loss_mlp": 1.05779099, + "epoch": 0.037171377169055775, + "flos": 62004583336320.0, + "grad_norm": 3.5561833127341114, + "language_loss": 0.96670371, + "learning_rate": 3.999461168020593e-06, + "loss": 0.98983681, + "num_input_tokens_seen": 35626065, + "router_z_loss_clip": 1.48828125, + "router_z_loss_mlp": 0.24304199, + "step": 1281, + "time_per_iteration": 2.904053211212158 + }, + { + "auxiliary_loss_clip": 0.01203566, + "auxiliary_loss_mlp": 0.01057559, + "balance_loss_clip": 1.07819366, + "balance_loss_mlp": 1.03722179, + "epoch": 0.03720039463757182, + "flos": 51162685061760.0, + "grad_norm": 3.3341868269702837, + "language_loss": 0.90385246, + "learning_rate": 3.999456796374517e-06, + "loss": 0.92646372, + "num_input_tokens_seen": 35646075, + "router_z_loss_clip": 1.25390625, + "router_z_loss_mlp": 0.20349121, + "step": 1282, + "time_per_iteration": 2.816612482070923 + }, + { + "auxiliary_loss_clip": 0.01220035, + "auxiliary_loss_mlp": 0.01063312, + "balance_loss_clip": 1.08095813, + "balance_loss_mlp": 1.0418067, + "epoch": 0.037229412106087864, + "flos": 30912694680960.0, + "grad_norm": 2.3231423427438385, + "language_loss": 0.83694255, + "learning_rate": 3.9994524070684295e-06, + "loss": 0.85977602, + "num_input_tokens_seen": 35660425, + "router_z_loss_clip": 1.39160156, + "router_z_loss_mlp": 0.21496582, + "step": 1283, + "time_per_iteration": 2.6088614463806152 + }, + { + "auxiliary_loss_clip": 0.01057252, + "auxiliary_loss_mlp": 0.01004899, + "balance_loss_clip": 1.02289176, + "balance_loss_mlp": 1.00318193, + "epoch": 0.03725842957460391, + "flos": 74775389869440.0, + "grad_norm": 0.7280690104777929, + "language_loss": 0.53321493, + "learning_rate": 3.999448000102369e-06, + "loss": 0.55383641, + "num_input_tokens_seen": 35722125, + "router_z_loss_clip": 0.34375, + "router_z_loss_mlp": 0.01721191, + "step": 1284, + "time_per_iteration": 5.470884084701538 + }, + { + "auxiliary_loss_clip": 0.01211589, + "auxiliary_loss_mlp": 0.01049137, + "balance_loss_clip": 1.08199072, + "balance_loss_mlp": 1.03000426, + "epoch": 0.03728744704311996, + "flos": 27119343075840.0, + "grad_norm": 2.111552330989824, + "language_loss": 0.74412954, + "learning_rate": 3.999443575476374e-06, + "loss": 0.76673675, + "num_input_tokens_seen": 35738030, + "router_z_loss_clip": 1.29492188, + "router_z_loss_mlp": 0.19140625, + "step": 1285, + "time_per_iteration": 5.033479928970337 + }, + { + "auxiliary_loss_clip": 0.01056892, + "auxiliary_loss_mlp": 0.01000106, + "balance_loss_clip": 1.02221608, + "balance_loss_mlp": 0.99838966, + "epoch": 0.037316464511636005, + "flos": 62306775434880.0, + "grad_norm": 0.6736520417035738, + "language_loss": 0.51939476, + "learning_rate": 3.999439133190486e-06, + "loss": 0.53996468, + "num_input_tokens_seen": 35802555, + "router_z_loss_clip": 0.34765625, + "router_z_loss_mlp": 0.01721191, + "step": 1286, + "time_per_iteration": 5.403468370437622 + }, + { + "auxiliary_loss_clip": 0.01210244, + "auxiliary_loss_mlp": 0.01058594, + "balance_loss_clip": 1.07725739, + "balance_loss_mlp": 1.0373745, + "epoch": 0.03734548198015205, + "flos": 22304404759680.0, + "grad_norm": 2.119322561500502, + "language_loss": 0.71766531, + "learning_rate": 3.999434673244741e-06, + "loss": 0.7403537, + "num_input_tokens_seen": 35817845, + "router_z_loss_clip": 1.33203125, + "router_z_loss_mlp": 0.2121582, + "step": 1287, + "time_per_iteration": 5.039529323577881 + }, + { + "auxiliary_loss_clip": 0.01216808, + "auxiliary_loss_mlp": 0.01059487, + "balance_loss_clip": 1.0788852, + "balance_loss_mlp": 1.03509653, + "epoch": 0.0373744994486681, + "flos": 31245012334080.0, + "grad_norm": 1.8011668140340047, + "language_loss": 0.88666189, + "learning_rate": 3.99943019563918e-06, + "loss": 0.9094249, + "num_input_tokens_seen": 35838180, + "router_z_loss_clip": 1.37792969, + "router_z_loss_mlp": 0.24401855, + "step": 1288, + "time_per_iteration": 2.627676248550415 + }, + { + "auxiliary_loss_clip": 0.01216864, + "auxiliary_loss_mlp": 0.01063537, + "balance_loss_clip": 1.08255124, + "balance_loss_mlp": 1.04043412, + "epoch": 0.037403516917184146, + "flos": 74732437203840.0, + "grad_norm": 1.980387726923609, + "language_loss": 0.86343515, + "learning_rate": 3.999425700373843e-06, + "loss": 0.88623917, + "num_input_tokens_seen": 35865075, + "router_z_loss_clip": 1.34277344, + "router_z_loss_mlp": 0.23095703, + "step": 1289, + "time_per_iteration": 2.9297854900360107 + }, + { + "auxiliary_loss_clip": 0.01215319, + "auxiliary_loss_mlp": 0.01055053, + "balance_loss_clip": 1.08176243, + "balance_loss_mlp": 1.03338051, + "epoch": 0.03743253438570019, + "flos": 27922233830400.0, + "grad_norm": 2.732494285621679, + "language_loss": 0.80777973, + "learning_rate": 3.999421187448769e-06, + "loss": 0.83048344, + "num_input_tokens_seen": 35879290, + "router_z_loss_clip": 1.33496094, + "router_z_loss_mlp": 0.21655273, + "step": 1290, + "time_per_iteration": 2.5502541065216064 + }, + { + "auxiliary_loss_clip": 0.01226208, + "auxiliary_loss_mlp": 0.01066811, + "balance_loss_clip": 1.07844329, + "balance_loss_mlp": 1.04066801, + "epoch": 0.037461551854216235, + "flos": 25768778676480.0, + "grad_norm": 3.3845657687914636, + "language_loss": 1.0843358, + "learning_rate": 3.999416656863998e-06, + "loss": 1.10726595, + "num_input_tokens_seen": 35895065, + "router_z_loss_clip": 1.47753906, + "router_z_loss_mlp": 0.26135254, + "step": 1291, + "time_per_iteration": 2.570054531097412 + }, + { + "auxiliary_loss_clip": 0.01205177, + "auxiliary_loss_mlp": 0.01062876, + "balance_loss_clip": 1.0751431, + "balance_loss_mlp": 1.04327846, + "epoch": 0.03749056932273229, + "flos": 19124011768320.0, + "grad_norm": 3.184289689855913, + "language_loss": 1.01151013, + "learning_rate": 3.9994121086195695e-06, + "loss": 1.03419065, + "num_input_tokens_seen": 35905865, + "router_z_loss_clip": 1.30078125, + "router_z_loss_mlp": 0.19604492, + "step": 1292, + "time_per_iteration": 2.500863552093506 + }, + { + "auxiliary_loss_clip": 0.0105951, + "auxiliary_loss_mlp": 0.01017531, + "balance_loss_clip": 1.02411079, + "balance_loss_mlp": 1.01581478, + "epoch": 0.03751958679124833, + "flos": 69852001415040.0, + "grad_norm": 0.6774493561473469, + "language_loss": 0.47626507, + "learning_rate": 3.999407542715524e-06, + "loss": 0.49703547, + "num_input_tokens_seen": 35968590, + "router_z_loss_clip": 0.35351562, + "router_z_loss_mlp": 0.01721191, + "step": 1293, + "time_per_iteration": 3.1859352588653564 + }, + { + "auxiliary_loss_clip": 0.01207699, + "auxiliary_loss_mlp": 0.0105137, + "balance_loss_clip": 1.07438087, + "balance_loss_mlp": 1.03043103, + "epoch": 0.037548604259764376, + "flos": 20551640797440.0, + "grad_norm": 6.031495257764577, + "language_loss": 0.83127195, + "learning_rate": 3.999402959151903e-06, + "loss": 0.85386264, + "num_input_tokens_seen": 35982540, + "router_z_loss_clip": 1.33496094, + "router_z_loss_mlp": 0.20965576, + "step": 1294, + "time_per_iteration": 2.544203042984009 + }, + { + "auxiliary_loss_clip": 0.01206142, + "auxiliary_loss_mlp": 0.01050166, + "balance_loss_clip": 1.07536376, + "balance_loss_mlp": 1.0297575, + "epoch": 0.03757762172828043, + "flos": 23982869439360.0, + "grad_norm": 2.550957453900524, + "language_loss": 0.79802632, + "learning_rate": 3.9993983579287454e-06, + "loss": 0.82058942, + "num_input_tokens_seen": 35999585, + "router_z_loss_clip": 1.30859375, + "router_z_loss_mlp": 0.20410156, + "step": 1295, + "time_per_iteration": 2.6298763751983643 + }, + { + "auxiliary_loss_clip": 0.01203846, + "auxiliary_loss_mlp": 0.01056931, + "balance_loss_clip": 1.0734446, + "balance_loss_mlp": 1.03648043, + "epoch": 0.03760663919679647, + "flos": 32957412387840.0, + "grad_norm": 2.4519035254756605, + "language_loss": 0.89198029, + "learning_rate": 3.999393739046093e-06, + "loss": 0.91458803, + "num_input_tokens_seen": 36015895, + "router_z_loss_clip": 1.30322266, + "router_z_loss_mlp": 0.20452881, + "step": 1296, + "time_per_iteration": 2.5688636302948 + }, + { + "auxiliary_loss_clip": 0.01207791, + "auxiliary_loss_mlp": 0.01059726, + "balance_loss_clip": 1.07838464, + "balance_loss_mlp": 1.03820872, + "epoch": 0.03763565666531252, + "flos": 26649452332800.0, + "grad_norm": 2.759160043055139, + "language_loss": 0.90389669, + "learning_rate": 3.999389102503985e-06, + "loss": 0.92657191, + "num_input_tokens_seen": 36029950, + "router_z_loss_clip": 1.29394531, + "router_z_loss_mlp": 0.21514893, + "step": 1297, + "time_per_iteration": 2.5516529083251953 + }, + { + "auxiliary_loss_clip": 0.01207831, + "auxiliary_loss_mlp": 0.01058371, + "balance_loss_clip": 1.07985568, + "balance_loss_mlp": 1.03831959, + "epoch": 0.03766467413382856, + "flos": 14606091014400.0, + "grad_norm": 2.461738930507955, + "language_loss": 0.85288209, + "learning_rate": 3.999384448302464e-06, + "loss": 0.87554407, + "num_input_tokens_seen": 36041645, + "router_z_loss_clip": 1.27978516, + "router_z_loss_mlp": 0.20037842, + "step": 1298, + "time_per_iteration": 2.55718994140625 + }, + { + "auxiliary_loss_clip": 0.01209706, + "auxiliary_loss_mlp": 0.01051325, + "balance_loss_clip": 1.07684183, + "balance_loss_mlp": 1.03063059, + "epoch": 0.03769369160234461, + "flos": 29497922720640.0, + "grad_norm": 1.8103245867089632, + "language_loss": 0.85660124, + "learning_rate": 3.999379776441571e-06, + "loss": 0.87921154, + "num_input_tokens_seen": 36062860, + "router_z_loss_clip": 1.32714844, + "router_z_loss_mlp": 0.20703125, + "step": 1299, + "time_per_iteration": 2.618638038635254 + }, + { + "auxiliary_loss_clip": 0.01198411, + "auxiliary_loss_mlp": 0.01046053, + "balance_loss_clip": 1.0772264, + "balance_loss_mlp": 1.02775431, + "epoch": 0.03772270907086066, + "flos": 39998453114880.0, + "grad_norm": 3.10972006143479, + "language_loss": 0.66066694, + "learning_rate": 3.999375086921346e-06, + "loss": 0.68311155, + "num_input_tokens_seen": 36079885, + "router_z_loss_clip": 1.21289062, + "router_z_loss_mlp": 0.1829834, + "step": 1300, + "time_per_iteration": 2.6169464588165283 + }, + { + "auxiliary_loss_clip": 0.01206068, + "auxiliary_loss_mlp": 0.01047308, + "balance_loss_clip": 1.07540584, + "balance_loss_mlp": 1.02762699, + "epoch": 0.0377517265393767, + "flos": 32373864034560.0, + "grad_norm": 2.062899946033515, + "language_loss": 0.83303934, + "learning_rate": 3.999370379741831e-06, + "loss": 0.85557306, + "num_input_tokens_seen": 36097680, + "router_z_loss_clip": 1.30761719, + "router_z_loss_mlp": 0.19677734, + "step": 1301, + "time_per_iteration": 2.620513677597046 + }, + { + "auxiliary_loss_clip": 0.01059322, + "auxiliary_loss_mlp": 0.01004088, + "balance_loss_clip": 1.02353787, + "balance_loss_mlp": 1.00223982, + "epoch": 0.037780744007892754, + "flos": 62694717298560.0, + "grad_norm": 0.7070491280656985, + "language_loss": 0.50477535, + "learning_rate": 3.999365654903069e-06, + "loss": 0.52540946, + "num_input_tokens_seen": 36158395, + "router_z_loss_clip": 0.35742188, + "router_z_loss_mlp": 0.01843262, + "step": 1302, + "time_per_iteration": 3.103628158569336 + }, + { + "auxiliary_loss_clip": 0.01210373, + "auxiliary_loss_mlp": 0.0105266, + "balance_loss_clip": 1.0790025, + "balance_loss_mlp": 1.03203678, + "epoch": 0.0378097614764088, + "flos": 33393260016000.0, + "grad_norm": 3.066961291807902, + "language_loss": 0.73730242, + "learning_rate": 3.999360912405099e-06, + "loss": 0.75993276, + "num_input_tokens_seen": 36172330, + "router_z_loss_clip": 1.3125, + "router_z_loss_mlp": 0.20605469, + "step": 1303, + "time_per_iteration": 2.590296745300293 + }, + { + "auxiliary_loss_clip": 0.01218966, + "auxiliary_loss_mlp": 0.01056319, + "balance_loss_clip": 1.08077812, + "balance_loss_mlp": 1.03459907, + "epoch": 0.03783877894492484, + "flos": 26095062844800.0, + "grad_norm": 2.9966940033762293, + "language_loss": 1.0277226, + "learning_rate": 3.999356152247965e-06, + "loss": 1.05047548, + "num_input_tokens_seen": 36185345, + "router_z_loss_clip": 1.38183594, + "router_z_loss_mlp": 0.21716309, + "step": 1304, + "time_per_iteration": 2.6501622200012207 + }, + { + "auxiliary_loss_clip": 0.01222142, + "auxiliary_loss_mlp": 0.01068463, + "balance_loss_clip": 1.08021712, + "balance_loss_mlp": 1.04404855, + "epoch": 0.037867796413440895, + "flos": 40034363736960.0, + "grad_norm": 2.8367975294900263, + "language_loss": 0.87992275, + "learning_rate": 3.999351374431708e-06, + "loss": 0.90282881, + "num_input_tokens_seen": 36200395, + "router_z_loss_clip": 1.41992188, + "router_z_loss_mlp": 0.24401855, + "step": 1305, + "time_per_iteration": 2.592010259628296 + }, + { + "auxiliary_loss_clip": 0.01200562, + "auxiliary_loss_mlp": 0.01047293, + "balance_loss_clip": 1.07766914, + "balance_loss_mlp": 1.0261935, + "epoch": 0.03789681388195694, + "flos": 24602292501120.0, + "grad_norm": 2.856373509779279, + "language_loss": 0.89445978, + "learning_rate": 3.9993465789563715e-06, + "loss": 0.9169383, + "num_input_tokens_seen": 36215545, + "router_z_loss_clip": 1.22851562, + "router_z_loss_mlp": 0.21105957, + "step": 1306, + "time_per_iteration": 2.625382423400879 + }, + { + "auxiliary_loss_clip": 0.01213671, + "auxiliary_loss_mlp": 0.01064271, + "balance_loss_clip": 1.08208871, + "balance_loss_mlp": 1.03960633, + "epoch": 0.037925831350472984, + "flos": 17009914942080.0, + "grad_norm": 5.001652966613557, + "language_loss": 0.97724605, + "learning_rate": 3.999341765821997e-06, + "loss": 1.00002539, + "num_input_tokens_seen": 36227725, + "router_z_loss_clip": 1.31542969, + "router_z_loss_mlp": 0.24645996, + "step": 1307, + "time_per_iteration": 2.4988930225372314 + }, + { + "auxiliary_loss_clip": 0.01210399, + "auxiliary_loss_mlp": 0.01059107, + "balance_loss_clip": 1.07969737, + "balance_loss_mlp": 1.03764319, + "epoch": 0.03795484881898903, + "flos": 24017451258240.0, + "grad_norm": 2.6475324291305786, + "language_loss": 0.81591552, + "learning_rate": 3.999336935028626e-06, + "loss": 0.83861053, + "num_input_tokens_seen": 36243030, + "router_z_loss_clip": 1.30761719, + "router_z_loss_mlp": 0.21466064, + "step": 1308, + "time_per_iteration": 2.554453134536743 + }, + { + "auxiliary_loss_clip": 0.01201658, + "auxiliary_loss_mlp": 0.01050876, + "balance_loss_clip": 1.07763028, + "balance_loss_mlp": 1.03199291, + "epoch": 0.03798386628750508, + "flos": 18982344528000.0, + "grad_norm": 2.6898560699680676, + "language_loss": 0.71331197, + "learning_rate": 3.999332086576302e-06, + "loss": 0.73583734, + "num_input_tokens_seen": 36255530, + "router_z_loss_clip": 1.24121094, + "router_z_loss_mlp": 0.18896484, + "step": 1309, + "time_per_iteration": 2.4449002742767334 + }, + { + "auxiliary_loss_clip": 0.01210907, + "auxiliary_loss_mlp": 0.01062551, + "balance_loss_clip": 1.07564759, + "balance_loss_mlp": 1.04165339, + "epoch": 0.038012883756021125, + "flos": 34671536294400.0, + "grad_norm": 4.152359496788532, + "language_loss": 0.84641516, + "learning_rate": 3.999327220465069e-06, + "loss": 0.86914974, + "num_input_tokens_seen": 36273665, + "router_z_loss_clip": 1.35449219, + "router_z_loss_mlp": 0.2088623, + "step": 1310, + "time_per_iteration": 2.653035879135132 + }, + { + "auxiliary_loss_clip": 0.01203858, + "auxiliary_loss_mlp": 0.01058888, + "balance_loss_clip": 1.07348192, + "balance_loss_mlp": 1.03833652, + "epoch": 0.03804190122453717, + "flos": 32372822540160.0, + "grad_norm": 3.6459522680826737, + "language_loss": 0.767699, + "learning_rate": 3.999322336694969e-06, + "loss": 0.79032654, + "num_input_tokens_seen": 36288415, + "router_z_loss_clip": 1.30371094, + "router_z_loss_mlp": 0.20556641, + "step": 1311, + "time_per_iteration": 2.5988945960998535 + }, + { + "auxiliary_loss_clip": 0.01217402, + "auxiliary_loss_mlp": 0.01064361, + "balance_loss_clip": 1.07829785, + "balance_loss_mlp": 1.04140162, + "epoch": 0.03807091869305322, + "flos": 30218182237440.0, + "grad_norm": 2.8835907652691377, + "language_loss": 1.12372231, + "learning_rate": 3.9993174352660435e-06, + "loss": 1.14653993, + "num_input_tokens_seen": 36303815, + "router_z_loss_clip": 1.39160156, + "router_z_loss_mlp": 0.22949219, + "step": 1312, + "time_per_iteration": 2.589689254760742 + }, + { + "auxiliary_loss_clip": 0.0120642, + "auxiliary_loss_mlp": 0.01060865, + "balance_loss_clip": 1.07656574, + "balance_loss_mlp": 1.04090953, + "epoch": 0.038099936161569266, + "flos": 33941256883200.0, + "grad_norm": 4.856223560882642, + "language_loss": 0.98845911, + "learning_rate": 3.9993125161783395e-06, + "loss": 1.011132, + "num_input_tokens_seen": 36323515, + "router_z_loss_clip": 1.29980469, + "router_z_loss_mlp": 0.19976807, + "step": 1313, + "time_per_iteration": 2.5836055278778076 + }, + { + "auxiliary_loss_clip": 0.01218205, + "auxiliary_loss_mlp": 0.0106523, + "balance_loss_clip": 1.07861805, + "balance_loss_mlp": 1.04136443, + "epoch": 0.03812895363008531, + "flos": 13810778029440.0, + "grad_norm": 2.5049124046322335, + "language_loss": 0.96994317, + "learning_rate": 3.999307579431897e-06, + "loss": 0.99277753, + "num_input_tokens_seen": 36337335, + "router_z_loss_clip": 1.39746094, + "router_z_loss_mlp": 0.23876953, + "step": 1314, + "time_per_iteration": 2.5215444564819336 + }, + { + "auxiliary_loss_clip": 0.01202794, + "auxiliary_loss_mlp": 0.01052499, + "balance_loss_clip": 1.07845378, + "balance_loss_mlp": 1.0340457, + "epoch": 0.038157971098601355, + "flos": 11430043568640.0, + "grad_norm": 3.9921869765909688, + "language_loss": 0.70546836, + "learning_rate": 3.999302625026761e-06, + "loss": 0.72802126, + "num_input_tokens_seen": 36348865, + "router_z_loss_clip": 1.24414062, + "router_z_loss_mlp": 0.18444824, + "step": 1315, + "time_per_iteration": 2.452183485031128 + }, + { + "auxiliary_loss_clip": 0.01211856, + "auxiliary_loss_mlp": 0.01075423, + "balance_loss_clip": 1.07828188, + "balance_loss_mlp": 1.05385852, + "epoch": 0.03818698856711741, + "flos": 16830038609280.0, + "grad_norm": 4.674570399158507, + "language_loss": 1.03170323, + "learning_rate": 3.999297652962975e-06, + "loss": 1.05457616, + "num_input_tokens_seen": 36362765, + "router_z_loss_clip": 1.33886719, + "router_z_loss_mlp": 0.21557617, + "step": 1316, + "time_per_iteration": 2.562835693359375 + }, + { + "auxiliary_loss_clip": 0.01059616, + "auxiliary_loss_mlp": 0.01005516, + "balance_loss_clip": 1.02370262, + "balance_loss_mlp": 1.00362015, + "epoch": 0.03821600603563345, + "flos": 62588170581120.0, + "grad_norm": 0.6907211359794999, + "language_loss": 0.5168184, + "learning_rate": 3.999292663240584e-06, + "loss": 0.53746974, + "num_input_tokens_seen": 36425535, + "router_z_loss_clip": 0.359375, + "router_z_loss_mlp": 0.0189209, + "step": 1317, + "time_per_iteration": 3.1486623287200928 + }, + { + "auxiliary_loss_clip": 0.01060303, + "auxiliary_loss_mlp": 0.0100738, + "balance_loss_clip": 1.02438426, + "balance_loss_mlp": 1.00559223, + "epoch": 0.038245023504149496, + "flos": 60401390584320.0, + "grad_norm": 0.6961357195670826, + "language_loss": 0.52812803, + "learning_rate": 3.999287655859631e-06, + "loss": 0.54880488, + "num_input_tokens_seen": 36487280, + "router_z_loss_clip": 0.359375, + "router_z_loss_mlp": 0.01782227, + "step": 1318, + "time_per_iteration": 3.0859923362731934 + }, + { + "auxiliary_loss_clip": 0.01216252, + "auxiliary_loss_mlp": 0.01055226, + "balance_loss_clip": 1.07982695, + "balance_loss_mlp": 1.03213537, + "epoch": 0.03827404097266555, + "flos": 45659662836480.0, + "grad_norm": 1.7745326678671733, + "language_loss": 0.91192269, + "learning_rate": 3.99928263082016e-06, + "loss": 0.93463743, + "num_input_tokens_seen": 36513100, + "router_z_loss_clip": 1.36425781, + "router_z_loss_mlp": 0.23095703, + "step": 1319, + "time_per_iteration": 2.7749626636505127 + }, + { + "auxiliary_loss_clip": 0.01213026, + "auxiliary_loss_mlp": 0.01054603, + "balance_loss_clip": 1.07542872, + "balance_loss_mlp": 1.03058279, + "epoch": 0.03830305844118159, + "flos": 27081708600960.0, + "grad_norm": 2.7201722979425487, + "language_loss": 0.94063354, + "learning_rate": 3.999277588122215e-06, + "loss": 0.96330982, + "num_input_tokens_seen": 36530980, + "router_z_loss_clip": 1.37597656, + "router_z_loss_mlp": 0.24047852, + "step": 1320, + "time_per_iteration": 2.5730724334716797 + }, + { + "auxiliary_loss_clip": 0.0120301, + "auxiliary_loss_mlp": 0.01053363, + "balance_loss_clip": 1.0755477, + "balance_loss_mlp": 1.03195858, + "epoch": 0.03833207590969764, + "flos": 22777743208320.0, + "grad_norm": 2.262929746986888, + "language_loss": 0.86609721, + "learning_rate": 3.999272527765843e-06, + "loss": 0.88866091, + "num_input_tokens_seen": 36546405, + "router_z_loss_clip": 1.2734375, + "router_z_loss_mlp": 0.21417236, + "step": 1321, + "time_per_iteration": 2.575105905532837 + }, + { + "auxiliary_loss_clip": 0.01059055, + "auxiliary_loss_mlp": 0.0100501, + "balance_loss_clip": 1.02350509, + "balance_loss_mlp": 1.00334108, + "epoch": 0.03836109337821368, + "flos": 69488334167040.0, + "grad_norm": 0.7120809474343732, + "language_loss": 0.57226741, + "learning_rate": 3.999267449751085e-06, + "loss": 0.59290814, + "num_input_tokens_seen": 36608175, + "router_z_loss_clip": 0.35546875, + "router_z_loss_mlp": 0.01672363, + "step": 1322, + "time_per_iteration": 3.0810394287109375 + }, + { + "auxiliary_loss_clip": 0.01215974, + "auxiliary_loss_mlp": 0.01077911, + "balance_loss_clip": 1.07895207, + "balance_loss_mlp": 1.05405712, + "epoch": 0.03839011084672973, + "flos": 39376192878720.0, + "grad_norm": 3.1241288611738867, + "language_loss": 1.01068103, + "learning_rate": 3.99926235407799e-06, + "loss": 1.03361988, + "num_input_tokens_seen": 36625190, + "router_z_loss_clip": 1.37011719, + "router_z_loss_mlp": 0.23840332, + "step": 1323, + "time_per_iteration": 2.6770458221435547 + }, + { + "auxiliary_loss_clip": 0.01212314, + "auxiliary_loss_mlp": 0.01060317, + "balance_loss_clip": 1.07751453, + "balance_loss_mlp": 1.03906226, + "epoch": 0.03841912831524578, + "flos": 12013053217920.0, + "grad_norm": 2.891627744663203, + "language_loss": 0.65038019, + "learning_rate": 3.999257240746599e-06, + "loss": 0.67310655, + "num_input_tokens_seen": 36637880, + "router_z_loss_clip": 1.34863281, + "router_z_loss_mlp": 0.21240234, + "step": 1324, + "time_per_iteration": 2.467302083969116 + }, + { + "auxiliary_loss_clip": 0.01204327, + "auxiliary_loss_mlp": 0.01062953, + "balance_loss_clip": 1.07668853, + "balance_loss_mlp": 1.04305673, + "epoch": 0.03844814578376182, + "flos": 55869963338880.0, + "grad_norm": 3.097603085963016, + "language_loss": 0.74255961, + "learning_rate": 3.99925210975696e-06, + "loss": 0.76523238, + "num_input_tokens_seen": 36658220, + "router_z_loss_clip": 1.27734375, + "router_z_loss_mlp": 0.19909668, + "step": 1325, + "time_per_iteration": 2.8645031452178955 + }, + { + "auxiliary_loss_clip": 0.01207043, + "auxiliary_loss_mlp": 0.01061847, + "balance_loss_clip": 1.07777858, + "balance_loss_mlp": 1.03933978, + "epoch": 0.038477163252277874, + "flos": 18146416239360.0, + "grad_norm": 2.6386379613937034, + "language_loss": 0.83314186, + "learning_rate": 3.9992469611091175e-06, + "loss": 0.85583073, + "num_input_tokens_seen": 36670780, + "router_z_loss_clip": 1.29296875, + "router_z_loss_mlp": 0.22521973, + "step": 1326, + "time_per_iteration": 2.572557210922241 + }, + { + "auxiliary_loss_clip": 0.01059055, + "auxiliary_loss_mlp": 0.01006332, + "balance_loss_clip": 1.02330947, + "balance_loss_mlp": 1.0045321, + "epoch": 0.03850618072079392, + "flos": 74771187978240.0, + "grad_norm": 0.6740769614960422, + "language_loss": 0.50350267, + "learning_rate": 3.999241794803117e-06, + "loss": 0.52415657, + "num_input_tokens_seen": 36738790, + "router_z_loss_clip": 0.35742188, + "router_z_loss_mlp": 0.01794434, + "step": 1327, + "time_per_iteration": 3.449488878250122 + }, + { + "auxiliary_loss_clip": 0.0121999, + "auxiliary_loss_mlp": 0.0106312, + "balance_loss_clip": 1.08057737, + "balance_loss_mlp": 1.0380857, + "epoch": 0.03853519818930996, + "flos": 16832875783680.0, + "grad_norm": 3.111280656174302, + "language_loss": 0.85769409, + "learning_rate": 3.999236610839003e-06, + "loss": 0.88052523, + "num_input_tokens_seen": 36752300, + "router_z_loss_clip": 1.39355469, + "router_z_loss_mlp": 0.25048828, + "step": 1328, + "time_per_iteration": 2.5935826301574707 + }, + { + "auxiliary_loss_clip": 0.01059417, + "auxiliary_loss_mlp": 0.00999586, + "balance_loss_clip": 1.02379584, + "balance_loss_mlp": 0.9977625, + "epoch": 0.03856421565782601, + "flos": 60943605361920.0, + "grad_norm": 1.1120980555893487, + "language_loss": 0.48753503, + "learning_rate": 3.999231409216823e-06, + "loss": 0.50812507, + "num_input_tokens_seen": 36815850, + "router_z_loss_clip": 0.35546875, + "router_z_loss_mlp": 0.01818848, + "step": 1329, + "time_per_iteration": 3.2810027599334717 + }, + { + "auxiliary_loss_clip": 0.01058702, + "auxiliary_loss_mlp": 0.01001696, + "balance_loss_clip": 1.02291632, + "balance_loss_mlp": 0.99983615, + "epoch": 0.03859323312634206, + "flos": 64731929063040.0, + "grad_norm": 0.6983107950283245, + "language_loss": 0.53874052, + "learning_rate": 3.9992261899366226e-06, + "loss": 0.55934453, + "num_input_tokens_seen": 36873510, + "router_z_loss_clip": 0.35742188, + "router_z_loss_mlp": 0.01855469, + "step": 1330, + "time_per_iteration": 3.1083498001098633 + }, + { + "auxiliary_loss_clip": 0.01204621, + "auxiliary_loss_mlp": 0.01047021, + "balance_loss_clip": 1.07237649, + "balance_loss_mlp": 1.02489567, + "epoch": 0.038622250594858104, + "flos": 12597535324800.0, + "grad_norm": 2.849538721675183, + "language_loss": 0.93733144, + "learning_rate": 3.999220952998446e-06, + "loss": 0.95984787, + "num_input_tokens_seen": 36885815, + "router_z_loss_clip": 1.32324219, + "router_z_loss_mlp": 0.22119141, + "step": 1331, + "time_per_iteration": 2.6848702430725098 + }, + { + "auxiliary_loss_clip": 0.01205874, + "auxiliary_loss_mlp": 0.01059459, + "balance_loss_clip": 1.07504201, + "balance_loss_mlp": 1.03938437, + "epoch": 0.03865126806337415, + "flos": 36022783052160.0, + "grad_norm": 2.07288720936695, + "language_loss": 0.88575411, + "learning_rate": 3.999215698402342e-06, + "loss": 0.90840745, + "num_input_tokens_seen": 36911440, + "router_z_loss_clip": 1.30859375, + "router_z_loss_mlp": 0.20068359, + "step": 1332, + "time_per_iteration": 2.8788373470306396 + }, + { + "auxiliary_loss_clip": 0.01213195, + "auxiliary_loss_mlp": 0.01057792, + "balance_loss_clip": 1.0826211, + "balance_loss_mlp": 1.03596497, + "epoch": 0.0386802855318902, + "flos": 19931391722880.0, + "grad_norm": 3.3664020475914422, + "language_loss": 0.9215045, + "learning_rate": 3.999210426148356e-06, + "loss": 0.94421434, + "num_input_tokens_seen": 36924340, + "router_z_loss_clip": 1.30664062, + "router_z_loss_mlp": 0.21813965, + "step": 1333, + "time_per_iteration": 2.795396327972412 + }, + { + "auxiliary_loss_clip": 0.0105867, + "auxiliary_loss_mlp": 0.00999525, + "balance_loss_clip": 1.02332675, + "balance_loss_mlp": 0.99771255, + "epoch": 0.038709303000406245, + "flos": 74794495904640.0, + "grad_norm": 0.6276829225157037, + "language_loss": 0.5024327, + "learning_rate": 3.9992051362365346e-06, + "loss": 0.52301461, + "num_input_tokens_seen": 36997080, + "router_z_loss_clip": 0.35351562, + "router_z_loss_mlp": 0.01806641, + "step": 1334, + "time_per_iteration": 3.3629887104034424 + }, + { + "auxiliary_loss_clip": 0.01056576, + "auxiliary_loss_mlp": 0.01003026, + "balance_loss_clip": 1.0214839, + "balance_loss_mlp": 1.00130963, + "epoch": 0.03873832046892229, + "flos": 74769895088640.0, + "grad_norm": 0.8511304325615793, + "language_loss": 0.50728524, + "learning_rate": 3.9991998286669245e-06, + "loss": 0.52788126, + "num_input_tokens_seen": 37065425, + "router_z_loss_clip": 0.3515625, + "router_z_loss_mlp": 0.01721191, + "step": 1335, + "time_per_iteration": 3.3136818408966064 + }, + { + "auxiliary_loss_clip": 0.01210728, + "auxiliary_loss_mlp": 0.01050213, + "balance_loss_clip": 1.07783771, + "balance_loss_mlp": 1.02862382, + "epoch": 0.03876733793743834, + "flos": 38067033882240.0, + "grad_norm": 1.740016669223051, + "language_loss": 0.86371911, + "learning_rate": 3.999194503439572e-06, + "loss": 0.88632858, + "num_input_tokens_seen": 37096820, + "router_z_loss_clip": 1.32910156, + "router_z_loss_mlp": 0.21575928, + "step": 1336, + "time_per_iteration": 3.0136663913726807 + }, + { + "auxiliary_loss_clip": 0.01205684, + "auxiliary_loss_mlp": 0.01053853, + "balance_loss_clip": 1.07933962, + "balance_loss_mlp": 1.03277683, + "epoch": 0.038796355405954386, + "flos": 15408083928960.0, + "grad_norm": 2.6054925364148227, + "language_loss": 0.94342595, + "learning_rate": 3.999189160554525e-06, + "loss": 0.9660213, + "num_input_tokens_seen": 37110115, + "router_z_loss_clip": 1.26367188, + "router_z_loss_mlp": 0.21081543, + "step": 1337, + "time_per_iteration": 2.515408515930176 + }, + { + "auxiliary_loss_clip": 0.01220529, + "auxiliary_loss_mlp": 0.0106835, + "balance_loss_clip": 1.07803118, + "balance_loss_mlp": 1.04333973, + "epoch": 0.03882537287447043, + "flos": 16500450389760.0, + "grad_norm": 2.6222912568545493, + "language_loss": 0.78221935, + "learning_rate": 3.99918380001183e-06, + "loss": 0.80510813, + "num_input_tokens_seen": 37123825, + "router_z_loss_clip": 1.42382812, + "router_z_loss_mlp": 0.25012207, + "step": 1338, + "time_per_iteration": 2.5636403560638428 + }, + { + "auxiliary_loss_clip": 0.01053649, + "auxiliary_loss_mlp": 0.01001865, + "balance_loss_clip": 1.01867723, + "balance_loss_mlp": 1.00001752, + "epoch": 0.038854390342986475, + "flos": 74769607779840.0, + "grad_norm": 0.6922555887720329, + "language_loss": 0.51855618, + "learning_rate": 3.999178421811535e-06, + "loss": 0.53911132, + "num_input_tokens_seen": 37185365, + "router_z_loss_clip": 0.34960938, + "router_z_loss_mlp": 0.01843262, + "step": 1339, + "time_per_iteration": 3.104027271270752 + }, + { + "auxiliary_loss_clip": 0.01218989, + "auxiliary_loss_mlp": 0.01058748, + "balance_loss_clip": 1.07805145, + "balance_loss_mlp": 1.03707576, + "epoch": 0.03888340781150253, + "flos": 14934134949120.0, + "grad_norm": 3.17665866011052, + "language_loss": 0.86383641, + "learning_rate": 3.999173025953687e-06, + "loss": 0.88661373, + "num_input_tokens_seen": 37199420, + "router_z_loss_clip": 1.41015625, + "router_z_loss_mlp": 0.21655273, + "step": 1340, + "time_per_iteration": 2.5811784267425537 + }, + { + "auxiliary_loss_clip": 0.01210828, + "auxiliary_loss_mlp": 0.01069514, + "balance_loss_clip": 1.08138239, + "balance_loss_mlp": 1.04790115, + "epoch": 0.03891242528001857, + "flos": 40950588879360.0, + "grad_norm": 2.2628297469769563, + "language_loss": 0.87143236, + "learning_rate": 3.999167612438333e-06, + "loss": 0.89423573, + "num_input_tokens_seen": 37222680, + "router_z_loss_clip": 1.29394531, + "router_z_loss_mlp": 0.21606445, + "step": 1341, + "time_per_iteration": 2.7776687145233154 + }, + { + "auxiliary_loss_clip": 0.01198428, + "auxiliary_loss_mlp": 0.01060438, + "balance_loss_clip": 1.07637203, + "balance_loss_mlp": 1.04082775, + "epoch": 0.038941442748534616, + "flos": 42990386423040.0, + "grad_norm": 5.673270248043821, + "language_loss": 0.70439488, + "learning_rate": 3.999162181265523e-06, + "loss": 0.72698355, + "num_input_tokens_seen": 37241135, + "router_z_loss_clip": 1.21972656, + "router_z_loss_mlp": 0.19604492, + "step": 1342, + "time_per_iteration": 2.6361477375030518 + }, + { + "auxiliary_loss_clip": 0.01229865, + "auxiliary_loss_mlp": 0.01076637, + "balance_loss_clip": 1.0839901, + "balance_loss_mlp": 1.05266428, + "epoch": 0.03897046021705067, + "flos": 23072893263360.0, + "grad_norm": 2.7331946145524504, + "language_loss": 0.77235907, + "learning_rate": 3.999156732435304e-06, + "loss": 0.79542404, + "num_input_tokens_seen": 37255850, + "router_z_loss_clip": 1.45800781, + "router_z_loss_mlp": 0.23974609, + "step": 1343, + "time_per_iteration": 2.534757375717163 + }, + { + "auxiliary_loss_clip": 0.01234276, + "auxiliary_loss_mlp": 0.01084679, + "balance_loss_clip": 1.08154774, + "balance_loss_mlp": 1.05848908, + "epoch": 0.03899947768556671, + "flos": 25768527281280.0, + "grad_norm": 2.2010143543958516, + "language_loss": 0.89053464, + "learning_rate": 3.999151265947723e-06, + "loss": 0.91372418, + "num_input_tokens_seen": 37271465, + "router_z_loss_clip": 1.52734375, + "router_z_loss_mlp": 0.26196289, + "step": 1344, + "time_per_iteration": 2.5451467037200928 + }, + { + "auxiliary_loss_clip": 0.01200727, + "auxiliary_loss_mlp": 0.01062564, + "balance_loss_clip": 1.0731349, + "balance_loss_mlp": 1.04293013, + "epoch": 0.03902849515408276, + "flos": 10955555884800.0, + "grad_norm": 2.8661280979635575, + "language_loss": 0.82365251, + "learning_rate": 3.999145781802829e-06, + "loss": 0.84628546, + "num_input_tokens_seen": 37283240, + "router_z_loss_clip": 1.27539062, + "router_z_loss_mlp": 0.19628906, + "step": 1345, + "time_per_iteration": 2.4898674488067627 + }, + { + "auxiliary_loss_clip": 0.01203484, + "auxiliary_loss_mlp": 0.01055257, + "balance_loss_clip": 1.07838178, + "balance_loss_mlp": 1.03626704, + "epoch": 0.0390575126225988, + "flos": 34015412511360.0, + "grad_norm": 2.356414406982721, + "language_loss": 0.76595449, + "learning_rate": 3.99914028000067e-06, + "loss": 0.78854191, + "num_input_tokens_seen": 37302295, + "router_z_loss_clip": 1.25195312, + "router_z_loss_mlp": 0.18994141, + "step": 1346, + "time_per_iteration": 2.6444454193115234 + }, + { + "auxiliary_loss_clip": 0.01211046, + "auxiliary_loss_mlp": 0.01053457, + "balance_loss_clip": 1.07722032, + "balance_loss_mlp": 1.03080177, + "epoch": 0.03908653009111485, + "flos": 18398257470720.0, + "grad_norm": 3.408259498057151, + "language_loss": 1.04521632, + "learning_rate": 3.999134760541296e-06, + "loss": 1.06786132, + "num_input_tokens_seen": 37313935, + "router_z_loss_clip": 1.33789062, + "router_z_loss_mlp": 0.22662354, + "step": 1347, + "time_per_iteration": 2.4825966358184814 + }, + { + "auxiliary_loss_clip": 0.01217296, + "auxiliary_loss_mlp": 0.01065548, + "balance_loss_clip": 1.08007836, + "balance_loss_mlp": 1.04202819, + "epoch": 0.0391155475596309, + "flos": 31059246170880.0, + "grad_norm": 2.708882907285721, + "language_loss": 0.95557702, + "learning_rate": 3.999129223424754e-06, + "loss": 0.97840548, + "num_input_tokens_seen": 37329250, + "router_z_loss_clip": 1.37109375, + "router_z_loss_mlp": 0.23486328, + "step": 1348, + "time_per_iteration": 2.624781608581543 + }, + { + "auxiliary_loss_clip": 0.01202363, + "auxiliary_loss_mlp": 0.01050121, + "balance_loss_clip": 1.07978952, + "balance_loss_mlp": 1.03177428, + "epoch": 0.03914456502814694, + "flos": 23286238093440.0, + "grad_norm": 2.399275639497928, + "language_loss": 0.63433105, + "learning_rate": 3.999123668651094e-06, + "loss": 0.65685594, + "num_input_tokens_seen": 37343165, + "router_z_loss_clip": 1.22558594, + "router_z_loss_mlp": 0.18347168, + "step": 1349, + "time_per_iteration": 2.543505907058716 + }, + { + "auxiliary_loss_clip": 0.01214254, + "auxiliary_loss_mlp": 0.01070436, + "balance_loss_clip": 1.08098292, + "balance_loss_mlp": 1.0476079, + "epoch": 0.039173582496662994, + "flos": 15261532439040.0, + "grad_norm": 4.033440977984344, + "language_loss": 0.87598562, + "learning_rate": 3.999118096220366e-06, + "loss": 0.8988325, + "num_input_tokens_seen": 37356915, + "router_z_loss_clip": 1.33300781, + "router_z_loss_mlp": 0.22827148, + "step": 1350, + "time_per_iteration": 2.603749990463257 + }, + { + "auxiliary_loss_clip": 0.01213186, + "auxiliary_loss_mlp": 0.01061695, + "balance_loss_clip": 1.08318305, + "balance_loss_mlp": 1.03971314, + "epoch": 0.03920259996517904, + "flos": 16063094390400.0, + "grad_norm": 2.7438441062785492, + "language_loss": 0.89307427, + "learning_rate": 3.999112506132616e-06, + "loss": 0.91582322, + "num_input_tokens_seen": 37370485, + "router_z_loss_clip": 1.30175781, + "router_z_loss_mlp": 0.21984863, + "step": 1351, + "time_per_iteration": 2.491894006729126 + }, + { + "auxiliary_loss_clip": 0.01221649, + "auxiliary_loss_mlp": 0.01070649, + "balance_loss_clip": 1.08373559, + "balance_loss_mlp": 1.04828572, + "epoch": 0.03923161743369508, + "flos": 19968954370560.0, + "grad_norm": 2.816208050020733, + "language_loss": 0.965294, + "learning_rate": 3.999106898387897e-06, + "loss": 0.98821694, + "num_input_tokens_seen": 37385310, + "router_z_loss_clip": 1.37695312, + "router_z_loss_mlp": 0.22351074, + "step": 1352, + "time_per_iteration": 2.549044132232666 + }, + { + "auxiliary_loss_clip": 0.01208892, + "auxiliary_loss_mlp": 0.01065968, + "balance_loss_clip": 1.08266115, + "balance_loss_mlp": 1.04582191, + "epoch": 0.03926063490221113, + "flos": 25257949407360.0, + "grad_norm": 2.7359044648126343, + "language_loss": 0.7998879, + "learning_rate": 3.999101272986256e-06, + "loss": 0.8226366, + "num_input_tokens_seen": 37398455, + "router_z_loss_clip": 1.26171875, + "router_z_loss_mlp": 0.20166016, + "step": 1353, + "time_per_iteration": 2.5242514610290527 + }, + { + "auxiliary_loss_clip": 0.01057011, + "auxiliary_loss_mlp": 0.01006924, + "balance_loss_clip": 1.02178681, + "balance_loss_mlp": 1.00506389, + "epoch": 0.03928965237072718, + "flos": 74779232624640.0, + "grad_norm": 0.6890025913267495, + "language_loss": 0.5084002, + "learning_rate": 3.999095629927744e-06, + "loss": 0.5290395, + "num_input_tokens_seen": 37462715, + "router_z_loss_clip": 0.3515625, + "router_z_loss_mlp": 0.01855469, + "step": 1354, + "time_per_iteration": 3.217506170272827 + }, + { + "auxiliary_loss_clip": 0.01213345, + "auxiliary_loss_mlp": 0.01061543, + "balance_loss_clip": 1.07946157, + "balance_loss_mlp": 1.03819549, + "epoch": 0.039318669839243224, + "flos": 18655018865280.0, + "grad_norm": 5.601570854614982, + "language_loss": 0.92636055, + "learning_rate": 3.99908996921241e-06, + "loss": 0.94910949, + "num_input_tokens_seen": 37477995, + "router_z_loss_clip": 1.33789062, + "router_z_loss_mlp": 0.23370361, + "step": 1355, + "time_per_iteration": 4.853254556655884 + }, + { + "auxiliary_loss_clip": 0.01215172, + "auxiliary_loss_mlp": 0.0105042, + "balance_loss_clip": 1.0841974, + "balance_loss_mlp": 1.02893806, + "epoch": 0.03934768730775927, + "flos": 26030891197440.0, + "grad_norm": 2.069684051310867, + "language_loss": 0.81081557, + "learning_rate": 3.999084290840305e-06, + "loss": 0.83347148, + "num_input_tokens_seen": 37496900, + "router_z_loss_clip": 1.30859375, + "router_z_loss_mlp": 0.21459961, + "step": 1356, + "time_per_iteration": 5.007073640823364 + }, + { + "auxiliary_loss_clip": 0.01218487, + "auxiliary_loss_mlp": 0.01054328, + "balance_loss_clip": 1.08048749, + "balance_loss_mlp": 1.03203607, + "epoch": 0.03937670477627532, + "flos": 23870181496320.0, + "grad_norm": 8.33897727871385, + "language_loss": 0.79984653, + "learning_rate": 3.999078594811478e-06, + "loss": 0.82257468, + "num_input_tokens_seen": 37510390, + "router_z_loss_clip": 1.37988281, + "router_z_loss_mlp": 0.22290039, + "step": 1357, + "time_per_iteration": 4.785705327987671 + }, + { + "auxiliary_loss_clip": 0.01211353, + "auxiliary_loss_mlp": 0.01048991, + "balance_loss_clip": 1.07812679, + "balance_loss_mlp": 1.02792692, + "epoch": 0.039405722244791365, + "flos": 34324173820800.0, + "grad_norm": 1.6862101955533157, + "language_loss": 0.77111745, + "learning_rate": 3.999072881125981e-06, + "loss": 0.79372084, + "num_input_tokens_seen": 37535465, + "router_z_loss_clip": 1.33300781, + "router_z_loss_mlp": 0.21081543, + "step": 1358, + "time_per_iteration": 5.1971213817596436 + }, + { + "auxiliary_loss_clip": 0.01222848, + "auxiliary_loss_mlp": 0.01068, + "balance_loss_clip": 1.07926035, + "balance_loss_mlp": 1.0410465, + "epoch": 0.03943473971330741, + "flos": 39345741123840.0, + "grad_norm": 11.64167095169157, + "language_loss": 0.86303103, + "learning_rate": 3.999067149783863e-06, + "loss": 0.88593948, + "num_input_tokens_seen": 37555345, + "router_z_loss_clip": 1.43554688, + "router_z_loss_mlp": 0.26953125, + "step": 1359, + "time_per_iteration": 2.710794448852539 + }, + { + "auxiliary_loss_clip": 0.01224951, + "auxiliary_loss_mlp": 0.01067756, + "balance_loss_clip": 1.07977176, + "balance_loss_mlp": 1.04067135, + "epoch": 0.03946375718182346, + "flos": 16652460746880.0, + "grad_norm": 2.4589018180997373, + "language_loss": 0.82343841, + "learning_rate": 3.999061400785174e-06, + "loss": 0.84636545, + "num_input_tokens_seen": 37575710, + "router_z_loss_clip": 1.45214844, + "router_z_loss_mlp": 0.27075195, + "step": 1360, + "time_per_iteration": 2.5709710121154785 + }, + { + "auxiliary_loss_clip": 0.0105657, + "auxiliary_loss_mlp": 0.01004333, + "balance_loss_clip": 1.02054942, + "balance_loss_mlp": 1.00239015, + "epoch": 0.039492774650339506, + "flos": 58278997716480.0, + "grad_norm": 0.6928668265286867, + "language_loss": 0.51506495, + "learning_rate": 3.999055634129966e-06, + "loss": 0.53567398, + "num_input_tokens_seen": 37629910, + "router_z_loss_clip": 0.359375, + "router_z_loss_mlp": 0.01940918, + "step": 1361, + "time_per_iteration": 2.938403367996216 + }, + { + "auxiliary_loss_clip": 0.0120704, + "auxiliary_loss_mlp": 0.01059969, + "balance_loss_clip": 1.07748079, + "balance_loss_mlp": 1.03787911, + "epoch": 0.03952179211885555, + "flos": 74732688599040.0, + "grad_norm": 2.1344714958447715, + "language_loss": 0.73085958, + "learning_rate": 3.999049849818291e-06, + "loss": 0.75352967, + "num_input_tokens_seen": 37650935, + "router_z_loss_clip": 1.29492188, + "router_z_loss_mlp": 0.22070312, + "step": 1362, + "time_per_iteration": 2.9677908420562744 + }, + { + "auxiliary_loss_clip": 0.01205773, + "auxiliary_loss_mlp": 0.01062136, + "balance_loss_clip": 1.07624459, + "balance_loss_mlp": 1.04030848, + "epoch": 0.039550809587371595, + "flos": 12013448267520.0, + "grad_norm": 2.999145573706897, + "language_loss": 0.75140369, + "learning_rate": 3.999044047850198e-06, + "loss": 0.77408278, + "num_input_tokens_seen": 37662615, + "router_z_loss_clip": 1.29492188, + "router_z_loss_mlp": 0.21813965, + "step": 1363, + "time_per_iteration": 2.5107920169830322 + }, + { + "auxiliary_loss_clip": 0.01056718, + "auxiliary_loss_mlp": 0.01008709, + "balance_loss_clip": 1.0207181, + "balance_loss_mlp": 1.00707614, + "epoch": 0.03957982705588765, + "flos": 74774240634240.0, + "grad_norm": 0.7923520418321596, + "language_loss": 0.5207603, + "learning_rate": 3.999038228225739e-06, + "loss": 0.54141456, + "num_input_tokens_seen": 37721980, + "router_z_loss_clip": 0.359375, + "router_z_loss_mlp": 0.01635742, + "step": 1364, + "time_per_iteration": 3.093900442123413 + }, + { + "auxiliary_loss_clip": 0.01055797, + "auxiliary_loss_mlp": 0.01017878, + "balance_loss_clip": 1.02054667, + "balance_loss_mlp": 1.0161376, + "epoch": 0.03960884452440369, + "flos": 59865783909120.0, + "grad_norm": 0.6987144006819073, + "language_loss": 0.498689, + "learning_rate": 3.999032390944965e-06, + "loss": 0.51942575, + "num_input_tokens_seen": 37784685, + "router_z_loss_clip": 0.3515625, + "router_z_loss_mlp": 0.01745605, + "step": 1365, + "time_per_iteration": 3.157637596130371 + }, + { + "auxiliary_loss_clip": 0.01217117, + "auxiliary_loss_mlp": 0.01061966, + "balance_loss_clip": 1.08455181, + "balance_loss_mlp": 1.03974581, + "epoch": 0.039637861992919736, + "flos": 16099292321280.0, + "grad_norm": 2.667344546328145, + "language_loss": 0.74209917, + "learning_rate": 3.999026536007929e-06, + "loss": 0.76488996, + "num_input_tokens_seen": 37796360, + "router_z_loss_clip": 1.32421875, + "router_z_loss_mlp": 0.22241211, + "step": 1366, + "time_per_iteration": 2.515453338623047 + }, + { + "auxiliary_loss_clip": 0.01055622, + "auxiliary_loss_mlp": 0.01006333, + "balance_loss_clip": 1.02049589, + "balance_loss_mlp": 1.00459254, + "epoch": 0.03966687946143579, + "flos": 60842373857280.0, + "grad_norm": 0.6566663514186956, + "language_loss": 0.51477635, + "learning_rate": 3.999020663414681e-06, + "loss": 0.53539598, + "num_input_tokens_seen": 37862740, + "router_z_loss_clip": 0.3515625, + "router_z_loss_mlp": 0.01745605, + "step": 1367, + "time_per_iteration": 3.146608591079712 + }, + { + "auxiliary_loss_clip": 0.01055283, + "auxiliary_loss_mlp": 0.01002863, + "balance_loss_clip": 1.02027106, + "balance_loss_mlp": 1.00112271, + "epoch": 0.03969589692995183, + "flos": 67080344261760.0, + "grad_norm": 0.5963043224127345, + "language_loss": 0.46106434, + "learning_rate": 3.999014773165273e-06, + "loss": 0.48164582, + "num_input_tokens_seen": 37931500, + "router_z_loss_clip": 0.34960938, + "router_z_loss_mlp": 0.01745605, + "step": 1368, + "time_per_iteration": 3.207948923110962 + }, + { + "auxiliary_loss_clip": 0.01224623, + "auxiliary_loss_mlp": 0.01065476, + "balance_loss_clip": 1.08050096, + "balance_loss_mlp": 1.04064465, + "epoch": 0.03972491439846788, + "flos": 20075932051200.0, + "grad_norm": 3.3392545405068446, + "language_loss": 1.07483101, + "learning_rate": 3.999008865259759e-06, + "loss": 1.09773195, + "num_input_tokens_seen": 37944340, + "router_z_loss_clip": 1.44140625, + "router_z_loss_mlp": 0.24853516, + "step": 1369, + "time_per_iteration": 2.5213911533355713 + }, + { + "auxiliary_loss_clip": 0.01218472, + "auxiliary_loss_mlp": 0.01051723, + "balance_loss_clip": 1.08255982, + "balance_loss_mlp": 1.02996767, + "epoch": 0.03975393186698392, + "flos": 36729111070080.0, + "grad_norm": 2.509658517970948, + "language_loss": 0.83480823, + "learning_rate": 3.999002939698189e-06, + "loss": 0.85751021, + "num_input_tokens_seen": 37961550, + "router_z_loss_clip": 1.35742188, + "router_z_loss_mlp": 0.21765137, + "step": 1370, + "time_per_iteration": 2.6021816730499268 + }, + { + "auxiliary_loss_clip": 0.01055257, + "auxiliary_loss_mlp": 0.01001798, + "balance_loss_clip": 1.02041841, + "balance_loss_mlp": 0.99985534, + "epoch": 0.03978294933549997, + "flos": 74777041895040.0, + "grad_norm": 0.7204170618856071, + "language_loss": 0.52471936, + "learning_rate": 3.9989969964806165e-06, + "loss": 0.54528993, + "num_input_tokens_seen": 38024885, + "router_z_loss_clip": 0.34765625, + "router_z_loss_mlp": 0.01940918, + "step": 1371, + "time_per_iteration": 3.128058910369873 + }, + { + "auxiliary_loss_clip": 0.01053879, + "auxiliary_loss_mlp": 0.01002308, + "balance_loss_clip": 1.01927137, + "balance_loss_mlp": 1.00056744, + "epoch": 0.03981196680401602, + "flos": 69519432366720.0, + "grad_norm": 0.7198180696301454, + "language_loss": 0.52648866, + "learning_rate": 3.998991035607093e-06, + "loss": 0.54705048, + "num_input_tokens_seen": 38087115, + "router_z_loss_clip": 0.34570312, + "router_z_loss_mlp": 0.01745605, + "step": 1372, + "time_per_iteration": 3.121614694595337 + }, + { + "auxiliary_loss_clip": 0.01207769, + "auxiliary_loss_mlp": 0.01054463, + "balance_loss_clip": 1.08101857, + "balance_loss_mlp": 1.0340066, + "epoch": 0.03984098427253206, + "flos": 12268054846080.0, + "grad_norm": 3.0586640458529124, + "language_loss": 0.8478936, + "learning_rate": 3.9989850570776726e-06, + "loss": 0.87051594, + "num_input_tokens_seen": 38098110, + "router_z_loss_clip": 1.26855469, + "router_z_loss_mlp": 0.20440674, + "step": 1373, + "time_per_iteration": 2.5263023376464844 + }, + { + "auxiliary_loss_clip": 0.01217132, + "auxiliary_loss_mlp": 0.01067595, + "balance_loss_clip": 1.08233893, + "balance_loss_mlp": 1.0430975, + "epoch": 0.039870001741048114, + "flos": 28979767077120.0, + "grad_norm": 2.8755372552849168, + "language_loss": 0.91405284, + "learning_rate": 3.998979060892407e-06, + "loss": 0.93690014, + "num_input_tokens_seen": 38113445, + "router_z_loss_clip": 1.34863281, + "router_z_loss_mlp": 0.24511719, + "step": 1374, + "time_per_iteration": 2.642617702484131 + }, + { + "auxiliary_loss_clip": 0.01207543, + "auxiliary_loss_mlp": 0.01057272, + "balance_loss_clip": 1.07881689, + "balance_loss_mlp": 1.03505111, + "epoch": 0.03989901920956416, + "flos": 14533084621440.0, + "grad_norm": 2.770783249688816, + "language_loss": 0.84881371, + "learning_rate": 3.998973047051349e-06, + "loss": 0.87146187, + "num_input_tokens_seen": 38126325, + "router_z_loss_clip": 1.28515625, + "router_z_loss_mlp": 0.22229004, + "step": 1375, + "time_per_iteration": 2.558974266052246 + }, + { + "auxiliary_loss_clip": 0.01052534, + "auxiliary_loss_mlp": 0.01011152, + "balance_loss_clip": 1.01858068, + "balance_loss_mlp": 1.00932848, + "epoch": 0.0399280366780802, + "flos": 59225782354560.0, + "grad_norm": 0.7086986454504949, + "language_loss": 0.52729475, + "learning_rate": 3.998967015554552e-06, + "loss": 0.54793161, + "num_input_tokens_seen": 38182675, + "router_z_loss_clip": 0.33984375, + "router_z_loss_mlp": 0.01818848, + "step": 1376, + "time_per_iteration": 2.939802646636963 + }, + { + "auxiliary_loss_clip": 0.01052671, + "auxiliary_loss_mlp": 0.01009776, + "balance_loss_clip": 1.01818061, + "balance_loss_mlp": 1.00798738, + "epoch": 0.03995705414659625, + "flos": 74781064218240.0, + "grad_norm": 0.6244666539444903, + "language_loss": 0.52783781, + "learning_rate": 3.998960966402071e-06, + "loss": 0.54846227, + "num_input_tokens_seen": 38253010, + "router_z_loss_clip": 0.34375, + "router_z_loss_mlp": 0.01782227, + "step": 1377, + "time_per_iteration": 3.280937910079956 + }, + { + "auxiliary_loss_clip": 0.01214884, + "auxiliary_loss_mlp": 0.0106338, + "balance_loss_clip": 1.0780468, + "balance_loss_mlp": 1.03971732, + "epoch": 0.0399860716151123, + "flos": 16718499901440.0, + "grad_norm": 6.051384936530846, + "language_loss": 1.0628016, + "learning_rate": 3.998954899593956e-06, + "loss": 1.08558416, + "num_input_tokens_seen": 38263990, + "router_z_loss_clip": 1.36914062, + "router_z_loss_mlp": 0.23681641, + "step": 1378, + "time_per_iteration": 2.4801909923553467 + }, + { + "auxiliary_loss_clip": 0.01052275, + "auxiliary_loss_mlp": 0.01004727, + "balance_loss_clip": 1.01840591, + "balance_loss_mlp": 1.00299835, + "epoch": 0.040015089083628344, + "flos": 70716873087360.0, + "grad_norm": 0.7104738196495058, + "language_loss": 0.55720997, + "learning_rate": 3.998948815130263e-06, + "loss": 0.57777995, + "num_input_tokens_seen": 38332155, + "router_z_loss_clip": 0.33984375, + "router_z_loss_mlp": 0.01733398, + "step": 1379, + "time_per_iteration": 3.2342164516448975 + }, + { + "auxiliary_loss_clip": 0.01050804, + "auxiliary_loss_mlp": 0.0100798, + "balance_loss_clip": 1.0173521, + "balance_loss_mlp": 1.00623989, + "epoch": 0.04004410655214439, + "flos": 68667235217280.0, + "grad_norm": 0.6708818455908384, + "language_loss": 0.48893461, + "learning_rate": 3.9989427130110455e-06, + "loss": 0.5095225, + "num_input_tokens_seen": 38387570, + "router_z_loss_clip": 0.3359375, + "router_z_loss_mlp": 0.01745605, + "step": 1380, + "time_per_iteration": 3.0253963470458984 + }, + { + "auxiliary_loss_clip": 0.01219783, + "auxiliary_loss_mlp": 0.01069556, + "balance_loss_clip": 1.08047092, + "balance_loss_mlp": 1.04509413, + "epoch": 0.04007312402066044, + "flos": 21245865932160.0, + "grad_norm": 2.28821851786197, + "language_loss": 0.94894946, + "learning_rate": 3.998936593236356e-06, + "loss": 0.97184283, + "num_input_tokens_seen": 38403400, + "router_z_loss_clip": 1.39355469, + "router_z_loss_mlp": 0.24487305, + "step": 1381, + "time_per_iteration": 2.5128390789031982 + }, + { + "auxiliary_loss_clip": 0.01050583, + "auxiliary_loss_mlp": 0.01001619, + "balance_loss_clip": 1.01714444, + "balance_loss_mlp": 0.99962789, + "epoch": 0.040102141489176485, + "flos": 68606547189120.0, + "grad_norm": 0.6592431379743523, + "language_loss": 0.46153986, + "learning_rate": 3.998930455806251e-06, + "loss": 0.48206192, + "num_input_tokens_seen": 38462125, + "router_z_loss_clip": 0.33398438, + "router_z_loss_mlp": 0.01989746, + "step": 1382, + "time_per_iteration": 3.134857416152954 + }, + { + "auxiliary_loss_clip": 0.01209858, + "auxiliary_loss_mlp": 0.01061854, + "balance_loss_clip": 1.07732463, + "balance_loss_mlp": 1.03877473, + "epoch": 0.04013115895769253, + "flos": 34491264912000.0, + "grad_norm": 2.2029716004746906, + "language_loss": 0.93293083, + "learning_rate": 3.998924300720783e-06, + "loss": 0.95564795, + "num_input_tokens_seen": 38485450, + "router_z_loss_clip": 1.32617188, + "router_z_loss_mlp": 0.23095703, + "step": 1383, + "time_per_iteration": 2.6550676822662354 + }, + { + "auxiliary_loss_clip": 0.01210738, + "auxiliary_loss_mlp": 0.01062802, + "balance_loss_clip": 1.08061123, + "balance_loss_mlp": 1.04279876, + "epoch": 0.040160176426208574, + "flos": 74739835405440.0, + "grad_norm": 4.221152556012539, + "language_loss": 0.80426085, + "learning_rate": 3.998918127980006e-06, + "loss": 0.82699633, + "num_input_tokens_seen": 38514815, + "router_z_loss_clip": 1.30273438, + "router_z_loss_mlp": 0.20007324, + "step": 1384, + "time_per_iteration": 2.937615156173706 + }, + { + "auxiliary_loss_clip": 0.01211773, + "auxiliary_loss_mlp": 0.01059026, + "balance_loss_clip": 1.07859457, + "balance_loss_mlp": 1.03713953, + "epoch": 0.040189193894724626, + "flos": 27812885852160.0, + "grad_norm": 4.338238806768989, + "language_loss": 0.86880803, + "learning_rate": 3.998911937583976e-06, + "loss": 0.89151603, + "num_input_tokens_seen": 38529120, + "router_z_loss_clip": 1.33203125, + "router_z_loss_mlp": 0.21887207, + "step": 1385, + "time_per_iteration": 2.6031100749969482 + }, + { + "auxiliary_loss_clip": 0.01207213, + "auxiliary_loss_mlp": 0.01054363, + "balance_loss_clip": 1.07991624, + "balance_loss_mlp": 1.03202319, + "epoch": 0.04021821136324067, + "flos": 22995936374400.0, + "grad_norm": 2.3842387242823833, + "language_loss": 0.81395787, + "learning_rate": 3.998905729532746e-06, + "loss": 0.83657366, + "num_input_tokens_seen": 38543860, + "router_z_loss_clip": 1.2734375, + "router_z_loss_mlp": 0.22314453, + "step": 1386, + "time_per_iteration": 2.582611322402954 + }, + { + "auxiliary_loss_clip": 0.01219226, + "auxiliary_loss_mlp": 0.01067179, + "balance_loss_clip": 1.07924461, + "balance_loss_mlp": 1.04039288, + "epoch": 0.040247228831756715, + "flos": 31133545453440.0, + "grad_norm": 2.3012335778848434, + "language_loss": 0.92430663, + "learning_rate": 3.998899503826373e-06, + "loss": 0.94717073, + "num_input_tokens_seen": 38559860, + "router_z_loss_clip": 1.40039062, + "router_z_loss_mlp": 0.26794434, + "step": 1387, + "time_per_iteration": 2.590468168258667 + }, + { + "auxiliary_loss_clip": 0.01216644, + "auxiliary_loss_mlp": 0.0107143, + "balance_loss_clip": 1.07858014, + "balance_loss_mlp": 1.04588366, + "epoch": 0.04027624630027277, + "flos": 24453191145600.0, + "grad_norm": 2.923472444350308, + "language_loss": 0.87360972, + "learning_rate": 3.99889326046491e-06, + "loss": 0.89649045, + "num_input_tokens_seen": 38574205, + "router_z_loss_clip": 1.37988281, + "router_z_loss_mlp": 0.25524902, + "step": 1388, + "time_per_iteration": 2.5644314289093018 + }, + { + "auxiliary_loss_clip": 0.01212513, + "auxiliary_loss_mlp": 0.01059239, + "balance_loss_clip": 1.08040392, + "balance_loss_mlp": 1.03733993, + "epoch": 0.04030526376878881, + "flos": 35947945065600.0, + "grad_norm": 2.619001606906196, + "language_loss": 0.78783679, + "learning_rate": 3.998886999448413e-06, + "loss": 0.81055427, + "num_input_tokens_seen": 38590095, + "router_z_loss_clip": 1.32128906, + "router_z_loss_mlp": 0.21887207, + "step": 1389, + "time_per_iteration": 2.6447055339813232 + }, + { + "auxiliary_loss_clip": 0.01217356, + "auxiliary_loss_mlp": 0.01063883, + "balance_loss_clip": 1.0830574, + "balance_loss_mlp": 1.04169178, + "epoch": 0.040334281237304856, + "flos": 26287185715200.0, + "grad_norm": 3.7386805028852232, + "language_loss": 1.00852609, + "learning_rate": 3.998880720776937e-06, + "loss": 1.03133833, + "num_input_tokens_seen": 38607095, + "router_z_loss_clip": 1.34326172, + "router_z_loss_mlp": 0.22198486, + "step": 1390, + "time_per_iteration": 2.6134438514709473 + }, + { + "auxiliary_loss_clip": 0.01218127, + "auxiliary_loss_mlp": 0.01069301, + "balance_loss_clip": 1.08216786, + "balance_loss_mlp": 1.04448223, + "epoch": 0.04036329870582091, + "flos": 15261927488640.0, + "grad_norm": 2.9783490188057598, + "language_loss": 0.91826046, + "learning_rate": 3.998874424450538e-06, + "loss": 0.94113481, + "num_input_tokens_seen": 38621090, + "router_z_loss_clip": 1.359375, + "router_z_loss_mlp": 0.24816895, + "step": 1391, + "time_per_iteration": 2.5092153549194336 + }, + { + "auxiliary_loss_clip": 0.01213461, + "auxiliary_loss_mlp": 0.01061747, + "balance_loss_clip": 1.08166718, + "balance_loss_mlp": 1.03757095, + "epoch": 0.04039231617433695, + "flos": 35290959356160.0, + "grad_norm": 2.3400727227570584, + "language_loss": 0.97228509, + "learning_rate": 3.99886811046927e-06, + "loss": 0.99503714, + "num_input_tokens_seen": 38640300, + "router_z_loss_clip": 1.31640625, + "router_z_loss_mlp": 0.24182129, + "step": 1392, + "time_per_iteration": 2.6368298530578613 + }, + { + "auxiliary_loss_clip": 0.0120218, + "auxiliary_loss_mlp": 0.0106848, + "balance_loss_clip": 1.07962751, + "balance_loss_mlp": 1.0477612, + "epoch": 0.040421333642853, + "flos": 28870993716480.0, + "grad_norm": 3.121177417785791, + "language_loss": 0.69636369, + "learning_rate": 3.998861778833192e-06, + "loss": 0.7190702, + "num_input_tokens_seen": 38662360, + "router_z_loss_clip": 1.22509766, + "router_z_loss_mlp": 0.20739746, + "step": 1393, + "time_per_iteration": 2.7472898960113525 + }, + { + "auxiliary_loss_clip": 0.01223552, + "auxiliary_loss_mlp": 0.01070364, + "balance_loss_clip": 1.08780646, + "balance_loss_mlp": 1.05011058, + "epoch": 0.04045035111136904, + "flos": 16026501409920.0, + "grad_norm": 3.3420659024744284, + "language_loss": 0.97010237, + "learning_rate": 3.998855429542357e-06, + "loss": 0.99304152, + "num_input_tokens_seen": 38672665, + "router_z_loss_clip": 1.35546875, + "router_z_loss_mlp": 0.20263672, + "step": 1394, + "time_per_iteration": 2.4833574295043945 + }, + { + "auxiliary_loss_clip": 0.01058181, + "auxiliary_loss_mlp": 0.01003799, + "balance_loss_clip": 1.02445769, + "balance_loss_mlp": 1.00185609, + "epoch": 0.04047936857988509, + "flos": 57769676818560.0, + "grad_norm": 0.6923377783551218, + "language_loss": 0.50470173, + "learning_rate": 3.998849062596821e-06, + "loss": 0.52532148, + "num_input_tokens_seen": 38730725, + "router_z_loss_clip": 0.3359375, + "router_z_loss_mlp": 0.01940918, + "step": 1395, + "time_per_iteration": 3.060312509536743 + }, + { + "auxiliary_loss_clip": 0.01206934, + "auxiliary_loss_mlp": 0.01048164, + "balance_loss_clip": 1.07874751, + "balance_loss_mlp": 1.02774382, + "epoch": 0.04050838604840114, + "flos": 15590546040960.0, + "grad_norm": 3.2287050600833425, + "language_loss": 0.9686029, + "learning_rate": 3.998842677996642e-06, + "loss": 0.99115396, + "num_input_tokens_seen": 38743565, + "router_z_loss_clip": 1.28027344, + "router_z_loss_mlp": 0.20410156, + "step": 1396, + "time_per_iteration": 2.482658863067627 + }, + { + "auxiliary_loss_clip": 0.01215294, + "auxiliary_loss_mlp": 0.01062223, + "balance_loss_clip": 1.07889938, + "balance_loss_mlp": 1.0393883, + "epoch": 0.04053740351691718, + "flos": 27922521139200.0, + "grad_norm": 4.5486020333775645, + "language_loss": 0.90695179, + "learning_rate": 3.9988362757418765e-06, + "loss": 0.9297269, + "num_input_tokens_seen": 38759655, + "router_z_loss_clip": 1.36523438, + "router_z_loss_mlp": 0.22857666, + "step": 1397, + "time_per_iteration": 2.6099274158477783 + }, + { + "auxiliary_loss_clip": 0.01211233, + "auxiliary_loss_mlp": 0.01044443, + "balance_loss_clip": 1.0787456, + "balance_loss_mlp": 1.02169156, + "epoch": 0.040566420985433234, + "flos": 18944315003520.0, + "grad_norm": 2.9034546336283777, + "language_loss": 0.89017129, + "learning_rate": 3.9988298558325785e-06, + "loss": 0.91272807, + "num_input_tokens_seen": 38772545, + "router_z_loss_clip": 1.32324219, + "router_z_loss_mlp": 0.22747803, + "step": 1398, + "time_per_iteration": 2.4510412216186523 + }, + { + "auxiliary_loss_clip": 0.01209299, + "auxiliary_loss_mlp": 0.01054475, + "balance_loss_clip": 1.08261967, + "balance_loss_mlp": 1.03372121, + "epoch": 0.04059543845394928, + "flos": 25185769027200.0, + "grad_norm": 2.301914834841433, + "language_loss": 0.75762707, + "learning_rate": 3.998823418268807e-06, + "loss": 0.78026474, + "num_input_tokens_seen": 38789575, + "router_z_loss_clip": 1.26660156, + "router_z_loss_mlp": 0.20739746, + "step": 1399, + "time_per_iteration": 2.5699591636657715 + }, + { + "auxiliary_loss_clip": 0.01212915, + "auxiliary_loss_mlp": 0.01083769, + "balance_loss_clip": 1.08088851, + "balance_loss_mlp": 1.06047559, + "epoch": 0.04062445592246532, + "flos": 29605079969280.0, + "grad_norm": 2.2991294985552537, + "language_loss": 1.08498168, + "learning_rate": 3.998816963050619e-06, + "loss": 1.10794854, + "num_input_tokens_seen": 38811850, + "router_z_loss_clip": 1.31933594, + "router_z_loss_mlp": 0.23303223, + "step": 1400, + "time_per_iteration": 2.609675168991089 + }, + { + "auxiliary_loss_clip": 0.01196724, + "auxiliary_loss_mlp": 0.01055487, + "balance_loss_clip": 1.07723975, + "balance_loss_mlp": 1.03479242, + "epoch": 0.04065347339098137, + "flos": 17711897414400.0, + "grad_norm": 2.7578721093286873, + "language_loss": 0.9128654, + "learning_rate": 3.99881049017807e-06, + "loss": 0.93538755, + "num_input_tokens_seen": 38826270, + "router_z_loss_clip": 1.19628906, + "router_z_loss_mlp": 0.20690918, + "step": 1401, + "time_per_iteration": 2.5394322872161865 + }, + { + "auxiliary_loss_clip": 0.01196492, + "auxiliary_loss_mlp": 0.01050935, + "balance_loss_clip": 1.07754397, + "balance_loss_mlp": 1.0323149, + "epoch": 0.04068249085949742, + "flos": 27448500332160.0, + "grad_norm": 2.8109113325172936, + "language_loss": 1.0287354, + "learning_rate": 3.998803999651218e-06, + "loss": 1.05120969, + "num_input_tokens_seen": 38841340, + "router_z_loss_clip": 1.18945312, + "router_z_loss_mlp": 0.18615723, + "step": 1402, + "time_per_iteration": 2.6181530952453613 + }, + { + "auxiliary_loss_clip": 0.01059497, + "auxiliary_loss_mlp": 0.01002281, + "balance_loss_clip": 1.02563858, + "balance_loss_mlp": 1.00052893, + "epoch": 0.040711508328013464, + "flos": 57905238746880.0, + "grad_norm": 0.7516378691188709, + "language_loss": 0.47089112, + "learning_rate": 3.99879749147012e-06, + "loss": 0.4915089, + "num_input_tokens_seen": 38892920, + "router_z_loss_clip": 0.33984375, + "router_z_loss_mlp": 0.01757812, + "step": 1403, + "time_per_iteration": 3.0907225608825684 + }, + { + "auxiliary_loss_clip": 0.01212055, + "auxiliary_loss_mlp": 0.01055354, + "balance_loss_clip": 1.08257556, + "balance_loss_mlp": 1.03359866, + "epoch": 0.04074052579652951, + "flos": 56087581887360.0, + "grad_norm": 1.967256578594369, + "language_loss": 0.80933404, + "learning_rate": 3.998790965634835e-06, + "loss": 0.83200812, + "num_input_tokens_seen": 38913995, + "router_z_loss_clip": 1.29589844, + "router_z_loss_mlp": 0.21740723, + "step": 1404, + "time_per_iteration": 2.8380184173583984 + }, + { + "auxiliary_loss_clip": 0.01196484, + "auxiliary_loss_mlp": 0.01053352, + "balance_loss_clip": 1.07417035, + "balance_loss_mlp": 1.03361702, + "epoch": 0.04076954326504556, + "flos": 40947213000960.0, + "grad_norm": 2.3468244041385513, + "language_loss": 0.86009049, + "learning_rate": 3.998784422145418e-06, + "loss": 0.88258886, + "num_input_tokens_seen": 38930185, + "router_z_loss_clip": 1.22363281, + "router_z_loss_mlp": 0.19726562, + "step": 1405, + "time_per_iteration": 2.727649450302124 + }, + { + "auxiliary_loss_clip": 0.01216444, + "auxiliary_loss_mlp": 0.01059805, + "balance_loss_clip": 1.08052158, + "balance_loss_mlp": 1.03623724, + "epoch": 0.040798560733561605, + "flos": 47293130753280.0, + "grad_norm": 3.4900000524738966, + "language_loss": 1.01581919, + "learning_rate": 3.9987778610019285e-06, + "loss": 1.03858173, + "num_input_tokens_seen": 38944970, + "router_z_loss_clip": 1.35742188, + "router_z_loss_mlp": 0.23547363, + "step": 1406, + "time_per_iteration": 2.7550923824310303 + }, + { + "auxiliary_loss_clip": 0.01205791, + "auxiliary_loss_mlp": 0.01064055, + "balance_loss_clip": 1.07833433, + "balance_loss_mlp": 1.04320562, + "epoch": 0.04082757820207765, + "flos": 26796039736320.0, + "grad_norm": 3.460810608471456, + "language_loss": 1.05630803, + "learning_rate": 3.998771282204425e-06, + "loss": 1.07900655, + "num_input_tokens_seen": 38962650, + "router_z_loss_clip": 1.27441406, + "router_z_loss_mlp": 0.20861816, + "step": 1407, + "time_per_iteration": 2.5473618507385254 + }, + { + "auxiliary_loss_clip": 0.01191863, + "auxiliary_loss_mlp": 0.01044992, + "balance_loss_clip": 1.07551181, + "balance_loss_mlp": 1.02699184, + "epoch": 0.040856595670593694, + "flos": 12596386089600.0, + "grad_norm": 3.794237202556708, + "language_loss": 0.97125423, + "learning_rate": 3.9987646857529634e-06, + "loss": 0.99362278, + "num_input_tokens_seen": 38974215, + "router_z_loss_clip": 1.16259766, + "router_z_loss_mlp": 0.17999268, + "step": 1408, + "time_per_iteration": 2.493349313735962 + }, + { + "auxiliary_loss_clip": 0.01210226, + "auxiliary_loss_mlp": 0.01062432, + "balance_loss_clip": 1.07808661, + "balance_loss_mlp": 1.04126072, + "epoch": 0.040885613139109746, + "flos": 14499149247360.0, + "grad_norm": 2.604321347121971, + "language_loss": 0.86247659, + "learning_rate": 3.998758071647604e-06, + "loss": 0.88520318, + "num_input_tokens_seen": 38989350, + "router_z_loss_clip": 1.32128906, + "router_z_loss_mlp": 0.21166992, + "step": 1409, + "time_per_iteration": 2.4751572608947754 + }, + { + "auxiliary_loss_clip": 0.01205638, + "auxiliary_loss_mlp": 0.01064254, + "balance_loss_clip": 1.07986534, + "balance_loss_mlp": 1.04301047, + "epoch": 0.04091463060762579, + "flos": 16392179819520.0, + "grad_norm": 3.138846737808178, + "language_loss": 1.14004946, + "learning_rate": 3.998751439888404e-06, + "loss": 1.16274834, + "num_input_tokens_seen": 39002100, + "router_z_loss_clip": 1.25878906, + "router_z_loss_mlp": 0.21240234, + "step": 1410, + "time_per_iteration": 2.471220016479492 + }, + { + "auxiliary_loss_clip": 0.01204432, + "auxiliary_loss_mlp": 0.01062422, + "balance_loss_clip": 1.07400739, + "balance_loss_mlp": 1.04154897, + "epoch": 0.040943648076141835, + "flos": 33284917618560.0, + "grad_norm": 2.5673399824674212, + "language_loss": 0.95567906, + "learning_rate": 3.998744790475423e-06, + "loss": 0.97834754, + "num_input_tokens_seen": 39022125, + "router_z_loss_clip": 1.30273438, + "router_z_loss_mlp": 0.20874023, + "step": 1411, + "time_per_iteration": 2.6789870262145996 + }, + { + "auxiliary_loss_clip": 0.01213003, + "auxiliary_loss_mlp": 0.01058029, + "balance_loss_clip": 1.0808934, + "balance_loss_mlp": 1.03378808, + "epoch": 0.04097266554465789, + "flos": 25986900015360.0, + "grad_norm": 3.116080244935269, + "language_loss": 0.94845414, + "learning_rate": 3.998738123408719e-06, + "loss": 0.97116446, + "num_input_tokens_seen": 39037250, + "router_z_loss_clip": 1.32128906, + "router_z_loss_mlp": 0.24267578, + "step": 1412, + "time_per_iteration": 2.586550235748291 + }, + { + "auxiliary_loss_clip": 0.01210513, + "auxiliary_loss_mlp": 0.01050843, + "balance_loss_clip": 1.08354497, + "balance_loss_mlp": 1.03039861, + "epoch": 0.04100168301317393, + "flos": 35767853251200.0, + "grad_norm": 2.0476381262993764, + "language_loss": 0.85762501, + "learning_rate": 3.998731438688351e-06, + "loss": 0.88023859, + "num_input_tokens_seen": 39057600, + "router_z_loss_clip": 1.26953125, + "router_z_loss_mlp": 0.20458984, + "step": 1413, + "time_per_iteration": 2.617814779281616 + }, + { + "auxiliary_loss_clip": 0.01213603, + "auxiliary_loss_mlp": 0.01065781, + "balance_loss_clip": 1.08045816, + "balance_loss_mlp": 1.04394221, + "epoch": 0.041030700481689976, + "flos": 13909495582080.0, + "grad_norm": 4.0445185364071685, + "language_loss": 0.96858525, + "learning_rate": 3.998724736314378e-06, + "loss": 0.99137908, + "num_input_tokens_seen": 39068410, + "router_z_loss_clip": 1.33300781, + "router_z_loss_mlp": 0.21850586, + "step": 1414, + "time_per_iteration": 2.48819899559021 + }, + { + "auxiliary_loss_clip": 0.01210041, + "auxiliary_loss_mlp": 0.01054692, + "balance_loss_clip": 1.07885408, + "balance_loss_mlp": 1.03231072, + "epoch": 0.04105971795020603, + "flos": 30953633207040.0, + "grad_norm": 2.8463247164410417, + "language_loss": 0.98995566, + "learning_rate": 3.99871801628686e-06, + "loss": 1.01260304, + "num_input_tokens_seen": 39083135, + "router_z_loss_clip": 1.3125, + "router_z_loss_mlp": 0.22357178, + "step": 1415, + "time_per_iteration": 2.5666611194610596 + }, + { + "auxiliary_loss_clip": 0.01211704, + "auxiliary_loss_mlp": 0.01059574, + "balance_loss_clip": 1.07695556, + "balance_loss_mlp": 1.03693604, + "epoch": 0.04108873541872207, + "flos": 33542828248320.0, + "grad_norm": 2.1089898245819545, + "language_loss": 0.98784065, + "learning_rate": 3.998711278605855e-06, + "loss": 1.01055348, + "num_input_tokens_seen": 39103315, + "router_z_loss_clip": 1.34667969, + "router_z_loss_mlp": 0.22607422, + "step": 1416, + "time_per_iteration": 2.62650990486145 + }, + { + "auxiliary_loss_clip": 0.01061563, + "auxiliary_loss_mlp": 0.01022943, + "balance_loss_clip": 1.02659965, + "balance_loss_mlp": 1.02114296, + "epoch": 0.04111775288723812, + "flos": 68716793007360.0, + "grad_norm": 0.709195162069867, + "language_loss": 0.52530289, + "learning_rate": 3.998704523271423e-06, + "loss": 0.54614794, + "num_input_tokens_seen": 39164760, + "router_z_loss_clip": 0.34765625, + "router_z_loss_mlp": 0.01794434, + "step": 1417, + "time_per_iteration": 3.1040701866149902 + }, + { + "auxiliary_loss_clip": 0.01059882, + "auxiliary_loss_mlp": 0.01008151, + "balance_loss_clip": 1.02466345, + "balance_loss_mlp": 1.00629187, + "epoch": 0.04114677035575416, + "flos": 63010589500800.0, + "grad_norm": 0.6469330935048925, + "language_loss": 0.52008533, + "learning_rate": 3.998697750283624e-06, + "loss": 0.5407657, + "num_input_tokens_seen": 39233720, + "router_z_loss_clip": 0.3515625, + "router_z_loss_mlp": 0.01855469, + "step": 1418, + "time_per_iteration": 3.2579760551452637 + }, + { + "auxiliary_loss_clip": 0.01058834, + "auxiliary_loss_mlp": 0.00997546, + "balance_loss_clip": 1.02323651, + "balance_loss_mlp": 0.99565059, + "epoch": 0.04117578782427021, + "flos": 74332790484480.0, + "grad_norm": 0.6739781075583231, + "language_loss": 0.50469065, + "learning_rate": 3.998690959642519e-06, + "loss": 0.52525437, + "num_input_tokens_seen": 39299080, + "router_z_loss_clip": 0.35546875, + "router_z_loss_mlp": 0.0189209, + "step": 1419, + "time_per_iteration": 3.1902101039886475 + }, + { + "auxiliary_loss_clip": 0.01212683, + "auxiliary_loss_mlp": 0.01076408, + "balance_loss_clip": 1.07796764, + "balance_loss_mlp": 1.0547955, + "epoch": 0.04120480529278626, + "flos": 56819764719360.0, + "grad_norm": 2.3026856575774457, + "language_loss": 0.72111398, + "learning_rate": 3.9986841513481646e-06, + "loss": 0.74400491, + "num_input_tokens_seen": 39321450, + "router_z_loss_clip": 1.34667969, + "router_z_loss_mlp": 0.21618652, + "step": 1420, + "time_per_iteration": 2.8633694648742676 + }, + { + "auxiliary_loss_clip": 0.01211826, + "auxiliary_loss_mlp": 0.01054371, + "balance_loss_clip": 1.07868767, + "balance_loss_mlp": 1.0328176, + "epoch": 0.0412338227613023, + "flos": 45666450506880.0, + "grad_norm": 1.7342522546474128, + "language_loss": 0.61818826, + "learning_rate": 3.998677325400625e-06, + "loss": 0.64085025, + "num_input_tokens_seen": 39342110, + "router_z_loss_clip": 1.33007812, + "router_z_loss_mlp": 0.21557617, + "step": 1421, + "time_per_iteration": 2.708784580230713 + }, + { + "auxiliary_loss_clip": 0.01197681, + "auxiliary_loss_mlp": 0.01050075, + "balance_loss_clip": 1.07152736, + "balance_loss_mlp": 1.02955937, + "epoch": 0.041262840229818354, + "flos": 16173663431040.0, + "grad_norm": 2.809049654515847, + "language_loss": 0.97450483, + "learning_rate": 3.998670481799957e-06, + "loss": 0.99698246, + "num_input_tokens_seen": 39355520, + "router_z_loss_clip": 1.26171875, + "router_z_loss_mlp": 0.20501709, + "step": 1422, + "time_per_iteration": 2.493699550628662 + }, + { + "auxiliary_loss_clip": 0.01198411, + "auxiliary_loss_mlp": 0.01048141, + "balance_loss_clip": 1.07335019, + "balance_loss_mlp": 1.0276134, + "epoch": 0.0412918576983344, + "flos": 18656958199680.0, + "grad_norm": 2.469045253614721, + "language_loss": 0.76834667, + "learning_rate": 3.998663620546223e-06, + "loss": 0.79081219, + "num_input_tokens_seen": 39369135, + "router_z_loss_clip": 1.25, + "router_z_loss_mlp": 0.20532227, + "step": 1423, + "time_per_iteration": 2.5551300048828125 + }, + { + "auxiliary_loss_clip": 0.01209895, + "auxiliary_loss_mlp": 0.01065927, + "balance_loss_clip": 1.07515419, + "balance_loss_mlp": 1.04247856, + "epoch": 0.04132087516685044, + "flos": 19782505848960.0, + "grad_norm": 3.349599971739285, + "language_loss": 1.26319361, + "learning_rate": 3.998656741639484e-06, + "loss": 1.28595185, + "num_input_tokens_seen": 39380005, + "router_z_loss_clip": 1.34472656, + "router_z_loss_mlp": 0.234375, + "step": 1424, + "time_per_iteration": 2.5223805904388428 + }, + { + "auxiliary_loss_clip": 0.01059502, + "auxiliary_loss_mlp": 0.01017746, + "balance_loss_clip": 1.02353776, + "balance_loss_mlp": 1.01581478, + "epoch": 0.04134989263536649, + "flos": 74294976441600.0, + "grad_norm": 0.7441956317213756, + "language_loss": 0.50091261, + "learning_rate": 3.9986498450797986e-06, + "loss": 0.52168506, + "num_input_tokens_seen": 39436230, + "router_z_loss_clip": 0.359375, + "router_z_loss_mlp": 0.01928711, + "step": 1425, + "time_per_iteration": 3.082777261734009 + }, + { + "auxiliary_loss_clip": 0.01058939, + "auxiliary_loss_mlp": 0.01002332, + "balance_loss_clip": 1.02329946, + "balance_loss_mlp": 1.00032949, + "epoch": 0.04137891010388254, + "flos": 73132009799040.0, + "grad_norm": 0.7493065828456327, + "language_loss": 0.50889599, + "learning_rate": 3.9986429308672286e-06, + "loss": 0.52950871, + "num_input_tokens_seen": 39495390, + "router_z_loss_clip": 0.35546875, + "router_z_loss_mlp": 0.02001953, + "step": 1426, + "time_per_iteration": 5.430503845214844 + }, + { + "auxiliary_loss_clip": 0.01056442, + "auxiliary_loss_mlp": 0.01002123, + "balance_loss_clip": 1.02130723, + "balance_loss_mlp": 1.00004852, + "epoch": 0.041407927572398584, + "flos": 61763626903680.0, + "grad_norm": 0.6538076289556984, + "language_loss": 0.50567567, + "learning_rate": 3.998635999001837e-06, + "loss": 0.52626133, + "num_input_tokens_seen": 39560995, + "router_z_loss_clip": 0.3515625, + "router_z_loss_mlp": 0.02075195, + "step": 1427, + "time_per_iteration": 5.798093318939209 + }, + { + "auxiliary_loss_clip": 0.01195255, + "auxiliary_loss_mlp": 0.01055301, + "balance_loss_clip": 1.07158899, + "balance_loss_mlp": 1.03260338, + "epoch": 0.04143694504091463, + "flos": 12194581576320.0, + "grad_norm": 3.7285303062697044, + "language_loss": 0.69519198, + "learning_rate": 3.998629049483683e-06, + "loss": 0.71769762, + "num_input_tokens_seen": 39573280, + "router_z_loss_clip": 1.23730469, + "router_z_loss_mlp": 0.22680664, + "step": 1428, + "time_per_iteration": 4.822396755218506 + }, + { + "auxiliary_loss_clip": 0.01055435, + "auxiliary_loss_mlp": 0.01030577, + "balance_loss_clip": 1.02028465, + "balance_loss_mlp": 1.02845502, + "epoch": 0.04146596250943068, + "flos": 66562583667840.0, + "grad_norm": 0.748562733615555, + "language_loss": 0.56322986, + "learning_rate": 3.9986220823128275e-06, + "loss": 0.58408999, + "num_input_tokens_seen": 39633055, + "router_z_loss_clip": 0.3515625, + "router_z_loss_mlp": 0.02124023, + "step": 1429, + "time_per_iteration": 5.544557809829712 + }, + { + "auxiliary_loss_clip": 0.01199587, + "auxiliary_loss_mlp": 0.01060591, + "balance_loss_clip": 1.07111597, + "balance_loss_mlp": 1.03879905, + "epoch": 0.041494979977946725, + "flos": 23800335500160.0, + "grad_norm": 2.673077917591551, + "language_loss": 0.93275082, + "learning_rate": 3.998615097489334e-06, + "loss": 0.9553526, + "num_input_tokens_seen": 39649010, + "router_z_loss_clip": 1.28515625, + "router_z_loss_mlp": 0.21777344, + "step": 1430, + "time_per_iteration": 2.626298666000366 + }, + { + "auxiliary_loss_clip": 0.01202362, + "auxiliary_loss_mlp": 0.01054907, + "balance_loss_clip": 1.07430673, + "balance_loss_mlp": 1.03387213, + "epoch": 0.04152399744646277, + "flos": 74736315872640.0, + "grad_norm": 2.3674588376305485, + "language_loss": 0.82652098, + "learning_rate": 3.998608095013262e-06, + "loss": 0.84909368, + "num_input_tokens_seen": 39673240, + "router_z_loss_clip": 1.27929688, + "router_z_loss_mlp": 0.21026611, + "step": 1431, + "time_per_iteration": 2.912919044494629 + }, + { + "auxiliary_loss_clip": 0.01058933, + "auxiliary_loss_mlp": 0.01030921, + "balance_loss_clip": 1.02402472, + "balance_loss_mlp": 1.02897823, + "epoch": 0.041553014914978814, + "flos": 74782859898240.0, + "grad_norm": 0.7250056930328621, + "language_loss": 0.56138462, + "learning_rate": 3.998601074884676e-06, + "loss": 0.58228314, + "num_input_tokens_seen": 39743950, + "router_z_loss_clip": 0.34960938, + "router_z_loss_mlp": 0.01940918, + "step": 1432, + "time_per_iteration": 3.2558300495147705 + }, + { + "auxiliary_loss_clip": 0.01057168, + "auxiliary_loss_mlp": 0.01018704, + "balance_loss_clip": 1.02236485, + "balance_loss_mlp": 1.01683283, + "epoch": 0.041582032383494866, + "flos": 63290584016640.0, + "grad_norm": 0.7396227383780081, + "language_loss": 0.52667928, + "learning_rate": 3.998594037103637e-06, + "loss": 0.54743791, + "num_input_tokens_seen": 39803000, + "router_z_loss_clip": 0.34765625, + "router_z_loss_mlp": 0.01867676, + "step": 1433, + "time_per_iteration": 3.078404664993286 + }, + { + "auxiliary_loss_clip": 0.01212371, + "auxiliary_loss_mlp": 0.01052161, + "balance_loss_clip": 1.08012891, + "balance_loss_mlp": 1.03181195, + "epoch": 0.04161104985201091, + "flos": 13291185841920.0, + "grad_norm": 2.439337247927111, + "language_loss": 0.85586184, + "learning_rate": 3.998586981670206e-06, + "loss": 0.87850714, + "num_input_tokens_seen": 39815630, + "router_z_loss_clip": 1.32324219, + "router_z_loss_mlp": 0.20343018, + "step": 1434, + "time_per_iteration": 2.4658968448638916 + }, + { + "auxiliary_loss_clip": 0.01205539, + "auxiliary_loss_mlp": 0.01056892, + "balance_loss_clip": 1.07459497, + "balance_loss_mlp": 1.03433776, + "epoch": 0.041640067320526955, + "flos": 30222168647040.0, + "grad_norm": 2.5439129251638395, + "language_loss": 0.78544271, + "learning_rate": 3.998579908584445e-06, + "loss": 0.80806708, + "num_input_tokens_seen": 39833430, + "router_z_loss_clip": 1.31054688, + "router_z_loss_mlp": 0.22583008, + "step": 1435, + "time_per_iteration": 2.584566116333008 + }, + { + "auxiliary_loss_clip": 0.01206609, + "auxiliary_loss_mlp": 0.01059018, + "balance_loss_clip": 1.08219981, + "balance_loss_mlp": 1.03732157, + "epoch": 0.04166908478904301, + "flos": 34892063844480.0, + "grad_norm": 2.2656255517818122, + "language_loss": 0.91000831, + "learning_rate": 3.998572817846419e-06, + "loss": 0.93266457, + "num_input_tokens_seen": 39853670, + "router_z_loss_clip": 1.2421875, + "router_z_loss_mlp": 0.2166748, + "step": 1436, + "time_per_iteration": 2.610836982727051 + }, + { + "auxiliary_loss_clip": 0.0105662, + "auxiliary_loss_mlp": 0.01040332, + "balance_loss_clip": 1.02128935, + "balance_loss_mlp": 1.0385201, + "epoch": 0.04169810225755905, + "flos": 60440317948800.0, + "grad_norm": 0.7119584955188216, + "language_loss": 0.51871449, + "learning_rate": 3.998565709456188e-06, + "loss": 0.539684, + "num_input_tokens_seen": 39917985, + "router_z_loss_clip": 0.3515625, + "router_z_loss_mlp": 0.01806641, + "step": 1437, + "time_per_iteration": 3.1270923614501953 + }, + { + "auxiliary_loss_clip": 0.01208552, + "auxiliary_loss_mlp": 0.01054568, + "balance_loss_clip": 1.07661045, + "balance_loss_mlp": 1.0324074, + "epoch": 0.041727119726075096, + "flos": 20187291191040.0, + "grad_norm": 5.552855597827204, + "language_loss": 0.99188703, + "learning_rate": 3.998558583413817e-06, + "loss": 1.01451826, + "num_input_tokens_seen": 39932210, + "router_z_loss_clip": 1.31933594, + "router_z_loss_mlp": 0.22143555, + "step": 1438, + "time_per_iteration": 2.5049524307250977 + }, + { + "auxiliary_loss_clip": 0.0105548, + "auxiliary_loss_mlp": 0.01012624, + "balance_loss_clip": 1.02116501, + "balance_loss_mlp": 1.01083553, + "epoch": 0.04175613719459114, + "flos": 61525070726400.0, + "grad_norm": 0.790800198446391, + "language_loss": 0.56622326, + "learning_rate": 3.998551439719367e-06, + "loss": 0.58690429, + "num_input_tokens_seen": 39993345, + "router_z_loss_clip": 0.34375, + "router_z_loss_mlp": 0.01782227, + "step": 1439, + "time_per_iteration": 3.0759618282318115 + }, + { + "auxiliary_loss_clip": 0.01053744, + "auxiliary_loss_mlp": 0.00998892, + "balance_loss_clip": 1.02012277, + "balance_loss_mlp": 0.99713939, + "epoch": 0.04178515466310719, + "flos": 69122727584640.0, + "grad_norm": 0.6770344320085927, + "language_loss": 0.53914768, + "learning_rate": 3.998544278372902e-06, + "loss": 0.55967402, + "num_input_tokens_seen": 40056155, + "router_z_loss_clip": 0.3359375, + "router_z_loss_mlp": 0.01757812, + "step": 1440, + "time_per_iteration": 3.1279704570770264 + }, + { + "auxiliary_loss_clip": 0.01215065, + "auxiliary_loss_mlp": 0.01078901, + "balance_loss_clip": 1.07919002, + "balance_loss_mlp": 1.05698991, + "epoch": 0.04181417213162324, + "flos": 13509738144000.0, + "grad_norm": 3.2124428023508935, + "language_loss": 0.9704802, + "learning_rate": 3.998537099374486e-06, + "loss": 0.99341989, + "num_input_tokens_seen": 40069275, + "router_z_loss_clip": 1.359375, + "router_z_loss_mlp": 0.21887207, + "step": 1441, + "time_per_iteration": 2.5849945545196533 + }, + { + "auxiliary_loss_clip": 0.01052442, + "auxiliary_loss_mlp": 0.0103029, + "balance_loss_clip": 1.01868916, + "balance_loss_mlp": 1.02856147, + "epoch": 0.04184318960013928, + "flos": 63608895365760.0, + "grad_norm": 0.6392805618483336, + "language_loss": 0.52378052, + "learning_rate": 3.99852990272418e-06, + "loss": 0.54460782, + "num_input_tokens_seen": 40133275, + "router_z_loss_clip": 0.3359375, + "router_z_loss_mlp": 0.01733398, + "step": 1442, + "time_per_iteration": 3.1507678031921387 + }, + { + "auxiliary_loss_clip": 0.01206318, + "auxiliary_loss_mlp": 0.01060741, + "balance_loss_clip": 1.07642817, + "balance_loss_mlp": 1.04003406, + "epoch": 0.04187220706865533, + "flos": 74732437203840.0, + "grad_norm": 2.790788234608317, + "language_loss": 0.70093364, + "learning_rate": 3.998522688422051e-06, + "loss": 0.7236042, + "num_input_tokens_seen": 40158460, + "router_z_loss_clip": 1.29931641, + "router_z_loss_mlp": 0.20690918, + "step": 1443, + "time_per_iteration": 2.984386444091797 + }, + { + "auxiliary_loss_clip": 0.01200536, + "auxiliary_loss_mlp": 0.01063984, + "balance_loss_clip": 1.07761264, + "balance_loss_mlp": 1.04262173, + "epoch": 0.04190122453717138, + "flos": 16904122410240.0, + "grad_norm": 2.136096767193432, + "language_loss": 0.77143335, + "learning_rate": 3.99851545646816e-06, + "loss": 0.79407859, + "num_input_tokens_seen": 40172380, + "router_z_loss_clip": 1.22998047, + "router_z_loss_mlp": 0.21350098, + "step": 1444, + "time_per_iteration": 2.596344232559204 + }, + { + "auxiliary_loss_clip": 0.01205337, + "auxiliary_loss_mlp": 0.01068982, + "balance_loss_clip": 1.07788253, + "balance_loss_mlp": 1.04716671, + "epoch": 0.04193024200568742, + "flos": 29016431884800.0, + "grad_norm": 2.4395203923632875, + "language_loss": 0.79530442, + "learning_rate": 3.998508206862572e-06, + "loss": 0.81804764, + "num_input_tokens_seen": 40187140, + "router_z_loss_clip": 1.27490234, + "router_z_loss_mlp": 0.21826172, + "step": 1445, + "time_per_iteration": 2.612058162689209 + }, + { + "auxiliary_loss_clip": 0.01205892, + "auxiliary_loss_mlp": 0.01066371, + "balance_loss_clip": 1.0834558, + "balance_loss_mlp": 1.04662967, + "epoch": 0.041959259474203474, + "flos": 43829044145280.0, + "grad_norm": 2.5456216563915963, + "language_loss": 0.94843096, + "learning_rate": 3.998500939605351e-06, + "loss": 0.97115362, + "num_input_tokens_seen": 40206690, + "router_z_loss_clip": 1.22363281, + "router_z_loss_mlp": 0.1973877, + "step": 1446, + "time_per_iteration": 2.7552855014801025 + }, + { + "auxiliary_loss_clip": 0.01196986, + "auxiliary_loss_mlp": 0.0105962, + "balance_loss_clip": 1.07555699, + "balance_loss_mlp": 1.04153037, + "epoch": 0.04198827694271952, + "flos": 27699443723520.0, + "grad_norm": 3.4978916277525736, + "language_loss": 0.92147291, + "learning_rate": 3.998493654696561e-06, + "loss": 0.94403899, + "num_input_tokens_seen": 40219295, + "router_z_loss_clip": 1.21289062, + "router_z_loss_mlp": 0.18109131, + "step": 1447, + "time_per_iteration": 2.664030075073242 + }, + { + "auxiliary_loss_clip": 0.01210492, + "auxiliary_loss_mlp": 0.01069195, + "balance_loss_clip": 1.07709718, + "balance_loss_mlp": 1.04780912, + "epoch": 0.04201729441123556, + "flos": 19208761908480.0, + "grad_norm": 3.671897407531531, + "language_loss": 0.83892506, + "learning_rate": 3.998486352136265e-06, + "loss": 0.86172187, + "num_input_tokens_seen": 40235505, + "router_z_loss_clip": 1.33496094, + "router_z_loss_mlp": 0.21362305, + "step": 1448, + "time_per_iteration": 2.5430126190185547 + }, + { + "auxiliary_loss_clip": 0.01195445, + "auxiliary_loss_mlp": 0.01069918, + "balance_loss_clip": 1.07524431, + "balance_loss_mlp": 1.05141711, + "epoch": 0.04204631187975161, + "flos": 12233688508800.0, + "grad_norm": 3.2644019594457645, + "language_loss": 0.88433063, + "learning_rate": 3.99847903192453e-06, + "loss": 0.90698427, + "num_input_tokens_seen": 40247745, + "router_z_loss_clip": 1.20117188, + "router_z_loss_mlp": 0.18505859, + "step": 1449, + "time_per_iteration": 2.5167481899261475 + }, + { + "auxiliary_loss_clip": 0.01197553, + "auxiliary_loss_mlp": 0.01054075, + "balance_loss_clip": 1.07347417, + "balance_loss_mlp": 1.03260529, + "epoch": 0.04207532934826766, + "flos": 29313593101440.0, + "grad_norm": 2.1738130405890526, + "language_loss": 0.83062959, + "learning_rate": 3.99847169406142e-06, + "loss": 0.85314584, + "num_input_tokens_seen": 40267785, + "router_z_loss_clip": 1.2421875, + "router_z_loss_mlp": 0.21459961, + "step": 1450, + "time_per_iteration": 2.606861114501953 + }, + { + "auxiliary_loss_clip": 0.01196353, + "auxiliary_loss_mlp": 0.01058344, + "balance_loss_clip": 1.07355583, + "balance_loss_mlp": 1.03749382, + "epoch": 0.042104346816783704, + "flos": 17887679596800.0, + "grad_norm": 2.950916644077883, + "language_loss": 0.89007103, + "learning_rate": 3.9984643385469986e-06, + "loss": 0.91261798, + "num_input_tokens_seen": 40280300, + "router_z_loss_clip": 1.22851562, + "router_z_loss_mlp": 0.20861816, + "step": 1451, + "time_per_iteration": 2.50443696975708 + }, + { + "auxiliary_loss_clip": 0.01197836, + "auxiliary_loss_mlp": 0.01047575, + "balance_loss_clip": 1.075984, + "balance_loss_mlp": 1.02748275, + "epoch": 0.04213336428529975, + "flos": 34890806868480.0, + "grad_norm": 5.502520185034874, + "language_loss": 0.71230686, + "learning_rate": 3.998456965381331e-06, + "loss": 0.734761, + "num_input_tokens_seen": 40297640, + "router_z_loss_clip": 1.21875, + "router_z_loss_mlp": 0.20117188, + "step": 1452, + "time_per_iteration": 2.562218189239502 + }, + { + "auxiliary_loss_clip": 0.01204792, + "auxiliary_loss_mlp": 0.01048045, + "balance_loss_clip": 1.07610941, + "balance_loss_mlp": 1.02806568, + "epoch": 0.0421623817538158, + "flos": 11101640497920.0, + "grad_norm": 2.7046221456848882, + "language_loss": 0.76911509, + "learning_rate": 3.998449574564484e-06, + "loss": 0.79164344, + "num_input_tokens_seen": 40309930, + "router_z_loss_clip": 1.28564453, + "router_z_loss_mlp": 0.19995117, + "step": 1453, + "time_per_iteration": 2.451627016067505 + }, + { + "auxiliary_loss_clip": 0.01201419, + "auxiliary_loss_mlp": 0.0106102, + "balance_loss_clip": 1.07002294, + "balance_loss_mlp": 1.03919291, + "epoch": 0.042191399222331845, + "flos": 30258510232320.0, + "grad_norm": 2.6128660948262534, + "language_loss": 1.06160009, + "learning_rate": 3.998442166096521e-06, + "loss": 1.08422446, + "num_input_tokens_seen": 40327730, + "router_z_loss_clip": 1.31347656, + "router_z_loss_mlp": 0.21820068, + "step": 1454, + "time_per_iteration": 2.7381725311279297 + }, + { + "auxiliary_loss_clip": 0.01202065, + "auxiliary_loss_mlp": 0.01058768, + "balance_loss_clip": 1.07728553, + "balance_loss_mlp": 1.03843117, + "epoch": 0.04222041669084789, + "flos": 15005022439680.0, + "grad_norm": 3.1813470375355193, + "language_loss": 0.80742943, + "learning_rate": 3.998434739977508e-06, + "loss": 0.83003783, + "num_input_tokens_seen": 40338910, + "router_z_loss_clip": 1.24804688, + "router_z_loss_mlp": 0.20343018, + "step": 1455, + "time_per_iteration": 2.4605562686920166 + }, + { + "auxiliary_loss_clip": 0.01193289, + "auxiliary_loss_mlp": 0.01047301, + "balance_loss_clip": 1.07410562, + "balance_loss_mlp": 1.02829897, + "epoch": 0.042249434159363934, + "flos": 74734125143040.0, + "grad_norm": 2.1915892461295576, + "language_loss": 0.82824981, + "learning_rate": 3.99842729620751e-06, + "loss": 0.85065573, + "num_input_tokens_seen": 40362840, + "router_z_loss_clip": 1.19238281, + "router_z_loss_mlp": 0.18994141, + "step": 1456, + "time_per_iteration": 2.929537773132324 + }, + { + "auxiliary_loss_clip": 0.01217343, + "auxiliary_loss_mlp": 0.01066937, + "balance_loss_clip": 1.08130598, + "balance_loss_mlp": 1.04308367, + "epoch": 0.042278451627879986, + "flos": 31897688411520.0, + "grad_norm": 3.4335976216404847, + "language_loss": 0.87001014, + "learning_rate": 3.998419834786595e-06, + "loss": 0.89285302, + "num_input_tokens_seen": 40377525, + "router_z_loss_clip": 1.359375, + "router_z_loss_mlp": 0.23852539, + "step": 1457, + "time_per_iteration": 2.6477723121643066 + }, + { + "auxiliary_loss_clip": 0.01066054, + "auxiliary_loss_mlp": 0.01054963, + "balance_loss_clip": 1.03179991, + "balance_loss_mlp": 1.05329359, + "epoch": 0.04230746909639603, + "flos": 74774851165440.0, + "grad_norm": 0.7510904318492229, + "language_loss": 0.49162769, + "learning_rate": 3.998412355714826e-06, + "loss": 0.51283777, + "num_input_tokens_seen": 40442855, + "router_z_loss_clip": 0.34375, + "router_z_loss_mlp": 0.01672363, + "step": 1458, + "time_per_iteration": 3.345850706100464 + }, + { + "auxiliary_loss_clip": 0.01063579, + "auxiliary_loss_mlp": 0.0101823, + "balance_loss_clip": 1.02997732, + "balance_loss_mlp": 1.0166682, + "epoch": 0.042336486564912075, + "flos": 69230387623680.0, + "grad_norm": 0.7897805971539087, + "language_loss": 0.53203022, + "learning_rate": 3.998404858992271e-06, + "loss": 0.55284834, + "num_input_tokens_seen": 40507345, + "router_z_loss_clip": 0.3359375, + "router_z_loss_mlp": 0.015625, + "step": 1459, + "time_per_iteration": 3.2645883560180664 + }, + { + "auxiliary_loss_clip": 0.01063808, + "auxiliary_loss_mlp": 0.00997852, + "balance_loss_clip": 1.02976251, + "balance_loss_mlp": 0.99612331, + "epoch": 0.04236550403342813, + "flos": 74784906973440.0, + "grad_norm": 0.6458358728750594, + "language_loss": 0.52242398, + "learning_rate": 3.998397344618996e-06, + "loss": 0.54304063, + "num_input_tokens_seen": 40577040, + "router_z_loss_clip": 0.33984375, + "router_z_loss_mlp": 0.01733398, + "step": 1460, + "time_per_iteration": 3.2345097064971924 + }, + { + "auxiliary_loss_clip": 0.01213867, + "auxiliary_loss_mlp": 0.01081373, + "balance_loss_clip": 1.07893169, + "balance_loss_mlp": 1.05850911, + "epoch": 0.04239452150194417, + "flos": 37925366641920.0, + "grad_norm": 2.676154493580742, + "language_loss": 0.9056344, + "learning_rate": 3.9983898125950665e-06, + "loss": 0.92858678, + "num_input_tokens_seen": 40594925, + "router_z_loss_clip": 1.34765625, + "router_z_loss_mlp": 0.2286377, + "step": 1461, + "time_per_iteration": 2.7094528675079346 + }, + { + "auxiliary_loss_clip": 0.01063269, + "auxiliary_loss_mlp": 0.01020725, + "balance_loss_clip": 1.02933526, + "balance_loss_mlp": 1.0190556, + "epoch": 0.042423538970460216, + "flos": 64128882602880.0, + "grad_norm": 0.6988997351661944, + "language_loss": 0.4893364, + "learning_rate": 3.998382262920549e-06, + "loss": 0.5101763, + "num_input_tokens_seen": 40660760, + "router_z_loss_clip": 0.33984375, + "router_z_loss_mlp": 0.01672363, + "step": 1462, + "time_per_iteration": 3.219618797302246 + }, + { + "auxiliary_loss_clip": 0.01202381, + "auxiliary_loss_mlp": 0.01076169, + "balance_loss_clip": 1.07499146, + "balance_loss_mlp": 1.05605841, + "epoch": 0.04245255643897626, + "flos": 23616544584960.0, + "grad_norm": 3.719912895182669, + "language_loss": 0.87995398, + "learning_rate": 3.9983746955955115e-06, + "loss": 0.90273952, + "num_input_tokens_seen": 40673980, + "router_z_loss_clip": 1.27539062, + "router_z_loss_mlp": 0.20111084, + "step": 1463, + "time_per_iteration": 2.6402838230133057 + }, + { + "auxiliary_loss_clip": 0.01203668, + "auxiliary_loss_mlp": 0.01064713, + "balance_loss_clip": 1.07170033, + "balance_loss_mlp": 1.04287386, + "epoch": 0.04248157390749231, + "flos": 29572186089600.0, + "grad_norm": 2.793679820421585, + "language_loss": 0.81249011, + "learning_rate": 3.9983671106200205e-06, + "loss": 0.8351739, + "num_input_tokens_seen": 40689740, + "router_z_loss_clip": 1.3203125, + "router_z_loss_mlp": 0.21813965, + "step": 1464, + "time_per_iteration": 2.6645970344543457 + }, + { + "auxiliary_loss_clip": 0.01202815, + "auxiliary_loss_mlp": 0.01069292, + "balance_loss_clip": 1.07903385, + "balance_loss_mlp": 1.04869246, + "epoch": 0.04251059137600836, + "flos": 15662618680320.0, + "grad_norm": 2.6753245981395524, + "language_loss": 0.76018822, + "learning_rate": 3.998359507994142e-06, + "loss": 0.78290927, + "num_input_tokens_seen": 40702180, + "router_z_loss_clip": 1.23925781, + "router_z_loss_mlp": 0.20581055, + "step": 1465, + "time_per_iteration": 2.588998556137085 + }, + { + "auxiliary_loss_clip": 0.01063605, + "auxiliary_loss_mlp": 0.01009753, + "balance_loss_clip": 1.02997112, + "balance_loss_mlp": 1.00808418, + "epoch": 0.0425396088445244, + "flos": 57047226572160.0, + "grad_norm": 0.7448906935122017, + "language_loss": 0.52471542, + "learning_rate": 3.998351887717943e-06, + "loss": 0.54544896, + "num_input_tokens_seen": 40762105, + "router_z_loss_clip": 0.3359375, + "router_z_loss_mlp": 0.01672363, + "step": 1466, + "time_per_iteration": 3.115943670272827 + }, + { + "auxiliary_loss_clip": 0.01215294, + "auxiliary_loss_mlp": 0.01067973, + "balance_loss_clip": 1.07868552, + "balance_loss_mlp": 1.04533553, + "epoch": 0.04256862631304045, + "flos": 14348575434240.0, + "grad_norm": 4.020815123399099, + "language_loss": 0.90484583, + "learning_rate": 3.998344249791492e-06, + "loss": 0.92767853, + "num_input_tokens_seen": 40773220, + "router_z_loss_clip": 1.36621094, + "router_z_loss_mlp": 0.2265625, + "step": 1467, + "time_per_iteration": 2.550014019012451 + }, + { + "auxiliary_loss_clip": 0.01059867, + "auxiliary_loss_mlp": 0.01005749, + "balance_loss_clip": 1.02650642, + "balance_loss_mlp": 1.00404441, + "epoch": 0.0425976437815565, + "flos": 55071133799040.0, + "grad_norm": 0.6706276171747849, + "language_loss": 0.47932276, + "learning_rate": 3.998336594214856e-06, + "loss": 0.49997893, + "num_input_tokens_seen": 40833030, + "router_z_loss_clip": 0.33203125, + "router_z_loss_mlp": 0.01708984, + "step": 1468, + "time_per_iteration": 3.0652639865875244 + }, + { + "auxiliary_loss_clip": 0.01200596, + "auxiliary_loss_mlp": 0.01062311, + "balance_loss_clip": 1.07743979, + "balance_loss_mlp": 1.04211736, + "epoch": 0.04262666125007254, + "flos": 14897901104640.0, + "grad_norm": 3.081910614350983, + "language_loss": 0.84077197, + "learning_rate": 3.998328920988102e-06, + "loss": 0.86340111, + "num_input_tokens_seen": 40845060, + "router_z_loss_clip": 1.23144531, + "router_z_loss_mlp": 0.2020874, + "step": 1469, + "time_per_iteration": 2.5839176177978516 + }, + { + "auxiliary_loss_clip": 0.01213545, + "auxiliary_loss_mlp": 0.01069909, + "balance_loss_clip": 1.08033121, + "balance_loss_mlp": 1.0474143, + "epoch": 0.04265567871858859, + "flos": 27046193028480.0, + "grad_norm": 2.645502780886884, + "language_loss": 1.10787725, + "learning_rate": 3.9983212301113e-06, + "loss": 1.13071179, + "num_input_tokens_seen": 40863115, + "router_z_loss_clip": 1.33300781, + "router_z_loss_mlp": 0.22473145, + "step": 1470, + "time_per_iteration": 2.649674892425537 + }, + { + "auxiliary_loss_clip": 0.01210439, + "auxiliary_loss_mlp": 0.01043283, + "balance_loss_clip": 1.08149064, + "balance_loss_mlp": 1.02164698, + "epoch": 0.04268469618710464, + "flos": 13875560208000.0, + "grad_norm": 4.020023746714146, + "language_loss": 0.88906574, + "learning_rate": 3.998313521584514e-06, + "loss": 0.91160297, + "num_input_tokens_seen": 40877410, + "router_z_loss_clip": 1.28808594, + "router_z_loss_mlp": 0.21655273, + "step": 1471, + "time_per_iteration": 2.5783982276916504 + }, + { + "auxiliary_loss_clip": 0.01204458, + "auxiliary_loss_mlp": 0.01061804, + "balance_loss_clip": 1.07477033, + "balance_loss_mlp": 1.04078746, + "epoch": 0.04271371365562068, + "flos": 28469045548800.0, + "grad_norm": 2.3150119411677297, + "language_loss": 0.9307273, + "learning_rate": 3.998305795407816e-06, + "loss": 0.95338988, + "num_input_tokens_seen": 40895275, + "router_z_loss_clip": 1.29492188, + "router_z_loss_mlp": 0.21020508, + "step": 1472, + "time_per_iteration": 2.611499309539795 + }, + { + "auxiliary_loss_clip": 0.01206889, + "auxiliary_loss_mlp": 0.01053458, + "balance_loss_clip": 1.07928407, + "balance_loss_mlp": 1.03325188, + "epoch": 0.04274273112413673, + "flos": 31898370769920.0, + "grad_norm": 2.8425268669289183, + "language_loss": 0.82747179, + "learning_rate": 3.998298051581272e-06, + "loss": 0.85007524, + "num_input_tokens_seen": 40911995, + "router_z_loss_clip": 1.27734375, + "router_z_loss_mlp": 0.20214844, + "step": 1473, + "time_per_iteration": 2.6946473121643066 + }, + { + "auxiliary_loss_clip": 0.01201467, + "auxiliary_loss_mlp": 0.01049206, + "balance_loss_clip": 1.07835627, + "balance_loss_mlp": 1.02936924, + "epoch": 0.04277174859265278, + "flos": 16539162272640.0, + "grad_norm": 2.543672927159947, + "language_loss": 0.67697424, + "learning_rate": 3.998290290104951e-06, + "loss": 0.69948095, + "num_input_tokens_seen": 40925860, + "router_z_loss_clip": 1.22949219, + "router_z_loss_mlp": 0.19836426, + "step": 1474, + "time_per_iteration": 2.5285892486572266 + }, + { + "auxiliary_loss_clip": 0.01203841, + "auxiliary_loss_mlp": 0.01059708, + "balance_loss_clip": 1.07762003, + "balance_loss_mlp": 1.03912663, + "epoch": 0.042800766061168824, + "flos": 29893190958720.0, + "grad_norm": 2.448633491190291, + "language_loss": 0.90674901, + "learning_rate": 3.998282510978922e-06, + "loss": 0.92938447, + "num_input_tokens_seen": 40942415, + "router_z_loss_clip": 1.26074219, + "router_z_loss_mlp": 0.20562744, + "step": 1475, + "time_per_iteration": 2.6717123985290527 + }, + { + "auxiliary_loss_clip": 0.01092032, + "auxiliary_loss_mlp": 0.01142588, + "balance_loss_clip": 1.05601859, + "balance_loss_mlp": 1.14054918, + "epoch": 0.04282978352968487, + "flos": 55537074046080.0, + "grad_norm": 0.8109341494240854, + "language_loss": 0.5690341, + "learning_rate": 3.998274714203252e-06, + "loss": 0.59138024, + "num_input_tokens_seen": 40999330, + "router_z_loss_clip": 0.359375, + "router_z_loss_mlp": 0.02038574, + "step": 1476, + "time_per_iteration": 3.0759546756744385 + }, + { + "auxiliary_loss_clip": 0.01208776, + "auxiliary_loss_mlp": 0.01062508, + "balance_loss_clip": 1.08103144, + "balance_loss_mlp": 1.03964376, + "epoch": 0.04285880099820092, + "flos": 33978783617280.0, + "grad_norm": 2.5410179351941236, + "language_loss": 0.81926191, + "learning_rate": 3.998266899778012e-06, + "loss": 0.84197474, + "num_input_tokens_seen": 41014725, + "router_z_loss_clip": 1.27832031, + "router_z_loss_mlp": 0.22875977, + "step": 1477, + "time_per_iteration": 2.7092127799987793 + }, + { + "auxiliary_loss_clip": 0.01196101, + "auxiliary_loss_mlp": 0.01050402, + "balance_loss_clip": 1.07933152, + "balance_loss_mlp": 1.03185272, + "epoch": 0.042887818466716965, + "flos": 16136244437760.0, + "grad_norm": 2.9894552301005213, + "language_loss": 0.73231626, + "learning_rate": 3.9982590677032705e-06, + "loss": 0.75478137, + "num_input_tokens_seen": 41028050, + "router_z_loss_clip": 1.16796875, + "router_z_loss_mlp": 0.1854248, + "step": 1478, + "time_per_iteration": 2.5327446460723877 + }, + { + "auxiliary_loss_clip": 0.012065, + "auxiliary_loss_mlp": 0.01060583, + "balance_loss_clip": 1.07951164, + "balance_loss_mlp": 1.03839803, + "epoch": 0.04291683593523301, + "flos": 26642880144000.0, + "grad_norm": 2.7823366067826365, + "language_loss": 0.85066992, + "learning_rate": 3.998251217979095e-06, + "loss": 0.87334073, + "num_input_tokens_seen": 41041355, + "router_z_loss_clip": 1.26855469, + "router_z_loss_mlp": 0.22192383, + "step": 1479, + "time_per_iteration": 2.6845414638519287 + }, + { + "auxiliary_loss_clip": 0.01070684, + "auxiliary_loss_mlp": 0.01018937, + "balance_loss_clip": 1.03765559, + "balance_loss_mlp": 1.01738739, + "epoch": 0.042945853403749054, + "flos": 74781243786240.0, + "grad_norm": 0.663367926598786, + "language_loss": 0.55118454, + "learning_rate": 3.9982433506055574e-06, + "loss": 0.57208079, + "num_input_tokens_seen": 41110965, + "router_z_loss_clip": 0.33007812, + "router_z_loss_mlp": 0.01544189, + "step": 1480, + "time_per_iteration": 3.2423877716064453 + }, + { + "auxiliary_loss_clip": 0.01209577, + "auxiliary_loss_mlp": 0.0106414, + "balance_loss_clip": 1.08009374, + "balance_loss_mlp": 1.03811646, + "epoch": 0.042974870872265106, + "flos": 15517216425600.0, + "grad_norm": 2.4624539734189677, + "language_loss": 1.02643633, + "learning_rate": 3.998235465582726e-06, + "loss": 1.04917336, + "num_input_tokens_seen": 41124830, + "router_z_loss_clip": 1.29492188, + "router_z_loss_mlp": 0.26025391, + "step": 1481, + "time_per_iteration": 2.4759998321533203 + }, + { + "auxiliary_loss_clip": 0.01065934, + "auxiliary_loss_mlp": 0.01019829, + "balance_loss_clip": 1.03237391, + "balance_loss_mlp": 1.01818419, + "epoch": 0.04300388834078115, + "flos": 63347967993600.0, + "grad_norm": 0.7294995249099475, + "language_loss": 0.51730752, + "learning_rate": 3.99822756291067e-06, + "loss": 0.53816509, + "num_input_tokens_seen": 41177110, + "router_z_loss_clip": 0.3359375, + "router_z_loss_mlp": 0.01647949, + "step": 1482, + "time_per_iteration": 2.9173595905303955 + }, + { + "auxiliary_loss_clip": 0.01207296, + "auxiliary_loss_mlp": 0.01067677, + "balance_loss_clip": 1.07835937, + "balance_loss_mlp": 1.04583764, + "epoch": 0.043032905809297195, + "flos": 74734268797440.0, + "grad_norm": 2.720181341890416, + "language_loss": 0.93267202, + "learning_rate": 3.998219642589459e-06, + "loss": 0.95542181, + "num_input_tokens_seen": 41198220, + "router_z_loss_clip": 1.28808594, + "router_z_loss_mlp": 0.21850586, + "step": 1483, + "time_per_iteration": 2.8899855613708496 + }, + { + "auxiliary_loss_clip": 0.01202885, + "auxiliary_loss_mlp": 0.01057474, + "balance_loss_clip": 1.07770848, + "balance_loss_mlp": 1.03608179, + "epoch": 0.04306192327781325, + "flos": 15333748732800.0, + "grad_norm": 3.96698864869472, + "language_loss": 0.79171765, + "learning_rate": 3.998211704619164e-06, + "loss": 0.81432116, + "num_input_tokens_seen": 41210770, + "router_z_loss_clip": 1.25097656, + "router_z_loss_mlp": 0.21386719, + "step": 1484, + "time_per_iteration": 2.482961654663086 + }, + { + "auxiliary_loss_clip": 0.01060424, + "auxiliary_loss_mlp": 0.01006144, + "balance_loss_clip": 1.02752328, + "balance_loss_mlp": 1.00460649, + "epoch": 0.04309094074632929, + "flos": 57867604272000.0, + "grad_norm": 0.7332831033710709, + "language_loss": 0.4736886, + "learning_rate": 3.998203748999854e-06, + "loss": 0.49435428, + "num_input_tokens_seen": 41259835, + "router_z_loss_clip": 0.328125, + "router_z_loss_mlp": 0.01538086, + "step": 1485, + "time_per_iteration": 2.807767152786255 + }, + { + "auxiliary_loss_clip": 0.01212595, + "auxiliary_loss_mlp": 0.01061554, + "balance_loss_clip": 1.07683921, + "balance_loss_mlp": 1.03711629, + "epoch": 0.043119958214845336, + "flos": 27227649559680.0, + "grad_norm": 2.4346458523977756, + "language_loss": 0.87504387, + "learning_rate": 3.9981957757316015e-06, + "loss": 0.89778543, + "num_input_tokens_seen": 41274655, + "router_z_loss_clip": 1.35546875, + "router_z_loss_mlp": 0.24438477, + "step": 1486, + "time_per_iteration": 2.515072822570801 + }, + { + "auxiliary_loss_clip": 0.011925, + "auxiliary_loss_mlp": 0.01047357, + "balance_loss_clip": 1.07663894, + "balance_loss_mlp": 1.03077507, + "epoch": 0.04314897568336138, + "flos": 33504583242240.0, + "grad_norm": 3.0199640039866047, + "language_loss": 0.89976716, + "learning_rate": 3.998187784814474e-06, + "loss": 0.92216569, + "num_input_tokens_seen": 41288925, + "router_z_loss_clip": 1.15771484, + "router_z_loss_mlp": 0.16577148, + "step": 1487, + "time_per_iteration": 2.6137502193450928 + }, + { + "auxiliary_loss_clip": 0.01057795, + "auxiliary_loss_mlp": 0.01006824, + "balance_loss_clip": 1.02529228, + "balance_loss_mlp": 1.00523841, + "epoch": 0.04317799315187743, + "flos": 70547052562560.0, + "grad_norm": 0.6785028771170407, + "language_loss": 0.52888834, + "learning_rate": 3.998179776248544e-06, + "loss": 0.54953456, + "num_input_tokens_seen": 41355420, + "router_z_loss_clip": 0.32421875, + "router_z_loss_mlp": 0.01586914, + "step": 1488, + "time_per_iteration": 3.147158145904541 + }, + { + "auxiliary_loss_clip": 0.01205975, + "auxiliary_loss_mlp": 0.01068299, + "balance_loss_clip": 1.0746634, + "balance_loss_mlp": 1.04514885, + "epoch": 0.04320701062039348, + "flos": 18659615806080.0, + "grad_norm": 2.243903151873281, + "language_loss": 0.83908385, + "learning_rate": 3.998171750033881e-06, + "loss": 0.8618266, + "num_input_tokens_seen": 41370535, + "router_z_loss_clip": 1.3125, + "router_z_loss_mlp": 0.23144531, + "step": 1489, + "time_per_iteration": 2.4764645099639893 + }, + { + "auxiliary_loss_clip": 0.01202572, + "auxiliary_loss_mlp": 0.01055279, + "balance_loss_clip": 1.07825446, + "balance_loss_mlp": 1.03638995, + "epoch": 0.04323602808890952, + "flos": 26790293560320.0, + "grad_norm": 1.9764017827220228, + "language_loss": 0.77158314, + "learning_rate": 3.998163706170557e-06, + "loss": 0.79416162, + "num_input_tokens_seen": 41388665, + "router_z_loss_clip": 1.24316406, + "router_z_loss_mlp": 0.1887207, + "step": 1490, + "time_per_iteration": 2.6588783264160156 + }, + { + "auxiliary_loss_clip": 0.01059766, + "auxiliary_loss_mlp": 0.01021401, + "balance_loss_clip": 1.02661228, + "balance_loss_mlp": 1.019732, + "epoch": 0.04326504555742557, + "flos": 62441295868800.0, + "grad_norm": 0.624576836695596, + "language_loss": 0.48344475, + "learning_rate": 3.998155644658642e-06, + "loss": 0.50425643, + "num_input_tokens_seen": 41452970, + "router_z_loss_clip": 0.33203125, + "router_z_loss_mlp": 0.01672363, + "step": 1491, + "time_per_iteration": 3.10937762260437 + }, + { + "auxiliary_loss_clip": 0.01202505, + "auxiliary_loss_mlp": 0.01064885, + "balance_loss_clip": 1.07841873, + "balance_loss_mlp": 1.045156, + "epoch": 0.04329406302594162, + "flos": 36719809447680.0, + "grad_norm": 1.9645640877460315, + "language_loss": 0.99782681, + "learning_rate": 3.998147565498208e-06, + "loss": 1.02050078, + "num_input_tokens_seen": 41472605, + "router_z_loss_clip": 1.24121094, + "router_z_loss_mlp": 0.1973877, + "step": 1492, + "time_per_iteration": 2.6380293369293213 + }, + { + "auxiliary_loss_clip": 0.01198597, + "auxiliary_loss_mlp": 0.01048745, + "balance_loss_clip": 1.07526827, + "balance_loss_mlp": 1.02797914, + "epoch": 0.04332308049445766, + "flos": 11694849609600.0, + "grad_norm": 2.143971848495377, + "language_loss": 0.7951991, + "learning_rate": 3.998139468689327e-06, + "loss": 0.81767249, + "num_input_tokens_seen": 41485800, + "router_z_loss_clip": 1.23339844, + "router_z_loss_mlp": 0.20776367, + "step": 1493, + "time_per_iteration": 2.518038749694824 + }, + { + "auxiliary_loss_clip": 0.01201017, + "auxiliary_loss_mlp": 0.01050292, + "balance_loss_clip": 1.07859135, + "balance_loss_mlp": 1.03084898, + "epoch": 0.04335209796297371, + "flos": 39559193694720.0, + "grad_norm": 2.8869271096079507, + "language_loss": 0.93898696, + "learning_rate": 3.998131354232069e-06, + "loss": 0.96150005, + "num_input_tokens_seen": 41502675, + "router_z_loss_clip": 1.22558594, + "router_z_loss_mlp": 0.19421387, + "step": 1494, + "time_per_iteration": 2.656160593032837 + }, + { + "auxiliary_loss_clip": 0.01205166, + "auxiliary_loss_mlp": 0.01057246, + "balance_loss_clip": 1.07635975, + "balance_loss_mlp": 1.03428602, + "epoch": 0.04338111543148976, + "flos": 33620575236480.0, + "grad_norm": 1.8106881105697845, + "language_loss": 1.10259235, + "learning_rate": 3.998123222126506e-06, + "loss": 1.1252166, + "num_input_tokens_seen": 41525430, + "router_z_loss_clip": 1.28808594, + "router_z_loss_mlp": 0.2298584, + "step": 1495, + "time_per_iteration": 2.6497092247009277 + }, + { + "auxiliary_loss_clip": 0.01213801, + "auxiliary_loss_mlp": 0.01062824, + "balance_loss_clip": 1.08152342, + "balance_loss_mlp": 1.04026961, + "epoch": 0.0434101329000058, + "flos": 11905895969280.0, + "grad_norm": 2.639747488164423, + "language_loss": 0.89026058, + "learning_rate": 3.998115072372711e-06, + "loss": 0.91302687, + "num_input_tokens_seen": 41537700, + "router_z_loss_clip": 1.32128906, + "router_z_loss_mlp": 0.22583008, + "step": 1496, + "time_per_iteration": 2.45351505279541 + }, + { + "auxiliary_loss_clip": 0.01192858, + "auxiliary_loss_mlp": 0.0104784, + "balance_loss_clip": 1.07201207, + "balance_loss_mlp": 1.02869511, + "epoch": 0.04343915036852185, + "flos": 39961105948800.0, + "grad_norm": 5.272964685024911, + "language_loss": 1.0160358, + "learning_rate": 3.998106904970754e-06, + "loss": 1.03844285, + "num_input_tokens_seen": 41554365, + "router_z_loss_clip": 1.20800781, + "router_z_loss_mlp": 0.19152832, + "step": 1497, + "time_per_iteration": 2.598440408706665 + }, + { + "auxiliary_loss_clip": 0.01201611, + "auxiliary_loss_mlp": 0.0106471, + "balance_loss_clip": 1.07992303, + "balance_loss_mlp": 1.04400337, + "epoch": 0.0434681678370379, + "flos": 35035347196800.0, + "grad_norm": 2.697432207890562, + "language_loss": 0.91570544, + "learning_rate": 3.9980987199207096e-06, + "loss": 0.93836868, + "num_input_tokens_seen": 41571635, + "router_z_loss_clip": 1.21679688, + "router_z_loss_mlp": 0.20715332, + "step": 1498, + "time_per_iteration": 9.69427227973938 + }, + { + "auxiliary_loss_clip": 0.01059406, + "auxiliary_loss_mlp": 0.01027742, + "balance_loss_clip": 1.02614307, + "balance_loss_mlp": 1.02613294, + "epoch": 0.043497185305553944, + "flos": 74780058637440.0, + "grad_norm": 0.6742257879003287, + "language_loss": 0.47339216, + "learning_rate": 3.998090517222648e-06, + "loss": 0.49426365, + "num_input_tokens_seen": 41636485, + "router_z_loss_clip": 0.33203125, + "router_z_loss_mlp": 0.01611328, + "step": 1499, + "time_per_iteration": 3.1617562770843506 + }, + { + "auxiliary_loss_clip": 0.01212195, + "auxiliary_loss_mlp": 0.0107078, + "balance_loss_clip": 1.07889462, + "balance_loss_mlp": 1.04745054, + "epoch": 0.04352620277406999, + "flos": 10775140848000.0, + "grad_norm": 3.0456804989345367, + "language_loss": 0.9756372, + "learning_rate": 3.998082296876643e-06, + "loss": 0.99846691, + "num_input_tokens_seen": 41648245, + "router_z_loss_clip": 1.33203125, + "router_z_loss_mlp": 0.23303223, + "step": 1500, + "time_per_iteration": 4.938528776168823 + }, + { + "auxiliary_loss_clip": 0.01057759, + "auxiliary_loss_mlp": 0.0101033, + "balance_loss_clip": 1.02515483, + "balance_loss_mlp": 1.0084939, + "epoch": 0.04355522024258604, + "flos": 65849470957440.0, + "grad_norm": 0.678382537455724, + "language_loss": 0.52587485, + "learning_rate": 3.9980740588827655e-06, + "loss": 0.54655576, + "num_input_tokens_seen": 41716230, + "router_z_loss_clip": 0.32617188, + "router_z_loss_mlp": 0.01831055, + "step": 1501, + "time_per_iteration": 3.165630340576172 + }, + { + "auxiliary_loss_clip": 0.01210993, + "auxiliary_loss_mlp": 0.01063763, + "balance_loss_clip": 1.08152175, + "balance_loss_mlp": 1.04191208, + "epoch": 0.043584237711102085, + "flos": 28505423047680.0, + "grad_norm": 2.0248472909206416, + "language_loss": 0.83423704, + "learning_rate": 3.99806580324109e-06, + "loss": 0.85698462, + "num_input_tokens_seen": 41731535, + "router_z_loss_clip": 1.29492188, + "router_z_loss_mlp": 0.21844482, + "step": 1502, + "time_per_iteration": 2.557866334915161 + }, + { + "auxiliary_loss_clip": 0.01208094, + "auxiliary_loss_mlp": 0.01061263, + "balance_loss_clip": 1.07878077, + "balance_loss_mlp": 1.03929305, + "epoch": 0.04361325517961813, + "flos": 31425750593280.0, + "grad_norm": 2.3800498208381216, + "language_loss": 1.15437627, + "learning_rate": 3.998057529951688e-06, + "loss": 1.17706978, + "num_input_tokens_seen": 41754425, + "router_z_loss_clip": 1.29296875, + "router_z_loss_mlp": 0.21972656, + "step": 1503, + "time_per_iteration": 2.6660618782043457 + }, + { + "auxiliary_loss_clip": 0.01209293, + "auxiliary_loss_mlp": 0.01066169, + "balance_loss_clip": 1.07727027, + "balance_loss_mlp": 1.04298258, + "epoch": 0.043642272648134174, + "flos": 32086938193920.0, + "grad_norm": 6.150244651976813, + "language_loss": 1.07582426, + "learning_rate": 3.998049239014634e-06, + "loss": 1.09857893, + "num_input_tokens_seen": 41772655, + "router_z_loss_clip": 1.32128906, + "router_z_loss_mlp": 0.23181152, + "step": 1504, + "time_per_iteration": 2.6120176315307617 + }, + { + "auxiliary_loss_clip": 0.01193125, + "auxiliary_loss_mlp": 0.01050945, + "balance_loss_clip": 1.07547903, + "balance_loss_mlp": 1.03162074, + "epoch": 0.043671290116650226, + "flos": 15662331371520.0, + "grad_norm": 2.7332363238438098, + "language_loss": 0.90709931, + "learning_rate": 3.998040930430001e-06, + "loss": 0.92953998, + "num_input_tokens_seen": 41785700, + "router_z_loss_clip": 1.17773438, + "router_z_loss_mlp": 0.19335938, + "step": 1505, + "time_per_iteration": 2.490962266921997 + }, + { + "auxiliary_loss_clip": 0.01191055, + "auxiliary_loss_mlp": 0.01051985, + "balance_loss_clip": 1.07281578, + "balance_loss_mlp": 1.0329715, + "epoch": 0.04370030758516627, + "flos": 13727392606080.0, + "grad_norm": 2.5413341595681094, + "language_loss": 0.742751, + "learning_rate": 3.998032604197862e-06, + "loss": 0.76518142, + "num_input_tokens_seen": 41799735, + "router_z_loss_clip": 1.18212891, + "router_z_loss_mlp": 0.19018555, + "step": 1506, + "time_per_iteration": 2.482231378555298 + }, + { + "auxiliary_loss_clip": 0.01197932, + "auxiliary_loss_mlp": 0.01045268, + "balance_loss_clip": 1.07623577, + "balance_loss_mlp": 1.02588439, + "epoch": 0.043729325053682315, + "flos": 33286749212160.0, + "grad_norm": 3.3950856512561853, + "language_loss": 0.74941373, + "learning_rate": 3.99802426031829e-06, + "loss": 0.77184582, + "num_input_tokens_seen": 41816505, + "router_z_loss_clip": 1.21777344, + "router_z_loss_mlp": 0.1940918, + "step": 1507, + "time_per_iteration": 2.6475045680999756 + }, + { + "auxiliary_loss_clip": 0.01060398, + "auxiliary_loss_mlp": 0.01033896, + "balance_loss_clip": 1.02826834, + "balance_loss_mlp": 1.0319891, + "epoch": 0.04375834252219837, + "flos": 74775641264640.0, + "grad_norm": 0.6906430734638317, + "language_loss": 0.49893141, + "learning_rate": 3.9980158987913595e-06, + "loss": 0.51987433, + "num_input_tokens_seen": 41880970, + "router_z_loss_clip": 0.3203125, + "router_z_loss_mlp": 0.01904297, + "step": 1508, + "time_per_iteration": 3.137493133544922 + }, + { + "auxiliary_loss_clip": 0.01059958, + "auxiliary_loss_mlp": 0.01016652, + "balance_loss_clip": 1.02812481, + "balance_loss_mlp": 1.01480424, + "epoch": 0.04378735999071441, + "flos": 74766806519040.0, + "grad_norm": 0.7368562331793078, + "language_loss": 0.61000168, + "learning_rate": 3.998007519617144e-06, + "loss": 0.63076776, + "num_input_tokens_seen": 41936300, + "router_z_loss_clip": 0.31640625, + "router_z_loss_mlp": 0.01843262, + "step": 1509, + "time_per_iteration": 3.025902032852173 + }, + { + "auxiliary_loss_clip": 0.01206524, + "auxiliary_loss_mlp": 0.01062769, + "balance_loss_clip": 1.07898653, + "balance_loss_mlp": 1.04153776, + "epoch": 0.043816377459230456, + "flos": 74734089229440.0, + "grad_norm": 2.927442749238489, + "language_loss": 0.90522635, + "learning_rate": 3.997999122795718e-06, + "loss": 0.92791939, + "num_input_tokens_seen": 41959140, + "router_z_loss_clip": 1.27636719, + "router_z_loss_mlp": 0.21234131, + "step": 1510, + "time_per_iteration": 2.977712392807007 + }, + { + "auxiliary_loss_clip": 0.01205065, + "auxiliary_loss_mlp": 0.0107074, + "balance_loss_clip": 1.07906759, + "balance_loss_mlp": 1.0481379, + "epoch": 0.0438453949277465, + "flos": 13325516265600.0, + "grad_norm": 4.01352779527647, + "language_loss": 0.84031689, + "learning_rate": 3.997990708327154e-06, + "loss": 0.86307496, + "num_input_tokens_seen": 41969400, + "router_z_loss_clip": 1.26074219, + "router_z_loss_mlp": 0.22607422, + "step": 1511, + "time_per_iteration": 2.4653990268707275 + }, + { + "auxiliary_loss_clip": 0.01193854, + "auxiliary_loss_mlp": 0.01042354, + "balance_loss_clip": 1.07199466, + "balance_loss_mlp": 1.02343583, + "epoch": 0.04387441239626255, + "flos": 15771607522560.0, + "grad_norm": 2.288014060851349, + "language_loss": 0.66870832, + "learning_rate": 3.997982276211529e-06, + "loss": 0.69107044, + "num_input_tokens_seen": 41983435, + "router_z_loss_clip": 1.21875, + "router_z_loss_mlp": 0.18908691, + "step": 1512, + "time_per_iteration": 2.4826841354370117 + }, + { + "auxiliary_loss_clip": 0.01194724, + "auxiliary_loss_mlp": 0.01049997, + "balance_loss_clip": 1.07724535, + "balance_loss_mlp": 1.03169262, + "epoch": 0.0439034298647786, + "flos": 20626119648000.0, + "grad_norm": 2.701883593016089, + "language_loss": 0.74490309, + "learning_rate": 3.997973826448915e-06, + "loss": 0.76735032, + "num_input_tokens_seen": 41998705, + "router_z_loss_clip": 1.17578125, + "router_z_loss_mlp": 0.18304443, + "step": 1513, + "time_per_iteration": 2.5275096893310547 + }, + { + "auxiliary_loss_clip": 0.01207891, + "auxiliary_loss_mlp": 0.01058093, + "balance_loss_clip": 1.07880521, + "balance_loss_mlp": 1.03530037, + "epoch": 0.04393244733329464, + "flos": 36314557228800.0, + "grad_norm": 2.2506065203005963, + "language_loss": 0.85579824, + "learning_rate": 3.997965359039388e-06, + "loss": 0.87845814, + "num_input_tokens_seen": 42019220, + "router_z_loss_clip": 1.29101562, + "router_z_loss_mlp": 0.22802734, + "step": 1514, + "time_per_iteration": 2.647925853729248 + }, + { + "auxiliary_loss_clip": 0.01056975, + "auxiliary_loss_mlp": 0.01041843, + "balance_loss_clip": 1.02463722, + "balance_loss_mlp": 1.03993595, + "epoch": 0.04396146480181069, + "flos": 70506904135680.0, + "grad_norm": 0.7435237708854664, + "language_loss": 0.5350799, + "learning_rate": 3.997956873983023e-06, + "loss": 0.55606806, + "num_input_tokens_seen": 42079550, + "router_z_loss_clip": 0.32421875, + "router_z_loss_mlp": 0.01904297, + "step": 1515, + "time_per_iteration": 3.0761730670928955 + }, + { + "auxiliary_loss_clip": 0.01204245, + "auxiliary_loss_mlp": 0.01059118, + "balance_loss_clip": 1.07722521, + "balance_loss_mlp": 1.03630114, + "epoch": 0.04399048227032674, + "flos": 37372772833920.0, + "grad_norm": 2.199980265285963, + "language_loss": 0.93120277, + "learning_rate": 3.997948371279894e-06, + "loss": 0.95383644, + "num_input_tokens_seen": 42098940, + "router_z_loss_clip": 1.27050781, + "router_z_loss_mlp": 0.22814941, + "step": 1516, + "time_per_iteration": 2.665346622467041 + }, + { + "auxiliary_loss_clip": 0.01206711, + "auxiliary_loss_mlp": 0.01065089, + "balance_loss_clip": 1.0741663, + "balance_loss_mlp": 1.04130673, + "epoch": 0.04401949973884278, + "flos": 30659201424000.0, + "grad_norm": 3.189345449333686, + "language_loss": 1.00276136, + "learning_rate": 3.997939850930076e-06, + "loss": 1.02547932, + "num_input_tokens_seen": 42119350, + "router_z_loss_clip": 1.32714844, + "router_z_loss_mlp": 0.23791504, + "step": 1517, + "time_per_iteration": 2.6382415294647217 + }, + { + "auxiliary_loss_clip": 0.01054797, + "auxiliary_loss_mlp": 0.00998104, + "balance_loss_clip": 1.0232501, + "balance_loss_mlp": 0.99617308, + "epoch": 0.04404851720735883, + "flos": 57042737372160.0, + "grad_norm": 0.6938343203899838, + "language_loss": 0.49999282, + "learning_rate": 3.997931312933645e-06, + "loss": 0.52052188, + "num_input_tokens_seen": 42174115, + "router_z_loss_clip": 0.31640625, + "router_z_loss_mlp": 0.01928711, + "step": 1518, + "time_per_iteration": 3.0127246379852295 + }, + { + "auxiliary_loss_clip": 0.0105432, + "auxiliary_loss_mlp": 0.01005325, + "balance_loss_clip": 1.02271903, + "balance_loss_mlp": 1.00334644, + "epoch": 0.04407753467587488, + "flos": 47690915921280.0, + "grad_norm": 0.730570819318569, + "language_loss": 0.49862343, + "learning_rate": 3.997922757290677e-06, + "loss": 0.51921988, + "num_input_tokens_seen": 42222380, + "router_z_loss_clip": 0.31640625, + "router_z_loss_mlp": 0.01977539, + "step": 1519, + "time_per_iteration": 2.763251543045044 + }, + { + "auxiliary_loss_clip": 0.0120161, + "auxiliary_loss_mlp": 0.01054508, + "balance_loss_clip": 1.07552028, + "balance_loss_mlp": 1.03220367, + "epoch": 0.04410655214439092, + "flos": 30037731287040.0, + "grad_norm": 2.9099218664224353, + "language_loss": 0.84726536, + "learning_rate": 3.997914184001246e-06, + "loss": 0.86982656, + "num_input_tokens_seen": 42238210, + "router_z_loss_clip": 1.26074219, + "router_z_loss_mlp": 0.22302246, + "step": 1520, + "time_per_iteration": 2.636725902557373 + }, + { + "auxiliary_loss_clip": 0.01195523, + "auxiliary_loss_mlp": 0.01046241, + "balance_loss_clip": 1.07131755, + "balance_loss_mlp": 1.02444947, + "epoch": 0.04413556961290697, + "flos": 36569343375360.0, + "grad_norm": 1.9834380423502507, + "language_loss": 0.94941467, + "learning_rate": 3.997905593065429e-06, + "loss": 0.97183233, + "num_input_tokens_seen": 42258205, + "router_z_loss_clip": 1.24316406, + "router_z_loss_mlp": 0.21813965, + "step": 1521, + "time_per_iteration": 2.6284360885620117 + }, + { + "auxiliary_loss_clip": 0.0105449, + "auxiliary_loss_mlp": 0.01030867, + "balance_loss_clip": 1.02279902, + "balance_loss_mlp": 1.02893603, + "epoch": 0.04416458708142302, + "flos": 55799042912640.0, + "grad_norm": 0.7356461799001313, + "language_loss": 0.49756169, + "learning_rate": 3.9978969844833e-06, + "loss": 0.51841527, + "num_input_tokens_seen": 42316345, + "router_z_loss_clip": 0.31640625, + "router_z_loss_mlp": 0.01928711, + "step": 1522, + "time_per_iteration": 3.0695784091949463 + }, + { + "auxiliary_loss_clip": 0.01182616, + "auxiliary_loss_mlp": 0.01049823, + "balance_loss_clip": 1.06860662, + "balance_loss_mlp": 1.03139901, + "epoch": 0.044193604549939064, + "flos": 24671671620480.0, + "grad_norm": 3.600543733342778, + "language_loss": 0.8681618, + "learning_rate": 3.997888358254937e-06, + "loss": 0.89048624, + "num_input_tokens_seen": 42328330, + "router_z_loss_clip": 1.13867188, + "router_z_loss_mlp": 0.18426514, + "step": 1523, + "time_per_iteration": 2.4888410568237305 + }, + { + "auxiliary_loss_clip": 0.01053948, + "auxiliary_loss_mlp": 0.01014278, + "balance_loss_clip": 1.02264214, + "balance_loss_mlp": 1.01247835, + "epoch": 0.04422262201845511, + "flos": 74777831994240.0, + "grad_norm": 0.6871905555956479, + "language_loss": 0.52930969, + "learning_rate": 3.997879714380416e-06, + "loss": 0.54999191, + "num_input_tokens_seen": 42393765, + "router_z_loss_clip": 0.3125, + "router_z_loss_mlp": 0.01794434, + "step": 1524, + "time_per_iteration": 3.1790390014648438 + }, + { + "auxiliary_loss_clip": 0.01197727, + "auxiliary_loss_mlp": 0.01066169, + "balance_loss_clip": 1.07670689, + "balance_loss_mlp": 1.04522443, + "epoch": 0.04425163948697115, + "flos": 13765278476160.0, + "grad_norm": 3.4961794241908297, + "language_loss": 0.92982322, + "learning_rate": 3.997871052859813e-06, + "loss": 0.9524622, + "num_input_tokens_seen": 42405350, + "router_z_loss_clip": 1.20996094, + "router_z_loss_mlp": 0.20935059, + "step": 1525, + "time_per_iteration": 2.5013480186462402 + }, + { + "auxiliary_loss_clip": 0.01201509, + "auxiliary_loss_mlp": 0.01046679, + "balance_loss_clip": 1.07481146, + "balance_loss_mlp": 1.02811837, + "epoch": 0.044280656955487205, + "flos": 17741846378880.0, + "grad_norm": 2.1211907459326893, + "language_loss": 0.77501941, + "learning_rate": 3.997862373693203e-06, + "loss": 0.79750127, + "num_input_tokens_seen": 42418210, + "router_z_loss_clip": 1.26855469, + "router_z_loss_mlp": 0.18554688, + "step": 1526, + "time_per_iteration": 2.453317403793335 + }, + { + "auxiliary_loss_clip": 0.01190833, + "auxiliary_loss_mlp": 0.01051512, + "balance_loss_clip": 1.06865978, + "balance_loss_mlp": 1.02989888, + "epoch": 0.04430967442400325, + "flos": 16246167033600.0, + "grad_norm": 2.8790012754169245, + "language_loss": 0.89502978, + "learning_rate": 3.9978536768806665e-06, + "loss": 0.91745317, + "num_input_tokens_seen": 42430275, + "router_z_loss_clip": 1.22265625, + "router_z_loss_mlp": 0.21582031, + "step": 1527, + "time_per_iteration": 2.486518621444702 + }, + { + "auxiliary_loss_clip": 0.01204325, + "auxiliary_loss_mlp": 0.0105866, + "balance_loss_clip": 1.07286096, + "balance_loss_mlp": 1.03661871, + "epoch": 0.044338691892519294, + "flos": 16756278030720.0, + "grad_norm": 2.645042255134915, + "language_loss": 0.80623066, + "learning_rate": 3.997844962422277e-06, + "loss": 0.82886052, + "num_input_tokens_seen": 42442235, + "router_z_loss_clip": 1.31347656, + "router_z_loss_mlp": 0.22045898, + "step": 1528, + "time_per_iteration": 2.450498342514038 + }, + { + "auxiliary_loss_clip": 0.01052208, + "auxiliary_loss_mlp": 0.01028121, + "balance_loss_clip": 1.01999021, + "balance_loss_mlp": 1.02655983, + "epoch": 0.044367709361035346, + "flos": 67298321946240.0, + "grad_norm": 0.688676809379509, + "language_loss": 0.52602404, + "learning_rate": 3.997836230318111e-06, + "loss": 0.54682732, + "num_input_tokens_seen": 42506120, + "router_z_loss_clip": 0.32226562, + "router_z_loss_mlp": 0.01556396, + "step": 1529, + "time_per_iteration": 3.062610626220703 + }, + { + "auxiliary_loss_clip": 0.01051172, + "auxiliary_loss_mlp": 0.01010873, + "balance_loss_clip": 1.01948142, + "balance_loss_mlp": 1.00931108, + "epoch": 0.04439672682955139, + "flos": 67798987666560.0, + "grad_norm": 0.7545609277273283, + "language_loss": 0.49043384, + "learning_rate": 3.997827480568248e-06, + "loss": 0.51105428, + "num_input_tokens_seen": 42565935, + "router_z_loss_clip": 0.31640625, + "router_z_loss_mlp": 0.015625, + "step": 1530, + "time_per_iteration": 3.044445276260376 + }, + { + "auxiliary_loss_clip": 0.01050334, + "auxiliary_loss_mlp": 0.01001748, + "balance_loss_clip": 1.01892471, + "balance_loss_mlp": 1.00025165, + "epoch": 0.044425744298067435, + "flos": 53970507210240.0, + "grad_norm": 0.7050792427722314, + "language_loss": 0.49784598, + "learning_rate": 3.997818713172764e-06, + "loss": 0.51836681, + "num_input_tokens_seen": 42618385, + "router_z_loss_clip": 0.31445312, + "router_z_loss_mlp": 0.01495361, + "step": 1531, + "time_per_iteration": 2.9320945739746094 + }, + { + "auxiliary_loss_clip": 0.01202484, + "auxiliary_loss_mlp": 0.01058593, + "balance_loss_clip": 1.07901692, + "balance_loss_mlp": 1.03838694, + "epoch": 0.04445476176658349, + "flos": 25874104331520.0, + "grad_norm": 3.2553534400548445, + "language_loss": 0.88998491, + "learning_rate": 3.997809928131737e-06, + "loss": 0.91259563, + "num_input_tokens_seen": 42630755, + "router_z_loss_clip": 1.234375, + "router_z_loss_mlp": 0.20227051, + "step": 1532, + "time_per_iteration": 2.544437885284424 + }, + { + "auxiliary_loss_clip": 0.01052369, + "auxiliary_loss_mlp": 0.01028767, + "balance_loss_clip": 1.02081108, + "balance_loss_mlp": 1.02726483, + "epoch": 0.04448377923509953, + "flos": 65285743933440.0, + "grad_norm": 0.733363481350425, + "language_loss": 0.52774775, + "learning_rate": 3.997801125445244e-06, + "loss": 0.54855907, + "num_input_tokens_seen": 42689745, + "router_z_loss_clip": 0.31640625, + "router_z_loss_mlp": 0.01501465, + "step": 1533, + "time_per_iteration": 3.0013463497161865 + }, + { + "auxiliary_loss_clip": 0.01209256, + "auxiliary_loss_mlp": 0.01056567, + "balance_loss_clip": 1.0818882, + "balance_loss_mlp": 1.03637338, + "epoch": 0.044512796703615576, + "flos": 16319065685760.0, + "grad_norm": 2.575096561830262, + "language_loss": 0.644921, + "learning_rate": 3.997792305113363e-06, + "loss": 0.66757923, + "num_input_tokens_seen": 42702550, + "router_z_loss_clip": 1.2734375, + "router_z_loss_mlp": 0.2019043, + "step": 1534, + "time_per_iteration": 2.492389440536499 + }, + { + "auxiliary_loss_clip": 0.01191094, + "auxiliary_loss_mlp": 0.01060889, + "balance_loss_clip": 1.07382274, + "balance_loss_mlp": 1.04110003, + "epoch": 0.04454181417213162, + "flos": 12050148988800.0, + "grad_norm": 2.8342324246236865, + "language_loss": 0.78496659, + "learning_rate": 3.997783467136172e-06, + "loss": 0.80748641, + "num_input_tokens_seen": 42715200, + "router_z_loss_clip": 1.171875, + "router_z_loss_mlp": 0.19787598, + "step": 1535, + "time_per_iteration": 2.4535224437713623 + }, + { + "auxiliary_loss_clip": 0.01190284, + "auxiliary_loss_mlp": 0.01056584, + "balance_loss_clip": 1.06889248, + "balance_loss_mlp": 1.0335288, + "epoch": 0.04457083164064767, + "flos": 44594839128960.0, + "grad_norm": 3.1015272745027116, + "language_loss": 0.93616164, + "learning_rate": 3.99777461151375e-06, + "loss": 0.95863038, + "num_input_tokens_seen": 42734245, + "router_z_loss_clip": 1.21386719, + "router_z_loss_mlp": 0.23059082, + "step": 1536, + "time_per_iteration": 2.610023260116577 + }, + { + "auxiliary_loss_clip": 0.0119765, + "auxiliary_loss_mlp": 0.01072982, + "balance_loss_clip": 1.07415938, + "balance_loss_mlp": 1.05138135, + "epoch": 0.04459984910916372, + "flos": 26352865733760.0, + "grad_norm": 2.6241913633618843, + "language_loss": 0.91908795, + "learning_rate": 3.997765738246173e-06, + "loss": 0.94179422, + "num_input_tokens_seen": 42750425, + "router_z_loss_clip": 1.23486328, + "router_z_loss_mlp": 0.21600342, + "step": 1537, + "time_per_iteration": 2.5433340072631836 + }, + { + "auxiliary_loss_clip": 0.01053168, + "auxiliary_loss_mlp": 0.01045445, + "balance_loss_clip": 1.02208495, + "balance_loss_mlp": 1.04401445, + "epoch": 0.04462886657767976, + "flos": 57846490053120.0, + "grad_norm": 0.7072220965384205, + "language_loss": 0.47437659, + "learning_rate": 3.997756847333521e-06, + "loss": 0.49536273, + "num_input_tokens_seen": 42815870, + "router_z_loss_clip": 0.31054688, + "router_z_loss_mlp": 0.01428223, + "step": 1538, + "time_per_iteration": 3.1545612812042236 + }, + { + "auxiliary_loss_clip": 0.01203386, + "auxiliary_loss_mlp": 0.01064187, + "balance_loss_clip": 1.0773592, + "balance_loss_mlp": 1.04311132, + "epoch": 0.04465788404619581, + "flos": 31788304519680.0, + "grad_norm": 3.0012306993528943, + "language_loss": 1.18691421, + "learning_rate": 3.997747938775872e-06, + "loss": 1.20958996, + "num_input_tokens_seen": 42833515, + "router_z_loss_clip": 1.26074219, + "router_z_loss_mlp": 0.21057129, + "step": 1539, + "time_per_iteration": 2.6279866695404053 + }, + { + "auxiliary_loss_clip": 0.01196521, + "auxiliary_loss_mlp": 0.010506, + "balance_loss_clip": 1.07276297, + "balance_loss_mlp": 1.0293808, + "epoch": 0.04468690151471186, + "flos": 32264408315520.0, + "grad_norm": 2.0197648985120455, + "language_loss": 0.7558136, + "learning_rate": 3.997739012573305e-06, + "loss": 0.77828479, + "num_input_tokens_seen": 42852355, + "router_z_loss_clip": 1.23828125, + "router_z_loss_mlp": 0.21240234, + "step": 1540, + "time_per_iteration": 2.6152024269104004 + }, + { + "auxiliary_loss_clip": 0.01193617, + "auxiliary_loss_mlp": 0.01050576, + "balance_loss_clip": 1.07206905, + "balance_loss_mlp": 1.02946365, + "epoch": 0.0447159189832279, + "flos": 31424421790080.0, + "grad_norm": 3.7157413267990984, + "language_loss": 0.76455241, + "learning_rate": 3.997730068725898e-06, + "loss": 0.78699434, + "num_input_tokens_seen": 42867305, + "router_z_loss_clip": 1.21582031, + "router_z_loss_mlp": 0.21118164, + "step": 1541, + "time_per_iteration": 2.6276371479034424 + }, + { + "auxiliary_loss_clip": 0.01049465, + "auxiliary_loss_mlp": 0.00998229, + "balance_loss_clip": 1.01831889, + "balance_loss_mlp": 0.9968344, + "epoch": 0.04474493645174395, + "flos": 49594361437440.0, + "grad_norm": 0.6867441926831271, + "language_loss": 0.51690918, + "learning_rate": 3.997721107233731e-06, + "loss": 0.53738612, + "num_input_tokens_seen": 42925860, + "router_z_loss_clip": 0.31054688, + "router_z_loss_mlp": 0.01397705, + "step": 1542, + "time_per_iteration": 3.1853723526000977 + }, + { + "auxiliary_loss_clip": 0.01049867, + "auxiliary_loss_mlp": 0.01000393, + "balance_loss_clip": 1.01858425, + "balance_loss_mlp": 0.99906963, + "epoch": 0.04477395392026, + "flos": 73363598737920.0, + "grad_norm": 0.6607809159185841, + "language_loss": 0.51184404, + "learning_rate": 3.9977121280968834e-06, + "loss": 0.53234661, + "num_input_tokens_seen": 42995855, + "router_z_loss_clip": 0.3125, + "router_z_loss_mlp": 0.01324463, + "step": 1543, + "time_per_iteration": 3.3336637020111084 + }, + { + "auxiliary_loss_clip": 0.01049466, + "auxiliary_loss_mlp": 0.00999465, + "balance_loss_clip": 1.0180676, + "balance_loss_mlp": 0.99810582, + "epoch": 0.04480297138877604, + "flos": 57041157173760.0, + "grad_norm": 0.7228619761198057, + "language_loss": 0.51297867, + "learning_rate": 3.997703131315434e-06, + "loss": 0.53346795, + "num_input_tokens_seen": 43051915, + "router_z_loss_clip": 0.31445312, + "router_z_loss_mlp": 0.01361084, + "step": 1544, + "time_per_iteration": 2.928091287612915 + }, + { + "auxiliary_loss_clip": 0.01049021, + "auxiliary_loss_mlp": 0.01003713, + "balance_loss_clip": 1.01776648, + "balance_loss_mlp": 1.00241339, + "epoch": 0.04483198885729209, + "flos": 74771906250240.0, + "grad_norm": 0.6567199908408936, + "language_loss": 0.48264295, + "learning_rate": 3.997694116889461e-06, + "loss": 0.50317025, + "num_input_tokens_seen": 43114030, + "router_z_loss_clip": 0.3125, + "router_z_loss_mlp": 0.01300049, + "step": 1545, + "time_per_iteration": 3.17569637298584 + }, + { + "auxiliary_loss_clip": 0.01048249, + "auxiliary_loss_mlp": 0.01000583, + "balance_loss_clip": 1.01714957, + "balance_loss_mlp": 0.99928921, + "epoch": 0.04486100632580814, + "flos": 63316582485120.0, + "grad_norm": 0.670683547312686, + "language_loss": 0.49799353, + "learning_rate": 3.997685084819046e-06, + "loss": 0.51848179, + "num_input_tokens_seen": 43175515, + "router_z_loss_clip": 0.31054688, + "router_z_loss_mlp": 0.01293945, + "step": 1546, + "time_per_iteration": 3.181553363800049 + }, + { + "auxiliary_loss_clip": 0.01182221, + "auxiliary_loss_mlp": 0.01041786, + "balance_loss_clip": 1.07032597, + "balance_loss_mlp": 1.02384484, + "epoch": 0.044890023794324184, + "flos": 12561014171520.0, + "grad_norm": 2.5005884927480464, + "language_loss": 0.68542355, + "learning_rate": 3.9976760351042675e-06, + "loss": 0.70766366, + "num_input_tokens_seen": 43187810, + "router_z_loss_clip": 1.11865234, + "router_z_loss_mlp": 0.17932129, + "step": 1547, + "time_per_iteration": 2.4914417266845703 + }, + { + "auxiliary_loss_clip": 0.01048867, + "auxiliary_loss_mlp": 0.01013142, + "balance_loss_clip": 1.01764584, + "balance_loss_mlp": 1.01181889, + "epoch": 0.04491904126284023, + "flos": 74776610931840.0, + "grad_norm": 0.7251013784931187, + "language_loss": 0.57016391, + "learning_rate": 3.997666967745206e-06, + "loss": 0.59078401, + "num_input_tokens_seen": 43255190, + "router_z_loss_clip": 0.3125, + "router_z_loss_mlp": 0.01324463, + "step": 1548, + "time_per_iteration": 3.1975784301757812 + }, + { + "auxiliary_loss_clip": 0.01190997, + "auxiliary_loss_mlp": 0.01048981, + "balance_loss_clip": 1.07123709, + "balance_loss_mlp": 1.03054512, + "epoch": 0.04494805873135627, + "flos": 31023622857600.0, + "grad_norm": 2.202859591867566, + "language_loss": 0.83556676, + "learning_rate": 3.997657882741942e-06, + "loss": 0.85796648, + "num_input_tokens_seen": 43271420, + "router_z_loss_clip": 1.19921875, + "router_z_loss_mlp": 0.18426514, + "step": 1549, + "time_per_iteration": 2.576497793197632 + }, + { + "auxiliary_loss_clip": 0.0120352, + "auxiliary_loss_mlp": 0.01068393, + "balance_loss_clip": 1.07501698, + "balance_loss_mlp": 1.04642248, + "epoch": 0.044977076199872325, + "flos": 24784395477120.0, + "grad_norm": 4.4706886140773126, + "language_loss": 1.010849, + "learning_rate": 3.997648780094554e-06, + "loss": 1.03356814, + "num_input_tokens_seen": 43285405, + "router_z_loss_clip": 1.28613281, + "router_z_loss_mlp": 0.21972656, + "step": 1550, + "time_per_iteration": 2.549802541732788 + }, + { + "auxiliary_loss_clip": 0.01203266, + "auxiliary_loss_mlp": 0.0105647, + "balance_loss_clip": 1.07782221, + "balance_loss_mlp": 1.03542352, + "epoch": 0.04500609366838837, + "flos": 74737931984640.0, + "grad_norm": 1.9786513828409664, + "language_loss": 0.65042031, + "learning_rate": 3.997639659803124e-06, + "loss": 0.67301774, + "num_input_tokens_seen": 43311145, + "router_z_loss_clip": 1.25537109, + "router_z_loss_mlp": 0.21057129, + "step": 1551, + "time_per_iteration": 2.9493906497955322 + }, + { + "auxiliary_loss_clip": 0.011997, + "auxiliary_loss_mlp": 0.01059865, + "balance_loss_clip": 1.07605112, + "balance_loss_mlp": 1.03738248, + "epoch": 0.045035111136904414, + "flos": 31605267790080.0, + "grad_norm": 2.820540228375012, + "language_loss": 0.93264067, + "learning_rate": 3.9976305218677324e-06, + "loss": 0.95523632, + "num_input_tokens_seen": 43326720, + "router_z_loss_clip": 1.23632812, + "router_z_loss_mlp": 0.22460938, + "step": 1552, + "time_per_iteration": 2.597285270690918 + }, + { + "auxiliary_loss_clip": 0.01050927, + "auxiliary_loss_mlp": 0.01020837, + "balance_loss_clip": 1.01964283, + "balance_loss_mlp": 1.01951396, + "epoch": 0.045064128605420466, + "flos": 63425786808960.0, + "grad_norm": 0.7324461533397015, + "language_loss": 0.52637553, + "learning_rate": 3.997621366288461e-06, + "loss": 0.54709321, + "num_input_tokens_seen": 43388330, + "router_z_loss_clip": 0.3125, + "router_z_loss_mlp": 0.01324463, + "step": 1553, + "time_per_iteration": 3.013404130935669 + }, + { + "auxiliary_loss_clip": 0.01207588, + "auxiliary_loss_mlp": 0.01070999, + "balance_loss_clip": 1.07451463, + "balance_loss_mlp": 1.04671586, + "epoch": 0.04509314607393651, + "flos": 32301575913600.0, + "grad_norm": 1.8999815957018296, + "language_loss": 0.90163755, + "learning_rate": 3.997612193065388e-06, + "loss": 0.92442346, + "num_input_tokens_seen": 43410230, + "router_z_loss_clip": 1.33203125, + "router_z_loss_mlp": 0.24267578, + "step": 1554, + "time_per_iteration": 2.5799639225006104 + }, + { + "auxiliary_loss_clip": 0.01196679, + "auxiliary_loss_mlp": 0.01061414, + "balance_loss_clip": 1.07418787, + "balance_loss_mlp": 1.03915811, + "epoch": 0.045122163542452555, + "flos": 14350335200640.0, + "grad_norm": 3.287504245927468, + "language_loss": 0.82308918, + "learning_rate": 3.9976030021985955e-06, + "loss": 0.8456701, + "num_input_tokens_seen": 43422210, + "router_z_loss_clip": 1.22460938, + "router_z_loss_mlp": 0.22271729, + "step": 1555, + "time_per_iteration": 2.512226104736328 + }, + { + "auxiliary_loss_clip": 0.01209569, + "auxiliary_loss_mlp": 0.0106787, + "balance_loss_clip": 1.0793004, + "balance_loss_mlp": 1.04405224, + "epoch": 0.04515118101096861, + "flos": 20368891376640.0, + "grad_norm": 2.561598949331693, + "language_loss": 0.69777739, + "learning_rate": 3.9975937936881655e-06, + "loss": 0.72055185, + "num_input_tokens_seen": 43435700, + "router_z_loss_clip": 1.30175781, + "router_z_loss_mlp": 0.23803711, + "step": 1556, + "time_per_iteration": 2.4809348583221436 + }, + { + "auxiliary_loss_clip": 0.01049769, + "auxiliary_loss_mlp": 0.01008654, + "balance_loss_clip": 1.01839757, + "balance_loss_mlp": 1.00706863, + "epoch": 0.04518019847948465, + "flos": 74777903821440.0, + "grad_norm": 0.731701000047472, + "language_loss": 0.47930509, + "learning_rate": 3.997584567534178e-06, + "loss": 0.49988928, + "num_input_tokens_seen": 43495935, + "router_z_loss_clip": 0.31445312, + "router_z_loss_mlp": 0.01586914, + "step": 1557, + "time_per_iteration": 3.1613214015960693 + }, + { + "auxiliary_loss_clip": 0.01195165, + "auxiliary_loss_mlp": 0.01065869, + "balance_loss_clip": 1.07188869, + "balance_loss_mlp": 1.04619956, + "epoch": 0.045209215948000696, + "flos": 29195697686400.0, + "grad_norm": 2.4992078209934436, + "language_loss": 0.90935701, + "learning_rate": 3.997575323736717e-06, + "loss": 0.93196738, + "num_input_tokens_seen": 43512380, + "router_z_loss_clip": 1.23339844, + "router_z_loss_mlp": 0.19677734, + "step": 1558, + "time_per_iteration": 2.6216745376586914 + }, + { + "auxiliary_loss_clip": 0.01206249, + "auxiliary_loss_mlp": 0.0106346, + "balance_loss_clip": 1.07640421, + "balance_loss_mlp": 1.03910518, + "epoch": 0.04523823341651674, + "flos": 30115370534400.0, + "grad_norm": 2.6044981237895835, + "language_loss": 0.8246097, + "learning_rate": 3.9975660622958605e-06, + "loss": 0.84730673, + "num_input_tokens_seen": 43528600, + "router_z_loss_clip": 1.296875, + "router_z_loss_mlp": 0.24377441, + "step": 1559, + "time_per_iteration": 2.609624147415161 + }, + { + "auxiliary_loss_clip": 0.01204697, + "auxiliary_loss_mlp": 0.01052281, + "balance_loss_clip": 1.07919002, + "balance_loss_mlp": 1.02966738, + "epoch": 0.04526725088503279, + "flos": 10851989996160.0, + "grad_norm": 2.7497246603463834, + "language_loss": 0.94146872, + "learning_rate": 3.997556783211693e-06, + "loss": 0.96403843, + "num_input_tokens_seen": 43539370, + "router_z_loss_clip": 1.25585938, + "router_z_loss_mlp": 0.22631836, + "step": 1560, + "time_per_iteration": 2.4469714164733887 + }, + { + "auxiliary_loss_clip": 0.01191477, + "auxiliary_loss_mlp": 0.01056329, + "balance_loss_clip": 1.07160234, + "balance_loss_mlp": 1.03602803, + "epoch": 0.04529626835354884, + "flos": 10444762529280.0, + "grad_norm": 2.909422821588288, + "language_loss": 0.84246516, + "learning_rate": 3.997547486484296e-06, + "loss": 0.86494321, + "num_input_tokens_seen": 43550890, + "router_z_loss_clip": 1.19726562, + "router_z_loss_mlp": 0.20324707, + "step": 1561, + "time_per_iteration": 2.5328946113586426 + }, + { + "auxiliary_loss_clip": 0.01213397, + "auxiliary_loss_mlp": 0.01069156, + "balance_loss_clip": 1.07777309, + "balance_loss_mlp": 1.04508805, + "epoch": 0.04532528582206488, + "flos": 12012622254720.0, + "grad_norm": 3.3760944632584597, + "language_loss": 0.92703676, + "learning_rate": 3.997538172113751e-06, + "loss": 0.94986224, + "num_input_tokens_seen": 43561195, + "router_z_loss_clip": 1.35449219, + "router_z_loss_mlp": 0.24084473, + "step": 1562, + "time_per_iteration": 2.47013258934021 + }, + { + "auxiliary_loss_clip": 0.01049669, + "auxiliary_loss_mlp": 0.01000992, + "balance_loss_clip": 1.01860392, + "balance_loss_mlp": 0.99920398, + "epoch": 0.04535430329058093, + "flos": 71490209927040.0, + "grad_norm": 0.723158397769778, + "language_loss": 0.47815424, + "learning_rate": 3.99752884010014e-06, + "loss": 0.4986608, + "num_input_tokens_seen": 43624190, + "router_z_loss_clip": 0.31054688, + "router_z_loss_mlp": 0.01782227, + "step": 1563, + "time_per_iteration": 3.1272335052490234 + }, + { + "auxiliary_loss_clip": 0.01214292, + "auxiliary_loss_mlp": 0.01066013, + "balance_loss_clip": 1.07901466, + "balance_loss_mlp": 1.04299402, + "epoch": 0.04538332075909698, + "flos": 23033103972480.0, + "grad_norm": 2.284433658060149, + "language_loss": 0.72714233, + "learning_rate": 3.997519490443547e-06, + "loss": 0.7499454, + "num_input_tokens_seen": 43639145, + "router_z_loss_clip": 1.35351562, + "router_z_loss_mlp": 0.23022461, + "step": 1564, + "time_per_iteration": 2.485630750656128 + }, + { + "auxiliary_loss_clip": 0.0105026, + "auxiliary_loss_mlp": 0.0099792, + "balance_loss_clip": 1.01922822, + "balance_loss_mlp": 0.99622744, + "epoch": 0.04541233822761302, + "flos": 74782357107840.0, + "grad_norm": 0.6652758785679326, + "language_loss": 0.54066968, + "learning_rate": 3.997510123144053e-06, + "loss": 0.56115144, + "num_input_tokens_seen": 43708675, + "router_z_loss_clip": 0.31054688, + "router_z_loss_mlp": 0.01696777, + "step": 1565, + "time_per_iteration": 3.2112555503845215 + }, + { + "auxiliary_loss_clip": 0.01207079, + "auxiliary_loss_mlp": 0.01051658, + "balance_loss_clip": 1.08002985, + "balance_loss_mlp": 1.02952099, + "epoch": 0.04544135569612907, + "flos": 23326745656320.0, + "grad_norm": 2.457501379369961, + "language_loss": 0.97890151, + "learning_rate": 3.9975007382017406e-06, + "loss": 1.0014888, + "num_input_tokens_seen": 43727065, + "router_z_loss_clip": 1.26953125, + "router_z_loss_mlp": 0.22131348, + "step": 1566, + "time_per_iteration": 2.736218214035034 + }, + { + "auxiliary_loss_clip": 0.010506, + "auxiliary_loss_mlp": 0.01001273, + "balance_loss_clip": 1.01983035, + "balance_loss_mlp": 0.99950886, + "epoch": 0.04547037316464512, + "flos": 70906948882560.0, + "grad_norm": 0.6243803872433688, + "language_loss": 0.49938282, + "learning_rate": 3.997491335616694e-06, + "loss": 0.51990157, + "num_input_tokens_seen": 43794795, + "router_z_loss_clip": 0.30859375, + "router_z_loss_mlp": 0.0177002, + "step": 1567, + "time_per_iteration": 3.2637131214141846 + }, + { + "auxiliary_loss_clip": 0.01050222, + "auxiliary_loss_mlp": 0.00996586, + "balance_loss_clip": 1.0194447, + "balance_loss_mlp": 0.99483413, + "epoch": 0.04549939063316116, + "flos": 74768494458240.0, + "grad_norm": 0.7088353453893632, + "language_loss": 0.53506446, + "learning_rate": 3.997481915388996e-06, + "loss": 0.55553252, + "num_input_tokens_seen": 43854295, + "router_z_loss_clip": 0.30859375, + "router_z_loss_mlp": 0.01757812, + "step": 1568, + "time_per_iteration": 5.47186541557312 + }, + { + "auxiliary_loss_clip": 0.01050463, + "auxiliary_loss_mlp": 0.01003273, + "balance_loss_clip": 1.01962793, + "balance_loss_mlp": 1.00169933, + "epoch": 0.04552840810167721, + "flos": 62898152952960.0, + "grad_norm": 0.7026131779919892, + "language_loss": 0.50543487, + "learning_rate": 3.997472477518729e-06, + "loss": 0.52597219, + "num_input_tokens_seen": 43920915, + "router_z_loss_clip": 0.30859375, + "router_z_loss_mlp": 0.01574707, + "step": 1569, + "time_per_iteration": 7.834134340286255 + }, + { + "auxiliary_loss_clip": 0.01214965, + "auxiliary_loss_mlp": 0.01059122, + "balance_loss_clip": 1.08152127, + "balance_loss_mlp": 1.03491032, + "epoch": 0.04555742557019326, + "flos": 22594455083520.0, + "grad_norm": 2.3839478197084416, + "language_loss": 0.75077981, + "learning_rate": 3.997463022005977e-06, + "loss": 0.77352071, + "num_input_tokens_seen": 43934475, + "router_z_loss_clip": 1.33398438, + "router_z_loss_mlp": 0.24206543, + "step": 1570, + "time_per_iteration": 2.4435172080993652 + }, + { + "auxiliary_loss_clip": 0.01050299, + "auxiliary_loss_mlp": 0.01008029, + "balance_loss_clip": 1.0192132, + "balance_loss_mlp": 1.00631237, + "epoch": 0.045586443038709304, + "flos": 62006130599040.0, + "grad_norm": 0.6587392593798193, + "language_loss": 0.49481004, + "learning_rate": 3.997453548850823e-06, + "loss": 0.51539332, + "num_input_tokens_seen": 43999015, + "router_z_loss_clip": 0.31054688, + "router_z_loss_mlp": 0.01721191, + "step": 1571, + "time_per_iteration": 5.69406533241272 + }, + { + "auxiliary_loss_clip": 0.01209873, + "auxiliary_loss_mlp": 0.01065987, + "balance_loss_clip": 1.07945013, + "balance_loss_mlp": 1.04330146, + "epoch": 0.04561546050722535, + "flos": 26607149089920.0, + "grad_norm": 2.599752535354768, + "language_loss": 0.84277272, + "learning_rate": 3.997444058053352e-06, + "loss": 0.86553138, + "num_input_tokens_seen": 44013305, + "router_z_loss_clip": 1.30566406, + "router_z_loss_mlp": 0.22705078, + "step": 1572, + "time_per_iteration": 2.541109323501587 + }, + { + "auxiliary_loss_clip": 0.01209562, + "auxiliary_loss_mlp": 0.0105627, + "balance_loss_clip": 1.08142161, + "balance_loss_mlp": 1.03503907, + "epoch": 0.04564447797574139, + "flos": 74731431623040.0, + "grad_norm": 2.4793324657413565, + "language_loss": 0.80109739, + "learning_rate": 3.997434549613646e-06, + "loss": 0.82375574, + "num_input_tokens_seen": 44035210, + "router_z_loss_clip": 1.28027344, + "router_z_loss_mlp": 0.21203613, + "step": 1573, + "time_per_iteration": 2.9473252296447754 + }, + { + "auxiliary_loss_clip": 0.01185078, + "auxiliary_loss_mlp": 0.01054728, + "balance_loss_clip": 1.07062304, + "balance_loss_mlp": 1.03420579, + "epoch": 0.045673495444257445, + "flos": 74739368528640.0, + "grad_norm": 1.7871750919262155, + "language_loss": 0.83425075, + "learning_rate": 3.997425023531789e-06, + "loss": 0.8566488, + "num_input_tokens_seen": 44067440, + "router_z_loss_clip": 1.14355469, + "router_z_loss_mlp": 0.2053833, + "step": 1574, + "time_per_iteration": 2.917278289794922 + }, + { + "auxiliary_loss_clip": 0.01196935, + "auxiliary_loss_mlp": 0.01073828, + "balance_loss_clip": 1.07774973, + "balance_loss_mlp": 1.05351508, + "epoch": 0.04570251291277349, + "flos": 9641584465920.0, + "grad_norm": 2.9657017132562977, + "language_loss": 0.85653102, + "learning_rate": 3.997415479807867e-06, + "loss": 0.87923872, + "num_input_tokens_seen": 44078225, + "router_z_loss_clip": 1.19140625, + "router_z_loss_mlp": 0.203125, + "step": 1575, + "time_per_iteration": 2.5148441791534424 + }, + { + "auxiliary_loss_clip": 0.01050754, + "auxiliary_loss_mlp": 0.01011056, + "balance_loss_clip": 1.01982176, + "balance_loss_mlp": 1.00939894, + "epoch": 0.045731530381289534, + "flos": 72121304908800.0, + "grad_norm": 0.6844621454903494, + "language_loss": 0.50654846, + "learning_rate": 3.997405918441963e-06, + "loss": 0.5271666, + "num_input_tokens_seen": 44146605, + "router_z_loss_clip": 0.30859375, + "router_z_loss_mlp": 0.01660156, + "step": 1576, + "time_per_iteration": 3.167924165725708 + }, + { + "auxiliary_loss_clip": 0.01050343, + "auxiliary_loss_mlp": 0.01003628, + "balance_loss_clip": 1.01950622, + "balance_loss_mlp": 1.0018878, + "epoch": 0.045760547849805586, + "flos": 68724155295360.0, + "grad_norm": 0.6637795841574432, + "language_loss": 0.50323629, + "learning_rate": 3.9973963394341616e-06, + "loss": 0.52377605, + "num_input_tokens_seen": 44211840, + "router_z_loss_clip": 0.30859375, + "router_z_loss_mlp": 0.01745605, + "step": 1577, + "time_per_iteration": 3.093076705932617 + }, + { + "auxiliary_loss_clip": 0.01050431, + "auxiliary_loss_mlp": 0.0100398, + "balance_loss_clip": 1.0195024, + "balance_loss_mlp": 1.0024662, + "epoch": 0.04578956531832163, + "flos": 67179744172800.0, + "grad_norm": 0.7029552679089853, + "language_loss": 0.51695216, + "learning_rate": 3.997386742784547e-06, + "loss": 0.53749627, + "num_input_tokens_seen": 44271135, + "router_z_loss_clip": 0.30859375, + "router_z_loss_mlp": 0.01513672, + "step": 1578, + "time_per_iteration": 2.9558565616607666 + }, + { + "auxiliary_loss_clip": 0.01205071, + "auxiliary_loss_mlp": 0.01054401, + "balance_loss_clip": 1.07560253, + "balance_loss_mlp": 1.03294373, + "epoch": 0.045818582786837675, + "flos": 41458473233280.0, + "grad_norm": 1.8256198477226468, + "language_loss": 0.82029152, + "learning_rate": 3.997377128493205e-06, + "loss": 0.84288633, + "num_input_tokens_seen": 44296225, + "router_z_loss_clip": 1.296875, + "router_z_loss_mlp": 0.21484375, + "step": 1579, + "time_per_iteration": 2.894242525100708 + }, + { + "auxiliary_loss_clip": 0.01191463, + "auxiliary_loss_mlp": 0.01067745, + "balance_loss_clip": 1.07094026, + "balance_loss_mlp": 1.04340303, + "epoch": 0.04584760025535372, + "flos": 37588631616000.0, + "grad_norm": 3.6503391744297127, + "language_loss": 0.94995993, + "learning_rate": 3.99736749656022e-06, + "loss": 0.97255206, + "num_input_tokens_seen": 44313700, + "router_z_loss_clip": 1.20507812, + "router_z_loss_mlp": 0.24310303, + "step": 1580, + "time_per_iteration": 2.7016406059265137 + }, + { + "auxiliary_loss_clip": 0.0104801, + "auxiliary_loss_mlp": 0.01000644, + "balance_loss_clip": 1.01725531, + "balance_loss_mlp": 0.99901122, + "epoch": 0.04587661772386977, + "flos": 74795321917440.0, + "grad_norm": 0.6999859708479684, + "language_loss": 0.50244749, + "learning_rate": 3.997357846985677e-06, + "loss": 0.52293402, + "num_input_tokens_seen": 44382450, + "router_z_loss_clip": 0.30859375, + "router_z_loss_mlp": 0.01635742, + "step": 1581, + "time_per_iteration": 3.2641520500183105 + }, + { + "auxiliary_loss_clip": 0.01180151, + "auxiliary_loss_mlp": 0.01052748, + "balance_loss_clip": 1.06804323, + "balance_loss_mlp": 1.03385293, + "epoch": 0.045905635192385816, + "flos": 20041709368320.0, + "grad_norm": 4.130198704972614, + "language_loss": 1.09012258, + "learning_rate": 3.997348179769661e-06, + "loss": 1.11245155, + "num_input_tokens_seen": 44398985, + "router_z_loss_clip": 1.12109375, + "router_z_loss_mlp": 0.18890381, + "step": 1582, + "time_per_iteration": 2.493239402770996 + }, + { + "auxiliary_loss_clip": 0.01047358, + "auxiliary_loss_mlp": 0.01010231, + "balance_loss_clip": 1.01691997, + "balance_loss_mlp": 1.00862205, + "epoch": 0.04593465266090186, + "flos": 71807910744960.0, + "grad_norm": 0.7384007649823531, + "language_loss": 0.49593294, + "learning_rate": 3.997338494912258e-06, + "loss": 0.51650882, + "num_input_tokens_seen": 44452490, + "router_z_loss_clip": 0.3046875, + "router_z_loss_mlp": 0.01611328, + "step": 1583, + "time_per_iteration": 2.9221739768981934 + }, + { + "auxiliary_loss_clip": 0.01208291, + "auxiliary_loss_mlp": 0.01057407, + "balance_loss_clip": 1.07772279, + "balance_loss_mlp": 1.03449535, + "epoch": 0.04596367012941791, + "flos": 21359523542400.0, + "grad_norm": 6.421168915373509, + "language_loss": 0.87911576, + "learning_rate": 3.997328792413552e-06, + "loss": 0.90177274, + "num_input_tokens_seen": 44467060, + "router_z_loss_clip": 1.30566406, + "router_z_loss_mlp": 0.22888184, + "step": 1584, + "time_per_iteration": 2.622950315475464 + }, + { + "auxiliary_loss_clip": 0.01200228, + "auxiliary_loss_mlp": 0.01077299, + "balance_loss_clip": 1.07175076, + "balance_loss_mlp": 1.05546021, + "epoch": 0.04599268759793396, + "flos": 29455260341760.0, + "grad_norm": 2.4422624138161195, + "language_loss": 0.99712199, + "learning_rate": 3.997319072273631e-06, + "loss": 1.01989734, + "num_input_tokens_seen": 44483245, + "router_z_loss_clip": 1.28417969, + "router_z_loss_mlp": 0.21838379, + "step": 1585, + "time_per_iteration": 2.661879777908325 + }, + { + "auxiliary_loss_clip": 0.01190981, + "auxiliary_loss_mlp": 0.01061949, + "balance_loss_clip": 1.07029521, + "balance_loss_mlp": 1.04133737, + "epoch": 0.04602170506645, + "flos": 31973172842880.0, + "grad_norm": 2.7036461007779025, + "language_loss": 0.88279599, + "learning_rate": 3.997309334492579e-06, + "loss": 0.90532541, + "num_input_tokens_seen": 44499005, + "router_z_loss_clip": 1.20703125, + "router_z_loss_mlp": 0.20617676, + "step": 1586, + "time_per_iteration": 2.623253107070923 + }, + { + "auxiliary_loss_clip": 0.01048266, + "auxiliary_loss_mlp": 0.010018, + "balance_loss_clip": 1.01784277, + "balance_loss_mlp": 1.00033402, + "epoch": 0.04605072253496605, + "flos": 74776467277440.0, + "grad_norm": 0.7246292190049636, + "language_loss": 0.49347892, + "learning_rate": 3.997299579070483e-06, + "loss": 0.51397955, + "num_input_tokens_seen": 44562570, + "router_z_loss_clip": 0.3046875, + "router_z_loss_mlp": 0.01464844, + "step": 1587, + "time_per_iteration": 3.0996086597442627 + }, + { + "auxiliary_loss_clip": 0.01047868, + "auxiliary_loss_mlp": 0.0100612, + "balance_loss_clip": 1.0174737, + "balance_loss_mlp": 1.00448632, + "epoch": 0.0460797400034821, + "flos": 74783865479040.0, + "grad_norm": 0.6847168293997559, + "language_loss": 0.52563357, + "learning_rate": 3.99728980600743e-06, + "loss": 0.54617345, + "num_input_tokens_seen": 44633675, + "router_z_loss_clip": 0.3046875, + "router_z_loss_mlp": 0.01635742, + "step": 1588, + "time_per_iteration": 3.1775259971618652 + }, + { + "auxiliary_loss_clip": 0.01204629, + "auxiliary_loss_mlp": 0.01073108, + "balance_loss_clip": 1.0744071, + "balance_loss_mlp": 1.04865837, + "epoch": 0.04610875747199814, + "flos": 27855584144640.0, + "grad_norm": 3.3369264070080344, + "language_loss": 0.94657838, + "learning_rate": 3.997280015303504e-06, + "loss": 0.96935576, + "num_input_tokens_seen": 44652845, + "router_z_loss_clip": 1.30273438, + "router_z_loss_mlp": 0.24462891, + "step": 1589, + "time_per_iteration": 2.6098239421844482 + }, + { + "auxiliary_loss_clip": 0.01191881, + "auxiliary_loss_mlp": 0.01046126, + "balance_loss_clip": 1.07060504, + "balance_loss_mlp": 1.02594364, + "epoch": 0.04613777494051419, + "flos": 24017379431040.0, + "grad_norm": 2.422705409358909, + "language_loss": 0.80063093, + "learning_rate": 3.997270206958793e-06, + "loss": 0.82301092, + "num_input_tokens_seen": 44666580, + "router_z_loss_clip": 1.21289062, + "router_z_loss_mlp": 0.20178223, + "step": 1590, + "time_per_iteration": 2.465461254119873 + }, + { + "auxiliary_loss_clip": 0.01215384, + "auxiliary_loss_mlp": 0.01068601, + "balance_loss_clip": 1.0802108, + "balance_loss_mlp": 1.04267263, + "epoch": 0.04616679240903024, + "flos": 13031982322560.0, + "grad_norm": 3.1473218841366855, + "language_loss": 0.93918991, + "learning_rate": 3.997260380973384e-06, + "loss": 0.96202981, + "num_input_tokens_seen": 44676330, + "router_z_loss_clip": 1.3515625, + "router_z_loss_mlp": 0.2590332, + "step": 1591, + "time_per_iteration": 2.5277493000030518 + }, + { + "auxiliary_loss_clip": 0.0104865, + "auxiliary_loss_mlp": 0.01011968, + "balance_loss_clip": 1.0181663, + "balance_loss_mlp": 1.01014447, + "epoch": 0.04619580987754628, + "flos": 74768171235840.0, + "grad_norm": 0.6650005629955033, + "language_loss": 0.52565527, + "learning_rate": 3.9972505373473626e-06, + "loss": 0.54626143, + "num_input_tokens_seen": 44737345, + "router_z_loss_clip": 0.3046875, + "router_z_loss_mlp": 0.01818848, + "step": 1592, + "time_per_iteration": 3.0504822731018066 + }, + { + "auxiliary_loss_clip": 0.01048567, + "auxiliary_loss_mlp": 0.01005605, + "balance_loss_clip": 1.01804972, + "balance_loss_mlp": 1.00397205, + "epoch": 0.04622482734606233, + "flos": 62226442667520.0, + "grad_norm": 0.672071057106687, + "language_loss": 0.48877382, + "learning_rate": 3.997240676080816e-06, + "loss": 0.50931555, + "num_input_tokens_seen": 44799230, + "router_z_loss_clip": 0.3046875, + "router_z_loss_mlp": 0.01635742, + "step": 1593, + "time_per_iteration": 3.069427490234375 + }, + { + "auxiliary_loss_clip": 0.0104819, + "auxiliary_loss_mlp": 0.00999618, + "balance_loss_clip": 1.0177983, + "balance_loss_mlp": 0.99808007, + "epoch": 0.04625384481457838, + "flos": 68769615957120.0, + "grad_norm": 0.7554312635171133, + "language_loss": 0.56568694, + "learning_rate": 3.997230797173831e-06, + "loss": 0.58616501, + "num_input_tokens_seen": 44849465, + "router_z_loss_clip": 0.3046875, + "router_z_loss_mlp": 0.01538086, + "step": 1594, + "time_per_iteration": 2.8869152069091797 + }, + { + "auxiliary_loss_clip": 0.01199607, + "auxiliary_loss_mlp": 0.01063383, + "balance_loss_clip": 1.07578588, + "balance_loss_mlp": 1.04222322, + "epoch": 0.046282862283094424, + "flos": 12452097156480.0, + "grad_norm": 4.37465113738667, + "language_loss": 0.92918843, + "learning_rate": 3.9972209006264965e-06, + "loss": 0.95181835, + "num_input_tokens_seen": 44860745, + "router_z_loss_clip": 1.23632812, + "router_z_loss_mlp": 0.21154785, + "step": 1595, + "time_per_iteration": 2.441148042678833 + }, + { + "auxiliary_loss_clip": 0.01196506, + "auxiliary_loss_mlp": 0.01052265, + "balance_loss_clip": 1.07148135, + "balance_loss_mlp": 1.03208292, + "epoch": 0.04631187975161047, + "flos": 12380311825920.0, + "grad_norm": 3.4018086043127074, + "language_loss": 0.90086508, + "learning_rate": 3.997210986438898e-06, + "loss": 0.92335278, + "num_input_tokens_seen": 44874910, + "router_z_loss_clip": 1.24902344, + "router_z_loss_mlp": 0.20178223, + "step": 1596, + "time_per_iteration": 2.4938852787017822 + }, + { + "auxiliary_loss_clip": 0.01215176, + "auxiliary_loss_mlp": 0.01065373, + "balance_loss_clip": 1.08272195, + "balance_loss_mlp": 1.04256821, + "epoch": 0.04634089722012651, + "flos": 24568105731840.0, + "grad_norm": 2.706508622318338, + "language_loss": 0.96101737, + "learning_rate": 3.997201054611124e-06, + "loss": 0.98382294, + "num_input_tokens_seen": 44890885, + "router_z_loss_clip": 1.32714844, + "router_z_loss_mlp": 0.22814941, + "step": 1597, + "time_per_iteration": 2.553067684173584 + }, + { + "auxiliary_loss_clip": 0.01205243, + "auxiliary_loss_mlp": 0.01058734, + "balance_loss_clip": 1.07807148, + "balance_loss_mlp": 1.03605413, + "epoch": 0.046369914688642565, + "flos": 29233870865280.0, + "grad_norm": 2.3961989471995695, + "language_loss": 1.04609847, + "learning_rate": 3.997191105143263e-06, + "loss": 1.0687381, + "num_input_tokens_seen": 44905015, + "router_z_loss_clip": 1.27148438, + "router_z_loss_mlp": 0.22692871, + "step": 1598, + "time_per_iteration": 2.5983493328094482 + }, + { + "auxiliary_loss_clip": 0.0120578, + "auxiliary_loss_mlp": 0.01063413, + "balance_loss_clip": 1.07792032, + "balance_loss_mlp": 1.04077506, + "epoch": 0.04639893215715861, + "flos": 29934057657600.0, + "grad_norm": 2.371097221554246, + "language_loss": 0.9908787, + "learning_rate": 3.997181138035401e-06, + "loss": 1.01357055, + "num_input_tokens_seen": 44927515, + "router_z_loss_clip": 1.27832031, + "router_z_loss_mlp": 0.2265625, + "step": 1599, + "time_per_iteration": 2.6636312007904053 + }, + { + "auxiliary_loss_clip": 0.01196066, + "auxiliary_loss_mlp": 0.01060731, + "balance_loss_clip": 1.07238841, + "balance_loss_mlp": 1.03760481, + "epoch": 0.046427949625674654, + "flos": 29752134249600.0, + "grad_norm": 2.8557239380474395, + "language_loss": 0.89043486, + "learning_rate": 3.997171153287627e-06, + "loss": 0.91300273, + "num_input_tokens_seen": 44946205, + "router_z_loss_clip": 1.234375, + "router_z_loss_mlp": 0.23132324, + "step": 1600, + "time_per_iteration": 2.6656765937805176 + }, + { + "auxiliary_loss_clip": 0.01200132, + "auxiliary_loss_mlp": 0.01054544, + "balance_loss_clip": 1.07914472, + "balance_loss_mlp": 1.03200102, + "epoch": 0.046456967094190706, + "flos": 47368327875840.0, + "grad_norm": 2.5974962592032624, + "language_loss": 0.82323956, + "learning_rate": 3.9971611509000305e-06, + "loss": 0.84578639, + "num_input_tokens_seen": 44964950, + "router_z_loss_clip": 1.2109375, + "router_z_loss_mlp": 0.22558594, + "step": 1601, + "time_per_iteration": 2.7001636028289795 + }, + { + "auxiliary_loss_clip": 0.01207293, + "auxiliary_loss_mlp": 0.01068249, + "balance_loss_clip": 1.07803559, + "balance_loss_mlp": 1.04507434, + "epoch": 0.04648598456270675, + "flos": 14568600193920.0, + "grad_norm": 2.936529942536502, + "language_loss": 0.74388635, + "learning_rate": 3.997151130872697e-06, + "loss": 0.76664186, + "num_input_tokens_seen": 44977325, + "router_z_loss_clip": 1.29296875, + "router_z_loss_mlp": 0.23193359, + "step": 1602, + "time_per_iteration": 2.4995553493499756 + }, + { + "auxiliary_loss_clip": 0.01203916, + "auxiliary_loss_mlp": 0.01067427, + "balance_loss_clip": 1.07946563, + "balance_loss_mlp": 1.0450511, + "epoch": 0.046515002031222795, + "flos": 12160322979840.0, + "grad_norm": 3.9100052165004824, + "language_loss": 0.85778242, + "learning_rate": 3.997141093205717e-06, + "loss": 0.88049591, + "num_input_tokens_seen": 44990715, + "router_z_loss_clip": 1.24414062, + "router_z_loss_mlp": 0.22375488, + "step": 1603, + "time_per_iteration": 2.534205436706543 + }, + { + "auxiliary_loss_clip": 0.01190507, + "auxiliary_loss_mlp": 0.01055213, + "balance_loss_clip": 1.07396579, + "balance_loss_mlp": 1.03492308, + "epoch": 0.04654401949973884, + "flos": 26206924775040.0, + "grad_norm": 2.482258280720382, + "language_loss": 0.77585924, + "learning_rate": 3.997131037899179e-06, + "loss": 0.79831648, + "num_input_tokens_seen": 45006980, + "router_z_loss_clip": 1.16503906, + "router_z_loss_mlp": 0.20288086, + "step": 1604, + "time_per_iteration": 2.639665126800537 + }, + { + "auxiliary_loss_clip": 0.01055836, + "auxiliary_loss_mlp": 0.01035977, + "balance_loss_clip": 1.02523065, + "balance_loss_mlp": 1.03398597, + "epoch": 0.04657303696825489, + "flos": 67159527793920.0, + "grad_norm": 0.771175998081534, + "language_loss": 0.52632427, + "learning_rate": 3.997120964953171e-06, + "loss": 0.5472424, + "num_input_tokens_seen": 45068160, + "router_z_loss_clip": 0.30664062, + "router_z_loss_mlp": 0.01989746, + "step": 1605, + "time_per_iteration": 3.141751289367676 + }, + { + "auxiliary_loss_clip": 0.01052848, + "auxiliary_loss_mlp": 0.01018116, + "balance_loss_clip": 1.02236605, + "balance_loss_mlp": 1.01626813, + "epoch": 0.046602054436770936, + "flos": 60433673932800.0, + "grad_norm": 0.7599428105716878, + "language_loss": 0.52229786, + "learning_rate": 3.997110874367784e-06, + "loss": 0.54300755, + "num_input_tokens_seen": 45128660, + "router_z_loss_clip": 0.3046875, + "router_z_loss_mlp": 0.01843262, + "step": 1606, + "time_per_iteration": 3.1183254718780518 + }, + { + "auxiliary_loss_clip": 0.01206386, + "auxiliary_loss_mlp": 0.01069509, + "balance_loss_clip": 1.07682538, + "balance_loss_mlp": 1.04782462, + "epoch": 0.04663107190528698, + "flos": 16102991422080.0, + "grad_norm": 3.2657536230076634, + "language_loss": 0.80197644, + "learning_rate": 3.997100766143104e-06, + "loss": 0.8247354, + "num_input_tokens_seen": 45142810, + "router_z_loss_clip": 1.296875, + "router_z_loss_mlp": 0.2164917, + "step": 1607, + "time_per_iteration": 2.5067079067230225 + }, + { + "auxiliary_loss_clip": 0.01197647, + "auxiliary_loss_mlp": 0.01050609, + "balance_loss_clip": 1.07462406, + "balance_loss_mlp": 1.0299139, + "epoch": 0.04666008937380303, + "flos": 16359106371840.0, + "grad_norm": 2.7991486302617274, + "language_loss": 0.91027629, + "learning_rate": 3.997090640279222e-06, + "loss": 0.93275881, + "num_input_tokens_seen": 45157025, + "router_z_loss_clip": 1.22949219, + "router_z_loss_mlp": 0.20690918, + "step": 1608, + "time_per_iteration": 2.5850250720977783 + }, + { + "auxiliary_loss_clip": 0.01210073, + "auxiliary_loss_mlp": 0.01060264, + "balance_loss_clip": 1.08236098, + "balance_loss_mlp": 1.0388062, + "epoch": 0.04668910684231908, + "flos": 52849769005440.0, + "grad_norm": 1.9775972619058109, + "language_loss": 0.89683491, + "learning_rate": 3.9970804967762276e-06, + "loss": 0.91953832, + "num_input_tokens_seen": 45185775, + "router_z_loss_clip": 1.27734375, + "router_z_loss_mlp": 0.21459961, + "step": 1609, + "time_per_iteration": 2.8347976207733154 + }, + { + "auxiliary_loss_clip": 0.01204689, + "auxiliary_loss_mlp": 0.01061582, + "balance_loss_clip": 1.07675958, + "balance_loss_mlp": 1.03982639, + "epoch": 0.04671812431083512, + "flos": 27191595283200.0, + "grad_norm": 2.6327968950125102, + "language_loss": 1.02934492, + "learning_rate": 3.997070335634211e-06, + "loss": 1.05200768, + "num_input_tokens_seen": 45198860, + "router_z_loss_clip": 1.27832031, + "router_z_loss_mlp": 0.21740723, + "step": 1610, + "time_per_iteration": 2.559732437133789 + }, + { + "auxiliary_loss_clip": 0.01203119, + "auxiliary_loss_mlp": 0.01064963, + "balance_loss_clip": 1.07825363, + "balance_loss_mlp": 1.0438453, + "epoch": 0.046747141779351166, + "flos": 18763073953920.0, + "grad_norm": 2.7714179591766306, + "language_loss": 0.72639477, + "learning_rate": 3.99706015685326e-06, + "loss": 0.74907559, + "num_input_tokens_seen": 45212515, + "router_z_loss_clip": 1.24951172, + "router_z_loss_mlp": 0.21142578, + "step": 1611, + "time_per_iteration": 2.544804334640503 + }, + { + "auxiliary_loss_clip": 0.01208456, + "auxiliary_loss_mlp": 0.01062784, + "balance_loss_clip": 1.08139956, + "balance_loss_mlp": 1.03951478, + "epoch": 0.04677615924786722, + "flos": 23692208584320.0, + "grad_norm": 2.0164721126387297, + "language_loss": 0.77093244, + "learning_rate": 3.997049960433466e-06, + "loss": 0.79364491, + "num_input_tokens_seen": 45230125, + "router_z_loss_clip": 1.27148438, + "router_z_loss_mlp": 0.23266602, + "step": 1612, + "time_per_iteration": 2.572195291519165 + }, + { + "auxiliary_loss_clip": 0.01210884, + "auxiliary_loss_mlp": 0.01078064, + "balance_loss_clip": 1.08416152, + "balance_loss_mlp": 1.05631971, + "epoch": 0.04680517671638326, + "flos": 26389746023040.0, + "grad_norm": 2.6540004500340575, + "language_loss": 0.97954071, + "learning_rate": 3.997039746374918e-06, + "loss": 1.0024302, + "num_input_tokens_seen": 45244655, + "router_z_loss_clip": 1.26660156, + "router_z_loss_mlp": 0.2175293, + "step": 1613, + "time_per_iteration": 2.587846517562866 + }, + { + "auxiliary_loss_clip": 0.01207306, + "auxiliary_loss_mlp": 0.0107058, + "balance_loss_clip": 1.08179331, + "balance_loss_mlp": 1.05052924, + "epoch": 0.04683419418489931, + "flos": 41274969626880.0, + "grad_norm": 2.6327567243868137, + "language_loss": 0.71927661, + "learning_rate": 3.997029514677708e-06, + "loss": 0.74205548, + "num_input_tokens_seen": 45263760, + "router_z_loss_clip": 1.25488281, + "router_z_loss_mlp": 0.20068359, + "step": 1614, + "time_per_iteration": 2.7477807998657227 + }, + { + "auxiliary_loss_clip": 0.0119853, + "auxiliary_loss_mlp": 0.01071261, + "balance_loss_clip": 1.08095169, + "balance_loss_mlp": 1.05185366, + "epoch": 0.04686321165341536, + "flos": 15888964233600.0, + "grad_norm": 3.2200779693036696, + "language_loss": 0.68684322, + "learning_rate": 3.997019265341924e-06, + "loss": 0.70954114, + "num_input_tokens_seen": 45283035, + "router_z_loss_clip": 1.17578125, + "router_z_loss_mlp": 0.19421387, + "step": 1615, + "time_per_iteration": 2.565824270248413 + }, + { + "auxiliary_loss_clip": 0.01207931, + "auxiliary_loss_mlp": 0.01063573, + "balance_loss_clip": 1.08573771, + "balance_loss_mlp": 1.04485095, + "epoch": 0.0468922291219314, + "flos": 32052859165440.0, + "grad_norm": 2.177343542461303, + "language_loss": 0.98451, + "learning_rate": 3.997008998367658e-06, + "loss": 1.00722504, + "num_input_tokens_seen": 45302530, + "router_z_loss_clip": 1.22363281, + "router_z_loss_mlp": 0.18707275, + "step": 1616, + "time_per_iteration": 2.584343194961548 + }, + { + "auxiliary_loss_clip": 0.01073117, + "auxiliary_loss_mlp": 0.01150676, + "balance_loss_clip": 1.04260135, + "balance_loss_mlp": 1.14818466, + "epoch": 0.04692124659044745, + "flos": 74785373850240.0, + "grad_norm": 0.7323581297512627, + "language_loss": 0.46118498, + "learning_rate": 3.996998713755001e-06, + "loss": 0.48342288, + "num_input_tokens_seen": 45364585, + "router_z_loss_clip": 0.3046875, + "router_z_loss_mlp": 0.02490234, + "step": 1617, + "time_per_iteration": 3.21779727935791 + }, + { + "auxiliary_loss_clip": 0.01066991, + "auxiliary_loss_mlp": 0.01064566, + "balance_loss_clip": 1.03662157, + "balance_loss_mlp": 1.06246781, + "epoch": 0.0469502640589635, + "flos": 65657419914240.0, + "grad_norm": 0.6523794122100057, + "language_loss": 0.51785254, + "learning_rate": 3.9969884115040435e-06, + "loss": 0.53916812, + "num_input_tokens_seen": 45429370, + "router_z_loss_clip": 0.3046875, + "router_z_loss_mlp": 0.02099609, + "step": 1618, + "time_per_iteration": 3.162813901901245 + }, + { + "auxiliary_loss_clip": 0.01191756, + "auxiliary_loss_mlp": 0.01056387, + "balance_loss_clip": 1.07386279, + "balance_loss_mlp": 1.03442883, + "epoch": 0.046979281527479544, + "flos": 25587142577280.0, + "grad_norm": 2.6313403416166024, + "language_loss": 0.92462623, + "learning_rate": 3.996978091614875e-06, + "loss": 0.94710767, + "num_input_tokens_seen": 45446955, + "router_z_loss_clip": 1.17871094, + "router_z_loss_mlp": 0.21948242, + "step": 1619, + "time_per_iteration": 2.572735071182251 + }, + { + "auxiliary_loss_clip": 0.01052298, + "auxiliary_loss_mlp": 0.01076015, + "balance_loss_clip": 1.02126372, + "balance_loss_mlp": 1.07429838, + "epoch": 0.04700829899599559, + "flos": 61664583150720.0, + "grad_norm": 0.7843244015967828, + "language_loss": 0.5608561, + "learning_rate": 3.996967754087589e-06, + "loss": 0.58213919, + "num_input_tokens_seen": 45497800, + "router_z_loss_clip": 0.31054688, + "router_z_loss_mlp": 0.01721191, + "step": 1620, + "time_per_iteration": 2.8667585849761963 + }, + { + "auxiliary_loss_clip": 0.01188936, + "auxiliary_loss_mlp": 0.01072091, + "balance_loss_clip": 1.07138467, + "balance_loss_mlp": 1.05060911, + "epoch": 0.04703731646451163, + "flos": 33576655881600.0, + "grad_norm": 3.3700072013219273, + "language_loss": 0.94637072, + "learning_rate": 3.996957398922275e-06, + "loss": 0.96898103, + "num_input_tokens_seen": 45516155, + "router_z_loss_clip": 1.17675781, + "router_z_loss_mlp": 0.21472168, + "step": 1621, + "time_per_iteration": 2.614245653152466 + }, + { + "auxiliary_loss_clip": 0.01197272, + "auxiliary_loss_mlp": 0.01086651, + "balance_loss_clip": 1.07715023, + "balance_loss_mlp": 1.06627822, + "epoch": 0.047066333933027685, + "flos": 11282881547520.0, + "grad_norm": 3.202907914956559, + "language_loss": 0.91376048, + "learning_rate": 3.996947026119026e-06, + "loss": 0.93659979, + "num_input_tokens_seen": 45528870, + "router_z_loss_clip": 1.203125, + "router_z_loss_mlp": 0.20373535, + "step": 1622, + "time_per_iteration": 2.604245901107788 + }, + { + "auxiliary_loss_clip": 0.01067336, + "auxiliary_loss_mlp": 0.01132703, + "balance_loss_clip": 1.03619909, + "balance_loss_mlp": 1.13099813, + "epoch": 0.04709535140154373, + "flos": 74780381859840.0, + "grad_norm": 0.7131032718632854, + "language_loss": 0.49313819, + "learning_rate": 3.996936635677932e-06, + "loss": 0.51513857, + "num_input_tokens_seen": 45589075, + "router_z_loss_clip": 0.3125, + "router_z_loss_mlp": 0.01708984, + "step": 1623, + "time_per_iteration": 3.1475605964660645 + }, + { + "auxiliary_loss_clip": 0.01068483, + "auxiliary_loss_mlp": 0.01126321, + "balance_loss_clip": 1.03754973, + "balance_loss_mlp": 1.12469947, + "epoch": 0.047124368870059774, + "flos": 60322853496960.0, + "grad_norm": 0.7219059611309773, + "language_loss": 0.47014648, + "learning_rate": 3.996926227599085e-06, + "loss": 0.49209452, + "num_input_tokens_seen": 45645060, + "router_z_loss_clip": 0.30859375, + "router_z_loss_mlp": 0.01623535, + "step": 1624, + "time_per_iteration": 3.033684730529785 + }, + { + "auxiliary_loss_clip": 0.01187957, + "auxiliary_loss_mlp": 0.01086589, + "balance_loss_clip": 1.07578015, + "balance_loss_mlp": 1.06774163, + "epoch": 0.047153386338575826, + "flos": 17230981196160.0, + "grad_norm": 3.485655560184229, + "language_loss": 0.76600111, + "learning_rate": 3.996915801882579e-06, + "loss": 0.78874654, + "num_input_tokens_seen": 45656125, + "router_z_loss_clip": 1.12109375, + "router_z_loss_mlp": 0.18835449, + "step": 1625, + "time_per_iteration": 2.548041582107544 + }, + { + "auxiliary_loss_clip": 0.0119049, + "auxiliary_loss_mlp": 0.0107293, + "balance_loss_clip": 1.07283592, + "balance_loss_mlp": 1.05384469, + "epoch": 0.04718240380709187, + "flos": 11756938268160.0, + "grad_norm": 3.4311349282407937, + "language_loss": 1.01208317, + "learning_rate": 3.996905358528504e-06, + "loss": 1.03471744, + "num_input_tokens_seen": 45666565, + "router_z_loss_clip": 1.17529297, + "router_z_loss_mlp": 0.1907959, + "step": 1626, + "time_per_iteration": 2.4701623916625977 + }, + { + "auxiliary_loss_clip": 0.01056641, + "auxiliary_loss_mlp": 0.01036655, + "balance_loss_clip": 1.02610958, + "balance_loss_mlp": 1.03520095, + "epoch": 0.047211421275607915, + "flos": 70267022133120.0, + "grad_norm": 0.7138660862235916, + "language_loss": 0.55232787, + "learning_rate": 3.996894897536953e-06, + "loss": 0.5732609, + "num_input_tokens_seen": 45735045, + "router_z_loss_clip": 0.3046875, + "router_z_loss_mlp": 0.01452637, + "step": 1627, + "time_per_iteration": 3.251659631729126 + }, + { + "auxiliary_loss_clip": 0.01190484, + "auxiliary_loss_mlp": 0.01054847, + "balance_loss_clip": 1.07010019, + "balance_loss_mlp": 1.03518963, + "epoch": 0.04724043874412396, + "flos": 37370941240320.0, + "grad_norm": 2.867059217237847, + "language_loss": 0.93928432, + "learning_rate": 3.9968844189080174e-06, + "loss": 0.96173763, + "num_input_tokens_seen": 45750460, + "router_z_loss_clip": 1.20410156, + "router_z_loss_mlp": 0.19641113, + "step": 1628, + "time_per_iteration": 2.6627182960510254 + }, + { + "auxiliary_loss_clip": 0.01198806, + "auxiliary_loss_mlp": 0.0106314, + "balance_loss_clip": 1.07490599, + "balance_loss_mlp": 1.04230237, + "epoch": 0.04726945621264001, + "flos": 13918007105280.0, + "grad_norm": 3.2474042273209043, + "language_loss": 1.03211462, + "learning_rate": 3.996873922641791e-06, + "loss": 1.05473423, + "num_input_tokens_seen": 45763715, + "router_z_loss_clip": 1.24023438, + "router_z_loss_mlp": 0.20861816, + "step": 1629, + "time_per_iteration": 2.4813005924224854 + }, + { + "auxiliary_loss_clip": 0.01178619, + "auxiliary_loss_mlp": 0.01055036, + "balance_loss_clip": 1.07019567, + "balance_loss_mlp": 1.03758407, + "epoch": 0.047298473681156056, + "flos": 10552530309120.0, + "grad_norm": 3.530656454983885, + "language_loss": 0.97548288, + "learning_rate": 3.996863408738366e-06, + "loss": 0.99781942, + "num_input_tokens_seen": 45774190, + "router_z_loss_clip": 1.08447266, + "router_z_loss_mlp": 0.17456055, + "step": 1630, + "time_per_iteration": 2.43184494972229 + }, + { + "auxiliary_loss_clip": 0.01204968, + "auxiliary_loss_mlp": 0.01060518, + "balance_loss_clip": 1.07841754, + "balance_loss_mlp": 1.03777254, + "epoch": 0.0473274911496721, + "flos": 26099300649600.0, + "grad_norm": 1.8801246669259828, + "language_loss": 0.82228237, + "learning_rate": 3.996852877197835e-06, + "loss": 0.84493721, + "num_input_tokens_seen": 45795905, + "router_z_loss_clip": 1.26367188, + "router_z_loss_mlp": 0.22741699, + "step": 1631, + "time_per_iteration": 2.6351542472839355 + }, + { + "auxiliary_loss_clip": 0.0118491, + "auxiliary_loss_mlp": 0.01054262, + "balance_loss_clip": 1.07269216, + "balance_loss_mlp": 1.03650022, + "epoch": 0.04735650861818815, + "flos": 11027448956160.0, + "grad_norm": 2.8972787118382697, + "language_loss": 0.79031849, + "learning_rate": 3.996842328020292e-06, + "loss": 0.81271023, + "num_input_tokens_seen": 45809505, + "router_z_loss_clip": 1.12158203, + "router_z_loss_mlp": 0.1776123, + "step": 1632, + "time_per_iteration": 2.447805643081665 + }, + { + "auxiliary_loss_clip": 0.01066684, + "auxiliary_loss_mlp": 0.01094407, + "balance_loss_clip": 1.03300476, + "balance_loss_mlp": 1.09265482, + "epoch": 0.0473855260867042, + "flos": 59700270038400.0, + "grad_norm": 0.7671492843087377, + "language_loss": 0.5419066, + "learning_rate": 3.99683176120583e-06, + "loss": 0.56351757, + "num_input_tokens_seen": 45866390, + "router_z_loss_clip": 0.3359375, + "router_z_loss_mlp": 0.01757812, + "step": 1633, + "time_per_iteration": 2.993671417236328 + }, + { + "auxiliary_loss_clip": 0.01211939, + "auxiliary_loss_mlp": 0.01057187, + "balance_loss_clip": 1.08312345, + "balance_loss_mlp": 1.03514516, + "epoch": 0.04741454355522024, + "flos": 22668431143680.0, + "grad_norm": 2.568110635696357, + "language_loss": 0.99741101, + "learning_rate": 3.996821176754541e-06, + "loss": 1.02010226, + "num_input_tokens_seen": 45881810, + "router_z_loss_clip": 1.2890625, + "router_z_loss_mlp": 0.22058105, + "step": 1634, + "time_per_iteration": 2.5083529949188232 + }, + { + "auxiliary_loss_clip": 0.01066405, + "auxiliary_loss_mlp": 0.01034159, + "balance_loss_clip": 1.03491473, + "balance_loss_mlp": 1.03237057, + "epoch": 0.047443561023736286, + "flos": 74795321917440.0, + "grad_norm": 0.6001278201791098, + "language_loss": 0.50139105, + "learning_rate": 3.996810574666519e-06, + "loss": 0.52239668, + "num_input_tokens_seen": 45955650, + "router_z_loss_clip": 0.31640625, + "router_z_loss_mlp": 0.01782227, + "step": 1635, + "time_per_iteration": 3.3972713947296143 + }, + { + "auxiliary_loss_clip": 0.01190654, + "auxiliary_loss_mlp": 0.01053336, + "balance_loss_clip": 1.07637739, + "balance_loss_mlp": 1.03426802, + "epoch": 0.04747257849225234, + "flos": 43609450348800.0, + "grad_norm": 2.2033205398669136, + "language_loss": 0.88702625, + "learning_rate": 3.996799954941859e-06, + "loss": 0.90946615, + "num_input_tokens_seen": 45973715, + "router_z_loss_clip": 1.14355469, + "router_z_loss_mlp": 0.19061279, + "step": 1636, + "time_per_iteration": 2.6608128547668457 + }, + { + "auxiliary_loss_clip": 0.01183938, + "auxiliary_loss_mlp": 0.01054621, + "balance_loss_clip": 1.06886721, + "balance_loss_mlp": 1.03542793, + "epoch": 0.04750159596076838, + "flos": 16322728872960.0, + "grad_norm": 2.453360184341268, + "language_loss": 0.69826221, + "learning_rate": 3.9967893175806535e-06, + "loss": 0.72064781, + "num_input_tokens_seen": 45988600, + "router_z_loss_clip": 1.15087891, + "router_z_loss_mlp": 0.19189453, + "step": 1637, + "time_per_iteration": 2.500235080718994 + }, + { + "auxiliary_loss_clip": 0.01200412, + "auxiliary_loss_mlp": 0.01055934, + "balance_loss_clip": 1.07713807, + "balance_loss_mlp": 1.03538227, + "epoch": 0.04753061342928443, + "flos": 30845937254400.0, + "grad_norm": 2.322635203399113, + "language_loss": 0.85763896, + "learning_rate": 3.996778662582997e-06, + "loss": 0.88020241, + "num_input_tokens_seen": 46009030, + "router_z_loss_clip": 1.23144531, + "router_z_loss_mlp": 0.20556641, + "step": 1638, + "time_per_iteration": 2.6059491634368896 + }, + { + "auxiliary_loss_clip": 0.01060349, + "auxiliary_loss_mlp": 0.01085652, + "balance_loss_clip": 1.02946734, + "balance_loss_mlp": 1.08382845, + "epoch": 0.04755963089780048, + "flos": 74759408317440.0, + "grad_norm": 0.6823170639217876, + "language_loss": 0.45481491, + "learning_rate": 3.996767989948982e-06, + "loss": 0.47627491, + "num_input_tokens_seen": 46066725, + "router_z_loss_clip": 0.30859375, + "router_z_loss_mlp": 0.01818848, + "step": 1639, + "time_per_iteration": 5.3925981521606445 + }, + { + "auxiliary_loss_clip": 0.01057581, + "auxiliary_loss_mlp": 0.01079148, + "balance_loss_clip": 1.02670133, + "balance_loss_mlp": 1.07735944, + "epoch": 0.04758864836631652, + "flos": 73640289202560.0, + "grad_norm": 0.674795608262572, + "language_loss": 0.50599295, + "learning_rate": 3.996757299678705e-06, + "loss": 0.52736026, + "num_input_tokens_seen": 46123730, + "router_z_loss_clip": 0.30859375, + "router_z_loss_mlp": 0.01782227, + "step": 1640, + "time_per_iteration": 5.397092819213867 + }, + { + "auxiliary_loss_clip": 0.01199681, + "auxiliary_loss_mlp": 0.01058226, + "balance_loss_clip": 1.0733788, + "balance_loss_mlp": 1.0376265, + "epoch": 0.04761766583483257, + "flos": 22703372098560.0, + "grad_norm": 2.1072680483124455, + "language_loss": 0.79911083, + "learning_rate": 3.99674659177226e-06, + "loss": 0.8216899, + "num_input_tokens_seen": 46146960, + "router_z_loss_clip": 1.26269531, + "router_z_loss_mlp": 0.20581055, + "step": 1641, + "time_per_iteration": 5.022395133972168 + }, + { + "auxiliary_loss_clip": 0.01057404, + "auxiliary_loss_mlp": 0.01025368, + "balance_loss_clip": 1.02666974, + "balance_loss_mlp": 1.02362716, + "epoch": 0.04764668330334862, + "flos": 60946586190720.0, + "grad_norm": 0.7030431426881883, + "language_loss": 0.50256813, + "learning_rate": 3.99673586622974e-06, + "loss": 0.52339578, + "num_input_tokens_seen": 46204605, + "router_z_loss_clip": 0.30859375, + "router_z_loss_mlp": 0.01745605, + "step": 1642, + "time_per_iteration": 5.523186206817627 + }, + { + "auxiliary_loss_clip": 0.01059338, + "auxiliary_loss_mlp": 0.01013341, + "balance_loss_clip": 1.02824867, + "balance_loss_mlp": 1.01171935, + "epoch": 0.047675700771864664, + "flos": 67632255711360.0, + "grad_norm": 0.6521219656320851, + "language_loss": 0.50919151, + "learning_rate": 3.996725123051242e-06, + "loss": 0.52991825, + "num_input_tokens_seen": 46267920, + "router_z_loss_clip": 0.31054688, + "router_z_loss_mlp": 0.01623535, + "step": 1643, + "time_per_iteration": 3.10556697845459 + }, + { + "auxiliary_loss_clip": 0.01058297, + "auxiliary_loss_mlp": 0.01013844, + "balance_loss_clip": 1.02688968, + "balance_loss_mlp": 1.01200855, + "epoch": 0.04770471824038071, + "flos": 74778334784640.0, + "grad_norm": 0.6379288528863075, + "language_loss": 0.53753591, + "learning_rate": 3.996714362236859e-06, + "loss": 0.55825734, + "num_input_tokens_seen": 46335070, + "router_z_loss_clip": 0.31445312, + "router_z_loss_mlp": 0.01831055, + "step": 1644, + "time_per_iteration": 3.2083840370178223 + }, + { + "auxiliary_loss_clip": 0.01194142, + "auxiliary_loss_mlp": 0.01060309, + "balance_loss_clip": 1.07814562, + "balance_loss_mlp": 1.04065132, + "epoch": 0.04773373570889675, + "flos": 11539786596480.0, + "grad_norm": 2.365531829297608, + "language_loss": 0.73722094, + "learning_rate": 3.996703583786687e-06, + "loss": 0.75976539, + "num_input_tokens_seen": 46347560, + "router_z_loss_clip": 1.16210938, + "router_z_loss_mlp": 0.1965332, + "step": 1645, + "time_per_iteration": 2.4871182441711426 + }, + { + "auxiliary_loss_clip": 0.01054525, + "auxiliary_loss_mlp": 0.01018603, + "balance_loss_clip": 1.02333021, + "balance_loss_mlp": 1.0169338, + "epoch": 0.047762753177412805, + "flos": 74785050627840.0, + "grad_norm": 0.6476222483452487, + "language_loss": 0.51326352, + "learning_rate": 3.996692787700821e-06, + "loss": 0.53399479, + "num_input_tokens_seen": 46415375, + "router_z_loss_clip": 0.3125, + "router_z_loss_mlp": 0.01672363, + "step": 1646, + "time_per_iteration": 3.258739709854126 + }, + { + "auxiliary_loss_clip": 0.01185325, + "auxiliary_loss_mlp": 0.0104656, + "balance_loss_clip": 1.0668745, + "balance_loss_mlp": 1.02548385, + "epoch": 0.04779177064592885, + "flos": 30619268478720.0, + "grad_norm": 5.177192682970783, + "language_loss": 0.80767894, + "learning_rate": 3.996681973979356e-06, + "loss": 0.82999784, + "num_input_tokens_seen": 46430720, + "router_z_loss_clip": 1.18408203, + "router_z_loss_mlp": 0.21075439, + "step": 1647, + "time_per_iteration": 2.5534286499023438 + }, + { + "auxiliary_loss_clip": 0.01215186, + "auxiliary_loss_mlp": 0.01070882, + "balance_loss_clip": 1.0847007, + "balance_loss_mlp": 1.04891229, + "epoch": 0.047820788114444894, + "flos": 23214632330880.0, + "grad_norm": 2.782593513350456, + "language_loss": 0.80598897, + "learning_rate": 3.996671142622389e-06, + "loss": 0.82884967, + "num_input_tokens_seen": 46449005, + "router_z_loss_clip": 1.30664062, + "router_z_loss_mlp": 0.21984863, + "step": 1648, + "time_per_iteration": 2.647864818572998 + }, + { + "auxiliary_loss_clip": 0.01200327, + "auxiliary_loss_mlp": 0.01058558, + "balance_loss_clip": 1.07685423, + "balance_loss_mlp": 1.03623033, + "epoch": 0.047849805582960946, + "flos": 32960644611840.0, + "grad_norm": 2.148845206475005, + "language_loss": 0.87527883, + "learning_rate": 3.996660293630013e-06, + "loss": 0.89786774, + "num_input_tokens_seen": 46466575, + "router_z_loss_clip": 1.23291016, + "router_z_loss_mlp": 0.22332764, + "step": 1649, + "time_per_iteration": 2.6380767822265625 + }, + { + "auxiliary_loss_clip": 0.01063071, + "auxiliary_loss_mlp": 0.01004935, + "balance_loss_clip": 1.033113, + "balance_loss_mlp": 1.00318289, + "epoch": 0.04787882305147699, + "flos": 74776898240640.0, + "grad_norm": 0.6544208875528364, + "language_loss": 0.4800345, + "learning_rate": 3.996649427002326e-06, + "loss": 0.50071454, + "num_input_tokens_seen": 46534210, + "router_z_loss_clip": 0.30078125, + "router_z_loss_mlp": 0.01757812, + "step": 1650, + "time_per_iteration": 3.1480417251586914 + }, + { + "auxiliary_loss_clip": 0.01193856, + "auxiliary_loss_mlp": 0.01047204, + "balance_loss_clip": 1.07770371, + "balance_loss_mlp": 1.02724886, + "epoch": 0.047907840519993035, + "flos": 45835588673280.0, + "grad_norm": 2.8716464421297605, + "language_loss": 0.73345351, + "learning_rate": 3.996638542739423e-06, + "loss": 0.75586414, + "num_input_tokens_seen": 46551900, + "router_z_loss_clip": 1.1640625, + "router_z_loss_mlp": 0.19958496, + "step": 1651, + "time_per_iteration": 2.709648847579956 + }, + { + "auxiliary_loss_clip": 0.01062897, + "auxiliary_loss_mlp": 0.0100893, + "balance_loss_clip": 1.03278041, + "balance_loss_mlp": 1.00722563, + "epoch": 0.04793685798850908, + "flos": 70281276854400.0, + "grad_norm": 0.7443571886364412, + "language_loss": 0.5474875, + "learning_rate": 3.9966276408414005e-06, + "loss": 0.56820571, + "num_input_tokens_seen": 46611870, + "router_z_loss_clip": 0.30078125, + "router_z_loss_mlp": 0.01708984, + "step": 1652, + "time_per_iteration": 3.029003381729126 + }, + { + "auxiliary_loss_clip": 0.01200088, + "auxiliary_loss_mlp": 0.01061678, + "balance_loss_clip": 1.07466197, + "balance_loss_mlp": 1.03857517, + "epoch": 0.04796587545702513, + "flos": 26863551348480.0, + "grad_norm": 2.849981287249037, + "language_loss": 1.03773379, + "learning_rate": 3.996616721308355e-06, + "loss": 1.06035137, + "num_input_tokens_seen": 46629220, + "router_z_loss_clip": 1.25488281, + "router_z_loss_mlp": 0.23095703, + "step": 1653, + "time_per_iteration": 2.56294846534729 + }, + { + "auxiliary_loss_clip": 0.01189097, + "auxiliary_loss_mlp": 0.01052205, + "balance_loss_clip": 1.07111883, + "balance_loss_mlp": 1.03013885, + "epoch": 0.047994892925541176, + "flos": 31133258144640.0, + "grad_norm": 3.294500546345424, + "language_loss": 0.81797457, + "learning_rate": 3.996605784140383e-06, + "loss": 0.84038764, + "num_input_tokens_seen": 46643810, + "router_z_loss_clip": 1.18066406, + "router_z_loss_mlp": 0.2208252, + "step": 1654, + "time_per_iteration": 2.5760879516601562 + }, + { + "auxiliary_loss_clip": 0.01055297, + "auxiliary_loss_mlp": 0.01006327, + "balance_loss_clip": 1.02530026, + "balance_loss_mlp": 1.00449109, + "epoch": 0.04802391039405722, + "flos": 63960495644160.0, + "grad_norm": 0.8267475018237075, + "language_loss": 0.49350512, + "learning_rate": 3.99659482933758e-06, + "loss": 0.51412141, + "num_input_tokens_seen": 46697025, + "router_z_loss_clip": 0.30078125, + "router_z_loss_mlp": 0.01831055, + "step": 1655, + "time_per_iteration": 3.068694591522217 + }, + { + "auxiliary_loss_clip": 0.01053164, + "auxiliary_loss_mlp": 0.0100978, + "balance_loss_clip": 1.02296972, + "balance_loss_mlp": 1.00796819, + "epoch": 0.04805292786257327, + "flos": 74788498333440.0, + "grad_norm": 0.7731132258266813, + "language_loss": 0.53427672, + "learning_rate": 3.9965838569000435e-06, + "loss": 0.55490619, + "num_input_tokens_seen": 46765570, + "router_z_loss_clip": 0.30078125, + "router_z_loss_mlp": 0.01806641, + "step": 1656, + "time_per_iteration": 3.2298645973205566 + }, + { + "auxiliary_loss_clip": 0.01195209, + "auxiliary_loss_mlp": 0.01056878, + "balance_loss_clip": 1.0770967, + "balance_loss_mlp": 1.03796005, + "epoch": 0.04808194533108932, + "flos": 18909876839040.0, + "grad_norm": 2.765089537716844, + "language_loss": 0.71602505, + "learning_rate": 3.99657286682787e-06, + "loss": 0.73854589, + "num_input_tokens_seen": 46778620, + "router_z_loss_clip": 1.17773438, + "router_z_loss_mlp": 0.18933105, + "step": 1657, + "time_per_iteration": 2.4471123218536377 + }, + { + "auxiliary_loss_clip": 0.01211618, + "auxiliary_loss_mlp": 0.01057381, + "balance_loss_clip": 1.07966721, + "balance_loss_mlp": 1.03473103, + "epoch": 0.04811096279960536, + "flos": 31533374718720.0, + "grad_norm": 3.721747633598252, + "language_loss": 0.95157927, + "learning_rate": 3.9965618591211585e-06, + "loss": 0.97426933, + "num_input_tokens_seen": 46796975, + "router_z_loss_clip": 1.3203125, + "router_z_loss_mlp": 0.22668457, + "step": 1658, + "time_per_iteration": 2.6337099075317383 + }, + { + "auxiliary_loss_clip": 0.01053856, + "auxiliary_loss_mlp": 0.01000617, + "balance_loss_clip": 1.02332568, + "balance_loss_mlp": 0.99879259, + "epoch": 0.048139980268121406, + "flos": 68576418656640.0, + "grad_norm": 0.6423482777211088, + "language_loss": 0.46538615, + "learning_rate": 3.996550833780004e-06, + "loss": 0.48593086, + "num_input_tokens_seen": 46863770, + "router_z_loss_clip": 0.3046875, + "router_z_loss_mlp": 0.01818848, + "step": 1659, + "time_per_iteration": 3.1707983016967773 + }, + { + "auxiliary_loss_clip": 0.01198428, + "auxiliary_loss_mlp": 0.0106763, + "balance_loss_clip": 1.07546759, + "balance_loss_mlp": 1.04759061, + "epoch": 0.04816899773663746, + "flos": 13988104496640.0, + "grad_norm": 2.5644428303104814, + "language_loss": 1.01755607, + "learning_rate": 3.996539790804505e-06, + "loss": 1.04021668, + "num_input_tokens_seen": 46876295, + "router_z_loss_clip": 1.23144531, + "router_z_loss_mlp": 0.20043945, + "step": 1660, + "time_per_iteration": 2.5316967964172363 + }, + { + "auxiliary_loss_clip": 0.0119412, + "auxiliary_loss_mlp": 0.01063184, + "balance_loss_clip": 1.07240009, + "balance_loss_mlp": 1.04220271, + "epoch": 0.0481980152051535, + "flos": 11831093896320.0, + "grad_norm": 3.1147798756044542, + "language_loss": 0.91902971, + "learning_rate": 3.996528730194757e-06, + "loss": 0.94160277, + "num_input_tokens_seen": 46887810, + "router_z_loss_clip": 1.21582031, + "router_z_loss_mlp": 0.20996094, + "step": 1661, + "time_per_iteration": 2.4509198665618896 + }, + { + "auxiliary_loss_clip": 0.01200078, + "auxiliary_loss_mlp": 0.01065278, + "balance_loss_clip": 1.07752967, + "balance_loss_mlp": 1.04502416, + "epoch": 0.04822703267366955, + "flos": 24237799240320.0, + "grad_norm": 2.6353591937434064, + "language_loss": 0.80605125, + "learning_rate": 3.996517651950861e-06, + "loss": 0.82870477, + "num_input_tokens_seen": 46903070, + "router_z_loss_clip": 1.22460938, + "router_z_loss_mlp": 0.20239258, + "step": 1662, + "time_per_iteration": 2.5186023712158203 + }, + { + "auxiliary_loss_clip": 0.01203308, + "auxiliary_loss_mlp": 0.01067559, + "balance_loss_clip": 1.07573092, + "balance_loss_mlp": 1.04378891, + "epoch": 0.0482560501421856, + "flos": 16966752750720.0, + "grad_norm": 2.696384377732187, + "language_loss": 0.93888682, + "learning_rate": 3.996506556072913e-06, + "loss": 0.96159554, + "num_input_tokens_seen": 46922180, + "router_z_loss_clip": 1.27539062, + "router_z_loss_mlp": 0.23754883, + "step": 1663, + "time_per_iteration": 2.5630624294281006 + }, + { + "auxiliary_loss_clip": 0.01206448, + "auxiliary_loss_mlp": 0.01070649, + "balance_loss_clip": 1.08363354, + "balance_loss_mlp": 1.04892921, + "epoch": 0.04828506761070164, + "flos": 51160099282560.0, + "grad_norm": 3.179906583733601, + "language_loss": 0.96689153, + "learning_rate": 3.996495442561011e-06, + "loss": 0.98966241, + "num_input_tokens_seen": 46937330, + "router_z_loss_clip": 1.2265625, + "router_z_loss_mlp": 0.21728516, + "step": 1664, + "time_per_iteration": 2.7658917903900146 + }, + { + "auxiliary_loss_clip": 0.01182587, + "auxiliary_loss_mlp": 0.01058917, + "balance_loss_clip": 1.07352543, + "balance_loss_mlp": 1.0416081, + "epoch": 0.04831408507921769, + "flos": 33319930400640.0, + "grad_norm": 2.5977066160416746, + "language_loss": 0.82295316, + "learning_rate": 3.996484311415254e-06, + "loss": 0.84536821, + "num_input_tokens_seen": 46957075, + "router_z_loss_clip": 1.09130859, + "router_z_loss_mlp": 0.1730957, + "step": 1665, + "time_per_iteration": 2.643686056137085 + }, + { + "auxiliary_loss_clip": 0.01194225, + "auxiliary_loss_mlp": 0.01060036, + "balance_loss_clip": 1.07381797, + "balance_loss_mlp": 1.03931713, + "epoch": 0.04834310254773373, + "flos": 37700349891840.0, + "grad_norm": 6.958977478329371, + "language_loss": 0.72571826, + "learning_rate": 3.9964731626357385e-06, + "loss": 0.74826086, + "num_input_tokens_seen": 46975505, + "router_z_loss_clip": 1.20410156, + "router_z_loss_mlp": 0.20703125, + "step": 1666, + "time_per_iteration": 2.6586718559265137 + }, + { + "auxiliary_loss_clip": 0.0120312, + "auxiliary_loss_mlp": 0.0107367, + "balance_loss_clip": 1.07575774, + "balance_loss_mlp": 1.05174756, + "epoch": 0.048372120016249784, + "flos": 32628542440320.0, + "grad_norm": 3.860486116248037, + "language_loss": 1.04469264, + "learning_rate": 3.996461996222565e-06, + "loss": 1.06746054, + "num_input_tokens_seen": 46992010, + "router_z_loss_clip": 1.27441406, + "router_z_loss_mlp": 0.21936035, + "step": 1667, + "time_per_iteration": 2.612908124923706 + }, + { + "auxiliary_loss_clip": 0.01195149, + "auxiliary_loss_mlp": 0.01055262, + "balance_loss_clip": 1.07771349, + "balance_loss_mlp": 1.03664184, + "epoch": 0.04840113748476583, + "flos": 36312941116800.0, + "grad_norm": 3.5601070165762865, + "language_loss": 0.91411686, + "learning_rate": 3.996450812175831e-06, + "loss": 0.93662095, + "num_input_tokens_seen": 47007570, + "router_z_loss_clip": 1.17285156, + "router_z_loss_mlp": 0.18621826, + "step": 1668, + "time_per_iteration": 2.640901803970337 + }, + { + "auxiliary_loss_clip": 0.01198981, + "auxiliary_loss_mlp": 0.01056686, + "balance_loss_clip": 1.07439196, + "balance_loss_mlp": 1.03605115, + "epoch": 0.04843015495328187, + "flos": 36244675319040.0, + "grad_norm": 2.5954857003097835, + "language_loss": 0.86708289, + "learning_rate": 3.9964396104956344e-06, + "loss": 0.88963962, + "num_input_tokens_seen": 47033600, + "router_z_loss_clip": 1.24707031, + "router_z_loss_mlp": 0.20629883, + "step": 1669, + "time_per_iteration": 2.703547954559326 + }, + { + "auxiliary_loss_clip": 0.01064514, + "auxiliary_loss_mlp": 0.01001467, + "balance_loss_clip": 1.03389871, + "balance_loss_mlp": 0.99971479, + "epoch": 0.048459172421797925, + "flos": 63312380593920.0, + "grad_norm": 0.7409024256830489, + "language_loss": 0.49544489, + "learning_rate": 3.996428391182077e-06, + "loss": 0.5161047, + "num_input_tokens_seen": 47091355, + "router_z_loss_clip": 0.30664062, + "router_z_loss_mlp": 0.01757812, + "step": 1670, + "time_per_iteration": 3.066446542739868 + }, + { + "auxiliary_loss_clip": 0.01185012, + "auxiliary_loss_mlp": 0.01048733, + "balance_loss_clip": 1.07296729, + "balance_loss_mlp": 1.03110766, + "epoch": 0.04848818989031397, + "flos": 36878822956800.0, + "grad_norm": 2.5709864836367284, + "language_loss": 0.82435673, + "learning_rate": 3.9964171542352555e-06, + "loss": 0.84669411, + "num_input_tokens_seen": 47110510, + "router_z_loss_clip": 1.12060547, + "router_z_loss_mlp": 0.1763916, + "step": 1671, + "time_per_iteration": 2.68448543548584 + }, + { + "auxiliary_loss_clip": 0.01057066, + "auxiliary_loss_mlp": 0.00999033, + "balance_loss_clip": 1.02688444, + "balance_loss_mlp": 0.99745911, + "epoch": 0.048517207358830014, + "flos": 70063299169920.0, + "grad_norm": 0.716019882370947, + "language_loss": 0.52149636, + "learning_rate": 3.9964058996552705e-06, + "loss": 0.54205734, + "num_input_tokens_seen": 47169785, + "router_z_loss_clip": 0.30078125, + "router_z_loss_mlp": 0.01574707, + "step": 1672, + "time_per_iteration": 3.0272436141967773 + }, + { + "auxiliary_loss_clip": 0.01053026, + "auxiliary_loss_mlp": 0.00996682, + "balance_loss_clip": 1.02290893, + "balance_loss_mlp": 0.99504894, + "epoch": 0.048546224827346066, + "flos": 67575227892480.0, + "grad_norm": 0.7448766390076805, + "language_loss": 0.51897585, + "learning_rate": 3.9963946274422195e-06, + "loss": 0.53947288, + "num_input_tokens_seen": 47222165, + "router_z_loss_clip": 0.30078125, + "router_z_loss_mlp": 0.01635742, + "step": 1673, + "time_per_iteration": 2.925823926925659 + }, + { + "auxiliary_loss_clip": 0.01051648, + "auxiliary_loss_mlp": 0.00999797, + "balance_loss_clip": 1.02167821, + "balance_loss_mlp": 0.99818802, + "epoch": 0.04857524229586211, + "flos": 72490107801600.0, + "grad_norm": 0.731824128267796, + "language_loss": 0.50003576, + "learning_rate": 3.996383337596204e-06, + "loss": 0.52055025, + "num_input_tokens_seen": 47281300, + "router_z_loss_clip": 0.30078125, + "router_z_loss_mlp": 0.01611328, + "step": 1674, + "time_per_iteration": 3.1826486587524414 + }, + { + "auxiliary_loss_clip": 0.01188596, + "auxiliary_loss_mlp": 0.01056291, + "balance_loss_clip": 1.07226062, + "balance_loss_mlp": 1.03564417, + "epoch": 0.048604259764378155, + "flos": 28615561125120.0, + "grad_norm": 2.40067421240979, + "language_loss": 0.85944694, + "learning_rate": 3.9963720301173225e-06, + "loss": 0.88189584, + "num_input_tokens_seen": 47299615, + "router_z_loss_clip": 1.16308594, + "router_z_loss_mlp": 0.20654297, + "step": 1675, + "time_per_iteration": 2.566617250442505 + }, + { + "auxiliary_loss_clip": 0.01052491, + "auxiliary_loss_mlp": 0.01019068, + "balance_loss_clip": 1.02229786, + "balance_loss_mlp": 1.01743531, + "epoch": 0.0486332772328942, + "flos": 52099596437760.0, + "grad_norm": 0.7124131319208759, + "language_loss": 0.50786865, + "learning_rate": 3.996360705005676e-06, + "loss": 0.52858424, + "num_input_tokens_seen": 47349885, + "router_z_loss_clip": 0.30273438, + "router_z_loss_mlp": 0.01635742, + "step": 1676, + "time_per_iteration": 2.8925905227661133 + }, + { + "auxiliary_loss_clip": 0.01054364, + "auxiliary_loss_mlp": 0.0101624, + "balance_loss_clip": 1.02415299, + "balance_loss_mlp": 1.01457143, + "epoch": 0.04866229470141025, + "flos": 74771080237440.0, + "grad_norm": 0.6496122021334327, + "language_loss": 0.54365486, + "learning_rate": 3.996349362261364e-06, + "loss": 0.56436092, + "num_input_tokens_seen": 47415390, + "router_z_loss_clip": 0.30273438, + "router_z_loss_mlp": 0.01672363, + "step": 1677, + "time_per_iteration": 3.1585450172424316 + }, + { + "auxiliary_loss_clip": 0.01193889, + "auxiliary_loss_mlp": 0.01054186, + "balance_loss_clip": 1.07605875, + "balance_loss_mlp": 1.03347933, + "epoch": 0.048691312169926296, + "flos": 26027694887040.0, + "grad_norm": 1.8663313098846772, + "language_loss": 0.80339265, + "learning_rate": 3.9963380018844865e-06, + "loss": 0.82587337, + "num_input_tokens_seen": 47432260, + "router_z_loss_clip": 1.17871094, + "router_z_loss_mlp": 0.20703125, + "step": 1678, + "time_per_iteration": 2.545870304107666 + }, + { + "auxiliary_loss_clip": 0.01191667, + "auxiliary_loss_mlp": 0.01058144, + "balance_loss_clip": 1.07156217, + "balance_loss_mlp": 1.03616142, + "epoch": 0.04872032963844234, + "flos": 13436911319040.0, + "grad_norm": 2.7448426168109408, + "language_loss": 0.85425252, + "learning_rate": 3.996326623875143e-06, + "loss": 0.87675059, + "num_input_tokens_seen": 47443170, + "router_z_loss_clip": 1.20019531, + "router_z_loss_mlp": 0.21984863, + "step": 1679, + "time_per_iteration": 2.456655979156494 + }, + { + "auxiliary_loss_clip": 0.01055086, + "auxiliary_loss_mlp": 0.01003688, + "balance_loss_clip": 1.02492249, + "balance_loss_mlp": 1.00201941, + "epoch": 0.04874934710695839, + "flos": 72329837316480.0, + "grad_norm": 0.6920744512918013, + "language_loss": 0.54493642, + "learning_rate": 3.996315228233436e-06, + "loss": 0.5655241, + "num_input_tokens_seen": 47511555, + "router_z_loss_clip": 0.30078125, + "router_z_loss_mlp": 0.01672363, + "step": 1680, + "time_per_iteration": 3.2184698581695557 + }, + { + "auxiliary_loss_clip": 0.01200431, + "auxiliary_loss_mlp": 0.01061102, + "balance_loss_clip": 1.07452273, + "balance_loss_mlp": 1.03958499, + "epoch": 0.04877836457547444, + "flos": 13729439681280.0, + "grad_norm": 2.697037281406449, + "language_loss": 0.96172535, + "learning_rate": 3.996303814959465e-06, + "loss": 0.98434067, + "num_input_tokens_seen": 47523650, + "router_z_loss_clip": 1.25878906, + "router_z_loss_mlp": 0.21496582, + "step": 1681, + "time_per_iteration": 2.4483590126037598 + }, + { + "auxiliary_loss_clip": 0.01054619, + "auxiliary_loss_mlp": 0.00999558, + "balance_loss_clip": 1.02445722, + "balance_loss_mlp": 0.99784106, + "epoch": 0.04880738204399048, + "flos": 74771008410240.0, + "grad_norm": 0.6650744839087854, + "language_loss": 0.47424743, + "learning_rate": 3.9962923840533305e-06, + "loss": 0.49478921, + "num_input_tokens_seen": 47585840, + "router_z_loss_clip": 0.30078125, + "router_z_loss_mlp": 0.01721191, + "step": 1682, + "time_per_iteration": 3.1445188522338867 + }, + { + "auxiliary_loss_clip": 0.01201626, + "auxiliary_loss_mlp": 0.01054944, + "balance_loss_clip": 1.07719326, + "balance_loss_mlp": 1.03517938, + "epoch": 0.048836399512506526, + "flos": 32305849632000.0, + "grad_norm": 5.856006574573075, + "language_loss": 0.99118894, + "learning_rate": 3.996280935515134e-06, + "loss": 1.01375461, + "num_input_tokens_seen": 47603315, + "router_z_loss_clip": 1.24316406, + "router_z_loss_mlp": 0.19763184, + "step": 1683, + "time_per_iteration": 2.6465888023376465 + }, + { + "auxiliary_loss_clip": 0.01189868, + "auxiliary_loss_mlp": 0.0106038, + "balance_loss_clip": 1.07473004, + "balance_loss_mlp": 1.03937507, + "epoch": 0.04886541698102258, + "flos": 15516139017600.0, + "grad_norm": 2.774562381039683, + "language_loss": 1.01208925, + "learning_rate": 3.9962694693449765e-06, + "loss": 1.03459167, + "num_input_tokens_seen": 47614980, + "router_z_loss_clip": 1.15136719, + "router_z_loss_mlp": 0.21032715, + "step": 1684, + "time_per_iteration": 2.472263813018799 + }, + { + "auxiliary_loss_clip": 0.01205449, + "auxiliary_loss_mlp": 0.01068361, + "balance_loss_clip": 1.07640898, + "balance_loss_mlp": 1.04312468, + "epoch": 0.04889443444953862, + "flos": 13472714200320.0, + "grad_norm": 2.4329230866678437, + "language_loss": 0.86777496, + "learning_rate": 3.996257985542959e-06, + "loss": 0.89051306, + "num_input_tokens_seen": 47628135, + "router_z_loss_clip": 1.29101562, + "router_z_loss_mlp": 0.25244141, + "step": 1685, + "time_per_iteration": 2.5138678550720215 + }, + { + "auxiliary_loss_clip": 0.0119756, + "auxiliary_loss_mlp": 0.01064064, + "balance_loss_clip": 1.07731974, + "balance_loss_mlp": 1.04383993, + "epoch": 0.04892345191805467, + "flos": 29865468637440.0, + "grad_norm": 2.5282779378231957, + "language_loss": 0.72612166, + "learning_rate": 3.996246484109184e-06, + "loss": 0.74873799, + "num_input_tokens_seen": 47644315, + "router_z_loss_clip": 1.20214844, + "router_z_loss_mlp": 0.20263672, + "step": 1686, + "time_per_iteration": 2.608778715133667 + }, + { + "auxiliary_loss_clip": 0.0120473, + "auxiliary_loss_mlp": 0.0106592, + "balance_loss_clip": 1.08066571, + "balance_loss_mlp": 1.0428654, + "epoch": 0.04895246938657072, + "flos": 30220121571840.0, + "grad_norm": 3.0450510714264984, + "language_loss": 0.91746402, + "learning_rate": 3.9962349650437514e-06, + "loss": 0.94017047, + "num_input_tokens_seen": 47661035, + "router_z_loss_clip": 1.24023438, + "router_z_loss_mlp": 0.23046875, + "step": 1687, + "time_per_iteration": 2.6670846939086914 + }, + { + "auxiliary_loss_clip": 0.01194655, + "auxiliary_loss_mlp": 0.01048262, + "balance_loss_clip": 1.07716537, + "balance_loss_mlp": 1.02990985, + "epoch": 0.04898148685508676, + "flos": 52403147297280.0, + "grad_norm": 2.112081426839779, + "language_loss": 0.81830198, + "learning_rate": 3.996223428346764e-06, + "loss": 0.84073114, + "num_input_tokens_seen": 47682195, + "router_z_loss_clip": 1.17480469, + "router_z_loss_mlp": 0.18365479, + "step": 1688, + "time_per_iteration": 2.758979558944702 + }, + { + "auxiliary_loss_clip": 0.01050724, + "auxiliary_loss_mlp": 0.01002134, + "balance_loss_clip": 1.02035427, + "balance_loss_mlp": 1.00054836, + "epoch": 0.04901050432360281, + "flos": 67162508622720.0, + "grad_norm": 0.6948508009719694, + "language_loss": 0.5074532, + "learning_rate": 3.9962118740183235e-06, + "loss": 0.52798176, + "num_input_tokens_seen": 47747720, + "router_z_loss_clip": 0.3046875, + "router_z_loss_mlp": 0.01586914, + "step": 1689, + "time_per_iteration": 3.235200881958008 + }, + { + "auxiliary_loss_clip": 0.01202096, + "auxiliary_loss_mlp": 0.0105426, + "balance_loss_clip": 1.08106029, + "balance_loss_mlp": 1.03468585, + "epoch": 0.04903952179211885, + "flos": 33140341376640.0, + "grad_norm": 3.0652522597414817, + "language_loss": 0.82758832, + "learning_rate": 3.996200302058532e-06, + "loss": 0.8501519, + "num_input_tokens_seen": 47768645, + "router_z_loss_clip": 1.2109375, + "router_z_loss_mlp": 0.19567871, + "step": 1690, + "time_per_iteration": 2.6213746070861816 + }, + { + "auxiliary_loss_clip": 0.01197931, + "auxiliary_loss_mlp": 0.01057325, + "balance_loss_clip": 1.07556844, + "balance_loss_mlp": 1.03710747, + "epoch": 0.049068539260634904, + "flos": 16683594860160.0, + "grad_norm": 2.4465237994188076, + "language_loss": 0.83369702, + "learning_rate": 3.9961887124674916e-06, + "loss": 0.85624957, + "num_input_tokens_seen": 47781785, + "router_z_loss_clip": 1.22363281, + "router_z_loss_mlp": 0.20227051, + "step": 1691, + "time_per_iteration": 2.512082815170288 + }, + { + "auxiliary_loss_clip": 0.01199421, + "auxiliary_loss_mlp": 0.01055533, + "balance_loss_clip": 1.07968676, + "balance_loss_mlp": 1.03473139, + "epoch": 0.04909755672915095, + "flos": 25001690803200.0, + "grad_norm": 9.35147829920571, + "language_loss": 0.72831875, + "learning_rate": 3.996177105245304e-06, + "loss": 0.75086832, + "num_input_tokens_seen": 47796315, + "router_z_loss_clip": 1.19628906, + "router_z_loss_mlp": 0.20812988, + "step": 1692, + "time_per_iteration": 2.550976276397705 + }, + { + "auxiliary_loss_clip": 0.01196363, + "auxiliary_loss_mlp": 0.01055473, + "balance_loss_clip": 1.07557142, + "balance_loss_mlp": 1.03265655, + "epoch": 0.04912657419766699, + "flos": 13327886563200.0, + "grad_norm": 3.03073597305193, + "language_loss": 0.84420276, + "learning_rate": 3.996165480392074e-06, + "loss": 0.86672115, + "num_input_tokens_seen": 47808675, + "router_z_loss_clip": 1.20898438, + "router_z_loss_mlp": 0.22827148, + "step": 1693, + "time_per_iteration": 2.4969162940979004 + }, + { + "auxiliary_loss_clip": 0.01196979, + "auxiliary_loss_mlp": 0.01066745, + "balance_loss_clip": 1.07485032, + "balance_loss_mlp": 1.04556108, + "epoch": 0.049155591666183045, + "flos": 10699584589440.0, + "grad_norm": 4.782320150697475, + "language_loss": 0.80686235, + "learning_rate": 3.996153837907902e-06, + "loss": 0.82949966, + "num_input_tokens_seen": 47821645, + "router_z_loss_clip": 1.22070312, + "router_z_loss_mlp": 0.2121582, + "step": 1694, + "time_per_iteration": 2.4882400035858154 + }, + { + "auxiliary_loss_clip": 0.01197627, + "auxiliary_loss_mlp": 0.01052155, + "balance_loss_clip": 1.07937968, + "balance_loss_mlp": 1.03038764, + "epoch": 0.04918460913469909, + "flos": 17705397052800.0, + "grad_norm": 2.7369613161398365, + "language_loss": 0.98907018, + "learning_rate": 3.996142177792891e-06, + "loss": 1.01156807, + "num_input_tokens_seen": 47834530, + "router_z_loss_clip": 1.18359375, + "router_z_loss_mlp": 0.21765137, + "step": 1695, + "time_per_iteration": 2.4957730770111084 + }, + { + "auxiliary_loss_clip": 0.01198905, + "auxiliary_loss_mlp": 0.0105611, + "balance_loss_clip": 1.07486093, + "balance_loss_mlp": 1.03518271, + "epoch": 0.049213626603215134, + "flos": 28870957802880.0, + "grad_norm": 2.3884123002256925, + "language_loss": 0.93020892, + "learning_rate": 3.996130500047145e-06, + "loss": 0.95275903, + "num_input_tokens_seen": 47850805, + "router_z_loss_clip": 1.24023438, + "router_z_loss_mlp": 0.20935059, + "step": 1696, + "time_per_iteration": 2.5936107635498047 + }, + { + "auxiliary_loss_clip": 0.01199958, + "auxiliary_loss_mlp": 0.01058705, + "balance_loss_clip": 1.07719755, + "balance_loss_mlp": 1.03729546, + "epoch": 0.049242644071731186, + "flos": 26463614342400.0, + "grad_norm": 2.1860996023312262, + "language_loss": 0.75177956, + "learning_rate": 3.996118804670767e-06, + "loss": 0.77436614, + "num_input_tokens_seen": 47868295, + "router_z_loss_clip": 1.22851562, + "router_z_loss_mlp": 0.21417236, + "step": 1697, + "time_per_iteration": 2.5763707160949707 + }, + { + "auxiliary_loss_clip": 0.01053836, + "auxiliary_loss_mlp": 0.01012712, + "balance_loss_clip": 1.02348161, + "balance_loss_mlp": 1.01106739, + "epoch": 0.04927166154024723, + "flos": 48794379684480.0, + "grad_norm": 0.8035707709778382, + "language_loss": 0.51847607, + "learning_rate": 3.99610709166386e-06, + "loss": 0.53914154, + "num_input_tokens_seen": 47917785, + "router_z_loss_clip": 0.30273438, + "router_z_loss_mlp": 0.01647949, + "step": 1698, + "time_per_iteration": 2.931264877319336 + }, + { + "auxiliary_loss_clip": 0.01053809, + "auxiliary_loss_mlp": 0.01013572, + "balance_loss_clip": 1.02360332, + "balance_loss_mlp": 1.01205242, + "epoch": 0.049300679008763275, + "flos": 74771726682240.0, + "grad_norm": 0.755483848916453, + "language_loss": 0.50495309, + "learning_rate": 3.996095361026526e-06, + "loss": 0.5256269, + "num_input_tokens_seen": 47974730, + "router_z_loss_clip": 0.30078125, + "router_z_loss_mlp": 0.01519775, + "step": 1699, + "time_per_iteration": 3.113194227218628 + }, + { + "auxiliary_loss_clip": 0.01194056, + "auxiliary_loss_mlp": 0.01049687, + "balance_loss_clip": 1.07577324, + "balance_loss_mlp": 1.0308758, + "epoch": 0.04932969647727932, + "flos": 28686125393280.0, + "grad_norm": 2.353655310980659, + "language_loss": 0.92069685, + "learning_rate": 3.996083612758871e-06, + "loss": 0.94313431, + "num_input_tokens_seen": 47991065, + "router_z_loss_clip": 1.18261719, + "router_z_loss_mlp": 0.18798828, + "step": 1700, + "time_per_iteration": 2.6070199012756348 + }, + { + "auxiliary_loss_clip": 0.01190547, + "auxiliary_loss_mlp": 0.0106087, + "balance_loss_clip": 1.07305002, + "balance_loss_mlp": 1.04261911, + "epoch": 0.04935871394579537, + "flos": 20952798865920.0, + "grad_norm": 2.7486640556841815, + "language_loss": 0.73205465, + "learning_rate": 3.996071846860998e-06, + "loss": 0.75456882, + "num_input_tokens_seen": 48003495, + "router_z_loss_clip": 1.17480469, + "router_z_loss_mlp": 0.18261719, + "step": 1701, + "time_per_iteration": 2.4515492916107178 + }, + { + "auxiliary_loss_clip": 0.01196807, + "auxiliary_loss_mlp": 0.01066864, + "balance_loss_clip": 1.07688928, + "balance_loss_mlp": 1.04645526, + "epoch": 0.049387731414311416, + "flos": 30732279644160.0, + "grad_norm": 2.6939238469853075, + "language_loss": 0.88101006, + "learning_rate": 3.996060063333011e-06, + "loss": 0.90364671, + "num_input_tokens_seen": 48023340, + "router_z_loss_clip": 1.19824219, + "router_z_loss_mlp": 0.20422363, + "step": 1702, + "time_per_iteration": 2.7134382724761963 + }, + { + "auxiliary_loss_clip": 0.01054665, + "auxiliary_loss_mlp": 0.01003838, + "balance_loss_clip": 1.02392435, + "balance_loss_mlp": 1.00215709, + "epoch": 0.04941674888282746, + "flos": 66798087189120.0, + "grad_norm": 0.6867495269996298, + "language_loss": 0.55158037, + "learning_rate": 3.996048262175013e-06, + "loss": 0.57216531, + "num_input_tokens_seen": 48089515, + "router_z_loss_clip": 0.30859375, + "router_z_loss_mlp": 0.0168457, + "step": 1703, + "time_per_iteration": 3.191662073135376 + }, + { + "auxiliary_loss_clip": 0.0120876, + "auxiliary_loss_mlp": 0.01057394, + "balance_loss_clip": 1.07937241, + "balance_loss_mlp": 1.03250289, + "epoch": 0.04944576635134351, + "flos": 23178721708800.0, + "grad_norm": 2.856956196786458, + "language_loss": 0.99703544, + "learning_rate": 3.99603644338711e-06, + "loss": 1.01969695, + "num_input_tokens_seen": 48106025, + "router_z_loss_clip": 1.29589844, + "router_z_loss_mlp": 0.24890137, + "step": 1704, + "time_per_iteration": 2.585650682449341 + }, + { + "auxiliary_loss_clip": 0.01192402, + "auxiliary_loss_mlp": 0.01055332, + "balance_loss_clip": 1.07689595, + "balance_loss_mlp": 1.03627038, + "epoch": 0.04947478381985956, + "flos": 11319582268800.0, + "grad_norm": 2.467926161724509, + "language_loss": 0.82870656, + "learning_rate": 3.996024606969405e-06, + "loss": 0.85118395, + "num_input_tokens_seen": 48116120, + "router_z_loss_clip": 1.15380859, + "router_z_loss_mlp": 0.19042969, + "step": 1705, + "time_per_iteration": 2.4820261001586914 + }, + { + "auxiliary_loss_clip": 0.01053899, + "auxiliary_loss_mlp": 0.00998676, + "balance_loss_clip": 1.0231775, + "balance_loss_mlp": 0.99706668, + "epoch": 0.0495038012883756, + "flos": 70796990373120.0, + "grad_norm": 0.6771001215146908, + "language_loss": 0.49687716, + "learning_rate": 3.996012752922002e-06, + "loss": 0.51740289, + "num_input_tokens_seen": 48173955, + "router_z_loss_clip": 0.30859375, + "router_z_loss_mlp": 0.01611328, + "step": 1706, + "time_per_iteration": 3.0772719383239746 + }, + { + "auxiliary_loss_clip": 0.01190639, + "auxiliary_loss_mlp": 0.01042025, + "balance_loss_clip": 1.07346654, + "balance_loss_mlp": 1.02289796, + "epoch": 0.049532818756891646, + "flos": 21352699958400.0, + "grad_norm": 2.6293147512099866, + "language_loss": 0.92917258, + "learning_rate": 3.996000881245008e-06, + "loss": 0.95149922, + "num_input_tokens_seen": 48187125, + "router_z_loss_clip": 1.17089844, + "router_z_loss_mlp": 0.19134521, + "step": 1707, + "time_per_iteration": 2.4682939052581787 + }, + { + "auxiliary_loss_clip": 0.01187151, + "auxiliary_loss_mlp": 0.01067, + "balance_loss_clip": 1.07239187, + "balance_loss_mlp": 1.04441571, + "epoch": 0.0495618362254077, + "flos": 14567522785920.0, + "grad_norm": 3.1974341528100174, + "language_loss": 0.96274376, + "learning_rate": 3.995988991938526e-06, + "loss": 0.98528528, + "num_input_tokens_seen": 48198350, + "router_z_loss_clip": 1.14697266, + "router_z_loss_mlp": 0.22595215, + "step": 1708, + "time_per_iteration": 2.4777889251708984 + }, + { + "auxiliary_loss_clip": 0.0119783, + "auxiliary_loss_mlp": 0.01061249, + "balance_loss_clip": 1.07247829, + "balance_loss_mlp": 1.03769326, + "epoch": 0.04959085369392374, + "flos": 30219439213440.0, + "grad_norm": 2.391585251332959, + "language_loss": 0.85503852, + "learning_rate": 3.9959770850026615e-06, + "loss": 0.87762934, + "num_input_tokens_seen": 48212790, + "router_z_loss_clip": 1.25488281, + "router_z_loss_mlp": 0.23547363, + "step": 1709, + "time_per_iteration": 2.593780755996704 + }, + { + "auxiliary_loss_clip": 0.01194196, + "auxiliary_loss_mlp": 0.01062156, + "balance_loss_clip": 1.07653809, + "balance_loss_mlp": 1.04130661, + "epoch": 0.04961987116243979, + "flos": 12933659819520.0, + "grad_norm": 2.397427877509311, + "language_loss": 0.81961131, + "learning_rate": 3.99596516043752e-06, + "loss": 0.84217477, + "num_input_tokens_seen": 48225655, + "router_z_loss_clip": 1.17480469, + "router_z_loss_mlp": 0.20849609, + "step": 1710, + "time_per_iteration": 2.520961046218872 + }, + { + "auxiliary_loss_clip": 0.011982, + "auxiliary_loss_mlp": 0.01064596, + "balance_loss_clip": 1.07385468, + "balance_loss_mlp": 1.04245901, + "epoch": 0.04964888863095584, + "flos": 31094294866560.0, + "grad_norm": 2.6162967549446945, + "language_loss": 1.07526934, + "learning_rate": 3.995953218243206e-06, + "loss": 1.09789729, + "num_input_tokens_seen": 48240665, + "router_z_loss_clip": 1.24316406, + "router_z_loss_mlp": 0.22143555, + "step": 1711, + "time_per_iteration": 7.359845876693726 + }, + { + "auxiliary_loss_clip": 0.01206744, + "auxiliary_loss_mlp": 0.01064902, + "balance_loss_clip": 1.0787487, + "balance_loss_mlp": 1.04480362, + "epoch": 0.04967790609947188, + "flos": 16903296397440.0, + "grad_norm": 2.550984474133309, + "language_loss": 0.81122988, + "learning_rate": 3.995941258419826e-06, + "loss": 0.83394635, + "num_input_tokens_seen": 48254640, + "router_z_loss_clip": 1.28027344, + "router_z_loss_mlp": 0.2010498, + "step": 1712, + "time_per_iteration": 4.753640651702881 + }, + { + "auxiliary_loss_clip": 0.01206878, + "auxiliary_loss_mlp": 0.01080128, + "balance_loss_clip": 1.0790143, + "balance_loss_mlp": 1.05414057, + "epoch": 0.04970692356798793, + "flos": 12161328560640.0, + "grad_norm": 3.258805702016356, + "language_loss": 0.80196297, + "learning_rate": 3.995929280967485e-06, + "loss": 0.82483304, + "num_input_tokens_seen": 48266310, + "router_z_loss_clip": 1.27734375, + "router_z_loss_mlp": 0.2598877, + "step": 1713, + "time_per_iteration": 4.968615531921387 + }, + { + "auxiliary_loss_clip": 0.01059321, + "auxiliary_loss_mlp": 0.01002267, + "balance_loss_clip": 1.02842498, + "balance_loss_mlp": 1.00070572, + "epoch": 0.04973594103650397, + "flos": 64220201953920.0, + "grad_norm": 0.7374403202244578, + "language_loss": 0.55442011, + "learning_rate": 3.995917285886289e-06, + "loss": 0.57503599, + "num_input_tokens_seen": 48323180, + "router_z_loss_clip": 0.30859375, + "router_z_loss_mlp": 0.015625, + "step": 1714, + "time_per_iteration": 2.983957052230835 + }, + { + "auxiliary_loss_clip": 0.01197927, + "auxiliary_loss_mlp": 0.0106502, + "balance_loss_clip": 1.07723749, + "balance_loss_mlp": 1.04124999, + "epoch": 0.049764958505020024, + "flos": 15188705614080.0, + "grad_norm": 3.7840543554910715, + "language_loss": 0.77052653, + "learning_rate": 3.995905273176343e-06, + "loss": 0.79315603, + "num_input_tokens_seen": 48335835, + "router_z_loss_clip": 1.20751953, + "router_z_loss_mlp": 0.23754883, + "step": 1715, + "time_per_iteration": 2.5021135807037354 + }, + { + "auxiliary_loss_clip": 0.01198134, + "auxiliary_loss_mlp": 0.01057509, + "balance_loss_clip": 1.07551908, + "balance_loss_mlp": 1.03910315, + "epoch": 0.04979397597353607, + "flos": 26535615154560.0, + "grad_norm": 2.9893283908299155, + "language_loss": 0.73124194, + "learning_rate": 3.9958932428377545e-06, + "loss": 0.75379831, + "num_input_tokens_seen": 48350420, + "router_z_loss_clip": 1.2265625, + "router_z_loss_mlp": 0.18408203, + "step": 1716, + "time_per_iteration": 2.5869555473327637 + }, + { + "auxiliary_loss_clip": 0.01193384, + "auxiliary_loss_mlp": 0.01063845, + "balance_loss_clip": 1.07747078, + "balance_loss_mlp": 1.0440321, + "epoch": 0.04982299344205211, + "flos": 16282185396480.0, + "grad_norm": 2.3459691540988907, + "language_loss": 0.67195982, + "learning_rate": 3.99588119487063e-06, + "loss": 0.69453204, + "num_input_tokens_seen": 48363590, + "router_z_loss_clip": 1.16015625, + "router_z_loss_mlp": 0.19824219, + "step": 1717, + "time_per_iteration": 2.5001516342163086 + }, + { + "auxiliary_loss_clip": 0.01206522, + "auxiliary_loss_mlp": 0.01064864, + "balance_loss_clip": 1.07927132, + "balance_loss_mlp": 1.0420953, + "epoch": 0.049852010910568165, + "flos": 22267021680000.0, + "grad_norm": 2.873635823513704, + "language_loss": 0.93693483, + "learning_rate": 3.995869129275074e-06, + "loss": 0.95964873, + "num_input_tokens_seen": 48379160, + "router_z_loss_clip": 1.27246094, + "router_z_loss_mlp": 0.2277832, + "step": 1718, + "time_per_iteration": 2.5252599716186523 + }, + { + "auxiliary_loss_clip": 0.01191104, + "auxiliary_loss_mlp": 0.01049212, + "balance_loss_clip": 1.07487321, + "balance_loss_mlp": 1.0299952, + "epoch": 0.04988102837908421, + "flos": 18111403457280.0, + "grad_norm": 2.011104348899601, + "language_loss": 0.68100107, + "learning_rate": 3.995857046051195e-06, + "loss": 0.70340425, + "num_input_tokens_seen": 48396205, + "router_z_loss_clip": 1.16113281, + "router_z_loss_mlp": 0.19213867, + "step": 1719, + "time_per_iteration": 2.572551965713501 + }, + { + "auxiliary_loss_clip": 0.01191284, + "auxiliary_loss_mlp": 0.01062682, + "balance_loss_clip": 1.07363844, + "balance_loss_mlp": 1.04404974, + "epoch": 0.049910045847600254, + "flos": 27457227336960.0, + "grad_norm": 3.4118598290778155, + "language_loss": 0.9501285, + "learning_rate": 3.995844945199099e-06, + "loss": 0.97266817, + "num_input_tokens_seen": 48413235, + "router_z_loss_clip": 1.17529297, + "router_z_loss_mlp": 0.18615723, + "step": 1720, + "time_per_iteration": 2.521925449371338 + }, + { + "auxiliary_loss_clip": 0.01057455, + "auxiliary_loss_mlp": 0.01006947, + "balance_loss_clip": 1.02670455, + "balance_loss_mlp": 1.00539732, + "epoch": 0.0499390633161163, + "flos": 74775964487040.0, + "grad_norm": 0.6466885433531013, + "language_loss": 0.50312811, + "learning_rate": 3.995832826718892e-06, + "loss": 0.52377212, + "num_input_tokens_seen": 48483785, + "router_z_loss_clip": 0.30859375, + "router_z_loss_mlp": 0.01550293, + "step": 1721, + "time_per_iteration": 3.364751100540161 + }, + { + "auxiliary_loss_clip": 0.01056896, + "auxiliary_loss_mlp": 0.01005612, + "balance_loss_clip": 1.0260253, + "balance_loss_mlp": 1.00400293, + "epoch": 0.04996808078463235, + "flos": 61478209434240.0, + "grad_norm": 0.6888400422990544, + "language_loss": 0.51857805, + "learning_rate": 3.995820690610682e-06, + "loss": 0.53920317, + "num_input_tokens_seen": 48550950, + "router_z_loss_clip": 0.30859375, + "router_z_loss_mlp": 0.01611328, + "step": 1722, + "time_per_iteration": 3.2798075675964355 + }, + { + "auxiliary_loss_clip": 0.0120111, + "auxiliary_loss_mlp": 0.01065824, + "balance_loss_clip": 1.07509947, + "balance_loss_mlp": 1.04371059, + "epoch": 0.049997098253148395, + "flos": 13508912131200.0, + "grad_norm": 2.941061090986721, + "language_loss": 1.03088236, + "learning_rate": 3.995808536874577e-06, + "loss": 1.05355167, + "num_input_tokens_seen": 48562490, + "router_z_loss_clip": 1.26074219, + "router_z_loss_mlp": 0.2208252, + "step": 1723, + "time_per_iteration": 2.521040916442871 + }, + { + "auxiliary_loss_clip": 0.0119925, + "auxiliary_loss_mlp": 0.01061623, + "balance_loss_clip": 1.07609844, + "balance_loss_mlp": 1.03806686, + "epoch": 0.05002611572166444, + "flos": 18925176032640.0, + "grad_norm": 2.468617921604559, + "language_loss": 0.8106811, + "learning_rate": 3.995796365510682e-06, + "loss": 0.83328974, + "num_input_tokens_seen": 48578755, + "router_z_loss_clip": 1.23144531, + "router_z_loss_mlp": 0.23571777, + "step": 1724, + "time_per_iteration": 2.525818347930908 + }, + { + "auxiliary_loss_clip": 0.01189755, + "auxiliary_loss_mlp": 0.01059736, + "balance_loss_clip": 1.07096255, + "balance_loss_mlp": 1.03982794, + "epoch": 0.05005513319018049, + "flos": 27227541818880.0, + "grad_norm": 3.011359968019195, + "language_loss": 0.88614058, + "learning_rate": 3.995784176519107e-06, + "loss": 0.9086355, + "num_input_tokens_seen": 48595135, + "router_z_loss_clip": 1.1875, + "router_z_loss_mlp": 0.19921875, + "step": 1725, + "time_per_iteration": 2.596630096435547 + }, + { + "auxiliary_loss_clip": 0.0118131, + "auxiliary_loss_mlp": 0.01046026, + "balance_loss_clip": 1.06959152, + "balance_loss_mlp": 1.02740598, + "epoch": 0.050084150658696536, + "flos": 20588808395520.0, + "grad_norm": 2.7445448512269137, + "language_loss": 0.87180781, + "learning_rate": 3.995771969899958e-06, + "loss": 0.89408123, + "num_input_tokens_seen": 48610115, + "router_z_loss_clip": 1.11621094, + "router_z_loss_mlp": 0.1862793, + "step": 1726, + "time_per_iteration": 2.5064876079559326 + }, + { + "auxiliary_loss_clip": 0.01203506, + "auxiliary_loss_mlp": 0.01062578, + "balance_loss_clip": 1.0831461, + "balance_loss_mlp": 1.04160953, + "epoch": 0.05011316812721258, + "flos": 20807576179200.0, + "grad_norm": 2.812124996494577, + "language_loss": 0.8666023, + "learning_rate": 3.9957597456533435e-06, + "loss": 0.88926315, + "num_input_tokens_seen": 48625615, + "router_z_loss_clip": 1.203125, + "router_z_loss_mlp": 0.2097168, + "step": 1727, + "time_per_iteration": 2.593167781829834 + }, + { + "auxiliary_loss_clip": 0.01193498, + "auxiliary_loss_mlp": 0.01054353, + "balance_loss_clip": 1.07510757, + "balance_loss_mlp": 1.03396189, + "epoch": 0.05014218559572863, + "flos": 21536670441600.0, + "grad_norm": 3.676557694357948, + "language_loss": 1.03209674, + "learning_rate": 3.995747503779372e-06, + "loss": 1.05457532, + "num_input_tokens_seen": 48638880, + "router_z_loss_clip": 1.18457031, + "router_z_loss_mlp": 0.20397949, + "step": 1728, + "time_per_iteration": 2.5038862228393555 + }, + { + "auxiliary_loss_clip": 0.01196971, + "auxiliary_loss_mlp": 0.01059785, + "balance_loss_clip": 1.07792473, + "balance_loss_mlp": 1.0372541, + "epoch": 0.05017120306424468, + "flos": 23616077708160.0, + "grad_norm": 2.053736067205178, + "language_loss": 0.85213208, + "learning_rate": 3.9957352442781504e-06, + "loss": 0.87469959, + "num_input_tokens_seen": 48657505, + "router_z_loss_clip": 1.18945312, + "router_z_loss_mlp": 0.2253418, + "step": 1729, + "time_per_iteration": 2.5464210510253906 + }, + { + "auxiliary_loss_clip": 0.01054816, + "auxiliary_loss_mlp": 0.01000956, + "balance_loss_clip": 1.02445734, + "balance_loss_mlp": 0.99950141, + "epoch": 0.05020022053276072, + "flos": 67585750577280.0, + "grad_norm": 0.6843292192549804, + "language_loss": 0.52883476, + "learning_rate": 3.995722967149787e-06, + "loss": 0.54939252, + "num_input_tokens_seen": 48717525, + "router_z_loss_clip": 0.3046875, + "router_z_loss_mlp": 0.01452637, + "step": 1730, + "time_per_iteration": 3.0960679054260254 + }, + { + "auxiliary_loss_clip": 0.01192492, + "auxiliary_loss_mlp": 0.01066406, + "balance_loss_clip": 1.07166481, + "balance_loss_mlp": 1.04592562, + "epoch": 0.050229238001276766, + "flos": 20809623254400.0, + "grad_norm": 2.7183037812015733, + "language_loss": 0.9286859, + "learning_rate": 3.9957106723943915e-06, + "loss": 0.95127487, + "num_input_tokens_seen": 48735345, + "router_z_loss_clip": 1.20898438, + "router_z_loss_mlp": 0.20489502, + "step": 1731, + "time_per_iteration": 2.525317907333374 + }, + { + "auxiliary_loss_clip": 0.01192953, + "auxiliary_loss_mlp": 0.01049207, + "balance_loss_clip": 1.07855129, + "balance_loss_mlp": 1.03103924, + "epoch": 0.05025825546979282, + "flos": 40181597585280.0, + "grad_norm": 2.145676466117142, + "language_loss": 0.77794325, + "learning_rate": 3.995698360012072e-06, + "loss": 0.80036485, + "num_input_tokens_seen": 48754600, + "router_z_loss_clip": 1.14404297, + "router_z_loss_mlp": 0.18151855, + "step": 1732, + "time_per_iteration": 2.7086477279663086 + }, + { + "auxiliary_loss_clip": 0.0105365, + "auxiliary_loss_mlp": 0.01000389, + "balance_loss_clip": 1.02319777, + "balance_loss_mlp": 0.99877977, + "epoch": 0.05028727293830886, + "flos": 63321538561920.0, + "grad_norm": 0.6671506865899469, + "language_loss": 0.48483449, + "learning_rate": 3.995686030002936e-06, + "loss": 0.50537485, + "num_input_tokens_seen": 48816535, + "router_z_loss_clip": 0.3046875, + "router_z_loss_mlp": 0.01611328, + "step": 1733, + "time_per_iteration": 3.093319892883301 + }, + { + "auxiliary_loss_clip": 0.01191471, + "auxiliary_loss_mlp": 0.01054691, + "balance_loss_clip": 1.08150315, + "balance_loss_mlp": 1.03757238, + "epoch": 0.05031629040682491, + "flos": 18143327669760.0, + "grad_norm": 2.631337236757085, + "language_loss": 0.92865372, + "learning_rate": 3.995673682367094e-06, + "loss": 0.95111543, + "num_input_tokens_seen": 48829095, + "router_z_loss_clip": 1.09863281, + "router_z_loss_mlp": 0.17114258, + "step": 1734, + "time_per_iteration": 2.525308609008789 + }, + { + "auxiliary_loss_clip": 0.01052984, + "auxiliary_loss_mlp": 0.01005391, + "balance_loss_clip": 1.02285504, + "balance_loss_mlp": 1.00368679, + "epoch": 0.05034530787534096, + "flos": 74778981229440.0, + "grad_norm": 0.6775537479126885, + "language_loss": 0.53822583, + "learning_rate": 3.995661317104654e-06, + "loss": 0.55880958, + "num_input_tokens_seen": 48892975, + "router_z_loss_clip": 0.30078125, + "router_z_loss_mlp": 0.01708984, + "step": 1735, + "time_per_iteration": 3.1454200744628906 + }, + { + "auxiliary_loss_clip": 0.01051945, + "auxiliary_loss_mlp": 0.01003627, + "balance_loss_clip": 1.02200627, + "balance_loss_mlp": 1.00188661, + "epoch": 0.050374325343857, + "flos": 74485662768000.0, + "grad_norm": 0.6973200510490033, + "language_loss": 0.56052268, + "learning_rate": 3.995648934215726e-06, + "loss": 0.58107841, + "num_input_tokens_seen": 48959070, + "router_z_loss_clip": 0.30078125, + "router_z_loss_mlp": 0.01745605, + "step": 1736, + "time_per_iteration": 3.125980854034424 + }, + { + "auxiliary_loss_clip": 0.01188512, + "auxiliary_loss_mlp": 0.01053556, + "balance_loss_clip": 1.07321334, + "balance_loss_mlp": 1.03441095, + "epoch": 0.05040334281237305, + "flos": 16544764794240.0, + "grad_norm": 2.2351069055139683, + "language_loss": 0.70745325, + "learning_rate": 3.995636533700419e-06, + "loss": 0.72987396, + "num_input_tokens_seen": 48973280, + "router_z_loss_clip": 1.15185547, + "router_z_loss_mlp": 0.19152832, + "step": 1737, + "time_per_iteration": 2.4889743328094482 + }, + { + "auxiliary_loss_clip": 0.01200399, + "auxiliary_loss_mlp": 0.0106342, + "balance_loss_clip": 1.07555068, + "balance_loss_mlp": 1.03870797, + "epoch": 0.05043236028088909, + "flos": 18107381134080.0, + "grad_norm": 2.518788680588803, + "language_loss": 1.03979242, + "learning_rate": 3.995624115558843e-06, + "loss": 1.06243062, + "num_input_tokens_seen": 48989085, + "router_z_loss_clip": 1.24804688, + "router_z_loss_mlp": 0.24719238, + "step": 1738, + "time_per_iteration": 2.5335745811462402 + }, + { + "auxiliary_loss_clip": 0.01051616, + "auxiliary_loss_mlp": 0.01001672, + "balance_loss_clip": 1.02146161, + "balance_loss_mlp": 1.00008702, + "epoch": 0.050461377749405144, + "flos": 64722591527040.0, + "grad_norm": 0.757844774123165, + "language_loss": 0.4768914, + "learning_rate": 3.995611679791107e-06, + "loss": 0.4974243, + "num_input_tokens_seen": 49036660, + "router_z_loss_clip": 0.30078125, + "router_z_loss_mlp": 0.01586914, + "step": 1739, + "time_per_iteration": 2.8615872859954834 + }, + { + "auxiliary_loss_clip": 0.01196759, + "auxiliary_loss_mlp": 0.01067348, + "balance_loss_clip": 1.07516086, + "balance_loss_mlp": 1.04419756, + "epoch": 0.05049039521792119, + "flos": 11140280553600.0, + "grad_norm": 2.714901706161417, + "language_loss": 0.94337678, + "learning_rate": 3.995599226397321e-06, + "loss": 0.96601784, + "num_input_tokens_seen": 49049080, + "router_z_loss_clip": 1.21484375, + "router_z_loss_mlp": 0.23156738, + "step": 1740, + "time_per_iteration": 2.455662488937378 + }, + { + "auxiliary_loss_clip": 0.0118945, + "auxiliary_loss_mlp": 0.01053904, + "balance_loss_clip": 1.07378161, + "balance_loss_mlp": 1.03343546, + "epoch": 0.05051941268643723, + "flos": 21903138950400.0, + "grad_norm": 2.2875155764246737, + "language_loss": 0.84240854, + "learning_rate": 3.995586755377595e-06, + "loss": 0.864842, + "num_input_tokens_seen": 49063855, + "router_z_loss_clip": 1.15625, + "router_z_loss_mlp": 0.20458984, + "step": 1741, + "time_per_iteration": 2.674687385559082 + }, + { + "auxiliary_loss_clip": 0.01050024, + "auxiliary_loss_mlp": 0.0100152, + "balance_loss_clip": 1.01992798, + "balance_loss_mlp": 0.99995881, + "epoch": 0.050548430154953285, + "flos": 63696446766720.0, + "grad_norm": 0.6336689691189321, + "language_loss": 0.53679931, + "learning_rate": 3.99557426673204e-06, + "loss": 0.55731475, + "num_input_tokens_seen": 49133965, + "router_z_loss_clip": 0.30078125, + "router_z_loss_mlp": 0.015625, + "step": 1742, + "time_per_iteration": 3.2751612663269043 + }, + { + "auxiliary_loss_clip": 0.01205288, + "auxiliary_loss_mlp": 0.01063362, + "balance_loss_clip": 1.07930732, + "balance_loss_mlp": 1.04185629, + "epoch": 0.05057744762346933, + "flos": 16059503030400.0, + "grad_norm": 3.0674232021793038, + "language_loss": 0.78697151, + "learning_rate": 3.9955617604607644e-06, + "loss": 0.80965793, + "num_input_tokens_seen": 49145200, + "router_z_loss_clip": 1.2578125, + "router_z_loss_mlp": 0.21496582, + "step": 1743, + "time_per_iteration": 2.522610664367676 + }, + { + "auxiliary_loss_clip": 0.01201944, + "auxiliary_loss_mlp": 0.01065326, + "balance_loss_clip": 1.07996917, + "balance_loss_mlp": 1.04345155, + "epoch": 0.050606465091985374, + "flos": 74733227303040.0, + "grad_norm": 2.2761043724370698, + "language_loss": 0.69563806, + "learning_rate": 3.99554923656388e-06, + "loss": 0.71831077, + "num_input_tokens_seen": 49169180, + "router_z_loss_clip": 1.21875, + "router_z_loss_mlp": 0.21887207, + "step": 1744, + "time_per_iteration": 2.934464931488037 + }, + { + "auxiliary_loss_clip": 0.0119227, + "auxiliary_loss_mlp": 0.01060951, + "balance_loss_clip": 1.07594275, + "balance_loss_mlp": 1.03974319, + "epoch": 0.05063548256050142, + "flos": 18042096165120.0, + "grad_norm": 3.9357858551295273, + "language_loss": 0.86982977, + "learning_rate": 3.995536695041499e-06, + "loss": 0.892362, + "num_input_tokens_seen": 49183930, + "router_z_loss_clip": 1.16503906, + "router_z_loss_mlp": 0.21179199, + "step": 1745, + "time_per_iteration": 2.541919231414795 + }, + { + "auxiliary_loss_clip": 0.0118766, + "auxiliary_loss_mlp": 0.01052267, + "balance_loss_clip": 1.07213295, + "balance_loss_mlp": 1.03226399, + "epoch": 0.05066450002901747, + "flos": 31682691555840.0, + "grad_norm": 2.0632351551520514, + "language_loss": 0.86533815, + "learning_rate": 3.995524135893728e-06, + "loss": 0.88773739, + "num_input_tokens_seen": 49200215, + "router_z_loss_clip": 1.15332031, + "router_z_loss_mlp": 0.1998291, + "step": 1746, + "time_per_iteration": 2.677502393722534 + }, + { + "auxiliary_loss_clip": 0.01051637, + "auxiliary_loss_mlp": 0.01014779, + "balance_loss_clip": 1.02174973, + "balance_loss_mlp": 1.01334822, + "epoch": 0.050693517497533515, + "flos": 60108179863680.0, + "grad_norm": 0.6597876276045882, + "language_loss": 0.50066763, + "learning_rate": 3.995511559120681e-06, + "loss": 0.52133179, + "num_input_tokens_seen": 49264335, + "router_z_loss_clip": 0.29882812, + "router_z_loss_mlp": 0.01428223, + "step": 1747, + "time_per_iteration": 3.199213981628418 + }, + { + "auxiliary_loss_clip": 0.01052226, + "auxiliary_loss_mlp": 0.01006036, + "balance_loss_clip": 1.0220952, + "balance_loss_mlp": 1.00458777, + "epoch": 0.05072253496604956, + "flos": 61373850468480.0, + "grad_norm": 1.1714970696918192, + "language_loss": 0.51075947, + "learning_rate": 3.995498964722469e-06, + "loss": 0.53134209, + "num_input_tokens_seen": 49319605, + "router_z_loss_clip": 0.30078125, + "router_z_loss_mlp": 0.01446533, + "step": 1748, + "time_per_iteration": 3.0235776901245117 + }, + { + "auxiliary_loss_clip": 0.01185328, + "auxiliary_loss_mlp": 0.010498, + "balance_loss_clip": 1.07381094, + "balance_loss_mlp": 1.03168046, + "epoch": 0.05075155243456561, + "flos": 16392215733120.0, + "grad_norm": 2.934009387692858, + "language_loss": 0.81376433, + "learning_rate": 3.9954863526992026e-06, + "loss": 0.8361156, + "num_input_tokens_seen": 49333280, + "router_z_loss_clip": 1.11621094, + "router_z_loss_mlp": 0.18115234, + "step": 1749, + "time_per_iteration": 2.462038516998291 + }, + { + "auxiliary_loss_clip": 0.01204923, + "auxiliary_loss_mlp": 0.01059819, + "balance_loss_clip": 1.07846665, + "balance_loss_mlp": 1.03757429, + "epoch": 0.050780569903081656, + "flos": 20074818729600.0, + "grad_norm": 2.435438611713195, + "language_loss": 0.7366001, + "learning_rate": 3.995473723050993e-06, + "loss": 0.75924754, + "num_input_tokens_seen": 49347555, + "router_z_loss_clip": 1.26367188, + "router_z_loss_mlp": 0.22265625, + "step": 1750, + "time_per_iteration": 2.5130412578582764 + }, + { + "auxiliary_loss_clip": 0.01190019, + "auxiliary_loss_mlp": 0.01047209, + "balance_loss_clip": 1.07480216, + "balance_loss_mlp": 1.02761054, + "epoch": 0.0508095873715977, + "flos": 10809363530880.0, + "grad_norm": 2.835484691439645, + "language_loss": 0.79064345, + "learning_rate": 3.995461075777952e-06, + "loss": 0.8130157, + "num_input_tokens_seen": 49358300, + "router_z_loss_clip": 1.15429688, + "router_z_loss_mlp": 0.19604492, + "step": 1751, + "time_per_iteration": 2.4550185203552246 + }, + { + "auxiliary_loss_clip": 0.01184404, + "auxiliary_loss_mlp": 0.01053976, + "balance_loss_clip": 1.07233906, + "balance_loss_mlp": 1.0347358, + "epoch": 0.05083860484011375, + "flos": 74732329463040.0, + "grad_norm": 2.155438399921072, + "language_loss": 0.81918818, + "learning_rate": 3.995448410880192e-06, + "loss": 0.84157205, + "num_input_tokens_seen": 49378550, + "router_z_loss_clip": 1.12060547, + "router_z_loss_mlp": 0.19250488, + "step": 1752, + "time_per_iteration": 2.916825771331787 + }, + { + "auxiliary_loss_clip": 0.01190716, + "auxiliary_loss_mlp": 0.01056417, + "balance_loss_clip": 1.07489848, + "balance_loss_mlp": 1.03616309, + "epoch": 0.0508676223086298, + "flos": 24894461727360.0, + "grad_norm": 2.2994712183130606, + "language_loss": 0.7810725, + "learning_rate": 3.995435728357823e-06, + "loss": 0.80354381, + "num_input_tokens_seen": 49392820, + "router_z_loss_clip": 1.15917969, + "router_z_loss_mlp": 0.20233154, + "step": 1753, + "time_per_iteration": 2.5615406036376953 + }, + { + "auxiliary_loss_clip": 0.01206691, + "auxiliary_loss_mlp": 0.01064189, + "balance_loss_clip": 1.0815351, + "balance_loss_mlp": 1.03991807, + "epoch": 0.05089663977714584, + "flos": 16245448761600.0, + "grad_norm": 2.660159041875012, + "language_loss": 0.74406499, + "learning_rate": 3.995423028210959e-06, + "loss": 0.76677382, + "num_input_tokens_seen": 49406565, + "router_z_loss_clip": 1.24951172, + "router_z_loss_mlp": 0.24267578, + "step": 1754, + "time_per_iteration": 2.4750843048095703 + }, + { + "auxiliary_loss_clip": 0.011939, + "auxiliary_loss_mlp": 0.01053814, + "balance_loss_clip": 1.07433188, + "balance_loss_mlp": 1.0336678, + "epoch": 0.050925657245661886, + "flos": 27234042180480.0, + "grad_norm": 1.8675354101511208, + "language_loss": 0.65371287, + "learning_rate": 3.995410310439711e-06, + "loss": 0.67619002, + "num_input_tokens_seen": 49430015, + "router_z_loss_clip": 1.19628906, + "router_z_loss_mlp": 0.20141602, + "step": 1755, + "time_per_iteration": 2.655015468597412 + }, + { + "auxiliary_loss_clip": 0.01201361, + "auxiliary_loss_mlp": 0.01053605, + "balance_loss_clip": 1.07808483, + "balance_loss_mlp": 1.03294611, + "epoch": 0.05095467471417794, + "flos": 13296177832320.0, + "grad_norm": 2.9501729006085986, + "language_loss": 0.99529946, + "learning_rate": 3.9953975750441915e-06, + "loss": 1.01784909, + "num_input_tokens_seen": 49443105, + "router_z_loss_clip": 1.23242188, + "router_z_loss_mlp": 0.2064209, + "step": 1756, + "time_per_iteration": 2.465244770050049 + }, + { + "auxiliary_loss_clip": 0.01204933, + "auxiliary_loss_mlp": 0.01069856, + "balance_loss_clip": 1.08019757, + "balance_loss_mlp": 1.04761112, + "epoch": 0.05098369218269398, + "flos": 15735625073280.0, + "grad_norm": 3.111218370905101, + "language_loss": 0.78989607, + "learning_rate": 3.995384822024513e-06, + "loss": 0.81264395, + "num_input_tokens_seen": 49456990, + "router_z_loss_clip": 1.24609375, + "router_z_loss_mlp": 0.22253418, + "step": 1757, + "time_per_iteration": 2.463263988494873 + }, + { + "auxiliary_loss_clip": 0.01175093, + "auxiliary_loss_mlp": 0.0105012, + "balance_loss_clip": 1.07069802, + "balance_loss_mlp": 1.03170276, + "epoch": 0.05101270965121003, + "flos": 25440447432960.0, + "grad_norm": 2.9114026097077876, + "language_loss": 0.91784567, + "learning_rate": 3.995372051380789e-06, + "loss": 0.94009781, + "num_input_tokens_seen": 49470115, + "router_z_loss_clip": 1.04199219, + "router_z_loss_mlp": 0.184021, + "step": 1758, + "time_per_iteration": 2.590700626373291 + }, + { + "auxiliary_loss_clip": 0.0120842, + "auxiliary_loss_mlp": 0.01064458, + "balance_loss_clip": 1.07722974, + "balance_loss_mlp": 1.04070008, + "epoch": 0.05104172711972608, + "flos": 15261927488640.0, + "grad_norm": 3.0738067438601724, + "language_loss": 0.82619619, + "learning_rate": 3.9953592631131315e-06, + "loss": 0.84892499, + "num_input_tokens_seen": 49483675, + "router_z_loss_clip": 1.31152344, + "router_z_loss_mlp": 0.23742676, + "step": 1759, + "time_per_iteration": 2.474508285522461 + }, + { + "auxiliary_loss_clip": 0.01187715, + "auxiliary_loss_mlp": 0.01040785, + "balance_loss_clip": 1.07594645, + "balance_loss_mlp": 1.02486444, + "epoch": 0.05107074458824212, + "flos": 30219618781440.0, + "grad_norm": 2.188189312196203, + "language_loss": 0.77462929, + "learning_rate": 3.995346457221653e-06, + "loss": 0.79691434, + "num_input_tokens_seen": 49500015, + "router_z_loss_clip": 1.11767578, + "router_z_loss_mlp": 0.15930176, + "step": 1760, + "time_per_iteration": 2.6440231800079346 + }, + { + "auxiliary_loss_clip": 0.01058999, + "auxiliary_loss_mlp": 0.0099663, + "balance_loss_clip": 1.02806807, + "balance_loss_mlp": 0.99507999, + "epoch": 0.05109976205675817, + "flos": 57110787688320.0, + "grad_norm": 0.7290277416316661, + "language_loss": 0.56076849, + "learning_rate": 3.995333633706468e-06, + "loss": 0.58132482, + "num_input_tokens_seen": 49551920, + "router_z_loss_clip": 0.30859375, + "router_z_loss_mlp": 0.01550293, + "step": 1761, + "time_per_iteration": 2.9118332862854004 + }, + { + "auxiliary_loss_clip": 0.01057015, + "auxiliary_loss_mlp": 0.00997254, + "balance_loss_clip": 1.026227, + "balance_loss_mlp": 0.99565619, + "epoch": 0.05112877952527421, + "flos": 74779950896640.0, + "grad_norm": 0.7334410431262246, + "language_loss": 0.5125649, + "learning_rate": 3.995320792567688e-06, + "loss": 0.53310764, + "num_input_tokens_seen": 49620250, + "router_z_loss_clip": 0.30859375, + "router_z_loss_mlp": 0.01599121, + "step": 1762, + "time_per_iteration": 3.2768735885620117 + }, + { + "auxiliary_loss_clip": 0.01206337, + "auxiliary_loss_mlp": 0.01067305, + "balance_loss_clip": 1.08369255, + "balance_loss_mlp": 1.04602587, + "epoch": 0.051157796993790264, + "flos": 21173721465600.0, + "grad_norm": 2.1916890806600415, + "language_loss": 0.85158163, + "learning_rate": 3.995307933805428e-06, + "loss": 0.87431806, + "num_input_tokens_seen": 49634965, + "router_z_loss_clip": 1.2265625, + "router_z_loss_mlp": 0.21276855, + "step": 1763, + "time_per_iteration": 2.574139356613159 + }, + { + "auxiliary_loss_clip": 0.01053878, + "auxiliary_loss_mlp": 0.0100458, + "balance_loss_clip": 1.02367556, + "balance_loss_mlp": 1.00286329, + "epoch": 0.05118681446230631, + "flos": 74773989239040.0, + "grad_norm": 0.6543437199639005, + "language_loss": 0.53189707, + "learning_rate": 3.9952950574198e-06, + "loss": 0.55248165, + "num_input_tokens_seen": 49697380, + "router_z_loss_clip": 0.30078125, + "router_z_loss_mlp": 0.01721191, + "step": 1764, + "time_per_iteration": 3.1072728633880615 + }, + { + "auxiliary_loss_clip": 0.01200443, + "auxiliary_loss_mlp": 0.01061997, + "balance_loss_clip": 1.07713699, + "balance_loss_mlp": 1.03764248, + "epoch": 0.05121583193082235, + "flos": 15809170170240.0, + "grad_norm": 3.164844784978793, + "language_loss": 1.0479511, + "learning_rate": 3.99528216341092e-06, + "loss": 1.07057548, + "num_input_tokens_seen": 49712370, + "router_z_loss_clip": 1.234375, + "router_z_loss_mlp": 0.24365234, + "step": 1765, + "time_per_iteration": 2.5160510540008545 + }, + { + "auxiliary_loss_clip": 0.01052537, + "auxiliary_loss_mlp": 0.0101799, + "balance_loss_clip": 1.02213836, + "balance_loss_mlp": 1.01640439, + "epoch": 0.051244849399338405, + "flos": 51322671216000.0, + "grad_norm": 0.7657641852101512, + "language_loss": 0.52507645, + "learning_rate": 3.9952692517789e-06, + "loss": 0.54578173, + "num_input_tokens_seen": 49771515, + "router_z_loss_clip": 0.3046875, + "router_z_loss_mlp": 0.01586914, + "step": 1766, + "time_per_iteration": 3.0527865886688232 + }, + { + "auxiliary_loss_clip": 0.01051243, + "auxiliary_loss_mlp": 0.01026, + "balance_loss_clip": 1.02092957, + "balance_loss_mlp": 1.02431953, + "epoch": 0.05127386686785445, + "flos": 71886268264320.0, + "grad_norm": 0.7175888297214154, + "language_loss": 0.49713016, + "learning_rate": 3.995256322523854e-06, + "loss": 0.51790255, + "num_input_tokens_seen": 49827015, + "router_z_loss_clip": 0.30273438, + "router_z_loss_mlp": 0.0168457, + "step": 1767, + "time_per_iteration": 3.03136944770813 + }, + { + "auxiliary_loss_clip": 0.01195855, + "auxiliary_loss_mlp": 0.01062109, + "balance_loss_clip": 1.0762285, + "balance_loss_mlp": 1.04288077, + "epoch": 0.051302884336370494, + "flos": 14165466877440.0, + "grad_norm": 3.630359531574853, + "language_loss": 1.03818953, + "learning_rate": 3.995243375645898e-06, + "loss": 1.06076932, + "num_input_tokens_seen": 49838015, + "router_z_loss_clip": 1.1953125, + "router_z_loss_mlp": 0.19226074, + "step": 1768, + "time_per_iteration": 2.4961419105529785 + }, + { + "auxiliary_loss_clip": 0.01049463, + "auxiliary_loss_mlp": 0.01015867, + "balance_loss_clip": 1.01931679, + "balance_loss_mlp": 1.01421034, + "epoch": 0.05133190180488654, + "flos": 62036905576320.0, + "grad_norm": 0.644818984992438, + "language_loss": 0.48709983, + "learning_rate": 3.995230411145144e-06, + "loss": 0.50775307, + "num_input_tokens_seen": 49899960, + "router_z_loss_clip": 0.30078125, + "router_z_loss_mlp": 0.01660156, + "step": 1769, + "time_per_iteration": 3.0819251537323 + }, + { + "auxiliary_loss_clip": 0.01206195, + "auxiliary_loss_mlp": 0.01064928, + "balance_loss_clip": 1.0784452, + "balance_loss_mlp": 1.0418489, + "epoch": 0.05136091927340259, + "flos": 32700507338880.0, + "grad_norm": 2.7706769642007067, + "language_loss": 0.93956685, + "learning_rate": 3.995217429021708e-06, + "loss": 0.96227807, + "num_input_tokens_seen": 49918150, + "router_z_loss_clip": 1.27734375, + "router_z_loss_mlp": 0.23095703, + "step": 1770, + "time_per_iteration": 2.6886162757873535 + }, + { + "auxiliary_loss_clip": 0.01187379, + "auxiliary_loss_mlp": 0.01064824, + "balance_loss_clip": 1.07247734, + "balance_loss_mlp": 1.04542267, + "epoch": 0.051389936741918635, + "flos": 16062878908800.0, + "grad_norm": 2.766678381141891, + "language_loss": 0.73310125, + "learning_rate": 3.995204429275704e-06, + "loss": 0.75562322, + "num_input_tokens_seen": 49932735, + "router_z_loss_clip": 1.14941406, + "router_z_loss_mlp": 0.19396973, + "step": 1771, + "time_per_iteration": 2.4587764739990234 + }, + { + "auxiliary_loss_clip": 0.01205731, + "auxiliary_loss_mlp": 0.01077566, + "balance_loss_clip": 1.08074903, + "balance_loss_mlp": 1.05575097, + "epoch": 0.05141895421043468, + "flos": 29488980234240.0, + "grad_norm": 2.5747623143351217, + "language_loss": 0.85019863, + "learning_rate": 3.9951914119072466e-06, + "loss": 0.87303162, + "num_input_tokens_seen": 49947890, + "router_z_loss_clip": 1.25, + "router_z_loss_mlp": 0.21832275, + "step": 1772, + "time_per_iteration": 2.608149766921997 + }, + { + "auxiliary_loss_clip": 0.01193247, + "auxiliary_loss_mlp": 0.01063799, + "balance_loss_clip": 1.07553005, + "balance_loss_mlp": 1.04255629, + "epoch": 0.05144797167895073, + "flos": 14165287309440.0, + "grad_norm": 3.028156091977049, + "language_loss": 0.80174518, + "learning_rate": 3.995178376916453e-06, + "loss": 0.82431561, + "num_input_tokens_seen": 49960110, + "router_z_loss_clip": 1.17578125, + "router_z_loss_mlp": 0.21264648, + "step": 1773, + "time_per_iteration": 2.4703633785247803 + }, + { + "auxiliary_loss_clip": 0.01195369, + "auxiliary_loss_mlp": 0.01051583, + "balance_loss_clip": 1.07909882, + "balance_loss_mlp": 1.03379703, + "epoch": 0.051476989147466776, + "flos": 30476487916800.0, + "grad_norm": 5.373175113535293, + "language_loss": 1.10353911, + "learning_rate": 3.9951653243034355e-06, + "loss": 1.12600863, + "num_input_tokens_seen": 49979035, + "router_z_loss_clip": 1.16113281, + "router_z_loss_mlp": 0.17785645, + "step": 1774, + "time_per_iteration": 2.604649066925049 + }, + { + "auxiliary_loss_clip": 0.01053205, + "auxiliary_loss_mlp": 0.01011114, + "balance_loss_clip": 1.02217603, + "balance_loss_mlp": 1.00939739, + "epoch": 0.05150600661598282, + "flos": 74795321917440.0, + "grad_norm": 0.654045910981279, + "language_loss": 0.45754775, + "learning_rate": 3.99515225406831e-06, + "loss": 0.47819099, + "num_input_tokens_seen": 50040985, + "router_z_loss_clip": 0.31054688, + "router_z_loss_mlp": 0.01721191, + "step": 1775, + "time_per_iteration": 3.2988250255584717 + }, + { + "auxiliary_loss_clip": 0.01193532, + "auxiliary_loss_mlp": 0.01055568, + "balance_loss_clip": 1.07584786, + "balance_loss_mlp": 1.03352618, + "epoch": 0.051535024084498865, + "flos": 25332966961920.0, + "grad_norm": 2.2066052941883303, + "language_loss": 0.80329937, + "learning_rate": 3.995139166211193e-06, + "loss": 0.82579041, + "num_input_tokens_seen": 50058245, + "router_z_loss_clip": 1.17480469, + "router_z_loss_mlp": 0.22021484, + "step": 1776, + "time_per_iteration": 2.5613646507263184 + }, + { + "auxiliary_loss_clip": 0.01052914, + "auxiliary_loss_mlp": 0.01001577, + "balance_loss_clip": 1.0223105, + "balance_loss_mlp": 0.99993193, + "epoch": 0.05156404155301492, + "flos": 59536985788800.0, + "grad_norm": 0.7151004683444269, + "language_loss": 0.51963115, + "learning_rate": 3.9951260607322e-06, + "loss": 0.54017603, + "num_input_tokens_seen": 50120770, + "router_z_loss_clip": 0.30664062, + "router_z_loss_mlp": 0.01647949, + "step": 1777, + "time_per_iteration": 3.1045708656311035 + }, + { + "auxiliary_loss_clip": 0.01052378, + "auxiliary_loss_mlp": 0.00999449, + "balance_loss_clip": 1.02207732, + "balance_loss_mlp": 0.99779218, + "epoch": 0.05159305902153096, + "flos": 66640582051200.0, + "grad_norm": 0.642420620375206, + "language_loss": 0.50713134, + "learning_rate": 3.995112937631446e-06, + "loss": 0.52764958, + "num_input_tokens_seen": 50186035, + "router_z_loss_clip": 0.30273438, + "router_z_loss_mlp": 0.01660156, + "step": 1778, + "time_per_iteration": 3.214977979660034 + }, + { + "auxiliary_loss_clip": 0.01214059, + "auxiliary_loss_mlp": 0.01070184, + "balance_loss_clip": 1.08229399, + "balance_loss_mlp": 1.04719996, + "epoch": 0.051622076490047006, + "flos": 20407603259520.0, + "grad_norm": 2.037188389989393, + "language_loss": 0.78549516, + "learning_rate": 3.9950997969090465e-06, + "loss": 0.80833757, + "num_input_tokens_seen": 50203225, + "router_z_loss_clip": 1.31835938, + "router_z_loss_mlp": 0.22998047, + "step": 1779, + "time_per_iteration": 2.588412284851074 + }, + { + "auxiliary_loss_clip": 0.01195688, + "auxiliary_loss_mlp": 0.01048764, + "balance_loss_clip": 1.0755899, + "balance_loss_mlp": 1.0290947, + "epoch": 0.05165109395856306, + "flos": 10589518339200.0, + "grad_norm": 3.126926223938334, + "language_loss": 0.70587486, + "learning_rate": 3.995086638565119e-06, + "loss": 0.72831935, + "num_input_tokens_seen": 50214900, + "router_z_loss_clip": 1.20117188, + "router_z_loss_mlp": 0.19665527, + "step": 1780, + "time_per_iteration": 2.4455044269561768 + }, + { + "auxiliary_loss_clip": 0.01186881, + "auxiliary_loss_mlp": 0.01050364, + "balance_loss_clip": 1.07323074, + "balance_loss_mlp": 1.03267968, + "epoch": 0.0516801114270791, + "flos": 12597678979200.0, + "grad_norm": 2.698636278864604, + "language_loss": 0.79776835, + "learning_rate": 3.9950734625997795e-06, + "loss": 0.82014084, + "num_input_tokens_seen": 50227670, + "router_z_loss_clip": 1.13769531, + "router_z_loss_mlp": 0.17681885, + "step": 1781, + "time_per_iteration": 2.4552459716796875 + }, + { + "auxiliary_loss_clip": 0.01192732, + "auxiliary_loss_mlp": 0.01063848, + "balance_loss_clip": 1.07298994, + "balance_loss_mlp": 1.04270041, + "epoch": 0.05170912889559515, + "flos": 32190504082560.0, + "grad_norm": 2.502936679274979, + "language_loss": 0.9516871, + "learning_rate": 3.995060269013142e-06, + "loss": 0.97425288, + "num_input_tokens_seen": 50243495, + "router_z_loss_clip": 1.19775391, + "router_z_loss_mlp": 0.21142578, + "step": 1782, + "time_per_iteration": 7.339204549789429 + }, + { + "auxiliary_loss_clip": 0.01191219, + "auxiliary_loss_mlp": 0.0106047, + "balance_loss_clip": 1.07165766, + "balance_loss_mlp": 1.04021645, + "epoch": 0.0517381463641112, + "flos": 18034590222720.0, + "grad_norm": 3.2703784480404106, + "language_loss": 0.90765256, + "learning_rate": 3.9950470578053265e-06, + "loss": 0.93016946, + "num_input_tokens_seen": 50259360, + "router_z_loss_clip": 1.1953125, + "router_z_loss_mlp": 0.20263672, + "step": 1783, + "time_per_iteration": 2.4929962158203125 + }, + { + "auxiliary_loss_clip": 0.01186288, + "auxiliary_loss_mlp": 0.01073971, + "balance_loss_clip": 1.07219362, + "balance_loss_mlp": 1.05625057, + "epoch": 0.05176716383262724, + "flos": 36788003418240.0, + "grad_norm": 2.0137276640518404, + "language_loss": 0.77752143, + "learning_rate": 3.995033828976448e-06, + "loss": 0.80012405, + "num_input_tokens_seen": 50279695, + "router_z_loss_clip": 1.13964844, + "router_z_loss_mlp": 0.17706299, + "step": 1784, + "time_per_iteration": 4.956677198410034 + }, + { + "auxiliary_loss_clip": 0.01189812, + "auxiliary_loss_mlp": 0.01060151, + "balance_loss_clip": 1.07663107, + "balance_loss_mlp": 1.04205513, + "epoch": 0.05179618130114329, + "flos": 23616257276160.0, + "grad_norm": 2.408125598074265, + "language_loss": 0.88012624, + "learning_rate": 3.995020582526623e-06, + "loss": 0.90262586, + "num_input_tokens_seen": 50292505, + "router_z_loss_clip": 1.13037109, + "router_z_loss_mlp": 0.18115234, + "step": 1785, + "time_per_iteration": 2.5502123832702637 + }, + { + "auxiliary_loss_clip": 0.01198066, + "auxiliary_loss_mlp": 0.01067635, + "balance_loss_clip": 1.07607317, + "balance_loss_mlp": 1.04621303, + "epoch": 0.05182519876965933, + "flos": 29634993020160.0, + "grad_norm": 2.6986383204091733, + "language_loss": 1.0092659, + "learning_rate": 3.995007318455968e-06, + "loss": 1.03192294, + "num_input_tokens_seen": 50306520, + "router_z_loss_clip": 1.22167969, + "router_z_loss_mlp": 0.2142334, + "step": 1786, + "time_per_iteration": 2.584237575531006 + }, + { + "auxiliary_loss_clip": 0.01057242, + "auxiliary_loss_mlp": 0.01077608, + "balance_loss_clip": 1.02587426, + "balance_loss_mlp": 1.07574797, + "epoch": 0.051854216238175384, + "flos": 57224373471360.0, + "grad_norm": 0.7169527462174398, + "language_loss": 0.50604445, + "learning_rate": 3.994994036764603e-06, + "loss": 0.52739298, + "num_input_tokens_seen": 50364155, + "router_z_loss_clip": 0.31445312, + "router_z_loss_mlp": 0.01855469, + "step": 1787, + "time_per_iteration": 3.0971767902374268 + }, + { + "auxiliary_loss_clip": 0.01054744, + "auxiliary_loss_mlp": 0.01043211, + "balance_loss_clip": 1.02409673, + "balance_loss_mlp": 1.04147041, + "epoch": 0.05188323370669143, + "flos": 74791407335040.0, + "grad_norm": 0.6609879107354696, + "language_loss": 0.4780764, + "learning_rate": 3.994980737452642e-06, + "loss": 0.49905592, + "num_input_tokens_seen": 50428965, + "router_z_loss_clip": 0.30664062, + "router_z_loss_mlp": 0.01745605, + "step": 1788, + "time_per_iteration": 3.2858104705810547 + }, + { + "auxiliary_loss_clip": 0.01197181, + "auxiliary_loss_mlp": 0.01050099, + "balance_loss_clip": 1.07792544, + "balance_loss_mlp": 1.02717459, + "epoch": 0.05191225117520747, + "flos": 19162687737600.0, + "grad_norm": 3.02545933774612, + "language_loss": 0.92299974, + "learning_rate": 3.994967420520204e-06, + "loss": 0.9454726, + "num_input_tokens_seen": 50441310, + "router_z_loss_clip": 1.19287109, + "router_z_loss_mlp": 0.22937012, + "step": 1789, + "time_per_iteration": 2.481212615966797 + }, + { + "auxiliary_loss_clip": 0.01190358, + "auxiliary_loss_mlp": 0.01051376, + "balance_loss_clip": 1.07379389, + "balance_loss_mlp": 1.03275514, + "epoch": 0.051941268643723525, + "flos": 15589863682560.0, + "grad_norm": 2.918023463293398, + "language_loss": 0.85189521, + "learning_rate": 3.994954085967407e-06, + "loss": 0.87431264, + "num_input_tokens_seen": 50453875, + "router_z_loss_clip": 1.16601562, + "router_z_loss_mlp": 0.18652344, + "step": 1790, + "time_per_iteration": 2.4885683059692383 + }, + { + "auxiliary_loss_clip": 0.01054294, + "auxiliary_loss_mlp": 0.01013244, + "balance_loss_clip": 1.02274036, + "balance_loss_mlp": 1.01163507, + "epoch": 0.05197028611223957, + "flos": 61382290164480.0, + "grad_norm": 0.6980871873882708, + "language_loss": 0.51786959, + "learning_rate": 3.994940733794368e-06, + "loss": 0.53854489, + "num_input_tokens_seen": 50507665, + "router_z_loss_clip": 0.31640625, + "router_z_loss_mlp": 0.01611328, + "step": 1791, + "time_per_iteration": 2.9400370121002197 + }, + { + "auxiliary_loss_clip": 0.0119759, + "auxiliary_loss_mlp": 0.01062061, + "balance_loss_clip": 1.0799371, + "balance_loss_mlp": 1.04358339, + "epoch": 0.051999303580755614, + "flos": 17230406578560.0, + "grad_norm": 3.667173555647355, + "language_loss": 0.83325803, + "learning_rate": 3.9949273640012056e-06, + "loss": 0.85585463, + "num_input_tokens_seen": 50519505, + "router_z_loss_clip": 1.17578125, + "router_z_loss_mlp": 0.18469238, + "step": 1792, + "time_per_iteration": 2.4639413356781006 + }, + { + "auxiliary_loss_clip": 0.01202675, + "auxiliary_loss_mlp": 0.01079057, + "balance_loss_clip": 1.07523036, + "balance_loss_mlp": 1.05560815, + "epoch": 0.05202832104927166, + "flos": 25184152915200.0, + "grad_norm": 2.771952071892185, + "language_loss": 1.0346396, + "learning_rate": 3.994913976588036e-06, + "loss": 1.05745697, + "num_input_tokens_seen": 50535170, + "router_z_loss_clip": 1.2734375, + "router_z_loss_mlp": 0.23474121, + "step": 1793, + "time_per_iteration": 2.5159847736358643 + }, + { + "auxiliary_loss_clip": 0.01195299, + "auxiliary_loss_mlp": 0.01073409, + "balance_loss_clip": 1.07685959, + "balance_loss_mlp": 1.05418038, + "epoch": 0.05205733851778771, + "flos": 11683357257600.0, + "grad_norm": 3.6555449103821136, + "language_loss": 0.97756028, + "learning_rate": 3.99490057155498e-06, + "loss": 1.00024736, + "num_input_tokens_seen": 50546945, + "router_z_loss_clip": 1.18359375, + "router_z_loss_mlp": 0.19238281, + "step": 1794, + "time_per_iteration": 2.5012218952178955 + }, + { + "auxiliary_loss_clip": 0.01196522, + "auxiliary_loss_mlp": 0.0107202, + "balance_loss_clip": 1.07447481, + "balance_loss_mlp": 1.05138516, + "epoch": 0.052086355986303755, + "flos": 28577459773440.0, + "grad_norm": 2.3173061552124166, + "language_loss": 0.9098177, + "learning_rate": 3.994887148902155e-06, + "loss": 0.93250322, + "num_input_tokens_seen": 50562205, + "router_z_loss_clip": 1.22070312, + "router_z_loss_mlp": 0.2064209, + "step": 1795, + "time_per_iteration": 2.5735998153686523 + }, + { + "auxiliary_loss_clip": 0.01055817, + "auxiliary_loss_mlp": 0.01028263, + "balance_loss_clip": 1.02285314, + "balance_loss_mlp": 1.02666557, + "epoch": 0.0521153734548198, + "flos": 67034772881280.0, + "grad_norm": 0.7258968206202896, + "language_loss": 0.54031086, + "learning_rate": 3.99487370862968e-06, + "loss": 0.56115168, + "num_input_tokens_seen": 50617275, + "router_z_loss_clip": 0.33007812, + "router_z_loss_mlp": 0.01599121, + "step": 1796, + "time_per_iteration": 3.133246660232544 + }, + { + "auxiliary_loss_clip": 0.01188848, + "auxiliary_loss_mlp": 0.01049897, + "balance_loss_clip": 1.07216978, + "balance_loss_mlp": 1.03132391, + "epoch": 0.05214439092333585, + "flos": 12707098784640.0, + "grad_norm": 2.840571222812208, + "language_loss": 0.87814349, + "learning_rate": 3.994860250737673e-06, + "loss": 0.90053099, + "num_input_tokens_seen": 50630635, + "router_z_loss_clip": 1.16503906, + "router_z_loss_mlp": 0.18566895, + "step": 1797, + "time_per_iteration": 2.456064224243164 + }, + { + "auxiliary_loss_clip": 0.01197608, + "auxiliary_loss_mlp": 0.01051537, + "balance_loss_clip": 1.07761335, + "balance_loss_mlp": 1.03263068, + "epoch": 0.052173408391851896, + "flos": 52222516778880.0, + "grad_norm": 2.173122035633328, + "language_loss": 0.8463341, + "learning_rate": 3.994846775226252e-06, + "loss": 0.86882555, + "num_input_tokens_seen": 50652455, + "router_z_loss_clip": 1.20019531, + "router_z_loss_mlp": 0.18896484, + "step": 1798, + "time_per_iteration": 2.7629857063293457 + }, + { + "auxiliary_loss_clip": 0.01205995, + "auxiliary_loss_mlp": 0.0107314, + "balance_loss_clip": 1.0777241, + "balance_loss_mlp": 1.0499177, + "epoch": 0.05220242586036794, + "flos": 23689586891520.0, + "grad_norm": 3.5222318925957112, + "language_loss": 1.17945635, + "learning_rate": 3.9948332820955365e-06, + "loss": 1.20224786, + "num_input_tokens_seen": 50668495, + "router_z_loss_clip": 1.28125, + "router_z_loss_mlp": 0.23217773, + "step": 1799, + "time_per_iteration": 2.570039987564087 + }, + { + "auxiliary_loss_clip": 0.01191446, + "auxiliary_loss_mlp": 0.01057856, + "balance_loss_clip": 1.07192385, + "balance_loss_mlp": 1.0361954, + "epoch": 0.052231443328883985, + "flos": 26937240099840.0, + "grad_norm": 3.587189373151435, + "language_loss": 0.79654741, + "learning_rate": 3.994819771345648e-06, + "loss": 0.81904042, + "num_input_tokens_seen": 50686755, + "router_z_loss_clip": 1.1953125, + "router_z_loss_mlp": 0.21643066, + "step": 1800, + "time_per_iteration": 2.5160133838653564 + }, + { + "auxiliary_loss_clip": 0.01194748, + "auxiliary_loss_mlp": 0.01059123, + "balance_loss_clip": 1.07455635, + "balance_loss_mlp": 1.03751028, + "epoch": 0.052260460797400037, + "flos": 19461070016640.0, + "grad_norm": 2.5350689348637405, + "language_loss": 0.76005328, + "learning_rate": 3.994806242976703e-06, + "loss": 0.78259206, + "num_input_tokens_seen": 50703635, + "router_z_loss_clip": 1.20117188, + "router_z_loss_mlp": 0.21630859, + "step": 1801, + "time_per_iteration": 2.566197395324707 + }, + { + "auxiliary_loss_clip": 0.01051646, + "auxiliary_loss_mlp": 0.01042896, + "balance_loss_clip": 1.02163315, + "balance_loss_mlp": 1.04134655, + "epoch": 0.05228947826591608, + "flos": 57139084627200.0, + "grad_norm": 0.6892573933536094, + "language_loss": 0.48865455, + "learning_rate": 3.994792696988822e-06, + "loss": 0.50959998, + "num_input_tokens_seen": 50759755, + "router_z_loss_clip": 0.30078125, + "router_z_loss_mlp": 0.01544189, + "step": 1802, + "time_per_iteration": 3.0444226264953613 + }, + { + "auxiliary_loss_clip": 0.01052175, + "auxiliary_loss_mlp": 0.0104068, + "balance_loss_clip": 1.02189517, + "balance_loss_mlp": 1.03905857, + "epoch": 0.052318495734432126, + "flos": 69942785040000.0, + "grad_norm": 0.6770778774431709, + "language_loss": 0.51403362, + "learning_rate": 3.994779133382125e-06, + "loss": 0.53496218, + "num_input_tokens_seen": 50823880, + "router_z_loss_clip": 0.30273438, + "router_z_loss_mlp": 0.01623535, + "step": 1803, + "time_per_iteration": 3.28434419631958 + }, + { + "auxiliary_loss_clip": 0.0105222, + "auxiliary_loss_mlp": 0.01025896, + "balance_loss_clip": 1.021999, + "balance_loss_mlp": 1.02441204, + "epoch": 0.05234751320294818, + "flos": 66099480595200.0, + "grad_norm": 0.6505509938598018, + "language_loss": 0.5216186, + "learning_rate": 3.994765552156731e-06, + "loss": 0.54239976, + "num_input_tokens_seen": 50888845, + "router_z_loss_clip": 0.30273438, + "router_z_loss_mlp": 0.01483154, + "step": 1804, + "time_per_iteration": 3.129894733428955 + }, + { + "auxiliary_loss_clip": 0.01212884, + "auxiliary_loss_mlp": 0.01063639, + "balance_loss_clip": 1.08454216, + "balance_loss_mlp": 1.0417881, + "epoch": 0.05237653067146422, + "flos": 32336014078080.0, + "grad_norm": 2.732404273535526, + "language_loss": 0.74364239, + "learning_rate": 3.994751953312762e-06, + "loss": 0.76640761, + "num_input_tokens_seen": 50903530, + "router_z_loss_clip": 1.28320312, + "router_z_loss_mlp": 0.21862793, + "step": 1805, + "time_per_iteration": 2.545290470123291 + }, + { + "auxiliary_loss_clip": 0.01052995, + "auxiliary_loss_mlp": 0.01003224, + "balance_loss_clip": 1.02302742, + "balance_loss_mlp": 1.00170445, + "epoch": 0.05240554813998027, + "flos": 72440334529920.0, + "grad_norm": 0.6022493710091692, + "language_loss": 0.51647753, + "learning_rate": 3.994738336850336e-06, + "loss": 0.5370397, + "num_input_tokens_seen": 50970050, + "router_z_loss_clip": 0.30078125, + "router_z_loss_mlp": 0.01519775, + "step": 1806, + "time_per_iteration": 3.1709678173065186 + }, + { + "auxiliary_loss_clip": 0.01176662, + "auxiliary_loss_mlp": 0.01053462, + "balance_loss_clip": 1.07140434, + "balance_loss_mlp": 1.03649879, + "epoch": 0.05243456560849631, + "flos": 30182810319360.0, + "grad_norm": 2.846802942048656, + "language_loss": 0.88652444, + "learning_rate": 3.994724702769573e-06, + "loss": 0.90882576, + "num_input_tokens_seen": 50984425, + "router_z_loss_clip": 1.05224609, + "router_z_loss_mlp": 0.16973877, + "step": 1807, + "time_per_iteration": 2.6649560928344727 + }, + { + "auxiliary_loss_clip": 0.01198927, + "auxiliary_loss_mlp": 0.01077664, + "balance_loss_clip": 1.07457089, + "balance_loss_mlp": 1.05557513, + "epoch": 0.05246358307701236, + "flos": 23652886170240.0, + "grad_norm": 2.9087592782504714, + "language_loss": 0.7494055, + "learning_rate": 3.994711051070595e-06, + "loss": 0.77217144, + "num_input_tokens_seen": 51001550, + "router_z_loss_clip": 1.24365234, + "router_z_loss_mlp": 0.22088623, + "step": 1808, + "time_per_iteration": 2.6394670009613037 + }, + { + "auxiliary_loss_clip": 0.0105386, + "auxiliary_loss_mlp": 0.01036248, + "balance_loss_clip": 1.02351785, + "balance_loss_mlp": 1.03466213, + "epoch": 0.05249260054552841, + "flos": 65914935494400.0, + "grad_norm": 0.7560804430437283, + "language_loss": 0.54326439, + "learning_rate": 3.99469738175352e-06, + "loss": 0.56416547, + "num_input_tokens_seen": 51062220, + "router_z_loss_clip": 0.3046875, + "router_z_loss_mlp": 0.01586914, + "step": 1809, + "time_per_iteration": 3.140679121017456 + }, + { + "auxiliary_loss_clip": 0.01053583, + "auxiliary_loss_mlp": 0.01026759, + "balance_loss_clip": 1.02343512, + "balance_loss_mlp": 1.02518582, + "epoch": 0.05252161801404445, + "flos": 64670594590080.0, + "grad_norm": 0.6572927422939701, + "language_loss": 0.51157463, + "learning_rate": 3.994683694818472e-06, + "loss": 0.53237808, + "num_input_tokens_seen": 51127685, + "router_z_loss_clip": 0.30078125, + "router_z_loss_mlp": 0.01574707, + "step": 1810, + "time_per_iteration": 3.1705870628356934 + }, + { + "auxiliary_loss_clip": 0.01193694, + "auxiliary_loss_mlp": 0.01056994, + "balance_loss_clip": 1.07539892, + "balance_loss_mlp": 1.0359056, + "epoch": 0.052550635482560504, + "flos": 22267955433600.0, + "grad_norm": 2.0393257970336474, + "language_loss": 0.95630789, + "learning_rate": 3.994669990265571e-06, + "loss": 0.97881472, + "num_input_tokens_seen": 51150190, + "router_z_loss_clip": 1.18261719, + "router_z_loss_mlp": 0.21081543, + "step": 1811, + "time_per_iteration": 2.7887446880340576 + }, + { + "auxiliary_loss_clip": 0.01208824, + "auxiliary_loss_mlp": 0.01054809, + "balance_loss_clip": 1.08002281, + "balance_loss_mlp": 1.03168201, + "epoch": 0.05257965295107655, + "flos": 34088059768320.0, + "grad_norm": 2.708452108025248, + "language_loss": 0.95160776, + "learning_rate": 3.994656268094937e-06, + "loss": 0.97424412, + "num_input_tokens_seen": 51164500, + "router_z_loss_clip": 1.29003906, + "router_z_loss_mlp": 0.23120117, + "step": 1812, + "time_per_iteration": 2.635946750640869 + }, + { + "auxiliary_loss_clip": 0.01191865, + "auxiliary_loss_mlp": 0.01066202, + "balance_loss_clip": 1.07806361, + "balance_loss_mlp": 1.046229, + "epoch": 0.05260867041959259, + "flos": 25948403614080.0, + "grad_norm": 4.255284919474476, + "language_loss": 1.08222842, + "learning_rate": 3.994642528306691e-06, + "loss": 1.10480905, + "num_input_tokens_seen": 51176405, + "router_z_loss_clip": 1.13867188, + "router_z_loss_mlp": 0.19958496, + "step": 1813, + "time_per_iteration": 2.567044734954834 + }, + { + "auxiliary_loss_clip": 0.01206176, + "auxiliary_loss_mlp": 0.01078484, + "balance_loss_clip": 1.08059967, + "balance_loss_mlp": 1.05627561, + "epoch": 0.052637687888108645, + "flos": 11463655720320.0, + "grad_norm": 3.0585264852628296, + "language_loss": 1.10119581, + "learning_rate": 3.994628770900956e-06, + "loss": 1.12404239, + "num_input_tokens_seen": 51186080, + "router_z_loss_clip": 1.25390625, + "router_z_loss_mlp": 0.2220459, + "step": 1814, + "time_per_iteration": 2.4394938945770264 + }, + { + "auxiliary_loss_clip": 0.01202787, + "auxiliary_loss_mlp": 0.01075362, + "balance_loss_clip": 1.0776633, + "balance_loss_mlp": 1.05305791, + "epoch": 0.05266670535662469, + "flos": 33181243989120.0, + "grad_norm": 2.6085014376844478, + "language_loss": 0.87778461, + "learning_rate": 3.994614995877852e-06, + "loss": 0.9005661, + "num_input_tokens_seen": 51209075, + "router_z_loss_clip": 1.25, + "router_z_loss_mlp": 0.22314453, + "step": 1815, + "time_per_iteration": 2.629397392272949 + }, + { + "auxiliary_loss_clip": 0.01205128, + "auxiliary_loss_mlp": 0.01062641, + "balance_loss_clip": 1.08059549, + "balance_loss_mlp": 1.0402534, + "epoch": 0.052695722825140734, + "flos": 18910056407040.0, + "grad_norm": 3.809998451121776, + "language_loss": 0.90289056, + "learning_rate": 3.994601203237501e-06, + "loss": 0.92556822, + "num_input_tokens_seen": 51221510, + "router_z_loss_clip": 1.24609375, + "router_z_loss_mlp": 0.22387695, + "step": 1816, + "time_per_iteration": 2.4674758911132812 + }, + { + "auxiliary_loss_clip": 0.01190779, + "auxiliary_loss_mlp": 0.01062551, + "balance_loss_clip": 1.07404065, + "balance_loss_mlp": 1.04316807, + "epoch": 0.05272474029365678, + "flos": 13109118779520.0, + "grad_norm": 2.5966057911337406, + "language_loss": 0.87409562, + "learning_rate": 3.994587392980026e-06, + "loss": 0.89662898, + "num_input_tokens_seen": 51234150, + "router_z_loss_clip": 1.16699219, + "router_z_loss_mlp": 0.19384766, + "step": 1817, + "time_per_iteration": 2.476471185684204 + }, + { + "auxiliary_loss_clip": 0.01200904, + "auxiliary_loss_mlp": 0.01084618, + "balance_loss_clip": 1.07824802, + "balance_loss_mlp": 1.06112218, + "epoch": 0.05275375776217283, + "flos": 14129412600960.0, + "grad_norm": 2.875054215580665, + "language_loss": 0.94143629, + "learning_rate": 3.9945735651055475e-06, + "loss": 0.96429145, + "num_input_tokens_seen": 51245270, + "router_z_loss_clip": 1.2265625, + "router_z_loss_mlp": 0.23510742, + "step": 1818, + "time_per_iteration": 2.532940149307251 + }, + { + "auxiliary_loss_clip": 0.01192638, + "auxiliary_loss_mlp": 0.01061928, + "balance_loss_clip": 1.07393551, + "balance_loss_mlp": 1.0419724, + "epoch": 0.052782775230688875, + "flos": 39018990078720.0, + "grad_norm": 1.647470207967642, + "language_loss": 0.81562304, + "learning_rate": 3.994559719614189e-06, + "loss": 0.83816874, + "num_input_tokens_seen": 51270730, + "router_z_loss_clip": 1.18652344, + "router_z_loss_mlp": 0.19934082, + "step": 1819, + "time_per_iteration": 2.7611451148986816 + }, + { + "auxiliary_loss_clip": 0.01054474, + "auxiliary_loss_mlp": 0.01069685, + "balance_loss_clip": 1.0239327, + "balance_loss_mlp": 1.06806362, + "epoch": 0.05281179269920492, + "flos": 74791766471040.0, + "grad_norm": 0.6788732731910943, + "language_loss": 0.48168856, + "learning_rate": 3.99454585650607e-06, + "loss": 0.50293016, + "num_input_tokens_seen": 51339545, + "router_z_loss_clip": 0.3046875, + "router_z_loss_mlp": 0.01623535, + "step": 1820, + "time_per_iteration": 3.3529531955718994 + }, + { + "auxiliary_loss_clip": 0.0105267, + "auxiliary_loss_mlp": 0.01057849, + "balance_loss_clip": 1.02257156, + "balance_loss_mlp": 1.05627537, + "epoch": 0.05284081016772097, + "flos": 67230594852480.0, + "grad_norm": 0.6817816867716814, + "language_loss": 0.52184325, + "learning_rate": 3.994531975781316e-06, + "loss": 0.54294848, + "num_input_tokens_seen": 51407870, + "router_z_loss_clip": 0.30078125, + "router_z_loss_mlp": 0.01574707, + "step": 1821, + "time_per_iteration": 3.185150384902954 + }, + { + "auxiliary_loss_clip": 0.01187181, + "auxiliary_loss_mlp": 0.01057463, + "balance_loss_clip": 1.07361329, + "balance_loss_mlp": 1.0373044, + "epoch": 0.052869827636237016, + "flos": 16391246065920.0, + "grad_norm": 2.969600263856774, + "language_loss": 0.98576593, + "learning_rate": 3.994518077440049e-06, + "loss": 1.00821233, + "num_input_tokens_seen": 51419305, + "router_z_loss_clip": 1.13476562, + "router_z_loss_mlp": 0.20166016, + "step": 1822, + "time_per_iteration": 2.465562343597412 + }, + { + "auxiliary_loss_clip": 0.01191193, + "auxiliary_loss_mlp": 0.01055267, + "balance_loss_clip": 1.07040429, + "balance_loss_mlp": 1.03347516, + "epoch": 0.05289884510475306, + "flos": 32264444229120.0, + "grad_norm": 2.3453570813625344, + "language_loss": 0.75585735, + "learning_rate": 3.99450416148239e-06, + "loss": 0.77832192, + "num_input_tokens_seen": 51435855, + "router_z_loss_clip": 1.20751953, + "router_z_loss_mlp": 0.21777344, + "step": 1823, + "time_per_iteration": 2.675044536590576 + }, + { + "auxiliary_loss_clip": 0.01199849, + "auxiliary_loss_mlp": 0.01058207, + "balance_loss_clip": 1.07781398, + "balance_loss_mlp": 1.0362606, + "epoch": 0.052927862573269105, + "flos": 42042416636160.0, + "grad_norm": 2.584959563430236, + "language_loss": 1.03120136, + "learning_rate": 3.994490227908464e-06, + "loss": 1.05378199, + "num_input_tokens_seen": 51455110, + "router_z_loss_clip": 1.22119141, + "router_z_loss_mlp": 0.21936035, + "step": 1824, + "time_per_iteration": 2.6818594932556152 + }, + { + "auxiliary_loss_clip": 0.01052806, + "auxiliary_loss_mlp": 0.01000661, + "balance_loss_clip": 1.02256382, + "balance_loss_mlp": 0.99909896, + "epoch": 0.052956880041785157, + "flos": 58055130201600.0, + "grad_norm": 0.7132386220320684, + "language_loss": 0.5180971, + "learning_rate": 3.994476276718394e-06, + "loss": 0.5386318, + "num_input_tokens_seen": 51512655, + "router_z_loss_clip": 0.30273438, + "router_z_loss_mlp": 0.015625, + "step": 1825, + "time_per_iteration": 2.965122699737549 + }, + { + "auxiliary_loss_clip": 0.01201237, + "auxiliary_loss_mlp": 0.01060229, + "balance_loss_clip": 1.07495713, + "balance_loss_mlp": 1.03745961, + "epoch": 0.0529858975103012, + "flos": 33806090004480.0, + "grad_norm": 1.9271045013999144, + "language_loss": 0.9929626, + "learning_rate": 3.9944623079123004e-06, + "loss": 1.0155772, + "num_input_tokens_seen": 51534775, + "router_z_loss_clip": 1.26269531, + "router_z_loss_mlp": 0.22790527, + "step": 1826, + "time_per_iteration": 2.693915605545044 + }, + { + "auxiliary_loss_clip": 0.01198359, + "auxiliary_loss_mlp": 0.01068527, + "balance_loss_clip": 1.07923162, + "balance_loss_mlp": 1.04729021, + "epoch": 0.053014914978817246, + "flos": 25841641415040.0, + "grad_norm": 2.2076030365377846, + "language_loss": 0.89363915, + "learning_rate": 3.99444832149031e-06, + "loss": 0.91630805, + "num_input_tokens_seen": 51553795, + "router_z_loss_clip": 1.19140625, + "router_z_loss_mlp": 0.21240234, + "step": 1827, + "time_per_iteration": 2.5242600440979004 + }, + { + "auxiliary_loss_clip": 0.0118125, + "auxiliary_loss_mlp": 0.01062625, + "balance_loss_clip": 1.07075715, + "balance_loss_mlp": 1.04583383, + "epoch": 0.0530439324473333, + "flos": 38467437765120.0, + "grad_norm": 4.236162147102911, + "language_loss": 1.05524468, + "learning_rate": 3.994434317452545e-06, + "loss": 1.07768333, + "num_input_tokens_seen": 51572940, + "router_z_loss_clip": 1.10546875, + "router_z_loss_mlp": 0.16809082, + "step": 1828, + "time_per_iteration": 2.7198972702026367 + }, + { + "auxiliary_loss_clip": 0.01198436, + "auxiliary_loss_mlp": 0.0107262, + "balance_loss_clip": 1.07516694, + "balance_loss_mlp": 1.051103, + "epoch": 0.05307294991584934, + "flos": 51088637174400.0, + "grad_norm": 2.6822724869331367, + "language_loss": 1.01575899, + "learning_rate": 3.994420295799129e-06, + "loss": 1.03846955, + "num_input_tokens_seen": 51591620, + "router_z_loss_clip": 1.23242188, + "router_z_loss_mlp": 0.21520996, + "step": 1829, + "time_per_iteration": 2.790311336517334 + }, + { + "auxiliary_loss_clip": 0.01196902, + "auxiliary_loss_mlp": 0.01068297, + "balance_loss_clip": 1.07495487, + "balance_loss_mlp": 1.04757893, + "epoch": 0.05310196738436539, + "flos": 21433786911360.0, + "grad_norm": 1.9762349562987025, + "language_loss": 0.89026093, + "learning_rate": 3.994406256530185e-06, + "loss": 0.91291291, + "num_input_tokens_seen": 51607455, + "router_z_loss_clip": 1.21875, + "router_z_loss_mlp": 0.20727539, + "step": 1830, + "time_per_iteration": 2.5313398838043213 + }, + { + "auxiliary_loss_clip": 0.01182718, + "auxiliary_loss_mlp": 0.01076334, + "balance_loss_clip": 1.07047904, + "balance_loss_mlp": 1.05708146, + "epoch": 0.05313098485288143, + "flos": 17889367536000.0, + "grad_norm": 1.956583024932878, + "language_loss": 0.74362087, + "learning_rate": 3.9943921996458385e-06, + "loss": 0.76621139, + "num_input_tokens_seen": 51623000, + "router_z_loss_clip": 1.12207031, + "router_z_loss_mlp": 0.19262695, + "step": 1831, + "time_per_iteration": 2.506664514541626 + }, + { + "auxiliary_loss_clip": 0.01055818, + "auxiliary_loss_mlp": 0.01053524, + "balance_loss_clip": 1.02527857, + "balance_loss_mlp": 1.05185461, + "epoch": 0.05316000232139748, + "flos": 74781567008640.0, + "grad_norm": 0.6921282465040225, + "language_loss": 0.51141918, + "learning_rate": 3.9943781251462135e-06, + "loss": 0.53251255, + "num_input_tokens_seen": 51685760, + "router_z_loss_clip": 0.3046875, + "router_z_loss_mlp": 0.01672363, + "step": 1832, + "time_per_iteration": 3.22226619720459 + }, + { + "auxiliary_loss_clip": 0.01054706, + "auxiliary_loss_mlp": 0.01023303, + "balance_loss_clip": 1.02447915, + "balance_loss_mlp": 1.02169371, + "epoch": 0.05318901978991353, + "flos": 62591402805120.0, + "grad_norm": 0.6823441255378404, + "language_loss": 0.54897124, + "learning_rate": 3.994364033031433e-06, + "loss": 0.56975132, + "num_input_tokens_seen": 51750160, + "router_z_loss_clip": 0.30078125, + "router_z_loss_mlp": 0.01611328, + "step": 1833, + "time_per_iteration": 3.1704211235046387 + }, + { + "auxiliary_loss_clip": 0.01053584, + "auxiliary_loss_mlp": 0.01010993, + "balance_loss_clip": 1.02376962, + "balance_loss_mlp": 1.0096339, + "epoch": 0.05321803725842957, + "flos": 71418604164480.0, + "grad_norm": 0.680112989517755, + "language_loss": 0.52005345, + "learning_rate": 3.994349923301623e-06, + "loss": 0.54069918, + "num_input_tokens_seen": 51812750, + "router_z_loss_clip": 0.296875, + "router_z_loss_mlp": 0.01361084, + "step": 1834, + "time_per_iteration": 3.1746997833251953 + }, + { + "auxiliary_loss_clip": 0.01200429, + "auxiliary_loss_mlp": 0.01062351, + "balance_loss_clip": 1.07300472, + "balance_loss_mlp": 1.0397253, + "epoch": 0.053247054726945624, + "flos": 29489626679040.0, + "grad_norm": 2.132563420668447, + "language_loss": 0.95857435, + "learning_rate": 3.994335795956907e-06, + "loss": 0.98120219, + "num_input_tokens_seen": 51830770, + "router_z_loss_clip": 1.2734375, + "router_z_loss_mlp": 0.22631836, + "step": 1835, + "time_per_iteration": 2.646468162536621 + }, + { + "auxiliary_loss_clip": 0.01196838, + "auxiliary_loss_mlp": 0.01065628, + "balance_loss_clip": 1.07903671, + "balance_loss_mlp": 1.04687643, + "epoch": 0.05327607219546167, + "flos": 20887082933760.0, + "grad_norm": 2.8772159921331015, + "language_loss": 0.99292409, + "learning_rate": 3.9943216509974105e-06, + "loss": 1.01554871, + "num_input_tokens_seen": 51849715, + "router_z_loss_clip": 1.17773438, + "router_z_loss_mlp": 0.1875, + "step": 1836, + "time_per_iteration": 2.482001304626465 + }, + { + "auxiliary_loss_clip": 0.01205527, + "auxiliary_loss_mlp": 0.01077, + "balance_loss_clip": 1.0773505, + "balance_loss_mlp": 1.05495787, + "epoch": 0.05330508966397771, + "flos": 20078050953600.0, + "grad_norm": 2.697747703119632, + "language_loss": 0.83402765, + "learning_rate": 3.994307488423258e-06, + "loss": 0.85685289, + "num_input_tokens_seen": 51862980, + "router_z_loss_clip": 1.28222656, + "router_z_loss_mlp": 0.22021484, + "step": 1837, + "time_per_iteration": 2.520695686340332 + }, + { + "auxiliary_loss_clip": 0.01206765, + "auxiliary_loss_mlp": 0.01069055, + "balance_loss_clip": 1.07693851, + "balance_loss_mlp": 1.04645252, + "epoch": 0.053334107132493765, + "flos": 30694429687680.0, + "grad_norm": 7.220870003792225, + "language_loss": 1.05349112, + "learning_rate": 3.994293308234575e-06, + "loss": 1.07624936, + "num_input_tokens_seen": 51880780, + "router_z_loss_clip": 1.29980469, + "router_z_loss_mlp": 0.22595215, + "step": 1838, + "time_per_iteration": 2.612030029296875 + }, + { + "auxiliary_loss_clip": 0.01203838, + "auxiliary_loss_mlp": 0.01086413, + "balance_loss_clip": 1.07478142, + "balance_loss_mlp": 1.06336975, + "epoch": 0.05336312460100981, + "flos": 22813725657600.0, + "grad_norm": 2.6025035707736266, + "language_loss": 1.04770875, + "learning_rate": 3.994279110431487e-06, + "loss": 1.07061136, + "num_input_tokens_seen": 51894595, + "router_z_loss_clip": 1.29199219, + "router_z_loss_mlp": 0.23046875, + "step": 1839, + "time_per_iteration": 2.5575945377349854 + }, + { + "auxiliary_loss_clip": 0.010586, + "auxiliary_loss_mlp": 0.01110021, + "balance_loss_clip": 1.02693641, + "balance_loss_mlp": 1.10829246, + "epoch": 0.053392142069525854, + "flos": 74773342794240.0, + "grad_norm": 0.6696598491055965, + "language_loss": 0.54437697, + "learning_rate": 3.994264895014118e-06, + "loss": 0.56606323, + "num_input_tokens_seen": 51958645, + "router_z_loss_clip": 0.31640625, + "router_z_loss_mlp": 0.01733398, + "step": 1840, + "time_per_iteration": 3.156235933303833 + }, + { + "auxiliary_loss_clip": 0.01205296, + "auxiliary_loss_mlp": 0.01087181, + "balance_loss_clip": 1.07739258, + "balance_loss_mlp": 1.06810164, + "epoch": 0.0534211595380419, + "flos": 37408970764800.0, + "grad_norm": 2.18479628840296, + "language_loss": 0.7868768, + "learning_rate": 3.994250661982594e-06, + "loss": 0.80980152, + "num_input_tokens_seen": 51976750, + "router_z_loss_clip": 1.27734375, + "router_z_loss_mlp": 0.19073486, + "step": 1841, + "time_per_iteration": 2.6757214069366455 + }, + { + "auxiliary_loss_clip": 0.01057068, + "auxiliary_loss_mlp": 0.01100358, + "balance_loss_clip": 1.02652049, + "balance_loss_mlp": 1.098701, + "epoch": 0.05345017700655795, + "flos": 74781459267840.0, + "grad_norm": 0.6245144506307478, + "language_loss": 0.52763116, + "learning_rate": 3.994236411337043e-06, + "loss": 0.54920542, + "num_input_tokens_seen": 52041235, + "router_z_loss_clip": 0.3046875, + "router_z_loss_mlp": 0.01660156, + "step": 1842, + "time_per_iteration": 3.1660985946655273 + }, + { + "auxiliary_loss_clip": 0.01054779, + "auxiliary_loss_mlp": 0.01082454, + "balance_loss_clip": 1.02496767, + "balance_loss_mlp": 1.08092785, + "epoch": 0.053479194475073995, + "flos": 65360474179200.0, + "grad_norm": 0.6681842113481966, + "language_loss": 0.50767589, + "learning_rate": 3.994222143077587e-06, + "loss": 0.52904826, + "num_input_tokens_seen": 52106420, + "router_z_loss_clip": 0.296875, + "router_z_loss_mlp": 0.01525879, + "step": 1843, + "time_per_iteration": 3.11673903465271 + }, + { + "auxiliary_loss_clip": 0.01186068, + "auxiliary_loss_mlp": 0.01061378, + "balance_loss_clip": 1.07372928, + "balance_loss_mlp": 1.04367518, + "epoch": 0.05350821194359004, + "flos": 11581802530560.0, + "grad_norm": 3.0611688302729307, + "language_loss": 0.87617469, + "learning_rate": 3.994207857204355e-06, + "loss": 0.89864916, + "num_input_tokens_seen": 52119330, + "router_z_loss_clip": 1.125, + "router_z_loss_mlp": 0.17700195, + "step": 1844, + "time_per_iteration": 2.5285732746124268 + }, + { + "auxiliary_loss_clip": 0.01197813, + "auxiliary_loss_mlp": 0.01067647, + "balance_loss_clip": 1.07596648, + "balance_loss_mlp": 1.04587948, + "epoch": 0.05353722941210609, + "flos": 11320623763200.0, + "grad_norm": 4.309681043158962, + "language_loss": 0.67698759, + "learning_rate": 3.994193553717472e-06, + "loss": 0.69964218, + "num_input_tokens_seen": 52131915, + "router_z_loss_clip": 1.21875, + "router_z_loss_mlp": 0.2175293, + "step": 1845, + "time_per_iteration": 2.472839593887329 + }, + { + "auxiliary_loss_clip": 0.01187503, + "auxiliary_loss_mlp": 0.01051955, + "balance_loss_clip": 1.07219636, + "balance_loss_mlp": 1.03179717, + "epoch": 0.053566246880622136, + "flos": 19422286306560.0, + "grad_norm": 2.5065082885488423, + "language_loss": 0.81990588, + "learning_rate": 3.994179232617065e-06, + "loss": 0.84230042, + "num_input_tokens_seen": 52145425, + "router_z_loss_clip": 1.15136719, + "router_z_loss_mlp": 0.20172119, + "step": 1846, + "time_per_iteration": 2.5150794982910156 + }, + { + "auxiliary_loss_clip": 0.01199356, + "auxiliary_loss_mlp": 0.010712, + "balance_loss_clip": 1.07569098, + "balance_loss_mlp": 1.04788291, + "epoch": 0.05359526434913818, + "flos": 35819996820480.0, + "grad_norm": 2.935909641921008, + "language_loss": 0.82315463, + "learning_rate": 3.994164893903259e-06, + "loss": 0.84586024, + "num_input_tokens_seen": 52165995, + "router_z_loss_clip": 1.23535156, + "router_z_loss_mlp": 0.23339844, + "step": 1847, + "time_per_iteration": 2.665952205657959 + }, + { + "auxiliary_loss_clip": 0.01194718, + "auxiliary_loss_mlp": 0.01061053, + "balance_loss_clip": 1.08002329, + "balance_loss_mlp": 1.04143107, + "epoch": 0.053624281817654225, + "flos": 14530750237440.0, + "grad_norm": 2.891707220038163, + "language_loss": 0.90742815, + "learning_rate": 3.9941505375761826e-06, + "loss": 0.92998588, + "num_input_tokens_seen": 52179215, + "router_z_loss_clip": 1.14746094, + "router_z_loss_mlp": 0.19628906, + "step": 1848, + "time_per_iteration": 2.549355983734131 + }, + { + "auxiliary_loss_clip": 0.01188353, + "auxiliary_loss_mlp": 0.0108104, + "balance_loss_clip": 1.07191539, + "balance_loss_mlp": 1.0596776, + "epoch": 0.053653299286170276, + "flos": 27593830759680.0, + "grad_norm": 2.5477165872251444, + "language_loss": 0.79504788, + "learning_rate": 3.994136163635962e-06, + "loss": 0.81774187, + "num_input_tokens_seen": 52195110, + "router_z_loss_clip": 1.16308594, + "router_z_loss_mlp": 0.21374512, + "step": 1849, + "time_per_iteration": 2.568202257156372 + }, + { + "auxiliary_loss_clip": 0.01060501, + "auxiliary_loss_mlp": 0.01059526, + "balance_loss_clip": 1.02672768, + "balance_loss_mlp": 1.05801225, + "epoch": 0.05368231675468632, + "flos": 74774240634240.0, + "grad_norm": 1.227167005641859, + "language_loss": 0.52218223, + "learning_rate": 3.994121772082724e-06, + "loss": 0.54338253, + "num_input_tokens_seen": 52264025, + "router_z_loss_clip": 0.33789062, + "router_z_loss_mlp": 0.01513672, + "step": 1850, + "time_per_iteration": 3.298321485519409 + }, + { + "auxiliary_loss_clip": 0.01201422, + "auxiliary_loss_mlp": 0.01069343, + "balance_loss_clip": 1.0835433, + "balance_loss_mlp": 1.04940486, + "epoch": 0.053711334223202366, + "flos": 31969761050880.0, + "grad_norm": 2.501119585176743, + "language_loss": 0.8696937, + "learning_rate": 3.9941073629165945e-06, + "loss": 0.89240134, + "num_input_tokens_seen": 52278120, + "router_z_loss_clip": 1.17871094, + "router_z_loss_mlp": 0.19952393, + "step": 1851, + "time_per_iteration": 2.5955557823181152 + }, + { + "auxiliary_loss_clip": 0.0118731, + "auxiliary_loss_mlp": 0.01062304, + "balance_loss_clip": 1.0722121, + "balance_loss_mlp": 1.04349279, + "epoch": 0.05374035169171842, + "flos": 30547698629760.0, + "grad_norm": 2.847109859409749, + "language_loss": 0.89181131, + "learning_rate": 3.994092936137702e-06, + "loss": 0.91430748, + "num_input_tokens_seen": 52295000, + "router_z_loss_clip": 1.15136719, + "router_z_loss_mlp": 0.18823242, + "step": 1852, + "time_per_iteration": 4.889440536499023 + }, + { + "auxiliary_loss_clip": 0.01057106, + "auxiliary_loss_mlp": 0.01020346, + "balance_loss_clip": 1.02525377, + "balance_loss_mlp": 1.01883841, + "epoch": 0.05376936916023446, + "flos": 64049734984320.0, + "grad_norm": 0.755642964204273, + "language_loss": 0.55380177, + "learning_rate": 3.994078491746175e-06, + "loss": 0.57457626, + "num_input_tokens_seen": 52351105, + "router_z_loss_clip": 0.31835938, + "router_z_loss_mlp": 0.01507568, + "step": 1853, + "time_per_iteration": 7.686108350753784 + }, + { + "auxiliary_loss_clip": 0.01054362, + "auxiliary_loss_mlp": 0.01001233, + "balance_loss_clip": 1.0236733, + "balance_loss_mlp": 0.99965918, + "epoch": 0.05379838662875051, + "flos": 64745360749440.0, + "grad_norm": 0.6788600586346052, + "language_loss": 0.49823752, + "learning_rate": 3.994064029742138e-06, + "loss": 0.51879346, + "num_input_tokens_seen": 52413490, + "router_z_loss_clip": 0.30664062, + "router_z_loss_mlp": 0.01574707, + "step": 1854, + "time_per_iteration": 3.1622912883758545 + }, + { + "auxiliary_loss_clip": 0.01190054, + "auxiliary_loss_mlp": 0.01065513, + "balance_loss_clip": 1.07664251, + "balance_loss_mlp": 1.04444289, + "epoch": 0.05382740409726655, + "flos": 66816792218880.0, + "grad_norm": 2.563030954572102, + "language_loss": 0.67967594, + "learning_rate": 3.994049550125722e-06, + "loss": 0.70223159, + "num_input_tokens_seen": 52435935, + "router_z_loss_clip": 1.13427734, + "router_z_loss_mlp": 0.21099854, + "step": 1855, + "time_per_iteration": 5.356945276260376 + }, + { + "auxiliary_loss_clip": 0.01179042, + "auxiliary_loss_mlp": 0.01051456, + "balance_loss_clip": 1.07081604, + "balance_loss_mlp": 1.03494, + "epoch": 0.0538564215657826, + "flos": 28833502896000.0, + "grad_norm": 2.374633694232795, + "language_loss": 0.63353527, + "learning_rate": 3.994035052897053e-06, + "loss": 0.65584028, + "num_input_tokens_seen": 52449910, + "router_z_loss_clip": 1.08203125, + "router_z_loss_mlp": 0.16497803, + "step": 1856, + "time_per_iteration": 2.560974359512329 + }, + { + "auxiliary_loss_clip": 0.01053444, + "auxiliary_loss_mlp": 0.01071697, + "balance_loss_clip": 1.02330256, + "balance_loss_mlp": 1.07031465, + "epoch": 0.05388543903429865, + "flos": 74779699501440.0, + "grad_norm": 0.756857165395188, + "language_loss": 0.5574131, + "learning_rate": 3.99402053805626e-06, + "loss": 0.57866454, + "num_input_tokens_seen": 52512785, + "router_z_loss_clip": 0.30078125, + "router_z_loss_mlp": 0.01385498, + "step": 1857, + "time_per_iteration": 3.1388518810272217 + }, + { + "auxiliary_loss_clip": 0.0105401, + "auxiliary_loss_mlp": 0.0106709, + "balance_loss_clip": 1.02355266, + "balance_loss_mlp": 1.06570137, + "epoch": 0.05391445650281469, + "flos": 59378151847680.0, + "grad_norm": 0.6436073798197153, + "language_loss": 0.49116066, + "learning_rate": 3.99400600560347e-06, + "loss": 0.51237166, + "num_input_tokens_seen": 52579040, + "router_z_loss_clip": 0.3046875, + "router_z_loss_mlp": 0.01391602, + "step": 1858, + "time_per_iteration": 3.1998798847198486 + }, + { + "auxiliary_loss_clip": 0.01196072, + "auxiliary_loss_mlp": 0.01071832, + "balance_loss_clip": 1.07717454, + "balance_loss_mlp": 1.05220985, + "epoch": 0.053943473971330744, + "flos": 11832674094720.0, + "grad_norm": 2.470969511166963, + "language_loss": 0.78692991, + "learning_rate": 3.993991455538812e-06, + "loss": 0.80960894, + "num_input_tokens_seen": 52590510, + "router_z_loss_clip": 1.1875, + "router_z_loss_mlp": 0.19628906, + "step": 1859, + "time_per_iteration": 2.500235080718994 + }, + { + "auxiliary_loss_clip": 0.01192677, + "auxiliary_loss_mlp": 0.01047304, + "balance_loss_clip": 1.07697952, + "balance_loss_mlp": 1.02933288, + "epoch": 0.05397249143984679, + "flos": 16281107988480.0, + "grad_norm": 2.4230042685381417, + "language_loss": 0.72322071, + "learning_rate": 3.993976887862415e-06, + "loss": 0.74562049, + "num_input_tokens_seen": 52602255, + "router_z_loss_clip": 1.15673828, + "router_z_loss_mlp": 0.17993164, + "step": 1860, + "time_per_iteration": 2.4623074531555176 + }, + { + "auxiliary_loss_clip": 0.01053429, + "auxiliary_loss_mlp": 0.01018155, + "balance_loss_clip": 1.02366805, + "balance_loss_mlp": 1.01667082, + "epoch": 0.05400150890836283, + "flos": 59709356179200.0, + "grad_norm": 0.6583324721902988, + "language_loss": 0.49945179, + "learning_rate": 3.993962302574407e-06, + "loss": 0.52016759, + "num_input_tokens_seen": 52664805, + "router_z_loss_clip": 0.296875, + "router_z_loss_mlp": 0.01483154, + "step": 1861, + "time_per_iteration": 3.085124969482422 + }, + { + "auxiliary_loss_clip": 0.01187212, + "auxiliary_loss_mlp": 0.01056466, + "balance_loss_clip": 1.06915069, + "balance_loss_mlp": 1.03543758, + "epoch": 0.05403052637687888, + "flos": 16355910061440.0, + "grad_norm": 4.221642838128936, + "language_loss": 0.82934022, + "learning_rate": 3.993947699674917e-06, + "loss": 0.85177708, + "num_input_tokens_seen": 52677270, + "router_z_loss_clip": 1.18164062, + "router_z_loss_mlp": 0.21044922, + "step": 1862, + "time_per_iteration": 2.4613840579986572 + }, + { + "auxiliary_loss_clip": 0.01198451, + "auxiliary_loss_mlp": 0.01051142, + "balance_loss_clip": 1.08062828, + "balance_loss_mlp": 1.03045869, + "epoch": 0.05405954384539493, + "flos": 18435927859200.0, + "grad_norm": 10.80148055256723, + "language_loss": 0.83002812, + "learning_rate": 3.993933079164075e-06, + "loss": 0.85252404, + "num_input_tokens_seen": 52690740, + "router_z_loss_clip": 1.17871094, + "router_z_loss_mlp": 0.20690918, + "step": 1863, + "time_per_iteration": 2.5185439586639404 + }, + { + "auxiliary_loss_clip": 0.01193143, + "auxiliary_loss_mlp": 0.01062883, + "balance_loss_clip": 1.07342649, + "balance_loss_mlp": 1.03910089, + "epoch": 0.054088561313910974, + "flos": 31715693176320.0, + "grad_norm": 2.588032809842869, + "language_loss": 0.84115303, + "learning_rate": 3.993918441042008e-06, + "loss": 0.86371326, + "num_input_tokens_seen": 52705105, + "router_z_loss_clip": 1.19628906, + "router_z_loss_mlp": 0.2376709, + "step": 1864, + "time_per_iteration": 2.6148359775543213 + }, + { + "auxiliary_loss_clip": 0.01187232, + "auxiliary_loss_mlp": 0.01046493, + "balance_loss_clip": 1.07621551, + "balance_loss_mlp": 1.02885032, + "epoch": 0.05411757878242702, + "flos": 20877673570560.0, + "grad_norm": 2.294990948200326, + "language_loss": 0.58963001, + "learning_rate": 3.993903785308847e-06, + "loss": 0.61196727, + "num_input_tokens_seen": 52716485, + "router_z_loss_clip": 1.10986328, + "router_z_loss_mlp": 0.17626953, + "step": 1865, + "time_per_iteration": 2.467095375061035 + }, + { + "auxiliary_loss_clip": 0.01190728, + "auxiliary_loss_mlp": 0.01061135, + "balance_loss_clip": 1.07543898, + "balance_loss_mlp": 1.04283607, + "epoch": 0.05414659625094307, + "flos": 11505563913600.0, + "grad_norm": 2.874221078354004, + "language_loss": 0.84660512, + "learning_rate": 3.993889111964721e-06, + "loss": 0.86912382, + "num_input_tokens_seen": 52729885, + "router_z_loss_clip": 1.15136719, + "router_z_loss_mlp": 0.18286133, + "step": 1866, + "time_per_iteration": 2.4825356006622314 + }, + { + "auxiliary_loss_clip": 0.01188381, + "auxiliary_loss_mlp": 0.0104524, + "balance_loss_clip": 1.07562363, + "balance_loss_mlp": 1.02658343, + "epoch": 0.054175613719459115, + "flos": 26096786697600.0, + "grad_norm": 2.8595542180031313, + "language_loss": 0.92491746, + "learning_rate": 3.993874421009759e-06, + "loss": 0.94725364, + "num_input_tokens_seen": 52744265, + "router_z_loss_clip": 1.12597656, + "router_z_loss_mlp": 0.18676758, + "step": 1867, + "time_per_iteration": 2.5695483684539795 + }, + { + "auxiliary_loss_clip": 0.01055111, + "auxiliary_loss_mlp": 0.01048024, + "balance_loss_clip": 1.02458119, + "balance_loss_mlp": 1.04630697, + "epoch": 0.05420463118797516, + "flos": 66428709678720.0, + "grad_norm": 0.6659314113281057, + "language_loss": 0.50431395, + "learning_rate": 3.993859712444092e-06, + "loss": 0.52534521, + "num_input_tokens_seen": 52810155, + "router_z_loss_clip": 0.3046875, + "router_z_loss_mlp": 0.01721191, + "step": 1868, + "time_per_iteration": 3.1920368671417236 + }, + { + "auxiliary_loss_clip": 0.01187801, + "auxiliary_loss_mlp": 0.01065873, + "balance_loss_clip": 1.07252789, + "balance_loss_mlp": 1.04539299, + "epoch": 0.05423364865649121, + "flos": 24567064237440.0, + "grad_norm": 2.2523991599448934, + "language_loss": 0.90313888, + "learning_rate": 3.993844986267849e-06, + "loss": 0.92567569, + "num_input_tokens_seen": 52826775, + "router_z_loss_clip": 1.15527344, + "router_z_loss_mlp": 0.20495605, + "step": 1869, + "time_per_iteration": 2.516413927078247 + }, + { + "auxiliary_loss_clip": 0.01182436, + "auxiliary_loss_mlp": 0.01053633, + "balance_loss_clip": 1.06968629, + "balance_loss_mlp": 1.03386784, + "epoch": 0.054262666125007256, + "flos": 16755954808320.0, + "grad_norm": 3.9874448779837546, + "language_loss": 0.88410437, + "learning_rate": 3.9938302424811605e-06, + "loss": 0.90646511, + "num_input_tokens_seen": 52838435, + "router_z_loss_clip": 1.12695312, + "router_z_loss_mlp": 0.19775391, + "step": 1870, + "time_per_iteration": 2.682727336883545 + }, + { + "auxiliary_loss_clip": 0.01192289, + "auxiliary_loss_mlp": 0.01047277, + "balance_loss_clip": 1.07515633, + "balance_loss_mlp": 1.02780998, + "epoch": 0.0542916835935233, + "flos": 15951483855360.0, + "grad_norm": 2.9214167126618946, + "language_loss": 0.61258161, + "learning_rate": 3.993815481084156e-06, + "loss": 0.63497722, + "num_input_tokens_seen": 52850670, + "router_z_loss_clip": 1.17089844, + "router_z_loss_mlp": 0.19470215, + "step": 1871, + "time_per_iteration": 2.4604878425598145 + }, + { + "auxiliary_loss_clip": 0.011953, + "auxiliary_loss_mlp": 0.01054092, + "balance_loss_clip": 1.07506084, + "balance_loss_mlp": 1.03408909, + "epoch": 0.054320701062039345, + "flos": 31362117649920.0, + "grad_norm": 1.9274996219407508, + "language_loss": 1.02330482, + "learning_rate": 3.9938007020769665e-06, + "loss": 1.04579878, + "num_input_tokens_seen": 52872325, + "router_z_loss_clip": 1.20214844, + "router_z_loss_mlp": 0.20019531, + "step": 1872, + "time_per_iteration": 2.5935614109039307 + }, + { + "auxiliary_loss_clip": 0.01054528, + "auxiliary_loss_mlp": 0.0099875, + "balance_loss_clip": 1.02538669, + "balance_loss_mlp": 0.997069, + "epoch": 0.054349718530555396, + "flos": 74777724253440.0, + "grad_norm": 0.6483823890951091, + "language_loss": 0.49514717, + "learning_rate": 3.993785905459722e-06, + "loss": 0.51567996, + "num_input_tokens_seen": 52940855, + "router_z_loss_clip": 0.29101562, + "router_z_loss_mlp": 0.0168457, + "step": 1873, + "time_per_iteration": 3.215771198272705 + }, + { + "auxiliary_loss_clip": 0.01054786, + "auxiliary_loss_mlp": 0.01008419, + "balance_loss_clip": 1.02541471, + "balance_loss_mlp": 1.00685716, + "epoch": 0.05437873599907144, + "flos": 74774707511040.0, + "grad_norm": 0.6802289728376238, + "language_loss": 0.51845497, + "learning_rate": 3.993771091232554e-06, + "loss": 0.53908706, + "num_input_tokens_seen": 53006680, + "router_z_loss_clip": 0.29296875, + "router_z_loss_mlp": 0.015625, + "step": 1874, + "time_per_iteration": 3.1544876098632812 + }, + { + "auxiliary_loss_clip": 0.01054226, + "auxiliary_loss_mlp": 0.01004422, + "balance_loss_clip": 1.02492464, + "balance_loss_mlp": 1.00290775, + "epoch": 0.054407753467587486, + "flos": 70074504213120.0, + "grad_norm": 0.6705439733653036, + "language_loss": 0.56436819, + "learning_rate": 3.993756259395593e-06, + "loss": 0.58495468, + "num_input_tokens_seen": 53073775, + "router_z_loss_clip": 0.29296875, + "router_z_loss_mlp": 0.01513672, + "step": 1875, + "time_per_iteration": 3.1394002437591553 + }, + { + "auxiliary_loss_clip": 0.01052993, + "auxiliary_loss_mlp": 0.0100246, + "balance_loss_clip": 1.02373862, + "balance_loss_mlp": 1.00069571, + "epoch": 0.05443677093610354, + "flos": 70764707024640.0, + "grad_norm": 0.6946352475930153, + "language_loss": 0.47162554, + "learning_rate": 3.993741409948969e-06, + "loss": 0.49218008, + "num_input_tokens_seen": 53134680, + "router_z_loss_clip": 0.29296875, + "router_z_loss_mlp": 0.0177002, + "step": 1876, + "time_per_iteration": 3.0778489112854004 + }, + { + "auxiliary_loss_clip": 0.01180004, + "auxiliary_loss_mlp": 0.01053317, + "balance_loss_clip": 1.06959248, + "balance_loss_mlp": 1.03588915, + "epoch": 0.05446578840461958, + "flos": 22014677658240.0, + "grad_norm": 2.103422267504379, + "language_loss": 0.70924592, + "learning_rate": 3.993726542892815e-06, + "loss": 0.73157912, + "num_input_tokens_seen": 53150195, + "router_z_loss_clip": 1.10351562, + "router_z_loss_mlp": 0.17419434, + "step": 1877, + "time_per_iteration": 2.5281474590301514 + }, + { + "auxiliary_loss_clip": 0.01052438, + "auxiliary_loss_mlp": 0.01002765, + "balance_loss_clip": 1.02340031, + "balance_loss_mlp": 1.00120306, + "epoch": 0.05449480587313563, + "flos": 62981319916800.0, + "grad_norm": 0.7346141870725013, + "language_loss": 0.51847941, + "learning_rate": 3.993711658227262e-06, + "loss": 0.53903145, + "num_input_tokens_seen": 53210565, + "router_z_loss_clip": 0.29101562, + "router_z_loss_mlp": 0.01556396, + "step": 1878, + "time_per_iteration": 3.0897343158721924 + }, + { + "auxiliary_loss_clip": 0.01186633, + "auxiliary_loss_mlp": 0.01062657, + "balance_loss_clip": 1.07489026, + "balance_loss_mlp": 1.04357123, + "epoch": 0.05452382334165167, + "flos": 74741631085440.0, + "grad_norm": 2.190547875331867, + "language_loss": 0.82174385, + "learning_rate": 3.99369675595244e-06, + "loss": 0.84423679, + "num_input_tokens_seen": 53234210, + "router_z_loss_clip": 1.1171875, + "router_z_loss_mlp": 0.19104004, + "step": 1879, + "time_per_iteration": 2.946765661239624 + }, + { + "auxiliary_loss_clip": 0.01180253, + "auxiliary_loss_mlp": 0.01051798, + "balance_loss_clip": 1.07319093, + "balance_loss_mlp": 1.03377366, + "epoch": 0.05455284081016772, + "flos": 11465092264320.0, + "grad_norm": 3.3709827017743774, + "language_loss": 0.84402454, + "learning_rate": 3.993681836068481e-06, + "loss": 0.86634499, + "num_input_tokens_seen": 53245480, + "router_z_loss_clip": 1.06933594, + "router_z_loss_mlp": 0.18017578, + "step": 1880, + "time_per_iteration": 2.453608751296997 + }, + { + "auxiliary_loss_clip": 0.01180775, + "auxiliary_loss_mlp": 0.01053996, + "balance_loss_clip": 1.06955457, + "balance_loss_mlp": 1.0357573, + "epoch": 0.05458185827868377, + "flos": 25476860845440.0, + "grad_norm": 2.6529941815311244, + "language_loss": 0.86912066, + "learning_rate": 3.993666898575518e-06, + "loss": 0.89146835, + "num_input_tokens_seen": 53259740, + "router_z_loss_clip": 1.11083984, + "router_z_loss_mlp": 0.18237305, + "step": 1881, + "time_per_iteration": 2.6073975563049316 + }, + { + "auxiliary_loss_clip": 0.01173899, + "auxiliary_loss_mlp": 0.01046115, + "balance_loss_clip": 1.071015, + "balance_loss_mlp": 1.03062344, + "epoch": 0.05461087574719981, + "flos": 17815068253440.0, + "grad_norm": 3.3168960233820894, + "language_loss": 0.78942901, + "learning_rate": 3.993651943473682e-06, + "loss": 0.81162918, + "num_input_tokens_seen": 53276345, + "router_z_loss_clip": 1.02880859, + "router_z_loss_mlp": 0.15490723, + "step": 1882, + "time_per_iteration": 2.4958207607269287 + }, + { + "auxiliary_loss_clip": 0.01196623, + "auxiliary_loss_mlp": 0.01062789, + "balance_loss_clip": 1.07711959, + "balance_loss_mlp": 1.04195142, + "epoch": 0.054639893215715864, + "flos": 11867363654400.0, + "grad_norm": 2.226850234419761, + "language_loss": 0.81376481, + "learning_rate": 3.993636970763106e-06, + "loss": 0.8363589, + "num_input_tokens_seen": 53288785, + "router_z_loss_clip": 1.1953125, + "router_z_loss_mlp": 0.20825195, + "step": 1883, + "time_per_iteration": 2.5123767852783203 + }, + { + "auxiliary_loss_clip": 0.0118882, + "auxiliary_loss_mlp": 0.01054062, + "balance_loss_clip": 1.07551849, + "balance_loss_mlp": 1.03413057, + "epoch": 0.05466891068423191, + "flos": 34563193896960.0, + "grad_norm": 2.326239632886691, + "language_loss": 1.05074191, + "learning_rate": 3.9936219804439205e-06, + "loss": 1.07317066, + "num_input_tokens_seen": 53309415, + "router_z_loss_clip": 1.13378906, + "router_z_loss_mlp": 0.19909668, + "step": 1884, + "time_per_iteration": 2.613316297531128 + }, + { + "auxiliary_loss_clip": 0.01188429, + "auxiliary_loss_mlp": 0.01054856, + "balance_loss_clip": 1.07457376, + "balance_loss_mlp": 1.03425717, + "epoch": 0.05469792815274795, + "flos": 35036855568000.0, + "grad_norm": 3.624470350893613, + "language_loss": 0.98153824, + "learning_rate": 3.9936069725162594e-06, + "loss": 1.0039711, + "num_input_tokens_seen": 53325620, + "router_z_loss_clip": 1.13769531, + "router_z_loss_mlp": 0.20605469, + "step": 1885, + "time_per_iteration": 2.5732176303863525 + }, + { + "auxiliary_loss_clip": 0.01183632, + "auxiliary_loss_mlp": 0.01056675, + "balance_loss_clip": 1.07338393, + "balance_loss_mlp": 1.03900838, + "epoch": 0.054726945621264, + "flos": 21573873953280.0, + "grad_norm": 2.850390859333898, + "language_loss": 0.61218166, + "learning_rate": 3.993591946980255e-06, + "loss": 0.63458467, + "num_input_tokens_seen": 53339170, + "router_z_loss_clip": 1.1015625, + "router_z_loss_mlp": 0.17675781, + "step": 1886, + "time_per_iteration": 2.4779317378997803 + }, + { + "auxiliary_loss_clip": 0.01053757, + "auxiliary_loss_mlp": 0.01006124, + "balance_loss_clip": 1.02449238, + "balance_loss_mlp": 1.0046277, + "epoch": 0.05475596308978005, + "flos": 62990980675200.0, + "grad_norm": 0.7417285461614967, + "language_loss": 0.52514905, + "learning_rate": 3.993576903836039e-06, + "loss": 0.54574788, + "num_input_tokens_seen": 53399620, + "router_z_loss_clip": 0.29296875, + "router_z_loss_mlp": 0.01495361, + "step": 1887, + "time_per_iteration": 3.070105791091919 + }, + { + "auxiliary_loss_clip": 0.01052685, + "auxiliary_loss_mlp": 0.01000859, + "balance_loss_clip": 1.02355278, + "balance_loss_mlp": 0.99940473, + "epoch": 0.054784980558296094, + "flos": 60082540531200.0, + "grad_norm": 0.7427306330495946, + "language_loss": 0.57670903, + "learning_rate": 3.993561843083745e-06, + "loss": 0.5972445, + "num_input_tokens_seen": 53464325, + "router_z_loss_clip": 0.29101562, + "router_z_loss_mlp": 0.01452637, + "step": 1888, + "time_per_iteration": 3.199958086013794 + }, + { + "auxiliary_loss_clip": 0.01051235, + "auxiliary_loss_mlp": 0.01002251, + "balance_loss_clip": 1.02207553, + "balance_loss_mlp": 1.00088584, + "epoch": 0.05481399802681214, + "flos": 74772229472640.0, + "grad_norm": 0.6876848793264996, + "language_loss": 0.54850298, + "learning_rate": 3.993546764723507e-06, + "loss": 0.5690378, + "num_input_tokens_seen": 53529965, + "router_z_loss_clip": 0.29101562, + "router_z_loss_mlp": 0.01367188, + "step": 1889, + "time_per_iteration": 3.2134499549865723 + }, + { + "auxiliary_loss_clip": 0.01198675, + "auxiliary_loss_mlp": 0.01066566, + "balance_loss_clip": 1.07846928, + "balance_loss_mlp": 1.04411864, + "epoch": 0.05484301549532819, + "flos": 34127274441600.0, + "grad_norm": 1.9047517083333063, + "language_loss": 0.91254908, + "learning_rate": 3.993531668755458e-06, + "loss": 0.93520141, + "num_input_tokens_seen": 53550155, + "router_z_loss_clip": 1.20214844, + "router_z_loss_mlp": 0.22436523, + "step": 1890, + "time_per_iteration": 2.624433994293213 + }, + { + "auxiliary_loss_clip": 0.01195656, + "auxiliary_loss_mlp": 0.01062747, + "balance_loss_clip": 1.07782245, + "balance_loss_mlp": 1.04099083, + "epoch": 0.054872032963844235, + "flos": 24894102591360.0, + "grad_norm": 2.109348294440468, + "language_loss": 0.77636033, + "learning_rate": 3.993516555179729e-06, + "loss": 0.79894435, + "num_input_tokens_seen": 53565570, + "router_z_loss_clip": 1.17871094, + "router_z_loss_mlp": 0.2175293, + "step": 1891, + "time_per_iteration": 2.547607183456421 + }, + { + "auxiliary_loss_clip": 0.0119182, + "auxiliary_loss_mlp": 0.01057359, + "balance_loss_clip": 1.07838595, + "balance_loss_mlp": 1.0391556, + "epoch": 0.05490105043236028, + "flos": 12853219311360.0, + "grad_norm": 2.9407573553995747, + "language_loss": 0.91333383, + "learning_rate": 3.993501423996456e-06, + "loss": 0.93582559, + "num_input_tokens_seen": 53577385, + "router_z_loss_clip": 1.13574219, + "router_z_loss_mlp": 0.18200684, + "step": 1892, + "time_per_iteration": 2.4805068969726562 + }, + { + "auxiliary_loss_clip": 0.011985, + "auxiliary_loss_mlp": 0.01068985, + "balance_loss_clip": 1.07810521, + "balance_loss_mlp": 1.04947603, + "epoch": 0.05493006790087633, + "flos": 20259507484800.0, + "grad_norm": 5.718338943489587, + "language_loss": 0.91250116, + "learning_rate": 3.993486275205771e-06, + "loss": 0.93517601, + "num_input_tokens_seen": 53591810, + "router_z_loss_clip": 1.20410156, + "router_z_loss_mlp": 0.19512939, + "step": 1893, + "time_per_iteration": 2.4840617179870605 + }, + { + "auxiliary_loss_clip": 0.01201555, + "auxiliary_loss_mlp": 0.01059058, + "balance_loss_clip": 1.08075416, + "balance_loss_mlp": 1.03714728, + "epoch": 0.054959085369392376, + "flos": 28614232321920.0, + "grad_norm": 2.507131625764128, + "language_loss": 0.87514657, + "learning_rate": 3.993471108807809e-06, + "loss": 0.89775264, + "num_input_tokens_seen": 53603890, + "router_z_loss_clip": 1.20898438, + "router_z_loss_mlp": 0.21923828, + "step": 1894, + "time_per_iteration": 2.540050983428955 + }, + { + "auxiliary_loss_clip": 0.01190504, + "auxiliary_loss_mlp": 0.01059472, + "balance_loss_clip": 1.07263136, + "balance_loss_mlp": 1.03921819, + "epoch": 0.05498810283790842, + "flos": 17121705045120.0, + "grad_norm": 3.0598074275961014, + "language_loss": 0.74024534, + "learning_rate": 3.993455924802703e-06, + "loss": 0.76274514, + "num_input_tokens_seen": 53617160, + "router_z_loss_clip": 1.17871094, + "router_z_loss_mlp": 0.20227051, + "step": 1895, + "time_per_iteration": 2.435877799987793 + }, + { + "auxiliary_loss_clip": 0.01190814, + "auxiliary_loss_mlp": 0.01072294, + "balance_loss_clip": 1.07388926, + "balance_loss_mlp": 1.05173063, + "epoch": 0.055017120306424465, + "flos": 35692225165440.0, + "grad_norm": 2.8654032137653873, + "language_loss": 0.81580824, + "learning_rate": 3.9934407231905885e-06, + "loss": 0.83843935, + "num_input_tokens_seen": 53633550, + "router_z_loss_clip": 1.16796875, + "router_z_loss_mlp": 0.20581055, + "step": 1896, + "time_per_iteration": 2.5835001468658447 + }, + { + "auxiliary_loss_clip": 0.01053534, + "auxiliary_loss_mlp": 0.01044343, + "balance_loss_clip": 1.02339458, + "balance_loss_mlp": 1.04300165, + "epoch": 0.055046137774940516, + "flos": 60583709041920.0, + "grad_norm": 0.6591479873758049, + "language_loss": 0.50203979, + "learning_rate": 3.993425503971598e-06, + "loss": 0.52301854, + "num_input_tokens_seen": 53694370, + "router_z_loss_clip": 0.30078125, + "router_z_loss_mlp": 0.01342773, + "step": 1897, + "time_per_iteration": 3.0467746257781982 + }, + { + "auxiliary_loss_clip": 0.011902, + "auxiliary_loss_mlp": 0.01071579, + "balance_loss_clip": 1.07449353, + "balance_loss_mlp": 1.05155206, + "epoch": 0.05507515524345656, + "flos": 14034609630720.0, + "grad_norm": 3.6184323733503985, + "language_loss": 0.92146194, + "learning_rate": 3.993410267145868e-06, + "loss": 0.94407976, + "num_input_tokens_seen": 53707870, + "router_z_loss_clip": 1.15820312, + "router_z_loss_mlp": 0.20031738, + "step": 1898, + "time_per_iteration": 2.475878953933716 + }, + { + "auxiliary_loss_clip": 0.01052179, + "auxiliary_loss_mlp": 0.01015376, + "balance_loss_clip": 1.02268648, + "balance_loss_mlp": 1.01399946, + "epoch": 0.055104172711972606, + "flos": 70361070917760.0, + "grad_norm": 0.7391324045673247, + "language_loss": 0.51566964, + "learning_rate": 3.99339501271353e-06, + "loss": 0.53634524, + "num_input_tokens_seen": 53772755, + "router_z_loss_clip": 0.29492188, + "router_z_loss_mlp": 0.01379395, + "step": 1899, + "time_per_iteration": 3.1016056537628174 + }, + { + "auxiliary_loss_clip": 0.01180957, + "auxiliary_loss_mlp": 0.01057333, + "balance_loss_clip": 1.07344997, + "balance_loss_mlp": 1.03883195, + "epoch": 0.05513319018048866, + "flos": 31168953285120.0, + "grad_norm": 2.33686440334376, + "language_loss": 0.6543594, + "learning_rate": 3.993379740674722e-06, + "loss": 0.67674232, + "num_input_tokens_seen": 53788175, + "router_z_loss_clip": 1.07275391, + "router_z_loss_mlp": 0.18505859, + "step": 1900, + "time_per_iteration": 2.5783135890960693 + }, + { + "auxiliary_loss_clip": 0.01050347, + "auxiliary_loss_mlp": 0.0100001, + "balance_loss_clip": 1.02117276, + "balance_loss_mlp": 0.99852574, + "epoch": 0.0551622076490047, + "flos": 59125628257920.0, + "grad_norm": 0.7030267864448746, + "language_loss": 0.52628839, + "learning_rate": 3.993364451029578e-06, + "loss": 0.54679191, + "num_input_tokens_seen": 53844825, + "router_z_loss_clip": 0.29296875, + "router_z_loss_mlp": 0.01483154, + "step": 1901, + "time_per_iteration": 3.022721290588379 + }, + { + "auxiliary_loss_clip": 0.01191025, + "auxiliary_loss_mlp": 0.01054922, + "balance_loss_clip": 1.07802773, + "balance_loss_mlp": 1.03721952, + "epoch": 0.05519122511752075, + "flos": 15589827768960.0, + "grad_norm": 2.559293449657416, + "language_loss": 0.77443367, + "learning_rate": 3.9933491437782314e-06, + "loss": 0.79689318, + "num_input_tokens_seen": 53859955, + "router_z_loss_clip": 1.12841797, + "router_z_loss_mlp": 0.17724609, + "step": 1902, + "time_per_iteration": 2.4969093799591064 + }, + { + "auxiliary_loss_clip": 0.01195573, + "auxiliary_loss_mlp": 0.01070512, + "balance_loss_clip": 1.0772562, + "balance_loss_mlp": 1.04868484, + "epoch": 0.05522024258603679, + "flos": 29345194091520.0, + "grad_norm": 2.404270961729682, + "language_loss": 0.88075751, + "learning_rate": 3.993333818920819e-06, + "loss": 0.90341836, + "num_input_tokens_seen": 53881455, + "router_z_loss_clip": 1.18261719, + "router_z_loss_mlp": 0.21838379, + "step": 1903, + "time_per_iteration": 2.728969097137451 + }, + { + "auxiliary_loss_clip": 0.01197278, + "auxiliary_loss_mlp": 0.01070165, + "balance_loss_clip": 1.07799482, + "balance_loss_mlp": 1.04775405, + "epoch": 0.05524926005455284, + "flos": 25586603873280.0, + "grad_norm": 2.392300309904191, + "language_loss": 0.77980328, + "learning_rate": 3.993318476457476e-06, + "loss": 0.80247772, + "num_input_tokens_seen": 53898610, + "router_z_loss_clip": 1.19287109, + "router_z_loss_mlp": 0.22412109, + "step": 1904, + "time_per_iteration": 2.5556044578552246 + }, + { + "auxiliary_loss_clip": 0.01173148, + "auxiliary_loss_mlp": 0.01050691, + "balance_loss_clip": 1.06576037, + "balance_loss_mlp": 1.03109312, + "epoch": 0.05527827752306889, + "flos": 11988599034240.0, + "grad_norm": 2.552325307064108, + "language_loss": 0.68866527, + "learning_rate": 3.993303116388336e-06, + "loss": 0.71090364, + "num_input_tokens_seen": 53911925, + "router_z_loss_clip": 1.07519531, + "router_z_loss_mlp": 0.19616699, + "step": 1905, + "time_per_iteration": 2.491826057434082 + }, + { + "auxiliary_loss_clip": 0.01197814, + "auxiliary_loss_mlp": 0.01060195, + "balance_loss_clip": 1.07982731, + "balance_loss_mlp": 1.04004884, + "epoch": 0.05530729499158493, + "flos": 11210162463360.0, + "grad_norm": 3.4501769480970856, + "language_loss": 0.88560998, + "learning_rate": 3.993287738713538e-06, + "loss": 0.90819013, + "num_input_tokens_seen": 53924455, + "router_z_loss_clip": 1.17919922, + "router_z_loss_mlp": 0.20153809, + "step": 1906, + "time_per_iteration": 2.4690518379211426 + }, + { + "auxiliary_loss_clip": 0.0118723, + "auxiliary_loss_mlp": 0.01061949, + "balance_loss_clip": 1.07605815, + "balance_loss_mlp": 1.04222, + "epoch": 0.055336312460100984, + "flos": 74731970327040.0, + "grad_norm": 2.890007529943676, + "language_loss": 0.82635677, + "learning_rate": 3.9932723434332155e-06, + "loss": 0.84884858, + "num_input_tokens_seen": 53947145, + "router_z_loss_clip": 1.11181641, + "router_z_loss_mlp": 0.19726562, + "step": 1907, + "time_per_iteration": 3.000900983810425 + }, + { + "auxiliary_loss_clip": 0.01190394, + "auxiliary_loss_mlp": 0.01063823, + "balance_loss_clip": 1.07368827, + "balance_loss_mlp": 1.04322338, + "epoch": 0.05536532992861703, + "flos": 46601850533760.0, + "grad_norm": 2.0457710665525988, + "language_loss": 0.93974322, + "learning_rate": 3.993256930547505e-06, + "loss": 0.9622854, + "num_input_tokens_seen": 53967215, + "router_z_loss_clip": 1.16699219, + "router_z_loss_mlp": 0.20605469, + "step": 1908, + "time_per_iteration": 2.7420060634613037 + }, + { + "auxiliary_loss_clip": 0.01202347, + "auxiliary_loss_mlp": 0.0105405, + "balance_loss_clip": 1.08017516, + "balance_loss_mlp": 1.03168619, + "epoch": 0.05539434739713307, + "flos": 30292014643200.0, + "grad_norm": 2.2964654054056894, + "language_loss": 0.94065648, + "learning_rate": 3.993241500056543e-06, + "loss": 0.96322048, + "num_input_tokens_seen": 53985930, + "router_z_loss_clip": 1.22070312, + "router_z_loss_mlp": 0.22375488, + "step": 1909, + "time_per_iteration": 2.6208109855651855 + }, + { + "auxiliary_loss_clip": 0.01189937, + "auxiliary_loss_mlp": 0.01052839, + "balance_loss_clip": 1.07597578, + "balance_loss_mlp": 1.03314567, + "epoch": 0.05542336486564912, + "flos": 16606853452800.0, + "grad_norm": 3.6380736530127766, + "language_loss": 0.97619545, + "learning_rate": 3.993226051960465e-06, + "loss": 0.99862325, + "num_input_tokens_seen": 53995850, + "router_z_loss_clip": 1.13867188, + "router_z_loss_mlp": 0.19689941, + "step": 1910, + "time_per_iteration": 2.4962172508239746 + }, + { + "auxiliary_loss_clip": 0.01202172, + "auxiliary_loss_mlp": 0.010558, + "balance_loss_clip": 1.07809544, + "balance_loss_mlp": 1.03313875, + "epoch": 0.05545238233416517, + "flos": 11503445011200.0, + "grad_norm": 2.9757164697609295, + "language_loss": 0.93536341, + "learning_rate": 3.993210586259408e-06, + "loss": 0.9579432, + "num_input_tokens_seen": 54007905, + "router_z_loss_clip": 1.24023438, + "router_z_loss_mlp": 0.22680664, + "step": 1911, + "time_per_iteration": 2.4872593879699707 + }, + { + "auxiliary_loss_clip": 0.0119411, + "auxiliary_loss_mlp": 0.01051392, + "balance_loss_clip": 1.0783236, + "balance_loss_mlp": 1.02955878, + "epoch": 0.055481399802681214, + "flos": 35694128586240.0, + "grad_norm": 2.0101619728211513, + "language_loss": 0.87615669, + "learning_rate": 3.993195102953509e-06, + "loss": 0.89861172, + "num_input_tokens_seen": 54027865, + "router_z_loss_clip": 1.15625, + "router_z_loss_mlp": 0.21856689, + "step": 1912, + "time_per_iteration": 2.76186203956604 + }, + { + "auxiliary_loss_clip": 0.01196, + "auxiliary_loss_mlp": 0.01052202, + "balance_loss_clip": 1.08022594, + "balance_loss_mlp": 1.03347433, + "epoch": 0.05551041727119726, + "flos": 25075128159360.0, + "grad_norm": 2.3353933654366545, + "language_loss": 0.9335165, + "learning_rate": 3.9931796020429036e-06, + "loss": 0.95599854, + "num_input_tokens_seen": 54043705, + "router_z_loss_clip": 1.15722656, + "router_z_loss_mlp": 0.18737793, + "step": 1913, + "time_per_iteration": 2.554744005203247 + }, + { + "auxiliary_loss_clip": 0.01057167, + "auxiliary_loss_mlp": 0.01001251, + "balance_loss_clip": 1.02799129, + "balance_loss_mlp": 0.99965376, + "epoch": 0.05553943473971331, + "flos": 74769500039040.0, + "grad_norm": 0.68541356800402, + "language_loss": 0.49397984, + "learning_rate": 3.99316408352773e-06, + "loss": 0.51456404, + "num_input_tokens_seen": 54104640, + "router_z_loss_clip": 0.29101562, + "router_z_loss_mlp": 0.01599121, + "step": 1914, + "time_per_iteration": 3.1256165504455566 + }, + { + "auxiliary_loss_clip": 0.01196621, + "auxiliary_loss_mlp": 0.0107113, + "balance_loss_clip": 1.07718492, + "balance_loss_mlp": 1.05023313, + "epoch": 0.055568452208229355, + "flos": 54556279228800.0, + "grad_norm": 2.765229097168187, + "language_loss": 0.93069106, + "learning_rate": 3.993148547408124e-06, + "loss": 0.9533686, + "num_input_tokens_seen": 54122490, + "router_z_loss_clip": 1.19433594, + "router_z_loss_mlp": 0.20910645, + "step": 1915, + "time_per_iteration": 2.723480224609375 + }, + { + "auxiliary_loss_clip": 0.01185798, + "auxiliary_loss_mlp": 0.01053825, + "balance_loss_clip": 1.07674623, + "balance_loss_mlp": 1.03489447, + "epoch": 0.0555974696767454, + "flos": 17047657157760.0, + "grad_norm": 3.265276805237645, + "language_loss": 0.89113337, + "learning_rate": 3.993132993684224e-06, + "loss": 0.91352952, + "num_input_tokens_seen": 54134150, + "router_z_loss_clip": 1.09033203, + "router_z_loss_mlp": 0.18920898, + "step": 1916, + "time_per_iteration": 2.4777445793151855 + }, + { + "auxiliary_loss_clip": 0.01051625, + "auxiliary_loss_mlp": 0.01004307, + "balance_loss_clip": 1.02289724, + "balance_loss_mlp": 1.0028466, + "epoch": 0.055626487145261444, + "flos": 60898859994240.0, + "grad_norm": 0.739670933926063, + "language_loss": 0.55830121, + "learning_rate": 3.993117422356168e-06, + "loss": 0.57886052, + "num_input_tokens_seen": 54182225, + "router_z_loss_clip": 0.28710938, + "router_z_loss_mlp": 0.0145874, + "step": 1917, + "time_per_iteration": 2.864866018295288 + }, + { + "auxiliary_loss_clip": 0.01197238, + "auxiliary_loss_mlp": 0.01055086, + "balance_loss_clip": 1.07791829, + "balance_loss_mlp": 1.03352129, + "epoch": 0.055655504613777496, + "flos": 20477197860480.0, + "grad_norm": 2.2459345143404223, + "language_loss": 0.81297171, + "learning_rate": 3.993101833424091e-06, + "loss": 0.835495, + "num_input_tokens_seen": 54196395, + "router_z_loss_clip": 1.19140625, + "router_z_loss_mlp": 0.21582031, + "step": 1918, + "time_per_iteration": 2.4589550495147705 + }, + { + "auxiliary_loss_clip": 0.01050028, + "auxiliary_loss_mlp": 0.01008581, + "balance_loss_clip": 1.02142644, + "balance_loss_mlp": 1.00720441, + "epoch": 0.05568452208229354, + "flos": 66782680254720.0, + "grad_norm": 0.7459008265431695, + "language_loss": 0.57316196, + "learning_rate": 3.993086226888132e-06, + "loss": 0.59374803, + "num_input_tokens_seen": 54253085, + "router_z_loss_clip": 0.28515625, + "router_z_loss_mlp": 0.01379395, + "step": 1919, + "time_per_iteration": 3.0111782550811768 + }, + { + "auxiliary_loss_clip": 0.01192615, + "auxiliary_loss_mlp": 0.01059067, + "balance_loss_clip": 1.07408142, + "balance_loss_mlp": 1.039433, + "epoch": 0.055713539550809585, + "flos": 16321256415360.0, + "grad_norm": 5.42017705137495, + "language_loss": 0.74521428, + "learning_rate": 3.993070602748429e-06, + "loss": 0.76773107, + "num_input_tokens_seen": 54268265, + "router_z_loss_clip": 1.18603516, + "router_z_loss_mlp": 0.1965332, + "step": 1920, + "time_per_iteration": 2.4974653720855713 + }, + { + "auxiliary_loss_clip": 0.01203266, + "auxiliary_loss_mlp": 0.01069473, + "balance_loss_clip": 1.08135355, + "balance_loss_mlp": 1.04620361, + "epoch": 0.055742557019325636, + "flos": 28247009627520.0, + "grad_norm": 4.092751421989611, + "language_loss": 0.90896308, + "learning_rate": 3.993054961005121e-06, + "loss": 0.93169045, + "num_input_tokens_seen": 54280450, + "router_z_loss_clip": 1.21875, + "router_z_loss_mlp": 0.23278809, + "step": 1921, + "time_per_iteration": 2.5422372817993164 + }, + { + "auxiliary_loss_clip": 0.01050436, + "auxiliary_loss_mlp": 0.00999666, + "balance_loss_clip": 1.02148676, + "balance_loss_mlp": 0.99810427, + "epoch": 0.05577157448784168, + "flos": 59524308288000.0, + "grad_norm": 0.6430034331177085, + "language_loss": 0.53405845, + "learning_rate": 3.9930393016583435e-06, + "loss": 0.55455947, + "num_input_tokens_seen": 54340800, + "router_z_loss_clip": 0.2890625, + "router_z_loss_mlp": 0.015625, + "step": 1922, + "time_per_iteration": 3.022324323654175 + }, + { + "auxiliary_loss_clip": 0.01050466, + "auxiliary_loss_mlp": 0.01002107, + "balance_loss_clip": 1.02185655, + "balance_loss_mlp": 1.00073004, + "epoch": 0.055800591956357726, + "flos": 52705267568640.0, + "grad_norm": 0.6807721883682333, + "language_loss": 0.50979567, + "learning_rate": 3.993023624708237e-06, + "loss": 0.53032136, + "num_input_tokens_seen": 54402060, + "router_z_loss_clip": 0.28515625, + "router_z_loss_mlp": 0.01379395, + "step": 1923, + "time_per_iteration": 5.390964031219482 + }, + { + "auxiliary_loss_clip": 0.01050814, + "auxiliary_loss_mlp": 0.01000078, + "balance_loss_clip": 1.02212381, + "balance_loss_mlp": 0.99846822, + "epoch": 0.05582960942487378, + "flos": 57486557819520.0, + "grad_norm": 0.863411270869671, + "language_loss": 0.52444237, + "learning_rate": 3.99300793015494e-06, + "loss": 0.54495126, + "num_input_tokens_seen": 54462010, + "router_z_loss_clip": 0.28710938, + "router_z_loss_mlp": 0.01611328, + "step": 1924, + "time_per_iteration": 5.334691047668457 + }, + { + "auxiliary_loss_clip": 0.01196988, + "auxiliary_loss_mlp": 0.01062614, + "balance_loss_clip": 1.07758665, + "balance_loss_mlp": 1.04201424, + "epoch": 0.05585862689338982, + "flos": 17194064993280.0, + "grad_norm": 3.2490484532435815, + "language_loss": 0.76025212, + "learning_rate": 3.99299221799859e-06, + "loss": 0.78284812, + "num_input_tokens_seen": 54476830, + "router_z_loss_clip": 1.19140625, + "router_z_loss_mlp": 0.20581055, + "step": 1925, + "time_per_iteration": 4.957145929336548 + }, + { + "auxiliary_loss_clip": 0.01188036, + "auxiliary_loss_mlp": 0.01052512, + "balance_loss_clip": 1.07550764, + "balance_loss_mlp": 1.03405857, + "epoch": 0.05588764436190587, + "flos": 20450876169600.0, + "grad_norm": 6.937661348842076, + "language_loss": 0.86925316, + "learning_rate": 3.992976488239327e-06, + "loss": 0.89165866, + "num_input_tokens_seen": 54491145, + "router_z_loss_clip": 1.12304688, + "router_z_loss_mlp": 0.18444824, + "step": 1926, + "time_per_iteration": 5.039806127548218 + }, + { + "auxiliary_loss_clip": 0.01203187, + "auxiliary_loss_mlp": 0.0105957, + "balance_loss_clip": 1.0847795, + "balance_loss_mlp": 1.03800488, + "epoch": 0.05591666183042191, + "flos": 34457868241920.0, + "grad_norm": 2.084762636652592, + "language_loss": 0.92224693, + "learning_rate": 3.992960740877287e-06, + "loss": 0.94487453, + "num_input_tokens_seen": 54509670, + "router_z_loss_clip": 1.18359375, + "router_z_loss_mlp": 0.21569824, + "step": 1927, + "time_per_iteration": 2.640568494796753 + }, + { + "auxiliary_loss_clip": 0.01193941, + "auxiliary_loss_mlp": 0.01051825, + "balance_loss_clip": 1.07729208, + "balance_loss_mlp": 1.03245306, + "epoch": 0.05594567929893796, + "flos": 28796945829120.0, + "grad_norm": 2.0712940467717793, + "language_loss": 0.68172467, + "learning_rate": 3.992944975912613e-06, + "loss": 0.70418227, + "num_input_tokens_seen": 54525320, + "router_z_loss_clip": 1.16699219, + "router_z_loss_mlp": 0.19348145, + "step": 1928, + "time_per_iteration": 2.5376393795013428 + }, + { + "auxiliary_loss_clip": 0.01192374, + "auxiliary_loss_mlp": 0.01062115, + "balance_loss_clip": 1.07774544, + "balance_loss_mlp": 1.04419172, + "epoch": 0.05597469676745401, + "flos": 27592681524480.0, + "grad_norm": 4.195470173716084, + "language_loss": 1.00014782, + "learning_rate": 3.992929193345443e-06, + "loss": 1.0226928, + "num_input_tokens_seen": 54542240, + "router_z_loss_clip": 1.14648438, + "router_z_loss_mlp": 0.17901611, + "step": 1929, + "time_per_iteration": 2.588367462158203 + }, + { + "auxiliary_loss_clip": 0.01197213, + "auxiliary_loss_mlp": 0.0106327, + "balance_loss_clip": 1.08062279, + "balance_loss_mlp": 1.04224145, + "epoch": 0.05600371423597005, + "flos": 17449605325440.0, + "grad_norm": 2.8454226105758744, + "language_loss": 0.73281097, + "learning_rate": 3.9929133931759145e-06, + "loss": 0.7554158, + "num_input_tokens_seen": 54554740, + "router_z_loss_clip": 1.16503906, + "router_z_loss_mlp": 0.21032715, + "step": 1930, + "time_per_iteration": 2.5005805492401123 + }, + { + "auxiliary_loss_clip": 0.01205144, + "auxiliary_loss_mlp": 0.01063445, + "balance_loss_clip": 1.08018661, + "balance_loss_mlp": 1.03980613, + "epoch": 0.056032731704486104, + "flos": 25406332490880.0, + "grad_norm": 2.670637297934534, + "language_loss": 1.03854299, + "learning_rate": 3.992897575404169e-06, + "loss": 1.06122887, + "num_input_tokens_seen": 54568675, + "router_z_loss_clip": 1.24853516, + "router_z_loss_mlp": 0.23632812, + "step": 1931, + "time_per_iteration": 2.582221508026123 + }, + { + "auxiliary_loss_clip": 0.01054284, + "auxiliary_loss_mlp": 0.0101176, + "balance_loss_clip": 1.02487683, + "balance_loss_mlp": 1.00987649, + "epoch": 0.05606174917300215, + "flos": 58218740651520.0, + "grad_norm": 0.6725724545065787, + "language_loss": 0.51531887, + "learning_rate": 3.9928817400303456e-06, + "loss": 0.53597927, + "num_input_tokens_seen": 54630900, + "router_z_loss_clip": 0.29296875, + "router_z_loss_mlp": 0.01879883, + "step": 1932, + "time_per_iteration": 3.1200757026672363 + }, + { + "auxiliary_loss_clip": 0.01053295, + "auxiliary_loss_mlp": 0.01005383, + "balance_loss_clip": 1.02373552, + "balance_loss_mlp": 1.00334454, + "epoch": 0.05609076664151819, + "flos": 73069777486080.0, + "grad_norm": 0.7538858434814907, + "language_loss": 0.54957849, + "learning_rate": 3.9928658870545844e-06, + "loss": 0.57016528, + "num_input_tokens_seen": 54697855, + "router_z_loss_clip": 0.296875, + "router_z_loss_mlp": 0.02038574, + "step": 1933, + "time_per_iteration": 3.1811084747314453 + }, + { + "auxiliary_loss_clip": 0.01195299, + "auxiliary_loss_mlp": 0.0106401, + "balance_loss_clip": 1.07769072, + "balance_loss_mlp": 1.04206336, + "epoch": 0.05611978411003424, + "flos": 24785149662720.0, + "grad_norm": 3.0186760636494965, + "language_loss": 0.8473419, + "learning_rate": 3.9928500164770255e-06, + "loss": 0.86993498, + "num_input_tokens_seen": 54713775, + "router_z_loss_clip": 1.17529297, + "router_z_loss_mlp": 0.21960449, + "step": 1934, + "time_per_iteration": 2.5167553424835205 + }, + { + "auxiliary_loss_clip": 0.01200185, + "auxiliary_loss_mlp": 0.01063217, + "balance_loss_clip": 1.0803355, + "balance_loss_mlp": 1.0397563, + "epoch": 0.05614880157855029, + "flos": 22340638604160.0, + "grad_norm": 2.8963966223794384, + "language_loss": 1.00313294, + "learning_rate": 3.9928341282978086e-06, + "loss": 1.02576697, + "num_input_tokens_seen": 54733215, + "router_z_loss_clip": 1.19921875, + "router_z_loss_mlp": 0.23449707, + "step": 1935, + "time_per_iteration": 2.74369740486145 + }, + { + "auxiliary_loss_clip": 0.01197294, + "auxiliary_loss_mlp": 0.01067861, + "balance_loss_clip": 1.07619071, + "balance_loss_mlp": 1.04604602, + "epoch": 0.056177819047066334, + "flos": 30075976293120.0, + "grad_norm": 2.1912063621192965, + "language_loss": 0.94748366, + "learning_rate": 3.992818222517074e-06, + "loss": 0.97013521, + "num_input_tokens_seen": 54750505, + "router_z_loss_clip": 1.21337891, + "router_z_loss_mlp": 0.21813965, + "step": 1936, + "time_per_iteration": 2.649780035018921 + }, + { + "auxiliary_loss_clip": 0.01198451, + "auxiliary_loss_mlp": 0.01062649, + "balance_loss_clip": 1.07642245, + "balance_loss_mlp": 1.03912902, + "epoch": 0.05620683651558238, + "flos": 29452890044160.0, + "grad_norm": 2.83567786417808, + "language_loss": 1.0202843, + "learning_rate": 3.992802299134963e-06, + "loss": 1.04289532, + "num_input_tokens_seen": 54765915, + "router_z_loss_clip": 1.21972656, + "router_z_loss_mlp": 0.23535156, + "step": 1937, + "time_per_iteration": 2.5408806800842285 + }, + { + "auxiliary_loss_clip": 0.01051641, + "auxiliary_loss_mlp": 0.01024425, + "balance_loss_clip": 1.0222168, + "balance_loss_mlp": 1.02290463, + "epoch": 0.05623585398409843, + "flos": 59157085593600.0, + "grad_norm": 0.7037727197615821, + "language_loss": 0.52287048, + "learning_rate": 3.9927863581516155e-06, + "loss": 0.54363114, + "num_input_tokens_seen": 54824975, + "router_z_loss_clip": 0.29492188, + "router_z_loss_mlp": 0.01519775, + "step": 1938, + "time_per_iteration": 3.0315959453582764 + }, + { + "auxiliary_loss_clip": 0.01193465, + "auxiliary_loss_mlp": 0.01060212, + "balance_loss_clip": 1.07634997, + "balance_loss_mlp": 1.04038739, + "epoch": 0.056264871452614475, + "flos": 24856468116480.0, + "grad_norm": 2.812325172757948, + "language_loss": 0.95969552, + "learning_rate": 3.992770399567172e-06, + "loss": 0.98223227, + "num_input_tokens_seen": 54839840, + "router_z_loss_clip": 1.17138672, + "router_z_loss_mlp": 0.19830322, + "step": 1939, + "time_per_iteration": 2.5268654823303223 + }, + { + "auxiliary_loss_clip": 0.01192086, + "auxiliary_loss_mlp": 0.01077432, + "balance_loss_clip": 1.07537603, + "balance_loss_mlp": 1.05580723, + "epoch": 0.05629388892113052, + "flos": 32664919939200.0, + "grad_norm": 4.225528257305587, + "language_loss": 0.73167491, + "learning_rate": 3.992754423381774e-06, + "loss": 0.75437015, + "num_input_tokens_seen": 54855545, + "router_z_loss_clip": 1.16699219, + "router_z_loss_mlp": 0.21606445, + "step": 1940, + "time_per_iteration": 2.593201160430908 + }, + { + "auxiliary_loss_clip": 0.01197845, + "auxiliary_loss_mlp": 0.01060088, + "balance_loss_clip": 1.07981706, + "balance_loss_mlp": 1.03899956, + "epoch": 0.056322906389646564, + "flos": 20622312806400.0, + "grad_norm": 2.4048107157265552, + "language_loss": 0.95214158, + "learning_rate": 3.9927384295955636e-06, + "loss": 0.97472095, + "num_input_tokens_seen": 54868360, + "router_z_loss_clip": 1.1796875, + "router_z_loss_mlp": 0.2109375, + "step": 1941, + "time_per_iteration": 2.566770553588867 + }, + { + "auxiliary_loss_clip": 0.01050645, + "auxiliary_loss_mlp": 0.01010915, + "balance_loss_clip": 1.02160883, + "balance_loss_mlp": 1.00936544, + "epoch": 0.056351923858162616, + "flos": 56550149233920.0, + "grad_norm": 0.741978373977638, + "language_loss": 0.5277437, + "learning_rate": 3.992722418208681e-06, + "loss": 0.54835927, + "num_input_tokens_seen": 54918355, + "router_z_loss_clip": 0.2890625, + "router_z_loss_mlp": 0.01544189, + "step": 1942, + "time_per_iteration": 2.883824348449707 + }, + { + "auxiliary_loss_clip": 0.01204604, + "auxiliary_loss_mlp": 0.0106874, + "balance_loss_clip": 1.0810684, + "balance_loss_mlp": 1.04434955, + "epoch": 0.05638094132667866, + "flos": 15443886810240.0, + "grad_norm": 2.207778685261524, + "language_loss": 0.78587997, + "learning_rate": 3.992706389221266e-06, + "loss": 0.80861342, + "num_input_tokens_seen": 54932020, + "router_z_loss_clip": 1.23632812, + "router_z_loss_mlp": 0.24389648, + "step": 1943, + "time_per_iteration": 2.508084297180176 + }, + { + "auxiliary_loss_clip": 0.01188346, + "auxiliary_loss_mlp": 0.01048182, + "balance_loss_clip": 1.07775474, + "balance_loss_mlp": 1.02963281, + "epoch": 0.056409958795194705, + "flos": 21718522022400.0, + "grad_norm": 2.273678293532874, + "language_loss": 0.90923536, + "learning_rate": 3.992690342633463e-06, + "loss": 0.93160063, + "num_input_tokens_seen": 54946710, + "router_z_loss_clip": 1.10693359, + "router_z_loss_mlp": 0.1854248, + "step": 1944, + "time_per_iteration": 2.6373376846313477 + }, + { + "auxiliary_loss_clip": 0.01202468, + "auxiliary_loss_mlp": 0.0106024, + "balance_loss_clip": 1.08013439, + "balance_loss_mlp": 1.03567076, + "epoch": 0.056438976263710756, + "flos": 26425405249920.0, + "grad_norm": 3.7095759785729094, + "language_loss": 0.90235829, + "learning_rate": 3.992674278445412e-06, + "loss": 0.92498541, + "num_input_tokens_seen": 54961465, + "router_z_loss_clip": 1.22167969, + "router_z_loss_mlp": 0.24572754, + "step": 1945, + "time_per_iteration": 2.623849630355835 + }, + { + "auxiliary_loss_clip": 0.01215122, + "auxiliary_loss_mlp": 0.01066555, + "balance_loss_clip": 1.08231974, + "balance_loss_mlp": 1.03868341, + "epoch": 0.0564679937322268, + "flos": 27009456393600.0, + "grad_norm": 1.9306755942004008, + "language_loss": 0.92424494, + "learning_rate": 3.992658196657256e-06, + "loss": 0.94706178, + "num_input_tokens_seen": 54978740, + "router_z_loss_clip": 1.328125, + "router_z_loss_mlp": 0.27880859, + "step": 1946, + "time_per_iteration": 2.647737741470337 + }, + { + "auxiliary_loss_clip": 0.01050576, + "auxiliary_loss_mlp": 0.00997187, + "balance_loss_clip": 1.0212357, + "balance_loss_mlp": 0.99562496, + "epoch": 0.056497011200742846, + "flos": 74790078531840.0, + "grad_norm": 0.7258312670803857, + "language_loss": 0.51176172, + "learning_rate": 3.992642097269136e-06, + "loss": 0.53223932, + "num_input_tokens_seen": 55046635, + "router_z_loss_clip": 0.29296875, + "router_z_loss_mlp": 0.015625, + "step": 1947, + "time_per_iteration": 3.2677502632141113 + }, + { + "auxiliary_loss_clip": 0.01181719, + "auxiliary_loss_mlp": 0.01048474, + "balance_loss_clip": 1.07114601, + "balance_loss_mlp": 1.03011596, + "epoch": 0.0565260286692589, + "flos": 27192062160000.0, + "grad_norm": 2.217142634771321, + "language_loss": 0.68839276, + "learning_rate": 3.992625980281195e-06, + "loss": 0.71069467, + "num_input_tokens_seen": 55060610, + "router_z_loss_clip": 1.10449219, + "router_z_loss_mlp": 0.18359375, + "step": 1948, + "time_per_iteration": 2.5960562229156494 + }, + { + "auxiliary_loss_clip": 0.01197634, + "auxiliary_loss_mlp": 0.01052184, + "balance_loss_clip": 1.07819057, + "balance_loss_mlp": 1.03185892, + "epoch": 0.05655504613777494, + "flos": 24491400238080.0, + "grad_norm": 2.414800420953646, + "language_loss": 0.80208045, + "learning_rate": 3.992609845693574e-06, + "loss": 0.82457864, + "num_input_tokens_seen": 55074200, + "router_z_loss_clip": 1.19433594, + "router_z_loss_mlp": 0.20336914, + "step": 1949, + "time_per_iteration": 2.5937483310699463 + }, + { + "auxiliary_loss_clip": 0.01195258, + "auxiliary_loss_mlp": 0.01058841, + "balance_loss_clip": 1.08070934, + "balance_loss_mlp": 1.03570294, + "epoch": 0.05658406360629099, + "flos": 40798865831040.0, + "grad_norm": 2.031262022165163, + "language_loss": 0.8448801, + "learning_rate": 3.992593693506418e-06, + "loss": 0.86742109, + "num_input_tokens_seen": 55094800, + "router_z_loss_clip": 1.14550781, + "router_z_loss_mlp": 0.23156738, + "step": 1950, + "time_per_iteration": 2.7515556812286377 + }, + { + "auxiliary_loss_clip": 0.01052108, + "auxiliary_loss_mlp": 0.00997299, + "balance_loss_clip": 1.02269673, + "balance_loss_mlp": 0.99585038, + "epoch": 0.05661308107480703, + "flos": 57775422994560.0, + "grad_norm": 0.7037529072385403, + "language_loss": 0.49821219, + "learning_rate": 3.9925775237198675e-06, + "loss": 0.5187062, + "num_input_tokens_seen": 55152805, + "router_z_loss_clip": 0.29492188, + "router_z_loss_mlp": 0.01446533, + "step": 1951, + "time_per_iteration": 2.9791946411132812 + }, + { + "auxiliary_loss_clip": 0.01051447, + "auxiliary_loss_mlp": 0.01003537, + "balance_loss_clip": 1.02221584, + "balance_loss_mlp": 1.00213075, + "epoch": 0.05664209854332308, + "flos": 52226901216000.0, + "grad_norm": 0.7157633826828389, + "language_loss": 0.50930285, + "learning_rate": 3.992561336334066e-06, + "loss": 0.52985263, + "num_input_tokens_seen": 55215420, + "router_z_loss_clip": 0.29296875, + "router_z_loss_mlp": 0.01403809, + "step": 1952, + "time_per_iteration": 3.1427252292633057 + }, + { + "auxiliary_loss_clip": 0.01050935, + "auxiliary_loss_mlp": 0.01004434, + "balance_loss_clip": 1.02187753, + "balance_loss_mlp": 1.003057, + "epoch": 0.05667111601183913, + "flos": 69378914361600.0, + "grad_norm": 0.6649233193773136, + "language_loss": 0.4948267, + "learning_rate": 3.992545131349156e-06, + "loss": 0.51538038, + "num_input_tokens_seen": 55279415, + "router_z_loss_clip": 0.29101562, + "router_z_loss_mlp": 0.01379395, + "step": 1953, + "time_per_iteration": 3.0811240673065186 + }, + { + "auxiliary_loss_clip": 0.01187421, + "auxiliary_loss_mlp": 0.01052923, + "balance_loss_clip": 1.07694972, + "balance_loss_mlp": 1.03508902, + "epoch": 0.05670013348035517, + "flos": 17303269317120.0, + "grad_norm": 2.3144368834413784, + "language_loss": 0.86736953, + "learning_rate": 3.992528908765282e-06, + "loss": 0.88977295, + "num_input_tokens_seen": 55292170, + "router_z_loss_clip": 1.10400391, + "router_z_loss_mlp": 0.17816162, + "step": 1954, + "time_per_iteration": 2.4804770946502686 + }, + { + "auxiliary_loss_clip": 0.01188802, + "auxiliary_loss_mlp": 0.01054065, + "balance_loss_clip": 1.07559001, + "balance_loss_mlp": 1.0354507, + "epoch": 0.056729150948871224, + "flos": 26826742886400.0, + "grad_norm": 2.5621402601877445, + "language_loss": 0.79677111, + "learning_rate": 3.992512668582586e-06, + "loss": 0.8191998, + "num_input_tokens_seen": 55307090, + "router_z_loss_clip": 1.12988281, + "router_z_loss_mlp": 0.18609619, + "step": 1955, + "time_per_iteration": 2.5828707218170166 + }, + { + "auxiliary_loss_clip": 0.01195234, + "auxiliary_loss_mlp": 0.0105187, + "balance_loss_clip": 1.0781672, + "balance_loss_mlp": 1.03223586, + "epoch": 0.05675816841738727, + "flos": 23579592468480.0, + "grad_norm": 2.3652002324632013, + "language_loss": 0.76645786, + "learning_rate": 3.992496410801212e-06, + "loss": 0.78892887, + "num_input_tokens_seen": 55322060, + "router_z_loss_clip": 1.17089844, + "router_z_loss_mlp": 0.19628906, + "step": 1956, + "time_per_iteration": 2.5629775524139404 + }, + { + "auxiliary_loss_clip": 0.01050307, + "auxiliary_loss_mlp": 0.01011088, + "balance_loss_clip": 1.02142811, + "balance_loss_mlp": 1.00977659, + "epoch": 0.05678718588590331, + "flos": 70212651920640.0, + "grad_norm": 0.6876865322999829, + "language_loss": 0.49773619, + "learning_rate": 3.992480135421303e-06, + "loss": 0.51835012, + "num_input_tokens_seen": 55384235, + "router_z_loss_clip": 0.2890625, + "router_z_loss_mlp": 0.01312256, + "step": 1957, + "time_per_iteration": 3.0812525749206543 + }, + { + "auxiliary_loss_clip": 0.01196303, + "auxiliary_loss_mlp": 0.0105931, + "balance_loss_clip": 1.07755232, + "balance_loss_mlp": 1.0379833, + "epoch": 0.05681620335441936, + "flos": 19820499459840.0, + "grad_norm": 18.746927564558717, + "language_loss": 0.91498923, + "learning_rate": 3.9924638424430035e-06, + "loss": 0.93754542, + "num_input_tokens_seen": 55395960, + "router_z_loss_clip": 1.18652344, + "router_z_loss_mlp": 0.21337891, + "step": 1958, + "time_per_iteration": 2.4904656410217285 + }, + { + "auxiliary_loss_clip": 0.01206366, + "auxiliary_loss_mlp": 0.01071894, + "balance_loss_clip": 1.07589841, + "balance_loss_mlp": 1.04803991, + "epoch": 0.05684522082293541, + "flos": 28834508476800.0, + "grad_norm": 2.2058550664814383, + "language_loss": 1.01541781, + "learning_rate": 3.992447531866457e-06, + "loss": 1.03820038, + "num_input_tokens_seen": 55415990, + "router_z_loss_clip": 1.30566406, + "router_z_loss_mlp": 0.23864746, + "step": 1959, + "time_per_iteration": 2.6083779335021973 + }, + { + "auxiliary_loss_clip": 0.01195103, + "auxiliary_loss_mlp": 0.01066607, + "balance_loss_clip": 1.07864678, + "balance_loss_mlp": 1.04410028, + "epoch": 0.056874238291451454, + "flos": 40289760414720.0, + "grad_norm": 2.230778910204442, + "language_loss": 0.76833379, + "learning_rate": 3.992431203691807e-06, + "loss": 0.79095089, + "num_input_tokens_seen": 55436675, + "router_z_loss_clip": 1.1640625, + "router_z_loss_mlp": 0.22521973, + "step": 1960, + "time_per_iteration": 2.704482316970825 + }, + { + "auxiliary_loss_clip": 0.01199795, + "auxiliary_loss_mlp": 0.01073337, + "balance_loss_clip": 1.07762051, + "balance_loss_mlp": 1.04758799, + "epoch": 0.0569032557599675, + "flos": 12596817052800.0, + "grad_norm": 2.4732351208192105, + "language_loss": 0.92714208, + "learning_rate": 3.992414857919199e-06, + "loss": 0.94987339, + "num_input_tokens_seen": 55449595, + "router_z_loss_clip": 1.22167969, + "router_z_loss_mlp": 0.25756836, + "step": 1961, + "time_per_iteration": 2.500631093978882 + }, + { + "auxiliary_loss_clip": 0.01206174, + "auxiliary_loss_mlp": 0.01063644, + "balance_loss_clip": 1.08384132, + "balance_loss_mlp": 1.04106569, + "epoch": 0.05693227322848355, + "flos": 31569824044800.0, + "grad_norm": 2.560541099988531, + "language_loss": 0.83560026, + "learning_rate": 3.992398494548777e-06, + "loss": 0.85829848, + "num_input_tokens_seen": 55467800, + "router_z_loss_clip": 1.22070312, + "router_z_loss_mlp": 0.22558594, + "step": 1962, + "time_per_iteration": 2.621561050415039 + }, + { + "auxiliary_loss_clip": 0.0104806, + "auxiliary_loss_mlp": 0.01005422, + "balance_loss_clip": 1.01894522, + "balance_loss_mlp": 1.00402689, + "epoch": 0.056961290696999595, + "flos": 74006973192960.0, + "grad_norm": 0.7004154660024641, + "language_loss": 0.5353182, + "learning_rate": 3.992382113580685e-06, + "loss": 0.55585301, + "num_input_tokens_seen": 55528745, + "router_z_loss_clip": 0.29101562, + "router_z_loss_mlp": 0.01397705, + "step": 1963, + "time_per_iteration": 3.1378862857818604 + }, + { + "auxiliary_loss_clip": 0.01199229, + "auxiliary_loss_mlp": 0.01067442, + "balance_loss_clip": 1.07734132, + "balance_loss_mlp": 1.04363585, + "epoch": 0.05699030816551564, + "flos": 35729751899520.0, + "grad_norm": 2.4943843675815747, + "language_loss": 0.96035081, + "learning_rate": 3.992365715015068e-06, + "loss": 0.98301756, + "num_input_tokens_seen": 55544640, + "router_z_loss_clip": 1.21582031, + "router_z_loss_mlp": 0.23815918, + "step": 1964, + "time_per_iteration": 2.654578447341919 + }, + { + "auxiliary_loss_clip": 0.01047644, + "auxiliary_loss_mlp": 0.0100201, + "balance_loss_clip": 1.01876676, + "balance_loss_mlp": 1.00065708, + "epoch": 0.057019325634031684, + "flos": 58569838139520.0, + "grad_norm": 0.7534754042785102, + "language_loss": 0.56229961, + "learning_rate": 3.992349298852071e-06, + "loss": 0.5827961, + "num_input_tokens_seen": 55601315, + "router_z_loss_clip": 0.2890625, + "router_z_loss_mlp": 0.0135498, + "step": 1965, + "time_per_iteration": 3.006031036376953 + }, + { + "auxiliary_loss_clip": 0.01191616, + "auxiliary_loss_mlp": 0.01064293, + "balance_loss_clip": 1.07318437, + "balance_loss_mlp": 1.04416466, + "epoch": 0.057048343102547736, + "flos": 19790658236160.0, + "grad_norm": 2.897975030920047, + "language_loss": 1.02438867, + "learning_rate": 3.992332865091838e-06, + "loss": 1.04694784, + "num_input_tokens_seen": 55615290, + "router_z_loss_clip": 1.18505859, + "router_z_loss_mlp": 0.20141602, + "step": 1966, + "time_per_iteration": 2.539914846420288 + }, + { + "auxiliary_loss_clip": 0.01193926, + "auxiliary_loss_mlp": 0.01056632, + "balance_loss_clip": 1.07474601, + "balance_loss_mlp": 1.03318369, + "epoch": 0.05707736057106378, + "flos": 19054955871360.0, + "grad_norm": 2.7059337619237276, + "language_loss": 0.92604184, + "learning_rate": 3.992316413734516e-06, + "loss": 0.94854736, + "num_input_tokens_seen": 55630915, + "router_z_loss_clip": 1.19238281, + "router_z_loss_mlp": 0.23449707, + "step": 1967, + "time_per_iteration": 2.492811918258667 + }, + { + "auxiliary_loss_clip": 0.0119321, + "auxiliary_loss_mlp": 0.01056627, + "balance_loss_clip": 1.07619929, + "balance_loss_mlp": 1.03376865, + "epoch": 0.057106378039579825, + "flos": 25788315674880.0, + "grad_norm": 2.9443721955063316, + "language_loss": 0.87677652, + "learning_rate": 3.992299944780248e-06, + "loss": 0.89927495, + "num_input_tokens_seen": 55651145, + "router_z_loss_clip": 1.17041016, + "router_z_loss_mlp": 0.22888184, + "step": 1968, + "time_per_iteration": 2.6347837448120117 + }, + { + "auxiliary_loss_clip": 0.0118901, + "auxiliary_loss_mlp": 0.01054201, + "balance_loss_clip": 1.07291865, + "balance_loss_mlp": 1.03491306, + "epoch": 0.057135395508095876, + "flos": 11102179201920.0, + "grad_norm": 5.667792984238826, + "language_loss": 0.89561278, + "learning_rate": 3.9922834582291815e-06, + "loss": 0.91804487, + "num_input_tokens_seen": 55662780, + "router_z_loss_clip": 1.16308594, + "router_z_loss_mlp": 0.19274902, + "step": 1969, + "time_per_iteration": 2.4880313873291016 + }, + { + "auxiliary_loss_clip": 0.01205097, + "auxiliary_loss_mlp": 0.01080779, + "balance_loss_clip": 1.08166873, + "balance_loss_mlp": 1.05514848, + "epoch": 0.05716441297661192, + "flos": 15297335320320.0, + "grad_norm": 3.3235005493266425, + "language_loss": 0.8807013, + "learning_rate": 3.992266954081461e-06, + "loss": 0.9035601, + "num_input_tokens_seen": 55675295, + "router_z_loss_clip": 1.234375, + "router_z_loss_mlp": 0.25610352, + "step": 1970, + "time_per_iteration": 2.4819278717041016 + }, + { + "auxiliary_loss_clip": 0.01202215, + "auxiliary_loss_mlp": 0.01061348, + "balance_loss_clip": 1.07965231, + "balance_loss_mlp": 1.03884172, + "epoch": 0.057193430445127966, + "flos": 34530371844480.0, + "grad_norm": 2.169543385198623, + "language_loss": 0.96395546, + "learning_rate": 3.992250432337233e-06, + "loss": 0.9865911, + "num_input_tokens_seen": 55700635, + "router_z_loss_clip": 1.22753906, + "router_z_loss_mlp": 0.22521973, + "step": 1971, + "time_per_iteration": 2.7777559757232666 + }, + { + "auxiliary_loss_clip": 0.01050827, + "auxiliary_loss_mlp": 0.01005812, + "balance_loss_clip": 1.02194571, + "balance_loss_mlp": 1.0043515, + "epoch": 0.05722244791364401, + "flos": 74772768176640.0, + "grad_norm": 0.6681080907115954, + "language_loss": 0.50920045, + "learning_rate": 3.992233892996642e-06, + "loss": 0.52976686, + "num_input_tokens_seen": 55765515, + "router_z_loss_clip": 0.2890625, + "router_z_loss_mlp": 0.0145874, + "step": 1972, + "time_per_iteration": 3.1720900535583496 + }, + { + "auxiliary_loss_clip": 0.0105069, + "auxiliary_loss_mlp": 0.01006368, + "balance_loss_clip": 1.02167106, + "balance_loss_mlp": 1.00489545, + "epoch": 0.05725146538216006, + "flos": 62150239964160.0, + "grad_norm": 0.6530395183938698, + "language_loss": 0.44705603, + "learning_rate": 3.992217336059836e-06, + "loss": 0.4676266, + "num_input_tokens_seen": 55828275, + "router_z_loss_clip": 0.29101562, + "router_z_loss_mlp": 0.01470947, + "step": 1973, + "time_per_iteration": 3.1065540313720703 + }, + { + "auxiliary_loss_clip": 0.01049698, + "auxiliary_loss_mlp": 0.01005939, + "balance_loss_clip": 1.020895, + "balance_loss_mlp": 1.00450897, + "epoch": 0.05728048285067611, + "flos": 63754584929280.0, + "grad_norm": 0.6695863141942502, + "language_loss": 0.52149749, + "learning_rate": 3.9922007615269606e-06, + "loss": 0.54205382, + "num_input_tokens_seen": 55887525, + "router_z_loss_clip": 0.2890625, + "router_z_loss_mlp": 0.01428223, + "step": 1974, + "time_per_iteration": 3.032728433609009 + }, + { + "auxiliary_loss_clip": 0.01186778, + "auxiliary_loss_mlp": 0.01066173, + "balance_loss_clip": 1.07200181, + "balance_loss_mlp": 1.04509652, + "epoch": 0.05730950031919215, + "flos": 52184702736000.0, + "grad_norm": 2.329088848435246, + "language_loss": 1.11010075, + "learning_rate": 3.992184169398161e-06, + "loss": 1.13263023, + "num_input_tokens_seen": 55907895, + "router_z_loss_clip": 1.1484375, + "router_z_loss_mlp": 0.21081543, + "step": 1975, + "time_per_iteration": 2.7688040733337402 + }, + { + "auxiliary_loss_clip": 0.01199103, + "auxiliary_loss_mlp": 0.0107284, + "balance_loss_clip": 1.07431197, + "balance_loss_mlp": 1.04854536, + "epoch": 0.0573385177877082, + "flos": 38252800045440.0, + "grad_norm": 2.057438450219748, + "language_loss": 0.90489525, + "learning_rate": 3.992167559673585e-06, + "loss": 0.92761469, + "num_input_tokens_seen": 55929165, + "router_z_loss_clip": 1.24707031, + "router_z_loss_mlp": 0.24291992, + "step": 1976, + "time_per_iteration": 2.651150941848755 + }, + { + "auxiliary_loss_clip": 0.01194628, + "auxiliary_loss_mlp": 0.0105831, + "balance_loss_clip": 1.0775125, + "balance_loss_mlp": 1.03561282, + "epoch": 0.05736753525622425, + "flos": 21500257029120.0, + "grad_norm": 2.2064862448367317, + "language_loss": 0.61069393, + "learning_rate": 3.9921509323533796e-06, + "loss": 0.6332233, + "num_input_tokens_seen": 55942155, + "router_z_loss_clip": 1.17089844, + "router_z_loss_mlp": 0.22692871, + "step": 1977, + "time_per_iteration": 2.499814510345459 + }, + { + "auxiliary_loss_clip": 0.01192907, + "auxiliary_loss_mlp": 0.01053463, + "balance_loss_clip": 1.07603526, + "balance_loss_mlp": 1.03317332, + "epoch": 0.05739655272474029, + "flos": 37663900565760.0, + "grad_norm": 2.155409817295062, + "language_loss": 0.81905329, + "learning_rate": 3.9921342874376906e-06, + "loss": 0.84151697, + "num_input_tokens_seen": 55961730, + "router_z_loss_clip": 1.16992188, + "router_z_loss_mlp": 0.203125, + "step": 1978, + "time_per_iteration": 2.570298433303833 + }, + { + "auxiliary_loss_clip": 0.01192733, + "auxiliary_loss_mlp": 0.01058589, + "balance_loss_clip": 1.07515681, + "balance_loss_mlp": 1.0370239, + "epoch": 0.057425570193256344, + "flos": 51054558145920.0, + "grad_norm": 2.584148454228864, + "language_loss": 0.76853323, + "learning_rate": 3.992117624926665e-06, + "loss": 0.79104644, + "num_input_tokens_seen": 55980090, + "router_z_loss_clip": 1.17480469, + "router_z_loss_mlp": 0.2154541, + "step": 1979, + "time_per_iteration": 2.8489723205566406 + }, + { + "auxiliary_loss_clip": 0.01193872, + "auxiliary_loss_mlp": 0.01051694, + "balance_loss_clip": 1.07448339, + "balance_loss_mlp": 1.03016472, + "epoch": 0.05745458766177239, + "flos": 12050867260800.0, + "grad_norm": 2.5200247017293664, + "language_loss": 0.76808894, + "learning_rate": 3.992100944820451e-06, + "loss": 0.79054463, + "num_input_tokens_seen": 55994030, + "router_z_loss_clip": 1.19433594, + "router_z_loss_mlp": 0.21520996, + "step": 1980, + "time_per_iteration": 2.5820717811584473 + }, + { + "auxiliary_loss_clip": 0.01199147, + "auxiliary_loss_mlp": 0.01064197, + "balance_loss_clip": 1.07895231, + "balance_loss_mlp": 1.0416131, + "epoch": 0.05748360513028843, + "flos": 11138413046400.0, + "grad_norm": 2.578125006045856, + "language_loss": 0.96340585, + "learning_rate": 3.992084247119194e-06, + "loss": 0.98603922, + "num_input_tokens_seen": 56006445, + "router_z_loss_clip": 1.20263672, + "router_z_loss_mlp": 0.22601318, + "step": 1981, + "time_per_iteration": 2.4677910804748535 + }, + { + "auxiliary_loss_clip": 0.01183336, + "auxiliary_loss_mlp": 0.01044966, + "balance_loss_clip": 1.07469845, + "balance_loss_mlp": 1.02552342, + "epoch": 0.05751262259880448, + "flos": 20916708675840.0, + "grad_norm": 3.0314513129618934, + "language_loss": 0.79728281, + "learning_rate": 3.9920675318230445e-06, + "loss": 0.81956583, + "num_input_tokens_seen": 56019140, + "router_z_loss_clip": 1.0859375, + "router_z_loss_mlp": 0.19445801, + "step": 1982, + "time_per_iteration": 2.4852280616760254 + }, + { + "auxiliary_loss_clip": 0.01186679, + "auxiliary_loss_mlp": 0.01058488, + "balance_loss_clip": 1.0727452, + "balance_loss_mlp": 1.03774512, + "epoch": 0.05754164006732053, + "flos": 32117569516800.0, + "grad_norm": 2.2482465697004033, + "language_loss": 0.73257542, + "learning_rate": 3.992050798932148e-06, + "loss": 0.75502706, + "num_input_tokens_seen": 56037580, + "router_z_loss_clip": 1.13867188, + "router_z_loss_mlp": 0.2074585, + "step": 1983, + "time_per_iteration": 2.6154139041900635 + }, + { + "auxiliary_loss_clip": 0.01194909, + "auxiliary_loss_mlp": 0.01051226, + "balance_loss_clip": 1.07878184, + "balance_loss_mlp": 1.03032839, + "epoch": 0.057570657535836574, + "flos": 20263062931200.0, + "grad_norm": 2.515343625028426, + "language_loss": 0.81374818, + "learning_rate": 3.992034048446652e-06, + "loss": 0.83620948, + "num_input_tokens_seen": 56056180, + "router_z_loss_clip": 1.16210938, + "router_z_loss_mlp": 0.20910645, + "step": 1984, + "time_per_iteration": 2.627358913421631 + }, + { + "auxiliary_loss_clip": 0.01054135, + "auxiliary_loss_mlp": 0.01005079, + "balance_loss_clip": 1.02482319, + "balance_loss_mlp": 1.00364268, + "epoch": 0.05759967500435262, + "flos": 60144521448960.0, + "grad_norm": 0.7384785671118618, + "language_loss": 0.5314849, + "learning_rate": 3.992017280366706e-06, + "loss": 0.55207705, + "num_input_tokens_seen": 56114660, + "router_z_loss_clip": 0.29296875, + "router_z_loss_mlp": 0.01434326, + "step": 1985, + "time_per_iteration": 3.015472650527954 + }, + { + "auxiliary_loss_clip": 0.01182954, + "auxiliary_loss_mlp": 0.01048187, + "balance_loss_clip": 1.07322097, + "balance_loss_mlp": 1.03010893, + "epoch": 0.05762869247286867, + "flos": 20184561757440.0, + "grad_norm": 2.6349935076980486, + "language_loss": 1.00384951, + "learning_rate": 3.9920004946924574e-06, + "loss": 1.02616096, + "num_input_tokens_seen": 56128535, + "router_z_loss_clip": 1.09912109, + "router_z_loss_mlp": 0.18060303, + "step": 1986, + "time_per_iteration": 2.5256545543670654 + }, + { + "auxiliary_loss_clip": 0.01053006, + "auxiliary_loss_mlp": 0.01001885, + "balance_loss_clip": 1.02354622, + "balance_loss_mlp": 1.00049043, + "epoch": 0.057657709941384715, + "flos": 63795882591360.0, + "grad_norm": 0.7049512823656997, + "language_loss": 0.50590849, + "learning_rate": 3.991983691424054e-06, + "loss": 0.52645743, + "num_input_tokens_seen": 56189080, + "router_z_loss_clip": 0.29296875, + "router_z_loss_mlp": 0.01397705, + "step": 1987, + "time_per_iteration": 3.0854556560516357 + }, + { + "auxiliary_loss_clip": 0.01192106, + "auxiliary_loss_mlp": 0.0106895, + "balance_loss_clip": 1.07750726, + "balance_loss_mlp": 1.04906607, + "epoch": 0.05768672740990076, + "flos": 13947596933760.0, + "grad_norm": 4.060742394875256, + "language_loss": 1.03385353, + "learning_rate": 3.991966870561644e-06, + "loss": 1.05646408, + "num_input_tokens_seen": 56202505, + "router_z_loss_clip": 1.14550781, + "router_z_loss_mlp": 0.19848633, + "step": 1988, + "time_per_iteration": 2.52612566947937 + }, + { + "auxiliary_loss_clip": 0.01182115, + "auxiliary_loss_mlp": 0.01054321, + "balance_loss_clip": 1.07258379, + "balance_loss_mlp": 1.03462815, + "epoch": 0.057715744878416804, + "flos": 45215231857920.0, + "grad_norm": 2.3457252250793155, + "language_loss": 1.02312779, + "learning_rate": 3.991950032105378e-06, + "loss": 1.04549217, + "num_input_tokens_seen": 56222880, + "router_z_loss_clip": 1.09521484, + "router_z_loss_mlp": 0.19665527, + "step": 1989, + "time_per_iteration": 2.748305559158325 + }, + { + "auxiliary_loss_clip": 0.01192459, + "auxiliary_loss_mlp": 0.01056984, + "balance_loss_clip": 1.0721519, + "balance_loss_mlp": 1.03236711, + "epoch": 0.057744762346932856, + "flos": 12341061239040.0, + "grad_norm": 2.643926771472112, + "language_loss": 0.93159622, + "learning_rate": 3.991933176055402e-06, + "loss": 0.95409071, + "num_input_tokens_seen": 56234730, + "router_z_loss_clip": 1.20263672, + "router_z_loss_mlp": 0.24609375, + "step": 1990, + "time_per_iteration": 2.5229554176330566 + }, + { + "auxiliary_loss_clip": 0.01195542, + "auxiliary_loss_mlp": 0.01060677, + "balance_loss_clip": 1.07577991, + "balance_loss_mlp": 1.03771114, + "epoch": 0.0577737798154489, + "flos": 25695808197120.0, + "grad_norm": 2.3012926711824138, + "language_loss": 0.93306988, + "learning_rate": 3.991916302411866e-06, + "loss": 0.95563209, + "num_input_tokens_seen": 56248735, + "router_z_loss_clip": 1.19824219, + "router_z_loss_mlp": 0.22937012, + "step": 1991, + "time_per_iteration": 2.54723858833313 + }, + { + "auxiliary_loss_clip": 0.010504, + "auxiliary_loss_mlp": 0.01001551, + "balance_loss_clip": 1.02109063, + "balance_loss_mlp": 1.00021636, + "epoch": 0.057802797283964945, + "flos": 58798194854400.0, + "grad_norm": 0.6750186525365154, + "language_loss": 0.53174055, + "learning_rate": 3.9918994111749194e-06, + "loss": 0.55226004, + "num_input_tokens_seen": 56310655, + "router_z_loss_clip": 0.29296875, + "router_z_loss_mlp": 0.0133667, + "step": 1992, + "time_per_iteration": 3.0460987091064453 + }, + { + "auxiliary_loss_clip": 0.01208417, + "auxiliary_loss_mlp": 0.01084607, + "balance_loss_clip": 1.08085704, + "balance_loss_mlp": 1.0612421, + "epoch": 0.057831814752480996, + "flos": 27484231386240.0, + "grad_norm": 2.3034699914525554, + "language_loss": 0.93900204, + "learning_rate": 3.991882502344712e-06, + "loss": 0.96193224, + "num_input_tokens_seen": 56328190, + "router_z_loss_clip": 1.27636719, + "router_z_loss_mlp": 0.23388672, + "step": 1993, + "time_per_iteration": 2.642744541168213 + }, + { + "auxiliary_loss_clip": 0.01050097, + "auxiliary_loss_mlp": 0.01003726, + "balance_loss_clip": 1.0209558, + "balance_loss_mlp": 1.00240278, + "epoch": 0.05786083222099704, + "flos": 63063807500160.0, + "grad_norm": 0.6636166296200489, + "language_loss": 0.4936842, + "learning_rate": 3.991865575921392e-06, + "loss": 0.51422238, + "num_input_tokens_seen": 56388670, + "router_z_loss_clip": 0.29296875, + "router_z_loss_mlp": 0.01324463, + "step": 1994, + "time_per_iteration": 2.9932050704956055 + }, + { + "auxiliary_loss_clip": 0.01049033, + "auxiliary_loss_mlp": 0.01002471, + "balance_loss_clip": 1.02007079, + "balance_loss_mlp": 1.00108194, + "epoch": 0.057889849689513086, + "flos": 66791155864320.0, + "grad_norm": 0.657791783842825, + "language_loss": 0.48284459, + "learning_rate": 3.9918486319051084e-06, + "loss": 0.50335962, + "num_input_tokens_seen": 56450730, + "router_z_loss_clip": 0.2890625, + "router_z_loss_mlp": 0.01391602, + "step": 1995, + "time_per_iteration": 7.8167643547058105 + }, + { + "auxiliary_loss_clip": 0.01048608, + "auxiliary_loss_mlp": 0.01006312, + "balance_loss_clip": 1.01984906, + "balance_loss_mlp": 1.00497115, + "epoch": 0.05791886715802913, + "flos": 70833260131200.0, + "grad_norm": 0.673743633444589, + "language_loss": 0.46442711, + "learning_rate": 3.991831670296013e-06, + "loss": 0.48497629, + "num_input_tokens_seen": 56507510, + "router_z_loss_clip": 0.2890625, + "router_z_loss_mlp": 0.01342773, + "step": 1996, + "time_per_iteration": 5.26514196395874 + }, + { + "auxiliary_loss_clip": 0.01205684, + "auxiliary_loss_mlp": 0.0106556, + "balance_loss_clip": 1.07617867, + "balance_loss_mlp": 1.04276729, + "epoch": 0.05794788462654518, + "flos": 22235492517120.0, + "grad_norm": 6.496350581093872, + "language_loss": 0.82952511, + "learning_rate": 3.991814691094253e-06, + "loss": 0.85223758, + "num_input_tokens_seen": 56524785, + "router_z_loss_clip": 1.29394531, + "router_z_loss_mlp": 0.22790527, + "step": 1997, + "time_per_iteration": 2.5331408977508545 + }, + { + "auxiliary_loss_clip": 0.01048256, + "auxiliary_loss_mlp": 0.01014208, + "balance_loss_clip": 1.01956058, + "balance_loss_mlp": 1.01284277, + "epoch": 0.05797690209506123, + "flos": 70284724560000.0, + "grad_norm": 0.7058094927194222, + "language_loss": 0.50366068, + "learning_rate": 3.99179769429998e-06, + "loss": 0.52428532, + "num_input_tokens_seen": 56592375, + "router_z_loss_clip": 0.28710938, + "router_z_loss_mlp": 0.01367188, + "step": 1998, + "time_per_iteration": 5.652668476104736 + }, + { + "auxiliary_loss_clip": 0.01195565, + "auxiliary_loss_mlp": 0.01058888, + "balance_loss_clip": 1.07662189, + "balance_loss_mlp": 1.03764534, + "epoch": 0.05800591956357727, + "flos": 30404702586240.0, + "grad_norm": 2.128608181247263, + "language_loss": 0.8947916, + "learning_rate": 3.991780679913344e-06, + "loss": 0.91733611, + "num_input_tokens_seen": 56610465, + "router_z_loss_clip": 1.18847656, + "router_z_loss_mlp": 0.21234131, + "step": 1999, + "time_per_iteration": 2.6222245693206787 + }, + { + "auxiliary_loss_clip": 0.01049082, + "auxiliary_loss_mlp": 0.0101126, + "balance_loss_clip": 1.02035081, + "balance_loss_mlp": 1.00989485, + "epoch": 0.05803493703209332, + "flos": 74777975648640.0, + "grad_norm": 0.7012160325496445, + "language_loss": 0.52106208, + "learning_rate": 3.991763647934495e-06, + "loss": 0.54166549, + "num_input_tokens_seen": 56669135, + "router_z_loss_clip": 0.28710938, + "router_z_loss_mlp": 0.01367188, + "step": 2000, + "time_per_iteration": 3.0966529846191406 + }, + { + "auxiliary_loss_clip": 0.01179358, + "auxiliary_loss_mlp": 0.01047278, + "balance_loss_clip": 1.07035434, + "balance_loss_mlp": 1.02715516, + "epoch": 0.05806395450060937, + "flos": 15595358463360.0, + "grad_norm": 2.8091083960485106, + "language_loss": 0.82700795, + "learning_rate": 3.991746598363583e-06, + "loss": 0.84927428, + "num_input_tokens_seen": 56683985, + "router_z_loss_clip": 1.08984375, + "router_z_loss_mlp": 0.2010498, + "step": 2001, + "time_per_iteration": 2.4816975593566895 + }, + { + "auxiliary_loss_clip": 0.01174407, + "auxiliary_loss_mlp": 0.01050179, + "balance_loss_clip": 1.06573427, + "balance_loss_mlp": 1.03139138, + "epoch": 0.05809297196912541, + "flos": 16466263620480.0, + "grad_norm": 2.8648377566187424, + "language_loss": 0.77654469, + "learning_rate": 3.99172953120076e-06, + "loss": 0.79879057, + "num_input_tokens_seen": 56698400, + "router_z_loss_clip": 1.08740234, + "router_z_loss_mlp": 0.18798828, + "step": 2002, + "time_per_iteration": 2.5053770542144775 + }, + { + "auxiliary_loss_clip": 0.01178949, + "auxiliary_loss_mlp": 0.01049626, + "balance_loss_clip": 1.07122123, + "balance_loss_mlp": 1.03075552, + "epoch": 0.05812198943764146, + "flos": 31606919815680.0, + "grad_norm": 1.9960417259630505, + "language_loss": 0.72617924, + "learning_rate": 3.991712446446175e-06, + "loss": 0.74846494, + "num_input_tokens_seen": 56712595, + "router_z_loss_clip": 1.07714844, + "router_z_loss_mlp": 0.18859863, + "step": 2003, + "time_per_iteration": 2.6124446392059326 + }, + { + "auxiliary_loss_clip": 0.0104842, + "auxiliary_loss_mlp": 0.00997934, + "balance_loss_clip": 1.01983857, + "balance_loss_mlp": 0.99658746, + "epoch": 0.05815100690615751, + "flos": 61938834468480.0, + "grad_norm": 0.6623961545684015, + "language_loss": 0.51053292, + "learning_rate": 3.991695344099981e-06, + "loss": 0.53099644, + "num_input_tokens_seen": 56782225, + "router_z_loss_clip": 0.28515625, + "router_z_loss_mlp": 0.01348877, + "step": 2004, + "time_per_iteration": 3.2227938175201416 + }, + { + "auxiliary_loss_clip": 0.01175847, + "auxiliary_loss_mlp": 0.01047353, + "balance_loss_clip": 1.07166505, + "balance_loss_mlp": 1.02939439, + "epoch": 0.05818002437467355, + "flos": 32955544880640.0, + "grad_norm": 4.05751051944043, + "language_loss": 0.95778555, + "learning_rate": 3.991678224162326e-06, + "loss": 0.9800176, + "num_input_tokens_seen": 56794750, + "router_z_loss_clip": 1.04199219, + "router_z_loss_mlp": 0.17974854, + "step": 2005, + "time_per_iteration": 2.637212038040161 + }, + { + "auxiliary_loss_clip": 0.01047943, + "auxiliary_loss_mlp": 0.00999431, + "balance_loss_clip": 1.01922226, + "balance_loss_mlp": 0.99807227, + "epoch": 0.0582090418431896, + "flos": 59817770403840.0, + "grad_norm": 0.6750620681021854, + "language_loss": 0.51691377, + "learning_rate": 3.991661086633364e-06, + "loss": 0.53738761, + "num_input_tokens_seen": 56855465, + "router_z_loss_clip": 0.28710938, + "router_z_loss_mlp": 0.01361084, + "step": 2006, + "time_per_iteration": 3.0346264839172363 + }, + { + "auxiliary_loss_clip": 0.011996, + "auxiliary_loss_mlp": 0.01066905, + "balance_loss_clip": 1.07552934, + "balance_loss_mlp": 1.04535234, + "epoch": 0.05823805931170565, + "flos": 52404260618880.0, + "grad_norm": 2.7047202539691897, + "language_loss": 0.85785323, + "learning_rate": 3.9916439315132455e-06, + "loss": 0.88051832, + "num_input_tokens_seen": 56873740, + "router_z_loss_clip": 1.24121094, + "router_z_loss_mlp": 0.21557617, + "step": 2007, + "time_per_iteration": 2.6849687099456787 + }, + { + "auxiliary_loss_clip": 0.01191841, + "auxiliary_loss_mlp": 0.01064528, + "balance_loss_clip": 1.07528281, + "balance_loss_mlp": 1.0435884, + "epoch": 0.058267076780221694, + "flos": 31277654818560.0, + "grad_norm": 2.6149199107634287, + "language_loss": 0.89501119, + "learning_rate": 3.9916267588021215e-06, + "loss": 0.91757482, + "num_input_tokens_seen": 56889550, + "router_z_loss_clip": 1.16455078, + "router_z_loss_mlp": 0.20953369, + "step": 2008, + "time_per_iteration": 2.590336322784424 + }, + { + "auxiliary_loss_clip": 0.01192015, + "auxiliary_loss_mlp": 0.01050747, + "balance_loss_clip": 1.07891488, + "balance_loss_mlp": 1.0287292, + "epoch": 0.05829609424873774, + "flos": 74733299130240.0, + "grad_norm": 2.71984759803704, + "language_loss": 0.86327034, + "learning_rate": 3.991609568500144e-06, + "loss": 0.88569796, + "num_input_tokens_seen": 56914560, + "router_z_loss_clip": 1.13085938, + "router_z_loss_mlp": 0.22045898, + "step": 2009, + "time_per_iteration": 2.9594757556915283 + }, + { + "auxiliary_loss_clip": 0.0118875, + "auxiliary_loss_mlp": 0.01050339, + "balance_loss_clip": 1.07343066, + "balance_loss_mlp": 1.03022814, + "epoch": 0.05832511171725379, + "flos": 11684470579200.0, + "grad_norm": 2.832667562368717, + "language_loss": 1.04087722, + "learning_rate": 3.991592360607465e-06, + "loss": 1.06326807, + "num_input_tokens_seen": 56928070, + "router_z_loss_clip": 1.15234375, + "router_z_loss_mlp": 0.20141602, + "step": 2010, + "time_per_iteration": 2.476227283477783 + }, + { + "auxiliary_loss_clip": 0.01198866, + "auxiliary_loss_mlp": 0.01060245, + "balance_loss_clip": 1.0809207, + "balance_loss_mlp": 1.0384233, + "epoch": 0.058354129185769835, + "flos": 10773057859200.0, + "grad_norm": 4.694476165752785, + "language_loss": 1.01598454, + "learning_rate": 3.991575135124236e-06, + "loss": 1.03857565, + "num_input_tokens_seen": 56938995, + "router_z_loss_clip": 1.1796875, + "router_z_loss_mlp": 0.21813965, + "step": 2011, + "time_per_iteration": 2.480182647705078 + }, + { + "auxiliary_loss_clip": 0.01047834, + "auxiliary_loss_mlp": 0.0100175, + "balance_loss_clip": 1.01935172, + "balance_loss_mlp": 1.00055206, + "epoch": 0.05838314665428588, + "flos": 74772552695040.0, + "grad_norm": 0.6651858256437381, + "language_loss": 0.53971392, + "learning_rate": 3.9915578920506095e-06, + "loss": 0.56020969, + "num_input_tokens_seen": 57002925, + "router_z_loss_clip": 0.28515625, + "router_z_loss_mlp": 0.01196289, + "step": 2012, + "time_per_iteration": 3.1492433547973633 + }, + { + "auxiliary_loss_clip": 0.01202688, + "auxiliary_loss_mlp": 0.01064062, + "balance_loss_clip": 1.07934487, + "balance_loss_mlp": 1.03955269, + "epoch": 0.058412164122801924, + "flos": 48973391112960.0, + "grad_norm": 2.49265763052959, + "language_loss": 0.76281446, + "learning_rate": 3.991540631386739e-06, + "loss": 0.78548193, + "num_input_tokens_seen": 57021115, + "router_z_loss_clip": 1.23291016, + "router_z_loss_mlp": 0.24523926, + "step": 2013, + "time_per_iteration": 2.696840763092041 + }, + { + "auxiliary_loss_clip": 0.01048259, + "auxiliary_loss_mlp": 0.01002493, + "balance_loss_clip": 1.01970434, + "balance_loss_mlp": 1.00121784, + "epoch": 0.058441181591317976, + "flos": 74783757738240.0, + "grad_norm": 0.6948997595778852, + "language_loss": 0.47850835, + "learning_rate": 3.991523353132774e-06, + "loss": 0.49901587, + "num_input_tokens_seen": 57090410, + "router_z_loss_clip": 0.28515625, + "router_z_loss_mlp": 0.01275635, + "step": 2014, + "time_per_iteration": 3.2549991607666016 + }, + { + "auxiliary_loss_clip": 0.01188096, + "auxiliary_loss_mlp": 0.01061101, + "balance_loss_clip": 1.07426214, + "balance_loss_mlp": 1.04075789, + "epoch": 0.05847019905983402, + "flos": 39925482635520.0, + "grad_norm": 2.288108697690154, + "language_loss": 1.00759232, + "learning_rate": 3.99150605728887e-06, + "loss": 1.03008425, + "num_input_tokens_seen": 57107745, + "router_z_loss_clip": 1.13769531, + "router_z_loss_mlp": 0.20330811, + "step": 2015, + "time_per_iteration": 2.6719937324523926 + }, + { + "auxiliary_loss_clip": 0.0118048, + "auxiliary_loss_mlp": 0.01033179, + "balance_loss_clip": 1.07308865, + "balance_loss_mlp": 1.01734173, + "epoch": 0.058499216528350065, + "flos": 13838428523520.0, + "grad_norm": 2.2241947099413433, + "language_loss": 0.62102962, + "learning_rate": 3.991488743855178e-06, + "loss": 0.64316618, + "num_input_tokens_seen": 57120560, + "router_z_loss_clip": 1.07519531, + "router_z_loss_mlp": 0.1583252, + "step": 2016, + "time_per_iteration": 2.443042278289795 + }, + { + "auxiliary_loss_clip": 0.01188562, + "auxiliary_loss_mlp": 0.01059845, + "balance_loss_clip": 1.0716691, + "balance_loss_mlp": 1.03885281, + "epoch": 0.058528233996866116, + "flos": 15882212476800.0, + "grad_norm": 1.955011123847395, + "language_loss": 0.66378605, + "learning_rate": 3.9914714128318515e-06, + "loss": 0.68627012, + "num_input_tokens_seen": 57139850, + "router_z_loss_clip": 1.16992188, + "router_z_loss_mlp": 0.20983887, + "step": 2017, + "time_per_iteration": 2.5583293437957764 + }, + { + "auxiliary_loss_clip": 0.01192221, + "auxiliary_loss_mlp": 0.01063521, + "balance_loss_clip": 1.07435191, + "balance_loss_mlp": 1.04331517, + "epoch": 0.05855725146538216, + "flos": 27270994296960.0, + "grad_norm": 1.835686333299172, + "language_loss": 0.95692694, + "learning_rate": 3.9914540642190445e-06, + "loss": 0.97948432, + "num_input_tokens_seen": 57160660, + "router_z_loss_clip": 1.17871094, + "router_z_loss_mlp": 0.20214844, + "step": 2018, + "time_per_iteration": 2.574443817138672 + }, + { + "auxiliary_loss_clip": 0.01048529, + "auxiliary_loss_mlp": 0.01002482, + "balance_loss_clip": 1.01998699, + "balance_loss_mlp": 1.00119424, + "epoch": 0.058586268933898206, + "flos": 70354893778560.0, + "grad_norm": 0.7082196654766246, + "language_loss": 0.50306791, + "learning_rate": 3.991436698016909e-06, + "loss": 0.52357799, + "num_input_tokens_seen": 57216190, + "router_z_loss_clip": 0.28515625, + "router_z_loss_mlp": 0.01287842, + "step": 2019, + "time_per_iteration": 3.0202863216400146 + }, + { + "auxiliary_loss_clip": 0.01047475, + "auxiliary_loss_mlp": 0.01003219, + "balance_loss_clip": 1.01910353, + "balance_loss_mlp": 1.00194955, + "epoch": 0.05861528640241425, + "flos": 74785625245440.0, + "grad_norm": 0.6967449300148971, + "language_loss": 0.51574898, + "learning_rate": 3.991419314225598e-06, + "loss": 0.53625584, + "num_input_tokens_seen": 57279550, + "router_z_loss_clip": 0.28320312, + "router_z_loss_mlp": 0.01269531, + "step": 2020, + "time_per_iteration": 3.1766397953033447 + }, + { + "auxiliary_loss_clip": 0.01187967, + "auxiliary_loss_mlp": 0.01064027, + "balance_loss_clip": 1.07783246, + "balance_loss_mlp": 1.04407144, + "epoch": 0.0586443038709303, + "flos": 18653438666880.0, + "grad_norm": 2.6035440500042304, + "language_loss": 0.90713549, + "learning_rate": 3.991401912845267e-06, + "loss": 0.92965543, + "num_input_tokens_seen": 57293315, + "router_z_loss_clip": 1.10107422, + "router_z_loss_mlp": 0.19946289, + "step": 2021, + "time_per_iteration": 2.453650712966919 + }, + { + "auxiliary_loss_clip": 0.01184747, + "auxiliary_loss_mlp": 0.01053262, + "balance_loss_clip": 1.07291245, + "balance_loss_mlp": 1.03471327, + "epoch": 0.05867332133944635, + "flos": 14713176435840.0, + "grad_norm": 2.6860460610899852, + "language_loss": 0.71120036, + "learning_rate": 3.9913844938760675e-06, + "loss": 0.73358047, + "num_input_tokens_seen": 57304815, + "router_z_loss_clip": 1.11767578, + "router_z_loss_mlp": 0.18548584, + "step": 2022, + "time_per_iteration": 2.4886951446533203 + }, + { + "auxiliary_loss_clip": 0.01203257, + "auxiliary_loss_mlp": 0.01070511, + "balance_loss_clip": 1.08108258, + "balance_loss_mlp": 1.04900551, + "epoch": 0.05870233880796239, + "flos": 37702145571840.0, + "grad_norm": 2.463006681379582, + "language_loss": 0.90646923, + "learning_rate": 3.991367057318155e-06, + "loss": 0.92920685, + "num_input_tokens_seen": 57324750, + "router_z_loss_clip": 1.22265625, + "router_z_loss_mlp": 0.21520996, + "step": 2023, + "time_per_iteration": 2.646519422531128 + }, + { + "auxiliary_loss_clip": 0.01048815, + "auxiliary_loss_mlp": 0.00998218, + "balance_loss_clip": 1.02048028, + "balance_loss_mlp": 0.99682885, + "epoch": 0.05873135627647844, + "flos": 67520250126720.0, + "grad_norm": 0.6438178224382469, + "language_loss": 0.54037839, + "learning_rate": 3.9913496031716816e-06, + "loss": 0.56084871, + "num_input_tokens_seen": 57389645, + "router_z_loss_clip": 0.28320312, + "router_z_loss_mlp": 0.01391602, + "step": 2024, + "time_per_iteration": 3.1083197593688965 + }, + { + "auxiliary_loss_clip": 0.01048425, + "auxiliary_loss_mlp": 0.01001995, + "balance_loss_clip": 1.02035642, + "balance_loss_mlp": 1.00065958, + "epoch": 0.05876037374499449, + "flos": 63055762853760.0, + "grad_norm": 0.7144989766867771, + "language_loss": 0.50240082, + "learning_rate": 3.991332131436804e-06, + "loss": 0.52290505, + "num_input_tokens_seen": 57445510, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.0133667, + "step": 2025, + "time_per_iteration": 2.9481043815612793 + }, + { + "auxiliary_loss_clip": 0.01047845, + "auxiliary_loss_mlp": 0.01004444, + "balance_loss_clip": 1.01966977, + "balance_loss_mlp": 1.00313902, + "epoch": 0.05878939121351053, + "flos": 59037717720960.0, + "grad_norm": 0.7297945319395004, + "language_loss": 0.51563293, + "learning_rate": 3.991314642113675e-06, + "loss": 0.53615582, + "num_input_tokens_seen": 57501545, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01306152, + "step": 2026, + "time_per_iteration": 3.0712451934814453 + }, + { + "auxiliary_loss_clip": 0.0104594, + "auxiliary_loss_mlp": 0.01002443, + "balance_loss_clip": 1.01791203, + "balance_loss_mlp": 1.00109005, + "epoch": 0.05881840868202658, + "flos": 62161265439360.0, + "grad_norm": 0.6372923639602764, + "language_loss": 0.47762883, + "learning_rate": 3.991297135202448e-06, + "loss": 0.49811268, + "num_input_tokens_seen": 57561480, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.0135498, + "step": 2027, + "time_per_iteration": 3.074622869491577 + }, + { + "auxiliary_loss_clip": 0.01175984, + "auxiliary_loss_mlp": 0.01047357, + "balance_loss_clip": 1.06974697, + "balance_loss_mlp": 1.02899885, + "epoch": 0.05884742615054263, + "flos": 37743874197120.0, + "grad_norm": 4.345310115836194, + "language_loss": 0.92843932, + "learning_rate": 3.991279610703281e-06, + "loss": 0.95067275, + "num_input_tokens_seen": 57577105, + "router_z_loss_clip": 1.06054688, + "router_z_loss_mlp": 0.18359375, + "step": 2028, + "time_per_iteration": 2.541198968887329 + }, + { + "auxiliary_loss_clip": 0.01195815, + "auxiliary_loss_mlp": 0.01051165, + "balance_loss_clip": 1.07627773, + "balance_loss_mlp": 1.03069687, + "epoch": 0.05887644361905867, + "flos": 39689299733760.0, + "grad_norm": 1.8792720102612466, + "language_loss": 0.97915834, + "learning_rate": 3.991262068616325e-06, + "loss": 1.00162816, + "num_input_tokens_seen": 57602400, + "router_z_loss_clip": 1.1953125, + "router_z_loss_mlp": 0.20471191, + "step": 2029, + "time_per_iteration": 2.713075876235962 + }, + { + "auxiliary_loss_clip": 0.01178567, + "auxiliary_loss_mlp": 0.01038677, + "balance_loss_clip": 1.07133794, + "balance_loss_mlp": 1.02292979, + "epoch": 0.05890546108757472, + "flos": 15916111937280.0, + "grad_norm": 2.7130690789946335, + "language_loss": 0.7364077, + "learning_rate": 3.991244508941737e-06, + "loss": 0.75858009, + "num_input_tokens_seen": 57616680, + "router_z_loss_clip": 1.07080078, + "router_z_loss_mlp": 0.1574707, + "step": 2030, + "time_per_iteration": 2.4494309425354004 + }, + { + "auxiliary_loss_clip": 0.01048954, + "auxiliary_loss_mlp": 0.01006414, + "balance_loss_clip": 1.02105021, + "balance_loss_mlp": 1.00517988, + "epoch": 0.05893447855609077, + "flos": 70474584873600.0, + "grad_norm": 0.6930083098200157, + "language_loss": 0.52351511, + "learning_rate": 3.991226931679672e-06, + "loss": 0.54406869, + "num_input_tokens_seen": 57680890, + "router_z_loss_clip": 0.27929688, + "router_z_loss_mlp": 0.0123291, + "step": 2031, + "time_per_iteration": 3.1725144386291504 + }, + { + "auxiliary_loss_clip": 0.01171473, + "auxiliary_loss_mlp": 0.01045961, + "balance_loss_clip": 1.07117021, + "balance_loss_mlp": 1.02964687, + "epoch": 0.058963496024606814, + "flos": 12158994176640.0, + "grad_norm": 2.95826688340625, + "language_loss": 0.76152527, + "learning_rate": 3.991209336830285e-06, + "loss": 0.78369963, + "num_input_tokens_seen": 57692135, + "router_z_loss_clip": 1.00244141, + "router_z_loss_mlp": 0.16308594, + "step": 2032, + "time_per_iteration": 2.469207763671875 + }, + { + "auxiliary_loss_clip": 0.01176765, + "auxiliary_loss_mlp": 0.01046426, + "balance_loss_clip": 1.07375276, + "balance_loss_mlp": 1.03051162, + "epoch": 0.05899251349312286, + "flos": 23213914058880.0, + "grad_norm": 2.1812678537592785, + "language_loss": 0.7605812, + "learning_rate": 3.991191724393732e-06, + "loss": 0.78281313, + "num_input_tokens_seen": 57705305, + "router_z_loss_clip": 1.03027344, + "router_z_loss_mlp": 0.15905762, + "step": 2033, + "time_per_iteration": 2.546025514602661 + }, + { + "auxiliary_loss_clip": 0.01190977, + "auxiliary_loss_mlp": 0.01053191, + "balance_loss_clip": 1.07531667, + "balance_loss_mlp": 1.03378356, + "epoch": 0.05902153096163891, + "flos": 23617011461760.0, + "grad_norm": 2.395577076154158, + "language_loss": 0.73329639, + "learning_rate": 3.991174094370167e-06, + "loss": 0.75573814, + "num_input_tokens_seen": 57718525, + "router_z_loss_clip": 1.15527344, + "router_z_loss_mlp": 0.19421387, + "step": 2034, + "time_per_iteration": 2.515435218811035 + }, + { + "auxiliary_loss_clip": 0.0118426, + "auxiliary_loss_mlp": 0.01048945, + "balance_loss_clip": 1.0771029, + "balance_loss_mlp": 1.03098571, + "epoch": 0.059050548430154955, + "flos": 24892378738560.0, + "grad_norm": 2.462494223605619, + "language_loss": 1.07617676, + "learning_rate": 3.991156446759747e-06, + "loss": 1.09850883, + "num_input_tokens_seen": 57734605, + "router_z_loss_clip": 1.07128906, + "router_z_loss_mlp": 0.17956543, + "step": 2035, + "time_per_iteration": 2.5449817180633545 + }, + { + "auxiliary_loss_clip": 0.01048765, + "auxiliary_loss_mlp": 0.0100627, + "balance_loss_clip": 1.02080524, + "balance_loss_mlp": 1.00501812, + "epoch": 0.059079565898671, + "flos": 71052099742080.0, + "grad_norm": 0.6679929765092161, + "language_loss": 0.51250845, + "learning_rate": 3.991138781562627e-06, + "loss": 0.53305876, + "num_input_tokens_seen": 57795595, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01251221, + "step": 2036, + "time_per_iteration": 3.0531375408172607 + }, + { + "auxiliary_loss_clip": 0.01194764, + "auxiliary_loss_mlp": 0.01054623, + "balance_loss_clip": 1.07358181, + "balance_loss_mlp": 1.03355908, + "epoch": 0.059108583367187044, + "flos": 74736818663040.0, + "grad_norm": 2.7153278240870096, + "language_loss": 0.92514896, + "learning_rate": 3.991121098778964e-06, + "loss": 0.94764286, + "num_input_tokens_seen": 57817550, + "router_z_loss_clip": 1.21337891, + "router_z_loss_mlp": 0.21069336, + "step": 2037, + "time_per_iteration": 2.888566017150879 + }, + { + "auxiliary_loss_clip": 0.01200717, + "auxiliary_loss_mlp": 0.01057798, + "balance_loss_clip": 1.07885313, + "balance_loss_mlp": 1.03481424, + "epoch": 0.059137600835703096, + "flos": 10916448952320.0, + "grad_norm": 2.788256166054023, + "language_loss": 0.99435782, + "learning_rate": 3.991103398408914e-06, + "loss": 1.01694298, + "num_input_tokens_seen": 57827940, + "router_z_loss_clip": 1.21972656, + "router_z_loss_mlp": 0.22973633, + "step": 2038, + "time_per_iteration": 2.4366555213928223 + }, + { + "auxiliary_loss_clip": 0.01047273, + "auxiliary_loss_mlp": 0.01001166, + "balance_loss_clip": 1.01952696, + "balance_loss_mlp": 1.00000918, + "epoch": 0.05916661830421914, + "flos": 74770757015040.0, + "grad_norm": 0.7903595075332392, + "language_loss": 0.51380277, + "learning_rate": 3.991085680452633e-06, + "loss": 0.53428721, + "num_input_tokens_seen": 57891715, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01153564, + "step": 2039, + "time_per_iteration": 3.08902645111084 + }, + { + "auxiliary_loss_clip": 0.01048213, + "auxiliary_loss_mlp": 0.0100062, + "balance_loss_clip": 1.02034843, + "balance_loss_mlp": 0.99943942, + "epoch": 0.059195635772735185, + "flos": 63313098865920.0, + "grad_norm": 0.9696114223939748, + "language_loss": 0.54499322, + "learning_rate": 3.991067944910277e-06, + "loss": 0.56548148, + "num_input_tokens_seen": 57947455, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01177979, + "step": 2040, + "time_per_iteration": 2.9227545261383057 + }, + { + "auxiliary_loss_clip": 0.0104858, + "auxiliary_loss_mlp": 0.01000548, + "balance_loss_clip": 1.02059579, + "balance_loss_mlp": 0.99923629, + "epoch": 0.059224653241251236, + "flos": 74775353955840.0, + "grad_norm": 0.7064438127943082, + "language_loss": 0.4879806, + "learning_rate": 3.991050191782004e-06, + "loss": 0.50847185, + "num_input_tokens_seen": 58006690, + "router_z_loss_clip": 0.27929688, + "router_z_loss_mlp": 0.01312256, + "step": 2041, + "time_per_iteration": 3.1306514739990234 + }, + { + "auxiliary_loss_clip": 0.01195235, + "auxiliary_loss_mlp": 0.01058235, + "balance_loss_clip": 1.07640219, + "balance_loss_mlp": 1.03731394, + "epoch": 0.05925367070976728, + "flos": 21354711120000.0, + "grad_norm": 2.977068011802807, + "language_loss": 0.94973719, + "learning_rate": 3.9910324210679695e-06, + "loss": 0.97227186, + "num_input_tokens_seen": 58020330, + "router_z_loss_clip": 1.18847656, + "router_z_loss_mlp": 0.20935059, + "step": 2042, + "time_per_iteration": 2.5416955947875977 + }, + { + "auxiliary_loss_clip": 0.01194789, + "auxiliary_loss_mlp": 0.01059103, + "balance_loss_clip": 1.07560849, + "balance_loss_mlp": 1.03675139, + "epoch": 0.059282688178283326, + "flos": 25622981372160.0, + "grad_norm": 2.6518611151896287, + "language_loss": 0.94817603, + "learning_rate": 3.991014632768331e-06, + "loss": 0.97071493, + "num_input_tokens_seen": 58035585, + "router_z_loss_clip": 1.19238281, + "router_z_loss_mlp": 0.22351074, + "step": 2043, + "time_per_iteration": 2.6229870319366455 + }, + { + "auxiliary_loss_clip": 0.01196069, + "auxiliary_loss_mlp": 0.01058773, + "balance_loss_clip": 1.0803225, + "balance_loss_mlp": 1.03859103, + "epoch": 0.05931170564679937, + "flos": 47806258492800.0, + "grad_norm": 2.620388064741013, + "language_loss": 0.82909805, + "learning_rate": 3.990996826883246e-06, + "loss": 0.85164654, + "num_input_tokens_seen": 58056960, + "router_z_loss_clip": 1.15917969, + "router_z_loss_mlp": 0.2019043, + "step": 2044, + "time_per_iteration": 2.7547049522399902 + }, + { + "auxiliary_loss_clip": 0.01177045, + "auxiliary_loss_mlp": 0.01046469, + "balance_loss_clip": 1.07082677, + "balance_loss_mlp": 1.02824807, + "epoch": 0.05934072311531542, + "flos": 19497447515520.0, + "grad_norm": 2.4291527268704667, + "language_loss": 0.76053798, + "learning_rate": 3.990979003412871e-06, + "loss": 0.78277308, + "num_input_tokens_seen": 58071820, + "router_z_loss_clip": 1.06201172, + "router_z_loss_mlp": 0.18231201, + "step": 2045, + "time_per_iteration": 2.489616870880127 + }, + { + "auxiliary_loss_clip": 0.01185761, + "auxiliary_loss_mlp": 0.01046717, + "balance_loss_clip": 1.07535589, + "balance_loss_mlp": 1.03059423, + "epoch": 0.059369740583831467, + "flos": 58167060981120.0, + "grad_norm": 2.3809349986209485, + "language_loss": 0.63555098, + "learning_rate": 3.990961162357363e-06, + "loss": 0.65787578, + "num_input_tokens_seen": 58092955, + "router_z_loss_clip": 1.10302734, + "router_z_loss_mlp": 0.16137695, + "step": 2046, + "time_per_iteration": 2.7492151260375977 + }, + { + "auxiliary_loss_clip": 0.01181139, + "auxiliary_loss_mlp": 0.01042729, + "balance_loss_clip": 1.07371593, + "balance_loss_mlp": 1.02457976, + "epoch": 0.05939875805234751, + "flos": 15186622625280.0, + "grad_norm": 2.7167929508942983, + "language_loss": 0.88183898, + "learning_rate": 3.9909433037168815e-06, + "loss": 0.90407765, + "num_input_tokens_seen": 58105050, + "router_z_loss_clip": 1.07470703, + "router_z_loss_mlp": 0.18139648, + "step": 2047, + "time_per_iteration": 2.5041255950927734 + }, + { + "auxiliary_loss_clip": 0.01187254, + "auxiliary_loss_mlp": 0.01063586, + "balance_loss_clip": 1.07253385, + "balance_loss_mlp": 1.04152036, + "epoch": 0.05942777552086356, + "flos": 51527465631360.0, + "grad_norm": 3.4739837013327493, + "language_loss": 0.6039905, + "learning_rate": 3.990925427491583e-06, + "loss": 0.62649888, + "num_input_tokens_seen": 58122150, + "router_z_loss_clip": 1.14746094, + "router_z_loss_mlp": 0.22070312, + "step": 2048, + "time_per_iteration": 2.7457644939422607 + }, + { + "auxiliary_loss_clip": 0.01183296, + "auxiliary_loss_mlp": 0.01051314, + "balance_loss_clip": 1.0722599, + "balance_loss_mlp": 1.03384936, + "epoch": 0.05945679298937961, + "flos": 20771198680320.0, + "grad_norm": 2.117211517675094, + "language_loss": 0.92470634, + "learning_rate": 3.990907533681625e-06, + "loss": 0.94705236, + "num_input_tokens_seen": 58138765, + "router_z_loss_clip": 1.11132812, + "router_z_loss_mlp": 0.17468262, + "step": 2049, + "time_per_iteration": 2.5406341552734375 + }, + { + "auxiliary_loss_clip": 0.01191837, + "auxiliary_loss_mlp": 0.01057186, + "balance_loss_clip": 1.07786345, + "balance_loss_mlp": 1.03515029, + "epoch": 0.05948581045789565, + "flos": 26023888045440.0, + "grad_norm": 2.534975136786739, + "language_loss": 0.94181764, + "learning_rate": 3.990889622287166e-06, + "loss": 0.9643079, + "num_input_tokens_seen": 58155710, + "router_z_loss_clip": 1.13964844, + "router_z_loss_mlp": 0.22015381, + "step": 2050, + "time_per_iteration": 2.5805068016052246 + }, + { + "auxiliary_loss_clip": 0.01195909, + "auxiliary_loss_mlp": 0.01049995, + "balance_loss_clip": 1.07829142, + "balance_loss_mlp": 1.02922893, + "epoch": 0.0595148279264117, + "flos": 13436516269440.0, + "grad_norm": 2.3698812718510847, + "language_loss": 0.78685677, + "learning_rate": 3.990871693308365e-06, + "loss": 0.8093158, + "num_input_tokens_seen": 58168795, + "router_z_loss_clip": 1.17724609, + "router_z_loss_mlp": 0.2076416, + "step": 2051, + "time_per_iteration": 2.4891765117645264 + }, + { + "auxiliary_loss_clip": 0.01180885, + "auxiliary_loss_mlp": 0.01049439, + "balance_loss_clip": 1.06832898, + "balance_loss_mlp": 1.02993083, + "epoch": 0.05954384539492775, + "flos": 74731072487040.0, + "grad_norm": 1.9348929412548506, + "language_loss": 1.05392349, + "learning_rate": 3.990853746745379e-06, + "loss": 1.07622671, + "num_input_tokens_seen": 58195535, + "router_z_loss_clip": 1.125, + "router_z_loss_mlp": 0.19500732, + "step": 2052, + "time_per_iteration": 2.9218525886535645 + }, + { + "auxiliary_loss_clip": 0.01177581, + "auxiliary_loss_mlp": 0.01045314, + "balance_loss_clip": 1.07250333, + "balance_loss_mlp": 1.02654457, + "epoch": 0.05957286286344379, + "flos": 12780356572800.0, + "grad_norm": 3.062082045244814, + "language_loss": 0.84006995, + "learning_rate": 3.990835782598367e-06, + "loss": 0.86229891, + "num_input_tokens_seen": 58207930, + "router_z_loss_clip": 1.05078125, + "router_z_loss_mlp": 0.18762207, + "step": 2053, + "time_per_iteration": 2.4915151596069336 + }, + { + "auxiliary_loss_clip": 0.01182096, + "auxiliary_loss_mlp": 0.01066614, + "balance_loss_clip": 1.07304716, + "balance_loss_mlp": 1.04780257, + "epoch": 0.05960188033195984, + "flos": 25804653384960.0, + "grad_norm": 2.32437296424891, + "language_loss": 0.96781445, + "learning_rate": 3.990817800867488e-06, + "loss": 0.99030155, + "num_input_tokens_seen": 58224775, + "router_z_loss_clip": 1.08886719, + "router_z_loss_mlp": 0.18798828, + "step": 2054, + "time_per_iteration": 2.566594362258911 + }, + { + "auxiliary_loss_clip": 0.0117959, + "auxiliary_loss_mlp": 0.01050038, + "balance_loss_clip": 1.07223892, + "balance_loss_mlp": 1.02946234, + "epoch": 0.05963089780047589, + "flos": 31462702709760.0, + "grad_norm": 1.936898267622696, + "language_loss": 0.8616215, + "learning_rate": 3.9907998015529e-06, + "loss": 0.88391781, + "num_input_tokens_seen": 58242425, + "router_z_loss_clip": 1.07373047, + "router_z_loss_mlp": 0.20568848, + "step": 2055, + "time_per_iteration": 2.6000874042510986 + }, + { + "auxiliary_loss_clip": 0.01053539, + "auxiliary_loss_mlp": 0.01004145, + "balance_loss_clip": 1.025442, + "balance_loss_mlp": 1.00272024, + "epoch": 0.059659915268991934, + "flos": 61743263892480.0, + "grad_norm": 0.7734786447483898, + "language_loss": 0.53867793, + "learning_rate": 3.990781784654763e-06, + "loss": 0.55925477, + "num_input_tokens_seen": 58300685, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01422119, + "step": 2056, + "time_per_iteration": 2.974613904953003 + }, + { + "auxiliary_loss_clip": 0.01052817, + "auxiliary_loss_mlp": 0.01005665, + "balance_loss_clip": 1.02473617, + "balance_loss_mlp": 1.0043242, + "epoch": 0.05968893273750798, + "flos": 64119114103680.0, + "grad_norm": 0.7343735843108089, + "language_loss": 0.5466643, + "learning_rate": 3.990763750173237e-06, + "loss": 0.56724912, + "num_input_tokens_seen": 58362210, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01342773, + "step": 2057, + "time_per_iteration": 3.098634958267212 + }, + { + "auxiliary_loss_clip": 0.01049068, + "auxiliary_loss_mlp": 0.01002711, + "balance_loss_clip": 1.02096152, + "balance_loss_mlp": 1.00135827, + "epoch": 0.05971795020602402, + "flos": 55433328589440.0, + "grad_norm": 0.7209659957161647, + "language_loss": 0.49944428, + "learning_rate": 3.990745698108478e-06, + "loss": 0.51996207, + "num_input_tokens_seen": 58419355, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.0135498, + "step": 2058, + "time_per_iteration": 2.9463088512420654 + }, + { + "auxiliary_loss_clip": 0.01189233, + "auxiliary_loss_mlp": 0.01073087, + "balance_loss_clip": 1.07253313, + "balance_loss_mlp": 1.05282164, + "epoch": 0.059746967674540075, + "flos": 34586681391360.0, + "grad_norm": 2.4391575561023915, + "language_loss": 0.83241236, + "learning_rate": 3.990727628460648e-06, + "loss": 0.85503554, + "num_input_tokens_seen": 58441350, + "router_z_loss_clip": 1.16601562, + "router_z_loss_mlp": 0.20288086, + "step": 2059, + "time_per_iteration": 2.703396797180176 + }, + { + "auxiliary_loss_clip": 0.01183087, + "auxiliary_loss_mlp": 0.01056358, + "balance_loss_clip": 1.07107115, + "balance_loss_mlp": 1.03647435, + "epoch": 0.05977598514305612, + "flos": 17232740962560.0, + "grad_norm": 3.7055102052828257, + "language_loss": 0.62310427, + "learning_rate": 3.990709541229906e-06, + "loss": 0.64549875, + "num_input_tokens_seen": 58456620, + "router_z_loss_clip": 1.12011719, + "router_z_loss_mlp": 0.19885254, + "step": 2060, + "time_per_iteration": 2.445565700531006 + }, + { + "auxiliary_loss_clip": 0.01202376, + "auxiliary_loss_mlp": 0.01067649, + "balance_loss_clip": 1.07910788, + "balance_loss_mlp": 1.04356909, + "epoch": 0.059805002611572164, + "flos": 19310819425920.0, + "grad_norm": 2.7650480534030124, + "language_loss": 1.05191946, + "learning_rate": 3.990691436416412e-06, + "loss": 1.07461965, + "num_input_tokens_seen": 58471395, + "router_z_loss_clip": 1.23144531, + "router_z_loss_mlp": 0.24072266, + "step": 2061, + "time_per_iteration": 2.52295184135437 + }, + { + "auxiliary_loss_clip": 0.01184347, + "auxiliary_loss_mlp": 0.01044283, + "balance_loss_clip": 1.07182264, + "balance_loss_mlp": 1.02520323, + "epoch": 0.059834020080088215, + "flos": 28285254633600.0, + "grad_norm": 2.3967270314011073, + "language_loss": 0.95180345, + "learning_rate": 3.990673314020326e-06, + "loss": 0.97408974, + "num_input_tokens_seen": 58485380, + "router_z_loss_clip": 1.125, + "router_z_loss_mlp": 0.19085693, + "step": 2062, + "time_per_iteration": 2.570518732070923 + }, + { + "auxiliary_loss_clip": 0.01051295, + "auxiliary_loss_mlp": 0.01022057, + "balance_loss_clip": 1.02321196, + "balance_loss_mlp": 1.02090061, + "epoch": 0.05986303754860426, + "flos": 63599450088960.0, + "grad_norm": 0.7291497912328538, + "language_loss": 0.51624101, + "learning_rate": 3.990655174041807e-06, + "loss": 0.53697455, + "num_input_tokens_seen": 58540830, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01153564, + "step": 2063, + "time_per_iteration": 2.997007369995117 + }, + { + "auxiliary_loss_clip": 0.01189248, + "auxiliary_loss_mlp": 0.01056194, + "balance_loss_clip": 1.07459092, + "balance_loss_mlp": 1.03681016, + "epoch": 0.059892055017120305, + "flos": 18071039548800.0, + "grad_norm": 2.384519043298407, + "language_loss": 0.73995221, + "learning_rate": 3.9906370164810164e-06, + "loss": 0.76240659, + "num_input_tokens_seen": 58556105, + "router_z_loss_clip": 1.14941406, + "router_z_loss_mlp": 0.19396973, + "step": 2064, + "time_per_iteration": 2.602895736694336 + }, + { + "auxiliary_loss_clip": 0.01048628, + "auxiliary_loss_mlp": 0.01006541, + "balance_loss_clip": 1.02062511, + "balance_loss_mlp": 1.00531888, + "epoch": 0.059921072485636356, + "flos": 65690421534720.0, + "grad_norm": 0.7234933627215501, + "language_loss": 0.49809521, + "learning_rate": 3.990618841338115e-06, + "loss": 0.5186469, + "num_input_tokens_seen": 58621085, + "router_z_loss_clip": 0.27929688, + "router_z_loss_mlp": 0.01220703, + "step": 2065, + "time_per_iteration": 5.443529844284058 + }, + { + "auxiliary_loss_clip": 0.01186831, + "auxiliary_loss_mlp": 0.01058377, + "balance_loss_clip": 1.07260048, + "balance_loss_mlp": 1.03892851, + "epoch": 0.0599500899541524, + "flos": 25146985317120.0, + "grad_norm": 2.141471338898945, + "language_loss": 0.80523759, + "learning_rate": 3.990600648613261e-06, + "loss": 0.82768971, + "num_input_tokens_seen": 58636500, + "router_z_loss_clip": 1.14111328, + "router_z_loss_mlp": 0.19451904, + "step": 2066, + "time_per_iteration": 4.983439207077026 + }, + { + "auxiliary_loss_clip": 0.01187749, + "auxiliary_loss_mlp": 0.01061479, + "balance_loss_clip": 1.07338262, + "balance_loss_mlp": 1.0390563, + "epoch": 0.059979107422668446, + "flos": 13799932122240.0, + "grad_norm": 2.7613576918099865, + "language_loss": 1.03018141, + "learning_rate": 3.990582438306617e-06, + "loss": 1.05267382, + "num_input_tokens_seen": 58647155, + "router_z_loss_clip": 1.14550781, + "router_z_loss_mlp": 0.22424316, + "step": 2067, + "time_per_iteration": 4.763803005218506 + }, + { + "auxiliary_loss_clip": 0.01185823, + "auxiliary_loss_mlp": 0.01059684, + "balance_loss_clip": 1.07370079, + "balance_loss_mlp": 1.0381788, + "epoch": 0.06000812489118449, + "flos": 26461746835200.0, + "grad_norm": 2.1627580263179302, + "language_loss": 1.10427856, + "learning_rate": 3.990564210418344e-06, + "loss": 1.12673354, + "num_input_tokens_seen": 58665710, + "router_z_loss_clip": 1.12060547, + "router_z_loss_mlp": 0.21520996, + "step": 2068, + "time_per_iteration": 5.072246551513672 + }, + { + "auxiliary_loss_clip": 0.01195712, + "auxiliary_loss_mlp": 0.01055896, + "balance_loss_clip": 1.07500005, + "balance_loss_mlp": 1.03410482, + "epoch": 0.06003714235970054, + "flos": 36679086403200.0, + "grad_norm": 2.2973225237567885, + "language_loss": 0.73118722, + "learning_rate": 3.990545964948602e-06, + "loss": 0.7537033, + "num_input_tokens_seen": 58686325, + "router_z_loss_clip": 1.20605469, + "router_z_loss_mlp": 0.21789551, + "step": 2069, + "time_per_iteration": 2.6431450843811035 + }, + { + "auxiliary_loss_clip": 0.01184729, + "auxiliary_loss_mlp": 0.01064095, + "balance_loss_clip": 1.07303691, + "balance_loss_mlp": 1.04150498, + "epoch": 0.060066159828216587, + "flos": 61500505824000.0, + "grad_norm": 2.242643018841888, + "language_loss": 0.75560731, + "learning_rate": 3.990527701897552e-06, + "loss": 0.7780956, + "num_input_tokens_seen": 58706060, + "router_z_loss_clip": 1.1171875, + "router_z_loss_mlp": 0.22589111, + "step": 2070, + "time_per_iteration": 2.8574485778808594 + }, + { + "auxiliary_loss_clip": 0.01194691, + "auxiliary_loss_mlp": 0.01060044, + "balance_loss_clip": 1.07475543, + "balance_loss_mlp": 1.03800786, + "epoch": 0.06009517729673263, + "flos": 30986778481920.0, + "grad_norm": 3.043387177107935, + "language_loss": 0.89463007, + "learning_rate": 3.990509421265356e-06, + "loss": 0.91717732, + "num_input_tokens_seen": 58720800, + "router_z_loss_clip": 1.19873047, + "router_z_loss_mlp": 0.22021484, + "step": 2071, + "time_per_iteration": 2.563035249710083 + }, + { + "auxiliary_loss_clip": 0.01189637, + "auxiliary_loss_mlp": 0.0105694, + "balance_loss_clip": 1.07320166, + "balance_loss_mlp": 1.03719926, + "epoch": 0.06012419476524868, + "flos": 49702557202560.0, + "grad_norm": 3.0854404589063544, + "language_loss": 0.6553942, + "learning_rate": 3.990491123052176e-06, + "loss": 0.67785996, + "num_input_tokens_seen": 58738545, + "router_z_loss_clip": 1.16455078, + "router_z_loss_mlp": 0.19750977, + "step": 2072, + "time_per_iteration": 2.821998119354248 + }, + { + "auxiliary_loss_clip": 0.01180029, + "auxiliary_loss_mlp": 0.01050768, + "balance_loss_clip": 1.07341957, + "balance_loss_mlp": 1.0313127, + "epoch": 0.06015321223376473, + "flos": 38469233445120.0, + "grad_norm": 2.401528785441907, + "language_loss": 1.02980375, + "learning_rate": 3.9904728072581726e-06, + "loss": 1.05211174, + "num_input_tokens_seen": 58755405, + "router_z_loss_clip": 1.06445312, + "router_z_loss_mlp": 0.19470215, + "step": 2073, + "time_per_iteration": 2.6841158866882324 + }, + { + "auxiliary_loss_clip": 0.01192044, + "auxiliary_loss_mlp": 0.01071106, + "balance_loss_clip": 1.07374668, + "balance_loss_mlp": 1.04961312, + "epoch": 0.06018222970228077, + "flos": 33691283159040.0, + "grad_norm": 2.323989979189591, + "language_loss": 0.66994065, + "learning_rate": 3.990454473883508e-06, + "loss": 0.69257218, + "num_input_tokens_seen": 58774900, + "router_z_loss_clip": 1.18261719, + "router_z_loss_mlp": 0.21472168, + "step": 2074, + "time_per_iteration": 2.6125328540802 + }, + { + "auxiliary_loss_clip": 0.01181324, + "auxiliary_loss_mlp": 0.01052249, + "balance_loss_clip": 1.07396317, + "balance_loss_mlp": 1.03486252, + "epoch": 0.06021124717079682, + "flos": 31605842407680.0, + "grad_norm": 3.447561398071, + "language_loss": 0.99672377, + "learning_rate": 3.990436122928344e-06, + "loss": 1.01905942, + "num_input_tokens_seen": 58790835, + "router_z_loss_clip": 1.07373047, + "router_z_loss_mlp": 0.1739502, + "step": 2075, + "time_per_iteration": 2.586482524871826 + }, + { + "auxiliary_loss_clip": 0.01190125, + "auxiliary_loss_mlp": 0.01060767, + "balance_loss_clip": 1.07570124, + "balance_loss_mlp": 1.04027462, + "epoch": 0.06024026463931287, + "flos": 33392685398400.0, + "grad_norm": 3.0511490801098504, + "language_loss": 1.0360707, + "learning_rate": 3.990417754392843e-06, + "loss": 1.05857956, + "num_input_tokens_seen": 58805345, + "router_z_loss_clip": 1.14453125, + "router_z_loss_mlp": 0.20495605, + "step": 2076, + "time_per_iteration": 2.6167244911193848 + }, + { + "auxiliary_loss_clip": 0.01175265, + "auxiliary_loss_mlp": 0.01054101, + "balance_loss_clip": 1.07129169, + "balance_loss_mlp": 1.03472984, + "epoch": 0.06026928210782891, + "flos": 25883334126720.0, + "grad_norm": 2.567636274742057, + "language_loss": 0.64816368, + "learning_rate": 3.990399368277166e-06, + "loss": 0.6704573, + "num_input_tokens_seen": 58821830, + "router_z_loss_clip": 1.04052734, + "router_z_loss_mlp": 0.19372559, + "step": 2077, + "time_per_iteration": 2.485736608505249 + }, + { + "auxiliary_loss_clip": 0.01181813, + "auxiliary_loss_mlp": 0.0105019, + "balance_loss_clip": 1.07316446, + "balance_loss_mlp": 1.032094, + "epoch": 0.06029829957634496, + "flos": 37704479955840.0, + "grad_norm": 3.4276497898958813, + "language_loss": 0.96416056, + "learning_rate": 3.990380964581477e-06, + "loss": 0.98648059, + "num_input_tokens_seen": 58837815, + "router_z_loss_clip": 1.0859375, + "router_z_loss_mlp": 0.18103027, + "step": 2078, + "time_per_iteration": 2.6594443321228027 + }, + { + "auxiliary_loss_clip": 0.01193139, + "auxiliary_loss_mlp": 0.0106508, + "balance_loss_clip": 1.07619834, + "balance_loss_mlp": 1.04302609, + "epoch": 0.06032731704486101, + "flos": 17887392288000.0, + "grad_norm": 2.8351858962664744, + "language_loss": 0.72892809, + "learning_rate": 3.990362543305938e-06, + "loss": 0.75151026, + "num_input_tokens_seen": 58850730, + "router_z_loss_clip": 1.16845703, + "router_z_loss_mlp": 0.22045898, + "step": 2079, + "time_per_iteration": 2.4470152854919434 + }, + { + "auxiliary_loss_clip": 0.01190061, + "auxiliary_loss_mlp": 0.01061673, + "balance_loss_clip": 1.07724524, + "balance_loss_mlp": 1.03951216, + "epoch": 0.060356334513377054, + "flos": 30950400983040.0, + "grad_norm": 3.4199610661352438, + "language_loss": 0.86581504, + "learning_rate": 3.990344104450711e-06, + "loss": 0.88833237, + "num_input_tokens_seen": 58869985, + "router_z_loss_clip": 1.12988281, + "router_z_loss_mlp": 0.22143555, + "step": 2080, + "time_per_iteration": 2.6764628887176514 + }, + { + "auxiliary_loss_clip": 0.0119882, + "auxiliary_loss_mlp": 0.0106604, + "balance_loss_clip": 1.07977712, + "balance_loss_mlp": 1.04417753, + "epoch": 0.0603853519818931, + "flos": 32231406695040.0, + "grad_norm": 1.7564312575761525, + "language_loss": 0.81468302, + "learning_rate": 3.99032564801596e-06, + "loss": 0.83733165, + "num_input_tokens_seen": 58889220, + "router_z_loss_clip": 1.19091797, + "router_z_loss_mlp": 0.21850586, + "step": 2081, + "time_per_iteration": 2.625770092010498 + }, + { + "auxiliary_loss_clip": 0.01180955, + "auxiliary_loss_mlp": 0.01054346, + "balance_loss_clip": 1.07255983, + "balance_loss_mlp": 1.03251863, + "epoch": 0.06041436945040914, + "flos": 28070257777920.0, + "grad_norm": 2.563826380105174, + "language_loss": 0.92186689, + "learning_rate": 3.990307174001848e-06, + "loss": 0.94421983, + "num_input_tokens_seen": 58906410, + "router_z_loss_clip": 1.08300781, + "router_z_loss_mlp": 0.21850586, + "step": 2082, + "time_per_iteration": 2.5359761714935303 + }, + { + "auxiliary_loss_clip": 0.01053828, + "auxiliary_loss_mlp": 0.01040281, + "balance_loss_clip": 1.02574015, + "balance_loss_mlp": 1.03880847, + "epoch": 0.060443386918925195, + "flos": 54843351701760.0, + "grad_norm": 0.7749704377087446, + "language_loss": 0.50933278, + "learning_rate": 3.9902886824085375e-06, + "loss": 0.5302738, + "num_input_tokens_seen": 58962195, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01470947, + "step": 2083, + "time_per_iteration": 3.017119884490967 + }, + { + "auxiliary_loss_clip": 0.01194131, + "auxiliary_loss_mlp": 0.0106639, + "balance_loss_clip": 1.07988501, + "balance_loss_mlp": 1.04385924, + "epoch": 0.06047240438744124, + "flos": 20003967152640.0, + "grad_norm": 2.273698165750122, + "language_loss": 0.85790658, + "learning_rate": 3.990270173236192e-06, + "loss": 0.88051182, + "num_input_tokens_seen": 58977855, + "router_z_loss_clip": 1.14160156, + "router_z_loss_mlp": 0.22497559, + "step": 2084, + "time_per_iteration": 2.5373356342315674 + }, + { + "auxiliary_loss_clip": 0.01052628, + "auxiliary_loss_mlp": 0.0100163, + "balance_loss_clip": 1.02453518, + "balance_loss_mlp": 1.00024116, + "epoch": 0.060501421855957284, + "flos": 74784296442240.0, + "grad_norm": 0.6853839313685781, + "language_loss": 0.48634884, + "learning_rate": 3.990251646484974e-06, + "loss": 0.50689143, + "num_input_tokens_seen": 59046595, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01391602, + "step": 2085, + "time_per_iteration": 3.2309179306030273 + }, + { + "auxiliary_loss_clip": 0.01187269, + "auxiliary_loss_mlp": 0.01060062, + "balance_loss_clip": 1.07593703, + "balance_loss_mlp": 1.03902161, + "epoch": 0.060530439324473335, + "flos": 56383234732800.0, + "grad_norm": 3.718268382431934, + "language_loss": 0.94296312, + "learning_rate": 3.990233102155048e-06, + "loss": 0.96543652, + "num_input_tokens_seen": 59068660, + "router_z_loss_clip": 1.11328125, + "router_z_loss_mlp": 0.21044922, + "step": 2086, + "time_per_iteration": 2.821621894836426 + }, + { + "auxiliary_loss_clip": 0.01184941, + "auxiliary_loss_mlp": 0.01074716, + "balance_loss_clip": 1.07554531, + "balance_loss_mlp": 1.05580389, + "epoch": 0.06055945679298938, + "flos": 29460503727360.0, + "grad_norm": 2.1017025239432128, + "language_loss": 0.70988333, + "learning_rate": 3.990214540246578e-06, + "loss": 0.73247993, + "num_input_tokens_seen": 59086640, + "router_z_loss_clip": 1.09277344, + "router_z_loss_mlp": 0.18908691, + "step": 2087, + "time_per_iteration": 2.6046838760375977 + }, + { + "auxiliary_loss_clip": 0.01190545, + "auxiliary_loss_mlp": 0.01082656, + "balance_loss_clip": 1.07971168, + "balance_loss_mlp": 1.063761, + "epoch": 0.060588474261505425, + "flos": 17668516763520.0, + "grad_norm": 2.395530335975013, + "language_loss": 0.87587988, + "learning_rate": 3.9901959607597285e-06, + "loss": 0.8986119, + "num_input_tokens_seen": 59099445, + "router_z_loss_clip": 1.10742188, + "router_z_loss_mlp": 0.18896484, + "step": 2088, + "time_per_iteration": 2.5135905742645264 + }, + { + "auxiliary_loss_clip": 0.0118525, + "auxiliary_loss_mlp": 0.01078946, + "balance_loss_clip": 1.0772531, + "balance_loss_mlp": 1.06005168, + "epoch": 0.060617491730021476, + "flos": 41123105902080.0, + "grad_norm": 1.8245247229292008, + "language_loss": 0.8765986, + "learning_rate": 3.990177363694662e-06, + "loss": 0.89924061, + "num_input_tokens_seen": 59124720, + "router_z_loss_clip": 1.07910156, + "router_z_loss_mlp": 0.18914795, + "step": 2089, + "time_per_iteration": 2.941129207611084 + }, + { + "auxiliary_loss_clip": 0.01190806, + "auxiliary_loss_mlp": 0.01081725, + "balance_loss_clip": 1.07799411, + "balance_loss_mlp": 1.06116152, + "epoch": 0.06064650919853752, + "flos": 27705620862720.0, + "grad_norm": 2.5636280410326497, + "language_loss": 0.8298496, + "learning_rate": 3.990158749051545e-06, + "loss": 0.85257494, + "num_input_tokens_seen": 59143295, + "router_z_loss_clip": 1.12792969, + "router_z_loss_mlp": 0.20556641, + "step": 2090, + "time_per_iteration": 2.604222059249878 + }, + { + "auxiliary_loss_clip": 0.0119288, + "auxiliary_loss_mlp": 0.01075835, + "balance_loss_clip": 1.07837284, + "balance_loss_mlp": 1.05412698, + "epoch": 0.060675526667053566, + "flos": 15588032088960.0, + "grad_norm": 3.7896299968015987, + "language_loss": 0.89787251, + "learning_rate": 3.99014011683054e-06, + "loss": 0.9205597, + "num_input_tokens_seen": 59154760, + "router_z_loss_clip": 1.14550781, + "router_z_loss_mlp": 0.21728516, + "step": 2091, + "time_per_iteration": 2.4631612300872803 + }, + { + "auxiliary_loss_clip": 0.0117932, + "auxiliary_loss_mlp": 0.01084078, + "balance_loss_clip": 1.07414746, + "balance_loss_mlp": 1.06389594, + "epoch": 0.06070454413556961, + "flos": 12123334949760.0, + "grad_norm": 2.8967468266544563, + "language_loss": 0.91598922, + "learning_rate": 3.990121467031812e-06, + "loss": 0.93862319, + "num_input_tokens_seen": 59166145, + "router_z_loss_clip": 1.05029297, + "router_z_loss_mlp": 0.2019043, + "step": 2092, + "time_per_iteration": 2.481226921081543 + }, + { + "auxiliary_loss_clip": 0.01055358, + "auxiliary_loss_mlp": 0.01056059, + "balance_loss_clip": 1.02682233, + "balance_loss_mlp": 1.05468225, + "epoch": 0.06073356160408566, + "flos": 69009824160000.0, + "grad_norm": 0.6813589188785897, + "language_loss": 0.50953865, + "learning_rate": 3.990102799655526e-06, + "loss": 0.53065288, + "num_input_tokens_seen": 59230900, + "router_z_loss_clip": 0.28515625, + "router_z_loss_mlp": 0.01379395, + "step": 2093, + "time_per_iteration": 3.2482333183288574 + }, + { + "auxiliary_loss_clip": 0.0119913, + "auxiliary_loss_mlp": 0.01065874, + "balance_loss_clip": 1.08005881, + "balance_loss_mlp": 1.04417825, + "epoch": 0.060762579072601706, + "flos": 13581954437760.0, + "grad_norm": 2.921440270035793, + "language_loss": 0.85874176, + "learning_rate": 3.990084114701847e-06, + "loss": 0.88139176, + "num_input_tokens_seen": 59242500, + "router_z_loss_clip": 1.18945312, + "router_z_loss_mlp": 0.21716309, + "step": 2094, + "time_per_iteration": 2.4631993770599365 + }, + { + "auxiliary_loss_clip": 0.01192298, + "auxiliary_loss_mlp": 0.01076588, + "balance_loss_clip": 1.0753454, + "balance_loss_mlp": 1.05458176, + "epoch": 0.06079159654111775, + "flos": 14167370298240.0, + "grad_norm": 3.0848788700048964, + "language_loss": 0.97050834, + "learning_rate": 3.990065412170939e-06, + "loss": 0.99319726, + "num_input_tokens_seen": 59254135, + "router_z_loss_clip": 1.16894531, + "router_z_loss_mlp": 0.22033691, + "step": 2095, + "time_per_iteration": 2.4568889141082764 + }, + { + "auxiliary_loss_clip": 0.01197208, + "auxiliary_loss_mlp": 0.01069866, + "balance_loss_clip": 1.07741261, + "balance_loss_mlp": 1.04450989, + "epoch": 0.0608206140096338, + "flos": 28266726193920.0, + "grad_norm": 4.663329279299406, + "language_loss": 0.90466791, + "learning_rate": 3.990046692062969e-06, + "loss": 0.9273386, + "num_input_tokens_seen": 59272665, + "router_z_loss_clip": 1.19628906, + "router_z_loss_mlp": 0.25354004, + "step": 2096, + "time_per_iteration": 2.613416910171509 + }, + { + "auxiliary_loss_clip": 0.01182161, + "auxiliary_loss_mlp": 0.01053343, + "balance_loss_clip": 1.07516313, + "balance_loss_mlp": 1.03661776, + "epoch": 0.06084963147814985, + "flos": 18406517598720.0, + "grad_norm": 2.4364939077378147, + "language_loss": 0.86075509, + "learning_rate": 3.990027954378101e-06, + "loss": 0.88311011, + "num_input_tokens_seen": 59287505, + "router_z_loss_clip": 1.06982422, + "router_z_loss_mlp": 0.16717529, + "step": 2097, + "time_per_iteration": 2.486163377761841 + }, + { + "auxiliary_loss_clip": 0.01052031, + "auxiliary_loss_mlp": 0.01007508, + "balance_loss_clip": 1.02429712, + "balance_loss_mlp": 1.00639307, + "epoch": 0.06087864894666589, + "flos": 63656334253440.0, + "grad_norm": 0.6993247548001955, + "language_loss": 0.51133674, + "learning_rate": 3.990009199116501e-06, + "loss": 0.53193212, + "num_input_tokens_seen": 59354730, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01116943, + "step": 2098, + "time_per_iteration": 3.222475290298462 + }, + { + "auxiliary_loss_clip": 0.01192845, + "auxiliary_loss_mlp": 0.01056668, + "balance_loss_clip": 1.08002448, + "balance_loss_mlp": 1.03647399, + "epoch": 0.06090766641518194, + "flos": 13617182701440.0, + "grad_norm": 2.992634857860363, + "language_loss": 1.0508796, + "learning_rate": 3.989990426278334e-06, + "loss": 1.07337475, + "num_input_tokens_seen": 59365210, + "router_z_loss_clip": 1.12890625, + "router_z_loss_mlp": 0.2019043, + "step": 2099, + "time_per_iteration": 2.4400429725646973 + }, + { + "auxiliary_loss_clip": 0.01188776, + "auxiliary_loss_mlp": 0.01057389, + "balance_loss_clip": 1.07645774, + "balance_loss_mlp": 1.0363133, + "epoch": 0.06093668388369799, + "flos": 25147057144320.0, + "grad_norm": 2.1706212404697554, + "language_loss": 0.85096085, + "learning_rate": 3.9899716358637665e-06, + "loss": 0.8734225, + "num_input_tokens_seen": 59384610, + "router_z_loss_clip": 1.12255859, + "router_z_loss_mlp": 0.21105957, + "step": 2100, + "time_per_iteration": 2.793086051940918 + }, + { + "auxiliary_loss_clip": 0.01182873, + "auxiliary_loss_mlp": 0.01042732, + "balance_loss_clip": 1.07413709, + "balance_loss_mlp": 1.02293146, + "epoch": 0.06096570135221403, + "flos": 16208496645120.0, + "grad_norm": 3.3869899644806063, + "language_loss": 0.91807932, + "learning_rate": 3.989952827872964e-06, + "loss": 0.94033533, + "num_input_tokens_seen": 59396515, + "router_z_loss_clip": 1.08740234, + "router_z_loss_mlp": 0.19787598, + "step": 2101, + "time_per_iteration": 2.4422690868377686 + }, + { + "auxiliary_loss_clip": 0.01184698, + "auxiliary_loss_mlp": 0.01056879, + "balance_loss_clip": 1.07410419, + "balance_loss_mlp": 1.03713834, + "epoch": 0.06099471882073008, + "flos": 29276784639360.0, + "grad_norm": 1.8222519763451568, + "language_loss": 0.8547442, + "learning_rate": 3.989934002306094e-06, + "loss": 0.87715995, + "num_input_tokens_seen": 59421870, + "router_z_loss_clip": 1.10595703, + "router_z_loss_mlp": 0.1973877, + "step": 2102, + "time_per_iteration": 2.6672933101654053 + }, + { + "auxiliary_loss_clip": 0.01056222, + "auxiliary_loss_mlp": 0.01034023, + "balance_loss_clip": 1.02639318, + "balance_loss_mlp": 1.03281927, + "epoch": 0.06102373628924613, + "flos": 58723141386240.0, + "grad_norm": 0.701744791197694, + "language_loss": 0.53539026, + "learning_rate": 3.989915159163321e-06, + "loss": 0.55629265, + "num_input_tokens_seen": 59485280, + "router_z_loss_clip": 0.296875, + "router_z_loss_mlp": 0.01202393, + "step": 2103, + "time_per_iteration": 3.069183349609375 + }, + { + "auxiliary_loss_clip": 0.01185902, + "auxiliary_loss_mlp": 0.01051761, + "balance_loss_clip": 1.0718056, + "balance_loss_mlp": 1.03056526, + "epoch": 0.061052753757762174, + "flos": 28068569838720.0, + "grad_norm": 2.419729882543086, + "language_loss": 0.82962841, + "learning_rate": 3.9898962984448105e-06, + "loss": 0.85200506, + "num_input_tokens_seen": 59502910, + "router_z_loss_clip": 1.13964844, + "router_z_loss_mlp": 0.21191406, + "step": 2104, + "time_per_iteration": 2.5899641513824463 + }, + { + "auxiliary_loss_clip": 0.0117602, + "auxiliary_loss_mlp": 0.01046664, + "balance_loss_clip": 1.06841898, + "balance_loss_mlp": 1.0276562, + "epoch": 0.06108177122627822, + "flos": 24892701960960.0, + "grad_norm": 2.263344275946216, + "language_loss": 0.8641662, + "learning_rate": 3.9898774201507324e-06, + "loss": 0.88639301, + "num_input_tokens_seen": 59517010, + "router_z_loss_clip": 1.07666016, + "router_z_loss_mlp": 0.19024658, + "step": 2105, + "time_per_iteration": 2.576781749725342 + }, + { + "auxiliary_loss_clip": 0.01174968, + "auxiliary_loss_mlp": 0.01054297, + "balance_loss_clip": 1.06910944, + "balance_loss_mlp": 1.03598595, + "epoch": 0.06111078869479426, + "flos": 17711646019200.0, + "grad_norm": 2.246965717711961, + "language_loss": 0.73818684, + "learning_rate": 3.989858524281252e-06, + "loss": 0.76047951, + "num_input_tokens_seen": 59533615, + "router_z_loss_clip": 1.06054688, + "router_z_loss_mlp": 0.18328857, + "step": 2106, + "time_per_iteration": 2.5376646518707275 + }, + { + "auxiliary_loss_clip": 0.01187367, + "auxiliary_loss_mlp": 0.01064904, + "balance_loss_clip": 1.07618141, + "balance_loss_mlp": 1.0432198, + "epoch": 0.061139806163310315, + "flos": 70574951473920.0, + "grad_norm": 1.9975862666279938, + "language_loss": 0.65906638, + "learning_rate": 3.989839610836535e-06, + "loss": 0.68158913, + "num_input_tokens_seen": 59557425, + "router_z_loss_clip": 1.11279297, + "router_z_loss_mlp": 0.21679688, + "step": 2107, + "time_per_iteration": 3.0078928470611572 + }, + { + "auxiliary_loss_clip": 0.01056583, + "auxiliary_loss_mlp": 0.01012513, + "balance_loss_clip": 1.02782607, + "balance_loss_mlp": 1.01135087, + "epoch": 0.06116882363182636, + "flos": 74160958798080.0, + "grad_norm": 0.6460308671047177, + "language_loss": 0.51228523, + "learning_rate": 3.9898206798167495e-06, + "loss": 0.53297621, + "num_input_tokens_seen": 59624055, + "router_z_loss_clip": 0.28710938, + "router_z_loss_mlp": 0.01159668, + "step": 2108, + "time_per_iteration": 3.219524383544922 + }, + { + "auxiliary_loss_clip": 0.01191235, + "auxiliary_loss_mlp": 0.01050786, + "balance_loss_clip": 1.07243454, + "balance_loss_mlp": 1.0287801, + "epoch": 0.061197841100342404, + "flos": 34196438079360.0, + "grad_norm": 2.049766145304452, + "language_loss": 1.00897586, + "learning_rate": 3.989801731222062e-06, + "loss": 1.03139615, + "num_input_tokens_seen": 59643100, + "router_z_loss_clip": 1.18847656, + "router_z_loss_mlp": 0.22009277, + "step": 2109, + "time_per_iteration": 2.6764132976531982 + }, + { + "auxiliary_loss_clip": 0.01181389, + "auxiliary_loss_mlp": 0.01071702, + "balance_loss_clip": 1.07244813, + "balance_loss_mlp": 1.05115592, + "epoch": 0.061226858568858455, + "flos": 17158513507200.0, + "grad_norm": 2.9090854404430537, + "language_loss": 1.00475419, + "learning_rate": 3.989782765052642e-06, + "loss": 1.02728498, + "num_input_tokens_seen": 59656035, + "router_z_loss_clip": 1.08837891, + "router_z_loss_mlp": 0.20544434, + "step": 2110, + "time_per_iteration": 2.4796721935272217 + }, + { + "auxiliary_loss_clip": 0.01179968, + "auxiliary_loss_mlp": 0.01058957, + "balance_loss_clip": 1.07509613, + "balance_loss_mlp": 1.03922772, + "epoch": 0.0612558760373745, + "flos": 32374043602560.0, + "grad_norm": 2.669910193440372, + "language_loss": 0.82638538, + "learning_rate": 3.989763781308654e-06, + "loss": 0.84877467, + "num_input_tokens_seen": 59671595, + "router_z_loss_clip": 1.04931641, + "router_z_loss_mlp": 0.19726562, + "step": 2111, + "time_per_iteration": 2.6436333656311035 + }, + { + "auxiliary_loss_clip": 0.01206218, + "auxiliary_loss_mlp": 0.01084678, + "balance_loss_clip": 1.07622254, + "balance_loss_mlp": 1.0593462, + "epoch": 0.061284893505890545, + "flos": 27201615177600.0, + "grad_norm": 2.231667538440581, + "language_loss": 0.90068394, + "learning_rate": 3.989744779990268e-06, + "loss": 0.92359287, + "num_input_tokens_seen": 59691365, + "router_z_loss_clip": 1.29882812, + "router_z_loss_mlp": 0.25341797, + "step": 2112, + "time_per_iteration": 2.553464651107788 + }, + { + "auxiliary_loss_clip": 0.01184545, + "auxiliary_loss_mlp": 0.01061803, + "balance_loss_clip": 1.07416415, + "balance_loss_mlp": 1.0419426, + "epoch": 0.06131391097440659, + "flos": 37809446474880.0, + "grad_norm": 2.0978028387461807, + "language_loss": 0.82400686, + "learning_rate": 3.989725761097651e-06, + "loss": 0.8464703, + "num_input_tokens_seen": 59708665, + "router_z_loss_clip": 1.10351562, + "router_z_loss_mlp": 0.19873047, + "step": 2113, + "time_per_iteration": 2.6787354946136475 + }, + { + "auxiliary_loss_clip": 0.01058425, + "auxiliary_loss_mlp": 0.0101324, + "balance_loss_clip": 1.02946675, + "balance_loss_mlp": 1.01210177, + "epoch": 0.06134292844292264, + "flos": 61155872784000.0, + "grad_norm": 0.6604517928171905, + "language_loss": 0.47234637, + "learning_rate": 3.98970672463097e-06, + "loss": 0.493063, + "num_input_tokens_seen": 59767355, + "router_z_loss_clip": 0.2890625, + "router_z_loss_mlp": 0.01141357, + "step": 2114, + "time_per_iteration": 3.025571823120117 + }, + { + "auxiliary_loss_clip": 0.0118725, + "auxiliary_loss_mlp": 0.01062797, + "balance_loss_clip": 1.07225287, + "balance_loss_mlp": 1.0405283, + "epoch": 0.061371945911438686, + "flos": 29308996160640.0, + "grad_norm": 2.795037695584413, + "language_loss": 1.00502026, + "learning_rate": 3.989687670590394e-06, + "loss": 1.02752078, + "num_input_tokens_seen": 59784495, + "router_z_loss_clip": 1.15039062, + "router_z_loss_mlp": 0.22277832, + "step": 2115, + "time_per_iteration": 2.604541540145874 + }, + { + "auxiliary_loss_clip": 0.01057604, + "auxiliary_loss_mlp": 0.01003221, + "balance_loss_clip": 1.02868831, + "balance_loss_mlp": 1.00190949, + "epoch": 0.06140096337995473, + "flos": 57120556187520.0, + "grad_norm": 0.6920053087552884, + "language_loss": 0.48687661, + "learning_rate": 3.989668598976092e-06, + "loss": 0.50748491, + "num_input_tokens_seen": 59841615, + "router_z_loss_clip": 0.2890625, + "router_z_loss_mlp": 0.01312256, + "step": 2116, + "time_per_iteration": 2.967064380645752 + }, + { + "auxiliary_loss_clip": 0.01183748, + "auxiliary_loss_mlp": 0.0104491, + "balance_loss_clip": 1.07194614, + "balance_loss_mlp": 1.02638495, + "epoch": 0.06142998084847078, + "flos": 19419808268160.0, + "grad_norm": 9.247619891975958, + "language_loss": 0.9930985, + "learning_rate": 3.989649509788232e-06, + "loss": 1.01538503, + "num_input_tokens_seen": 59854835, + "router_z_loss_clip": 1.11914062, + "router_z_loss_mlp": 0.18530273, + "step": 2117, + "time_per_iteration": 2.4920313358306885 + }, + { + "auxiliary_loss_clip": 0.01057296, + "auxiliary_loss_mlp": 0.00998302, + "balance_loss_clip": 1.02860641, + "balance_loss_mlp": 0.9970085, + "epoch": 0.061458998316986826, + "flos": 74771798509440.0, + "grad_norm": 0.7870267425959021, + "language_loss": 0.49505407, + "learning_rate": 3.9896304030269816e-06, + "loss": 0.5156101, + "num_input_tokens_seen": 59919840, + "router_z_loss_clip": 0.28710938, + "router_z_loss_mlp": 0.01293945, + "step": 2118, + "time_per_iteration": 3.1680121421813965 + }, + { + "auxiliary_loss_clip": 0.01057188, + "auxiliary_loss_mlp": 0.00998257, + "balance_loss_clip": 1.02871847, + "balance_loss_mlp": 0.99704665, + "epoch": 0.06148801578550287, + "flos": 74778622093440.0, + "grad_norm": 0.6499575732493819, + "language_loss": 0.53593922, + "learning_rate": 3.989611278692511e-06, + "loss": 0.55649364, + "num_input_tokens_seen": 59989600, + "router_z_loss_clip": 0.28515625, + "router_z_loss_mlp": 0.01208496, + "step": 2119, + "time_per_iteration": 3.200540781021118 + }, + { + "auxiliary_loss_clip": 0.01185087, + "auxiliary_loss_mlp": 0.0107014, + "balance_loss_clip": 1.07780218, + "balance_loss_mlp": 1.04921937, + "epoch": 0.06151703325401892, + "flos": 26936054951040.0, + "grad_norm": 2.122271338170677, + "language_loss": 0.87947893, + "learning_rate": 3.989592136784989e-06, + "loss": 0.90203118, + "num_input_tokens_seen": 60004390, + "router_z_loss_clip": 1.07177734, + "router_z_loss_mlp": 0.20910645, + "step": 2120, + "time_per_iteration": 2.581209897994995 + }, + { + "auxiliary_loss_clip": 0.01182493, + "auxiliary_loss_mlp": 0.0105444, + "balance_loss_clip": 1.0712738, + "balance_loss_mlp": 1.03347731, + "epoch": 0.06154605072253497, + "flos": 25988516127360.0, + "grad_norm": 3.0034291227853354, + "language_loss": 0.8484323, + "learning_rate": 3.9895729773045825e-06, + "loss": 0.87080169, + "num_input_tokens_seen": 60018875, + "router_z_loss_clip": 1.11181641, + "router_z_loss_mlp": 0.2097168, + "step": 2121, + "time_per_iteration": 2.552638530731201 + }, + { + "auxiliary_loss_clip": 0.01186361, + "auxiliary_loss_mlp": 0.01048196, + "balance_loss_clip": 1.07230973, + "balance_loss_mlp": 1.02777529, + "epoch": 0.06157506819105101, + "flos": 43427634681600.0, + "grad_norm": 2.2579982572520163, + "language_loss": 0.86411101, + "learning_rate": 3.989553800251464e-06, + "loss": 0.88645655, + "num_input_tokens_seen": 60036590, + "router_z_loss_clip": 1.140625, + "router_z_loss_mlp": 0.20397949, + "step": 2122, + "time_per_iteration": 2.7183916568756104 + }, + { + "auxiliary_loss_clip": 0.01184125, + "auxiliary_loss_mlp": 0.01049582, + "balance_loss_clip": 1.07297313, + "balance_loss_mlp": 1.03025866, + "epoch": 0.06160408565956706, + "flos": 34453666350720.0, + "grad_norm": 2.3407637026733026, + "language_loss": 0.80109537, + "learning_rate": 3.9895346056258e-06, + "loss": 0.82343239, + "num_input_tokens_seen": 60055145, + "router_z_loss_clip": 1.10986328, + "router_z_loss_mlp": 0.19335938, + "step": 2123, + "time_per_iteration": 2.6361358165740967 + }, + { + "auxiliary_loss_clip": 0.0119003, + "auxiliary_loss_mlp": 0.01069013, + "balance_loss_clip": 1.07242799, + "balance_loss_mlp": 1.04527807, + "epoch": 0.06163310312808311, + "flos": 17232130431360.0, + "grad_norm": 2.1382819335683094, + "language_loss": 0.87016475, + "learning_rate": 3.989515393427762e-06, + "loss": 0.89275515, + "num_input_tokens_seen": 60070500, + "router_z_loss_clip": 1.17480469, + "router_z_loss_mlp": 0.23706055, + "step": 2124, + "time_per_iteration": 2.4820544719696045 + }, + { + "auxiliary_loss_clip": 0.01181697, + "auxiliary_loss_mlp": 0.01053763, + "balance_loss_clip": 1.068591, + "balance_loss_mlp": 1.03300881, + "epoch": 0.06166212059659915, + "flos": 23361327475200.0, + "grad_norm": 3.055038146695014, + "language_loss": 0.80949008, + "learning_rate": 3.989496163657519e-06, + "loss": 0.83184469, + "num_input_tokens_seen": 60083605, + "router_z_loss_clip": 1.13085938, + "router_z_loss_mlp": 0.20758057, + "step": 2125, + "time_per_iteration": 2.4922845363616943 + }, + { + "auxiliary_loss_clip": 0.01055397, + "auxiliary_loss_mlp": 0.01009446, + "balance_loss_clip": 1.02697301, + "balance_loss_mlp": 1.00802696, + "epoch": 0.0616911380651152, + "flos": 70692598471680.0, + "grad_norm": 0.696529179096817, + "language_loss": 0.51934755, + "learning_rate": 3.9894769163152405e-06, + "loss": 0.53999603, + "num_input_tokens_seen": 60146745, + "router_z_loss_clip": 0.28515625, + "router_z_loss_mlp": 0.01416016, + "step": 2126, + "time_per_iteration": 3.1363205909729004 + }, + { + "auxiliary_loss_clip": 0.01177136, + "auxiliary_loss_mlp": 0.0104861, + "balance_loss_clip": 1.0696274, + "balance_loss_mlp": 1.02976918, + "epoch": 0.06172015553363125, + "flos": 26170367708160.0, + "grad_norm": 2.1184060766319908, + "language_loss": 0.69386935, + "learning_rate": 3.9894576514010975e-06, + "loss": 0.71612686, + "num_input_tokens_seen": 60160315, + "router_z_loss_clip": 1.07519531, + "router_z_loss_mlp": 0.1885376, + "step": 2127, + "time_per_iteration": 2.585939884185791 + }, + { + "auxiliary_loss_clip": 0.01195029, + "auxiliary_loss_mlp": 0.01061903, + "balance_loss_clip": 1.07511032, + "balance_loss_mlp": 1.03968239, + "epoch": 0.061749173002147294, + "flos": 33211659830400.0, + "grad_norm": 2.718648597395133, + "language_loss": 1.01408708, + "learning_rate": 3.989438368915259e-06, + "loss": 1.03665638, + "num_input_tokens_seen": 60175960, + "router_z_loss_clip": 1.20068359, + "router_z_loss_mlp": 0.22241211, + "step": 2128, + "time_per_iteration": 2.6197760105133057 + }, + { + "auxiliary_loss_clip": 0.01053973, + "auxiliary_loss_mlp": 0.01009644, + "balance_loss_clip": 1.02557373, + "balance_loss_mlp": 1.00812399, + "epoch": 0.06177819047066334, + "flos": 58207427867520.0, + "grad_norm": 0.7105801725925455, + "language_loss": 0.52773392, + "learning_rate": 3.989419068857896e-06, + "loss": 0.54837012, + "num_input_tokens_seen": 60238290, + "router_z_loss_clip": 0.28320312, + "router_z_loss_mlp": 0.01519775, + "step": 2129, + "time_per_iteration": 3.090977668762207 + }, + { + "auxiliary_loss_clip": 0.01184524, + "auxiliary_loss_mlp": 0.010466, + "balance_loss_clip": 1.07599199, + "balance_loss_mlp": 1.02694297, + "epoch": 0.06180720793917938, + "flos": 13727823569280.0, + "grad_norm": 3.782817927380421, + "language_loss": 1.03036308, + "learning_rate": 3.989399751229178e-06, + "loss": 1.05267429, + "num_input_tokens_seen": 60248770, + "router_z_loss_clip": 1.0859375, + "router_z_loss_mlp": 0.1965332, + "step": 2130, + "time_per_iteration": 2.5162951946258545 + }, + { + "auxiliary_loss_clip": 0.0117833, + "auxiliary_loss_mlp": 0.01052559, + "balance_loss_clip": 1.06770432, + "balance_loss_mlp": 1.032758, + "epoch": 0.061836225407695435, + "flos": 21135907422720.0, + "grad_norm": 3.0503566593021523, + "language_loss": 0.79437006, + "learning_rate": 3.989380416029276e-06, + "loss": 0.81667894, + "num_input_tokens_seen": 60263385, + "router_z_loss_clip": 1.10644531, + "router_z_loss_mlp": 0.19799805, + "step": 2131, + "time_per_iteration": 2.529890537261963 + }, + { + "auxiliary_loss_clip": 0.01190179, + "auxiliary_loss_mlp": 0.01066902, + "balance_loss_clip": 1.07326245, + "balance_loss_mlp": 1.04257166, + "epoch": 0.06186524287621148, + "flos": 15447047207040.0, + "grad_norm": 2.1716842302496495, + "language_loss": 0.85693836, + "learning_rate": 3.989361063258362e-06, + "loss": 0.87950915, + "num_input_tokens_seen": 60277190, + "router_z_loss_clip": 1.16894531, + "router_z_loss_mlp": 0.24328613, + "step": 2132, + "time_per_iteration": 2.4768295288085938 + }, + { + "auxiliary_loss_clip": 0.01182964, + "auxiliary_loss_mlp": 0.0105262, + "balance_loss_clip": 1.07085752, + "balance_loss_mlp": 1.03136528, + "epoch": 0.061894260344727524, + "flos": 45873079493760.0, + "grad_norm": 2.338406615995412, + "language_loss": 0.72928178, + "learning_rate": 3.989341692916607e-06, + "loss": 0.75163758, + "num_input_tokens_seen": 60295525, + "router_z_loss_clip": 1.12011719, + "router_z_loss_mlp": 0.21252441, + "step": 2133, + "time_per_iteration": 2.781378984451294 + }, + { + "auxiliary_loss_clip": 0.01179216, + "auxiliary_loss_mlp": 0.01043599, + "balance_loss_clip": 1.06869245, + "balance_loss_mlp": 1.02288032, + "epoch": 0.061923277813243575, + "flos": 44814899802240.0, + "grad_norm": 5.311678172183244, + "language_loss": 0.90644372, + "learning_rate": 3.98932230500418e-06, + "loss": 0.92867184, + "num_input_tokens_seen": 60315785, + "router_z_loss_clip": 1.10400391, + "router_z_loss_mlp": 0.20727539, + "step": 2134, + "time_per_iteration": 2.7300150394439697 + }, + { + "auxiliary_loss_clip": 0.01191682, + "auxiliary_loss_mlp": 0.01076855, + "balance_loss_clip": 1.07699227, + "balance_loss_mlp": 1.05321622, + "epoch": 0.06195229528175962, + "flos": 12926369358720.0, + "grad_norm": 2.556400214980158, + "language_loss": 1.06096601, + "learning_rate": 3.9893028995212544e-06, + "loss": 1.0836513, + "num_input_tokens_seen": 60327715, + "router_z_loss_clip": 1.14648438, + "router_z_loss_mlp": 0.23657227, + "step": 2135, + "time_per_iteration": 2.485990047454834 + }, + { + "auxiliary_loss_clip": 0.01177223, + "auxiliary_loss_mlp": 0.01054281, + "balance_loss_clip": 1.06734061, + "balance_loss_mlp": 1.03368187, + "epoch": 0.061981312750275665, + "flos": 21720030393600.0, + "grad_norm": 2.9781654414759906, + "language_loss": 0.84045792, + "learning_rate": 3.989283476467999e-06, + "loss": 0.86277294, + "num_input_tokens_seen": 60342010, + "router_z_loss_clip": 1.09863281, + "router_z_loss_mlp": 0.20593262, + "step": 2136, + "time_per_iteration": 4.859931468963623 + }, + { + "auxiliary_loss_clip": 0.01195459, + "auxiliary_loss_mlp": 0.01057879, + "balance_loss_clip": 1.0717268, + "balance_loss_mlp": 1.03307152, + "epoch": 0.06201033021879171, + "flos": 27411009511680.0, + "grad_norm": 3.719002157141579, + "language_loss": 0.90684295, + "learning_rate": 3.989264035844588e-06, + "loss": 0.92937636, + "num_input_tokens_seen": 60357235, + "router_z_loss_clip": 1.23681641, + "router_z_loss_mlp": 0.24804688, + "step": 2137, + "time_per_iteration": 2.5419116020202637 + }, + { + "auxiliary_loss_clip": 0.0105558, + "auxiliary_loss_mlp": 0.01005194, + "balance_loss_clip": 1.02720332, + "balance_loss_mlp": 1.00398445, + "epoch": 0.06203934768730776, + "flos": 71055331966080.0, + "grad_norm": 0.765097952923708, + "language_loss": 0.53474164, + "learning_rate": 3.989244577651192e-06, + "loss": 0.55534947, + "num_input_tokens_seen": 60425755, + "router_z_loss_clip": 0.28515625, + "router_z_loss_mlp": 0.01208496, + "step": 2138, + "time_per_iteration": 7.970232725143433 + }, + { + "auxiliary_loss_clip": 0.01179723, + "auxiliary_loss_mlp": 0.01055103, + "balance_loss_clip": 1.06922281, + "balance_loss_mlp": 1.03379989, + "epoch": 0.062068365155823806, + "flos": 41609442096000.0, + "grad_norm": 1.974146798737397, + "language_loss": 0.82482821, + "learning_rate": 3.989225101887983e-06, + "loss": 0.84717655, + "num_input_tokens_seen": 60453515, + "router_z_loss_clip": 1.10498047, + "router_z_loss_mlp": 0.2130127, + "step": 2139, + "time_per_iteration": 5.030135154724121 + }, + { + "auxiliary_loss_clip": 0.01170411, + "auxiliary_loss_mlp": 0.01052216, + "balance_loss_clip": 1.06611943, + "balance_loss_mlp": 1.03266573, + "epoch": 0.06209738262433985, + "flos": 25558522416000.0, + "grad_norm": 2.042895787536262, + "language_loss": 0.75348836, + "learning_rate": 3.9892056085551326e-06, + "loss": 0.77571464, + "num_input_tokens_seen": 60473675, + "router_z_loss_clip": 1.04199219, + "router_z_loss_mlp": 0.19580078, + "step": 2140, + "time_per_iteration": 2.5360589027404785 + }, + { + "auxiliary_loss_clip": 0.01055087, + "auxiliary_loss_mlp": 0.01000012, + "balance_loss_clip": 1.02666855, + "balance_loss_mlp": 0.99887961, + "epoch": 0.0621264000928559, + "flos": 68644361232000.0, + "grad_norm": 0.6916496994554693, + "language_loss": 0.51100004, + "learning_rate": 3.989186097652814e-06, + "loss": 0.531551, + "num_input_tokens_seen": 60535590, + "router_z_loss_clip": 0.28320312, + "router_z_loss_mlp": 0.01135254, + "step": 2141, + "time_per_iteration": 3.2589590549468994 + }, + { + "auxiliary_loss_clip": 0.01188544, + "auxiliary_loss_mlp": 0.0105771, + "balance_loss_clip": 1.07487977, + "balance_loss_mlp": 1.03474474, + "epoch": 0.062155417561371946, + "flos": 32081012449920.0, + "grad_norm": 2.2863584533960157, + "language_loss": 0.91806924, + "learning_rate": 3.989166569181198e-06, + "loss": 0.94053185, + "num_input_tokens_seen": 60550825, + "router_z_loss_clip": 1.13720703, + "router_z_loss_mlp": 0.22961426, + "step": 2142, + "time_per_iteration": 2.6131608486175537 + }, + { + "auxiliary_loss_clip": 0.01053286, + "auxiliary_loss_mlp": 0.00998238, + "balance_loss_clip": 1.0248251, + "balance_loss_mlp": 0.99716485, + "epoch": 0.06218443502988799, + "flos": 69166072321920.0, + "grad_norm": 0.6471149139973263, + "language_loss": 0.504044, + "learning_rate": 3.989147023140458e-06, + "loss": 0.52455926, + "num_input_tokens_seen": 60614170, + "router_z_loss_clip": 0.28515625, + "router_z_loss_mlp": 0.01074219, + "step": 2143, + "time_per_iteration": 3.1348323822021484 + }, + { + "auxiliary_loss_clip": 0.01175696, + "auxiliary_loss_mlp": 0.01057565, + "balance_loss_clip": 1.06785703, + "balance_loss_mlp": 1.03909981, + "epoch": 0.06221345249840404, + "flos": 48574782910080.0, + "grad_norm": 2.6484383907160742, + "language_loss": 0.96274412, + "learning_rate": 3.989127459530767e-06, + "loss": 0.98507679, + "num_input_tokens_seen": 60633555, + "router_z_loss_clip": 1.078125, + "router_z_loss_mlp": 0.18457031, + "step": 2144, + "time_per_iteration": 2.6867990493774414 + }, + { + "auxiliary_loss_clip": 0.01051556, + "auxiliary_loss_mlp": 0.01003713, + "balance_loss_clip": 1.02329588, + "balance_loss_mlp": 1.00262249, + "epoch": 0.06224246996692009, + "flos": 74782967639040.0, + "grad_norm": 0.6377285702009777, + "language_loss": 0.52668077, + "learning_rate": 3.989107878352297e-06, + "loss": 0.54723346, + "num_input_tokens_seen": 60701960, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01092529, + "step": 2145, + "time_per_iteration": 3.1955726146698 + }, + { + "auxiliary_loss_clip": 0.01192822, + "auxiliary_loss_mlp": 0.01064038, + "balance_loss_clip": 1.07216346, + "balance_loss_mlp": 1.03956389, + "epoch": 0.06227148743543613, + "flos": 74732113981440.0, + "grad_norm": 2.606597732516355, + "language_loss": 0.90807253, + "learning_rate": 3.989088279605222e-06, + "loss": 0.93064111, + "num_input_tokens_seen": 60722135, + "router_z_loss_clip": 1.20458984, + "router_z_loss_mlp": 0.24462891, + "step": 2146, + "time_per_iteration": 2.9152088165283203 + }, + { + "auxiliary_loss_clip": 0.01174029, + "auxiliary_loss_mlp": 0.01051717, + "balance_loss_clip": 1.06876493, + "balance_loss_mlp": 1.03235149, + "epoch": 0.06230050490395218, + "flos": 17341334755200.0, + "grad_norm": 2.120286124562728, + "language_loss": 0.82440221, + "learning_rate": 3.989068663289713e-06, + "loss": 0.84665966, + "num_input_tokens_seen": 60738100, + "router_z_loss_clip": 1.05273438, + "router_z_loss_mlp": 0.19348145, + "step": 2147, + "time_per_iteration": 2.5000863075256348 + }, + { + "auxiliary_loss_clip": 0.0118896, + "auxiliary_loss_mlp": 0.01062695, + "balance_loss_clip": 1.07535148, + "balance_loss_mlp": 1.04165483, + "epoch": 0.06232952237246823, + "flos": 28796838088320.0, + "grad_norm": 3.3316000337071348, + "language_loss": 0.80382466, + "learning_rate": 3.989049029405947e-06, + "loss": 0.82634127, + "num_input_tokens_seen": 60752345, + "router_z_loss_clip": 1.13671875, + "router_z_loss_mlp": 0.21032715, + "step": 2148, + "time_per_iteration": 2.6027536392211914 + }, + { + "auxiliary_loss_clip": 0.01050768, + "auxiliary_loss_mlp": 0.01004433, + "balance_loss_clip": 1.02239358, + "balance_loss_mlp": 1.00342607, + "epoch": 0.06235853984098427, + "flos": 74784368269440.0, + "grad_norm": 0.6943239235207244, + "language_loss": 0.53621674, + "learning_rate": 3.989029377954093e-06, + "loss": 0.55676877, + "num_input_tokens_seen": 60822275, + "router_z_loss_clip": 0.28320312, + "router_z_loss_mlp": 0.0100708, + "step": 2149, + "time_per_iteration": 3.27786922454834 + }, + { + "auxiliary_loss_clip": 0.01184873, + "auxiliary_loss_mlp": 0.01050042, + "balance_loss_clip": 1.07482147, + "balance_loss_mlp": 1.0306108, + "epoch": 0.06238755730950032, + "flos": 15590653781760.0, + "grad_norm": 2.5176351894354867, + "language_loss": 0.85790694, + "learning_rate": 3.989009708934328e-06, + "loss": 0.88025612, + "num_input_tokens_seen": 60836645, + "router_z_loss_clip": 1.10107422, + "router_z_loss_mlp": 0.19445801, + "step": 2150, + "time_per_iteration": 2.473872423171997 + }, + { + "auxiliary_loss_clip": 0.01180793, + "auxiliary_loss_mlp": 0.01067268, + "balance_loss_clip": 1.0712347, + "balance_loss_mlp": 1.04483247, + "epoch": 0.06241657477801637, + "flos": 30512003489280.0, + "grad_norm": 1.8938288418271858, + "language_loss": 0.91002512, + "learning_rate": 3.9889900223468234e-06, + "loss": 0.93250573, + "num_input_tokens_seen": 60855285, + "router_z_loss_clip": 1.09667969, + "router_z_loss_mlp": 0.22424316, + "step": 2151, + "time_per_iteration": 2.5925583839416504 + }, + { + "auxiliary_loss_clip": 0.0119228, + "auxiliary_loss_mlp": 0.01067413, + "balance_loss_clip": 1.07306623, + "balance_loss_mlp": 1.04394078, + "epoch": 0.062445592246532414, + "flos": 28180288114560.0, + "grad_norm": 2.0366913786436633, + "language_loss": 0.94555354, + "learning_rate": 3.988970318191753e-06, + "loss": 0.9681505, + "num_input_tokens_seen": 60875125, + "router_z_loss_clip": 1.19238281, + "router_z_loss_mlp": 0.23480225, + "step": 2152, + "time_per_iteration": 2.6129345893859863 + }, + { + "auxiliary_loss_clip": 0.01184729, + "auxiliary_loss_mlp": 0.01049973, + "balance_loss_clip": 1.07413363, + "balance_loss_mlp": 1.03081596, + "epoch": 0.06247460971504846, + "flos": 29857423991040.0, + "grad_norm": 2.294207531698927, + "language_loss": 0.87192583, + "learning_rate": 3.9889505964692946e-06, + "loss": 0.89427286, + "num_input_tokens_seen": 60891545, + "router_z_loss_clip": 1.10644531, + "router_z_loss_mlp": 0.19140625, + "step": 2153, + "time_per_iteration": 2.638577699661255 + }, + { + "auxiliary_loss_clip": 0.01050881, + "auxiliary_loss_mlp": 0.01007137, + "balance_loss_clip": 1.02285731, + "balance_loss_mlp": 1.00607038, + "epoch": 0.0625036271835645, + "flos": 74773773757440.0, + "grad_norm": 0.7147691447592391, + "language_loss": 0.51640236, + "learning_rate": 3.988930857179618e-06, + "loss": 0.53698254, + "num_input_tokens_seen": 60951225, + "router_z_loss_clip": 0.27929688, + "router_z_loss_mlp": 0.01068115, + "step": 2154, + "time_per_iteration": 3.0978171825408936 + }, + { + "auxiliary_loss_clip": 0.01184769, + "auxiliary_loss_mlp": 0.01050948, + "balance_loss_clip": 1.07155728, + "balance_loss_mlp": 1.02840531, + "epoch": 0.06253264465208055, + "flos": 14787691200000.0, + "grad_norm": 2.629711267032144, + "language_loss": 0.82619286, + "learning_rate": 3.988911100322899e-06, + "loss": 0.84854996, + "num_input_tokens_seen": 60966465, + "router_z_loss_clip": 1.13134766, + "router_z_loss_mlp": 0.2253418, + "step": 2155, + "time_per_iteration": 2.4909610748291016 + }, + { + "auxiliary_loss_clip": 0.01050779, + "auxiliary_loss_mlp": 0.01002085, + "balance_loss_clip": 1.02292955, + "balance_loss_mlp": 1.00097024, + "epoch": 0.06256166212059659, + "flos": 58315554783360.0, + "grad_norm": 0.6905771970654294, + "language_loss": 0.52414846, + "learning_rate": 3.988891325899313e-06, + "loss": 0.54467708, + "num_input_tokens_seen": 61022415, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01116943, + "step": 2156, + "time_per_iteration": 2.9215784072875977 + }, + { + "auxiliary_loss_clip": 0.01176108, + "auxiliary_loss_mlp": 0.01049074, + "balance_loss_clip": 1.07223582, + "balance_loss_mlp": 1.03178906, + "epoch": 0.06259067958911264, + "flos": 30257540565120.0, + "grad_norm": 2.2077530391942495, + "language_loss": 0.74291396, + "learning_rate": 3.988871533909035e-06, + "loss": 0.76516581, + "num_input_tokens_seen": 61040645, + "router_z_loss_clip": 1.04003906, + "router_z_loss_mlp": 0.17297363, + "step": 2157, + "time_per_iteration": 2.6077678203582764 + }, + { + "auxiliary_loss_clip": 0.0117399, + "auxiliary_loss_mlp": 0.01055777, + "balance_loss_clip": 1.07067466, + "balance_loss_mlp": 1.03700161, + "epoch": 0.0626196970576287, + "flos": 49120624961280.0, + "grad_norm": 2.5434613910717387, + "language_loss": 0.72352844, + "learning_rate": 3.988851724352237e-06, + "loss": 0.74582613, + "num_input_tokens_seen": 61061750, + "router_z_loss_clip": 1.03222656, + "router_z_loss_mlp": 0.18774414, + "step": 2158, + "time_per_iteration": 2.740274429321289 + }, + { + "auxiliary_loss_clip": 0.01187123, + "auxiliary_loss_mlp": 0.01055838, + "balance_loss_clip": 1.07536292, + "balance_loss_mlp": 1.03179336, + "epoch": 0.06264871452614473, + "flos": 11501469763200.0, + "grad_norm": 2.479176124899357, + "language_loss": 0.81062078, + "learning_rate": 3.988831897229097e-06, + "loss": 0.83305037, + "num_input_tokens_seen": 61073395, + "router_z_loss_clip": 1.11621094, + "router_z_loss_mlp": 0.24047852, + "step": 2159, + "time_per_iteration": 2.4558839797973633 + }, + { + "auxiliary_loss_clip": 0.01176699, + "auxiliary_loss_mlp": 0.01056595, + "balance_loss_clip": 1.07176113, + "balance_loss_mlp": 1.03619218, + "epoch": 0.06267773199466078, + "flos": 74734735674240.0, + "grad_norm": 3.1147718584974324, + "language_loss": 0.78847885, + "learning_rate": 3.988812052539788e-06, + "loss": 0.81081182, + "num_input_tokens_seen": 61100390, + "router_z_loss_clip": 1.04980469, + "router_z_loss_mlp": 0.20397949, + "step": 2160, + "time_per_iteration": 2.924741268157959 + }, + { + "auxiliary_loss_clip": 0.01051623, + "auxiliary_loss_mlp": 0.01004198, + "balance_loss_clip": 1.02356696, + "balance_loss_mlp": 1.00306594, + "epoch": 0.06270674946317684, + "flos": 74787672320640.0, + "grad_norm": 0.6508411464162811, + "language_loss": 0.50751674, + "learning_rate": 3.988792190284487e-06, + "loss": 0.52807498, + "num_input_tokens_seen": 61166900, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01135254, + "step": 2161, + "time_per_iteration": 3.2470266819000244 + }, + { + "auxiliary_loss_clip": 0.01176567, + "auxiliary_loss_mlp": 0.01047456, + "balance_loss_clip": 1.06545687, + "balance_loss_mlp": 1.02870417, + "epoch": 0.06273576693169287, + "flos": 16537079283840.0, + "grad_norm": 2.4449396847596403, + "language_loss": 0.69432735, + "learning_rate": 3.988772310463368e-06, + "loss": 0.71656758, + "num_input_tokens_seen": 61180240, + "router_z_loss_clip": 1.11083984, + "router_z_loss_mlp": 0.18756104, + "step": 2162, + "time_per_iteration": 2.461137056350708 + }, + { + "auxiliary_loss_clip": 0.01190483, + "auxiliary_loss_mlp": 0.01064852, + "balance_loss_clip": 1.07482576, + "balance_loss_mlp": 1.04124796, + "epoch": 0.06276478440020893, + "flos": 30365164690560.0, + "grad_norm": 2.8228475303189797, + "language_loss": 0.95009488, + "learning_rate": 3.988752413076607e-06, + "loss": 0.9726482, + "num_input_tokens_seen": 61194985, + "router_z_loss_clip": 1.15527344, + "router_z_loss_mlp": 0.23596191, + "step": 2163, + "time_per_iteration": 2.55749773979187 + }, + { + "auxiliary_loss_clip": 0.01182905, + "auxiliary_loss_mlp": 0.01043012, + "balance_loss_clip": 1.07373106, + "balance_loss_mlp": 1.02254999, + "epoch": 0.06279380186872498, + "flos": 43027374453120.0, + "grad_norm": 1.799279427696032, + "language_loss": 0.8497436, + "learning_rate": 3.98873249812438e-06, + "loss": 0.87200272, + "num_input_tokens_seen": 61215305, + "router_z_loss_clip": 1.09130859, + "router_z_loss_mlp": 0.20477295, + "step": 2164, + "time_per_iteration": 2.703428268432617 + }, + { + "auxiliary_loss_clip": 0.01186903, + "auxiliary_loss_mlp": 0.01051996, + "balance_loss_clip": 1.06980205, + "balance_loss_mlp": 1.03111017, + "epoch": 0.06282281933724101, + "flos": 20992049452800.0, + "grad_norm": 2.291623588654852, + "language_loss": 0.6915105, + "learning_rate": 3.988712565606864e-06, + "loss": 0.71389943, + "num_input_tokens_seen": 61231905, + "router_z_loss_clip": 1.16992188, + "router_z_loss_mlp": 0.20898438, + "step": 2165, + "time_per_iteration": 2.4815168380737305 + }, + { + "auxiliary_loss_clip": 0.01182001, + "auxiliary_loss_mlp": 0.01046668, + "balance_loss_clip": 1.0723207, + "balance_loss_mlp": 1.02629483, + "epoch": 0.06285183680575707, + "flos": 14824176439680.0, + "grad_norm": 2.227724586219037, + "language_loss": 0.67622447, + "learning_rate": 3.9886926155242325e-06, + "loss": 0.69851112, + "num_input_tokens_seen": 61244330, + "router_z_loss_clip": 1.09716797, + "router_z_loss_mlp": 0.20385742, + "step": 2166, + "time_per_iteration": 2.465911865234375 + }, + { + "auxiliary_loss_clip": 0.01177629, + "auxiliary_loss_mlp": 0.01046084, + "balance_loss_clip": 1.06683457, + "balance_loss_mlp": 1.02490032, + "epoch": 0.06288085427427312, + "flos": 23798396165760.0, + "grad_norm": 6.616369599836774, + "language_loss": 0.97840989, + "learning_rate": 3.988672647876664e-06, + "loss": 1.00064707, + "num_input_tokens_seen": 61259250, + "router_z_loss_clip": 1.10742188, + "router_z_loss_mlp": 0.21179199, + "step": 2167, + "time_per_iteration": 2.5048577785491943 + }, + { + "auxiliary_loss_clip": 0.01190954, + "auxiliary_loss_mlp": 0.01068308, + "balance_loss_clip": 1.07286334, + "balance_loss_mlp": 1.04613483, + "epoch": 0.06290987174278916, + "flos": 47624837875200.0, + "grad_norm": 1.8413295116141686, + "language_loss": 0.80228913, + "learning_rate": 3.988652662664333e-06, + "loss": 0.82488173, + "num_input_tokens_seen": 61281860, + "router_z_loss_clip": 1.18066406, + "router_z_loss_mlp": 0.22155762, + "step": 2168, + "time_per_iteration": 2.7397966384887695 + }, + { + "auxiliary_loss_clip": 0.01180076, + "auxiliary_loss_mlp": 0.01055285, + "balance_loss_clip": 1.07000911, + "balance_loss_mlp": 1.03418493, + "epoch": 0.06293888921130521, + "flos": 29928311481600.0, + "grad_norm": 3.0239949681397684, + "language_loss": 0.73458755, + "learning_rate": 3.988632659887417e-06, + "loss": 0.7569412, + "num_input_tokens_seen": 61296710, + "router_z_loss_clip": 1.1015625, + "router_z_loss_mlp": 0.21118164, + "step": 2169, + "time_per_iteration": 2.5523951053619385 + }, + { + "auxiliary_loss_clip": 0.01052749, + "auxiliary_loss_mlp": 0.00998861, + "balance_loss_clip": 1.02431107, + "balance_loss_mlp": 0.99762738, + "epoch": 0.06296790667982126, + "flos": 74773270967040.0, + "grad_norm": 0.9318251650041138, + "language_loss": 0.5382815, + "learning_rate": 3.988612639546093e-06, + "loss": 0.5587976, + "num_input_tokens_seen": 61365055, + "router_z_loss_clip": 0.28515625, + "router_z_loss_mlp": 0.0123291, + "step": 2170, + "time_per_iteration": 3.193026542663574 + }, + { + "auxiliary_loss_clip": 0.01177057, + "auxiliary_loss_mlp": 0.01056604, + "balance_loss_clip": 1.06672847, + "balance_loss_mlp": 1.03613615, + "epoch": 0.0629969241483373, + "flos": 12936209685120.0, + "grad_norm": 2.1538944143502117, + "language_loss": 0.84729415, + "learning_rate": 3.988592601640538e-06, + "loss": 0.86963075, + "num_input_tokens_seen": 61379445, + "router_z_loss_clip": 1.10253906, + "router_z_loss_mlp": 0.20458984, + "step": 2171, + "time_per_iteration": 2.450021266937256 + }, + { + "auxiliary_loss_clip": 0.01053643, + "auxiliary_loss_mlp": 0.01003225, + "balance_loss_clip": 1.0250442, + "balance_loss_mlp": 1.00201535, + "epoch": 0.06302594161685335, + "flos": 67621876680960.0, + "grad_norm": 0.6608660441535718, + "language_loss": 0.50124484, + "learning_rate": 3.988572546170928e-06, + "loss": 0.52181351, + "num_input_tokens_seen": 61436370, + "router_z_loss_clip": 0.28515625, + "router_z_loss_mlp": 0.01208496, + "step": 2172, + "time_per_iteration": 3.063725233078003 + }, + { + "auxiliary_loss_clip": 0.01053015, + "auxiliary_loss_mlp": 0.01004346, + "balance_loss_clip": 1.02463436, + "balance_loss_mlp": 1.00321341, + "epoch": 0.06305495908536939, + "flos": 62838718922880.0, + "grad_norm": 0.7362032281760582, + "language_loss": 0.53037244, + "learning_rate": 3.9885524731374405e-06, + "loss": 0.550946, + "num_input_tokens_seen": 61498475, + "router_z_loss_clip": 0.28320312, + "router_z_loss_mlp": 0.01135254, + "step": 2173, + "time_per_iteration": 3.0687596797943115 + }, + { + "auxiliary_loss_clip": 0.01192941, + "auxiliary_loss_mlp": 0.01056028, + "balance_loss_clip": 1.07423258, + "balance_loss_mlp": 1.03330648, + "epoch": 0.06308397655388544, + "flos": 29673058458240.0, + "grad_norm": 2.252476932816848, + "language_loss": 0.89343458, + "learning_rate": 3.988532382540253e-06, + "loss": 0.91592431, + "num_input_tokens_seen": 61518450, + "router_z_loss_clip": 1.18457031, + "router_z_loss_mlp": 0.22747803, + "step": 2174, + "time_per_iteration": 2.5724360942840576 + }, + { + "auxiliary_loss_clip": 0.011756, + "auxiliary_loss_mlp": 0.01055907, + "balance_loss_clip": 1.06768966, + "balance_loss_mlp": 1.03531933, + "epoch": 0.06311299402240149, + "flos": 29855089607040.0, + "grad_norm": 2.6010261395221734, + "language_loss": 0.91441715, + "learning_rate": 3.988512274379543e-06, + "loss": 0.93673217, + "num_input_tokens_seen": 61532740, + "router_z_loss_clip": 1.07763672, + "router_z_loss_mlp": 0.20599365, + "step": 2175, + "time_per_iteration": 2.6258366107940674 + }, + { + "auxiliary_loss_clip": 0.01159685, + "auxiliary_loss_mlp": 0.01051016, + "balance_loss_clip": 1.06631088, + "balance_loss_mlp": 1.03411257, + "epoch": 0.06314201149091753, + "flos": 20550132426240.0, + "grad_norm": 2.674951946613829, + "language_loss": 0.8075065, + "learning_rate": 3.988492148655487e-06, + "loss": 0.82961351, + "num_input_tokens_seen": 61545645, + "router_z_loss_clip": 0.93359375, + "router_z_loss_mlp": 0.16906738, + "step": 2176, + "time_per_iteration": 2.493229627609253 + }, + { + "auxiliary_loss_clip": 0.01178035, + "auxiliary_loss_mlp": 0.01054239, + "balance_loss_clip": 1.06985664, + "balance_loss_mlp": 1.03459311, + "epoch": 0.06317102895943358, + "flos": 19458161015040.0, + "grad_norm": 22.735430347922414, + "language_loss": 0.89310884, + "learning_rate": 3.9884720053682645e-06, + "loss": 0.9154315, + "num_input_tokens_seen": 61558855, + "router_z_loss_clip": 1.08300781, + "router_z_loss_mlp": 0.1965332, + "step": 2177, + "time_per_iteration": 2.5076682567596436 + }, + { + "auxiliary_loss_clip": 0.01181996, + "auxiliary_loss_mlp": 0.01053745, + "balance_loss_clip": 1.07119751, + "balance_loss_mlp": 1.03397954, + "epoch": 0.06320004642794963, + "flos": 25442099458560.0, + "grad_norm": 2.3655893988315766, + "language_loss": 0.90759856, + "learning_rate": 3.988451844518052e-06, + "loss": 0.92995596, + "num_input_tokens_seen": 61572720, + "router_z_loss_clip": 1.10742188, + "router_z_loss_mlp": 0.19763184, + "step": 2178, + "time_per_iteration": 2.4611735343933105 + }, + { + "auxiliary_loss_clip": 0.01055592, + "auxiliary_loss_mlp": 0.01010116, + "balance_loss_clip": 1.02687001, + "balance_loss_mlp": 1.00894821, + "epoch": 0.06322906389646567, + "flos": 60913943706240.0, + "grad_norm": 0.7681411495636159, + "language_loss": 0.49415651, + "learning_rate": 3.98843166610503e-06, + "loss": 0.5148136, + "num_input_tokens_seen": 61629855, + "router_z_loss_clip": 0.28710938, + "router_z_loss_mlp": 0.01165771, + "step": 2179, + "time_per_iteration": 3.1527695655822754 + }, + { + "auxiliary_loss_clip": 0.01055796, + "auxiliary_loss_mlp": 0.01010966, + "balance_loss_clip": 1.02727699, + "balance_loss_mlp": 1.00979161, + "epoch": 0.06325808136498172, + "flos": 59565246814080.0, + "grad_norm": 0.6923517436382981, + "language_loss": 0.49721116, + "learning_rate": 3.9884114701293725e-06, + "loss": 0.51787877, + "num_input_tokens_seen": 61695015, + "router_z_loss_clip": 0.28515625, + "router_z_loss_mlp": 0.01171875, + "step": 2180, + "time_per_iteration": 3.1474266052246094 + }, + { + "auxiliary_loss_clip": 0.01170056, + "auxiliary_loss_mlp": 0.01045232, + "balance_loss_clip": 1.06754577, + "balance_loss_mlp": 1.02854884, + "epoch": 0.06328709883349777, + "flos": 21027493198080.0, + "grad_norm": 2.248168117538295, + "language_loss": 0.88943076, + "learning_rate": 3.9883912565912614e-06, + "loss": 0.91158366, + "num_input_tokens_seen": 61714105, + "router_z_loss_clip": 1.02539062, + "router_z_loss_mlp": 0.16687012, + "step": 2181, + "time_per_iteration": 2.693582534790039 + }, + { + "auxiliary_loss_clip": 0.01055736, + "auxiliary_loss_mlp": 0.01003618, + "balance_loss_clip": 1.02688074, + "balance_loss_mlp": 1.0022707, + "epoch": 0.06331611630201381, + "flos": 66057177352320.0, + "grad_norm": 0.7043583355241936, + "language_loss": 0.50539684, + "learning_rate": 3.988371025490874e-06, + "loss": 0.52599037, + "num_input_tokens_seen": 61777675, + "router_z_loss_clip": 0.2890625, + "router_z_loss_mlp": 0.01348877, + "step": 2182, + "time_per_iteration": 3.051093339920044 + }, + { + "auxiliary_loss_clip": 0.01187455, + "auxiliary_loss_mlp": 0.01051854, + "balance_loss_clip": 1.0715003, + "balance_loss_mlp": 1.03152871, + "epoch": 0.06334513377052986, + "flos": 36896884519680.0, + "grad_norm": 2.4365896835746312, + "language_loss": 0.72715855, + "learning_rate": 3.98835077682839e-06, + "loss": 0.74955159, + "num_input_tokens_seen": 61795555, + "router_z_loss_clip": 1.15917969, + "router_z_loss_mlp": 0.20324707, + "step": 2183, + "time_per_iteration": 2.627446413040161 + }, + { + "auxiliary_loss_clip": 0.01179204, + "auxiliary_loss_mlp": 0.01055311, + "balance_loss_clip": 1.06783247, + "balance_loss_mlp": 1.03441334, + "epoch": 0.06337415123904591, + "flos": 15370162145280.0, + "grad_norm": 3.142987554639816, + "language_loss": 0.74077898, + "learning_rate": 3.988330510603986e-06, + "loss": 0.76312411, + "num_input_tokens_seen": 61807265, + "router_z_loss_clip": 1.11376953, + "router_z_loss_mlp": 0.2088623, + "step": 2184, + "time_per_iteration": 2.454838991165161 + }, + { + "auxiliary_loss_clip": 0.0118177, + "auxiliary_loss_mlp": 0.01051209, + "balance_loss_clip": 1.06601191, + "balance_loss_mlp": 1.03087759, + "epoch": 0.06340316870756195, + "flos": 19420885676160.0, + "grad_norm": 3.633573019656844, + "language_loss": 0.96098387, + "learning_rate": 3.9883102268178425e-06, + "loss": 0.98331368, + "num_input_tokens_seen": 61820340, + "router_z_loss_clip": 1.15820312, + "router_z_loss_mlp": 0.20330811, + "step": 2185, + "time_per_iteration": 2.4407174587249756 + }, + { + "auxiliary_loss_clip": 0.01185887, + "auxiliary_loss_mlp": 0.01075016, + "balance_loss_clip": 1.07143903, + "balance_loss_mlp": 1.05210352, + "epoch": 0.063432186176078, + "flos": 29896961886720.0, + "grad_norm": 2.1883248941284035, + "language_loss": 0.88807356, + "learning_rate": 3.988289925470138e-06, + "loss": 0.91068262, + "num_input_tokens_seen": 61839485, + "router_z_loss_clip": 1.14550781, + "router_z_loss_mlp": 0.22937012, + "step": 2186, + "time_per_iteration": 2.5856828689575195 + }, + { + "auxiliary_loss_clip": 0.01056836, + "auxiliary_loss_mlp": 0.01008203, + "balance_loss_clip": 1.02804613, + "balance_loss_mlp": 1.00692737, + "epoch": 0.06346120364459405, + "flos": 66752731290240.0, + "grad_norm": 0.6075105621461347, + "language_loss": 0.464618, + "learning_rate": 3.988269606561054e-06, + "loss": 0.48526838, + "num_input_tokens_seen": 61907555, + "router_z_loss_clip": 0.2890625, + "router_z_loss_mlp": 0.01275635, + "step": 2187, + "time_per_iteration": 3.1602261066436768 + }, + { + "auxiliary_loss_clip": 0.01182079, + "auxiliary_loss_mlp": 0.01057584, + "balance_loss_clip": 1.06861663, + "balance_loss_mlp": 1.03563714, + "epoch": 0.06349022111311009, + "flos": 27191738937600.0, + "grad_norm": 2.8526934216858213, + "language_loss": 0.85791045, + "learning_rate": 3.988249270090767e-06, + "loss": 0.88030708, + "num_input_tokens_seen": 61921040, + "router_z_loss_clip": 1.13525391, + "router_z_loss_mlp": 0.21948242, + "step": 2188, + "time_per_iteration": 2.572661876678467 + }, + { + "auxiliary_loss_clip": 0.01184063, + "auxiliary_loss_mlp": 0.01055176, + "balance_loss_clip": 1.07193172, + "balance_loss_mlp": 1.03452945, + "epoch": 0.06351923858162614, + "flos": 26294583916800.0, + "grad_norm": 3.763730445611042, + "language_loss": 1.02021778, + "learning_rate": 3.988228916059459e-06, + "loss": 1.04261017, + "num_input_tokens_seen": 61937520, + "router_z_loss_clip": 1.12109375, + "router_z_loss_mlp": 0.20654297, + "step": 2189, + "time_per_iteration": 2.5612540245056152 + }, + { + "auxiliary_loss_clip": 0.01184126, + "auxiliary_loss_mlp": 0.01052194, + "balance_loss_clip": 1.07169652, + "balance_loss_mlp": 1.02971077, + "epoch": 0.06354825605014218, + "flos": 25697208827520.0, + "grad_norm": 2.0933474317011336, + "language_loss": 0.83316267, + "learning_rate": 3.988208544467307e-06, + "loss": 0.85552591, + "num_input_tokens_seen": 61954590, + "router_z_loss_clip": 1.12402344, + "router_z_loss_mlp": 0.22485352, + "step": 2190, + "time_per_iteration": 2.4979515075683594 + }, + { + "auxiliary_loss_clip": 0.0117785, + "auxiliary_loss_mlp": 0.01070944, + "balance_loss_clip": 1.06891751, + "balance_loss_mlp": 1.04898548, + "epoch": 0.06357727351865823, + "flos": 12415288694400.0, + "grad_norm": 3.046068156029793, + "language_loss": 0.97894645, + "learning_rate": 3.988188155314494e-06, + "loss": 1.00143433, + "num_input_tokens_seen": 61965445, + "router_z_loss_clip": 1.08984375, + "router_z_loss_mlp": 0.21948242, + "step": 2191, + "time_per_iteration": 2.467557907104492 + }, + { + "auxiliary_loss_clip": 0.01188292, + "auxiliary_loss_mlp": 0.01058415, + "balance_loss_clip": 1.07304358, + "balance_loss_mlp": 1.03738618, + "epoch": 0.06360629098717428, + "flos": 34279608021120.0, + "grad_norm": 16.634706476916783, + "language_loss": 0.88741791, + "learning_rate": 3.988167748601198e-06, + "loss": 0.90988493, + "num_input_tokens_seen": 61986890, + "router_z_loss_clip": 1.15234375, + "router_z_loss_mlp": 0.21020508, + "step": 2192, + "time_per_iteration": 2.665741443634033 + }, + { + "auxiliary_loss_clip": 0.01185291, + "auxiliary_loss_mlp": 0.01050773, + "balance_loss_clip": 1.07526267, + "balance_loss_mlp": 1.03028119, + "epoch": 0.06363530845569032, + "flos": 24894784949760.0, + "grad_norm": 2.7015851092107823, + "language_loss": 0.85335428, + "learning_rate": 3.9881473243275994e-06, + "loss": 0.87571502, + "num_input_tokens_seen": 62001320, + "router_z_loss_clip": 1.10058594, + "router_z_loss_mlp": 0.20483398, + "step": 2193, + "time_per_iteration": 2.571838617324829 + }, + { + "auxiliary_loss_clip": 0.01061184, + "auxiliary_loss_mlp": 0.01000665, + "balance_loss_clip": 1.03240037, + "balance_loss_mlp": 0.9992938, + "epoch": 0.06366432592420637, + "flos": 67878709902720.0, + "grad_norm": 0.7663130024875928, + "language_loss": 0.5675863, + "learning_rate": 3.98812688249388e-06, + "loss": 0.58820486, + "num_input_tokens_seen": 62064250, + "router_z_loss_clip": 0.28710938, + "router_z_loss_mlp": 0.01373291, + "step": 2194, + "time_per_iteration": 3.1663594245910645 + }, + { + "auxiliary_loss_clip": 0.01061876, + "auxiliary_loss_mlp": 0.01001099, + "balance_loss_clip": 1.03277302, + "balance_loss_mlp": 0.99954939, + "epoch": 0.06369334339272242, + "flos": 71854379965440.0, + "grad_norm": 0.809611603097938, + "language_loss": 0.53014755, + "learning_rate": 3.988106423100219e-06, + "loss": 0.55077732, + "num_input_tokens_seen": 62124240, + "router_z_loss_clip": 0.29101562, + "router_z_loss_mlp": 0.01550293, + "step": 2195, + "time_per_iteration": 3.072422742843628 + }, + { + "auxiliary_loss_clip": 0.01184811, + "auxiliary_loss_mlp": 0.01062525, + "balance_loss_clip": 1.0762471, + "balance_loss_mlp": 1.04073322, + "epoch": 0.06372236086123846, + "flos": 31678848800640.0, + "grad_norm": 2.6535997768787327, + "language_loss": 0.93559468, + "learning_rate": 3.988085946146798e-06, + "loss": 0.95806801, + "num_input_tokens_seen": 62140860, + "router_z_loss_clip": 1.08496094, + "router_z_loss_mlp": 0.21813965, + "step": 2196, + "time_per_iteration": 2.6409811973571777 + }, + { + "auxiliary_loss_clip": 0.01061977, + "auxiliary_loss_mlp": 0.0099882, + "balance_loss_clip": 1.03281021, + "balance_loss_mlp": 0.99740094, + "epoch": 0.06375137832975451, + "flos": 58865203676160.0, + "grad_norm": 0.7115320654236656, + "language_loss": 0.53716558, + "learning_rate": 3.988065451633798e-06, + "loss": 0.55777353, + "num_input_tokens_seen": 62195735, + "router_z_loss_clip": 0.29101562, + "router_z_loss_mlp": 0.01416016, + "step": 2197, + "time_per_iteration": 2.9696648120880127 + }, + { + "auxiliary_loss_clip": 0.01189202, + "auxiliary_loss_mlp": 0.01056233, + "balance_loss_clip": 1.07389653, + "balance_loss_mlp": 1.03223586, + "epoch": 0.06378039579827056, + "flos": 21938726350080.0, + "grad_norm": 2.524354097699565, + "language_loss": 0.77965164, + "learning_rate": 3.9880449395613984e-06, + "loss": 0.80210602, + "num_input_tokens_seen": 62212915, + "router_z_loss_clip": 1.15429688, + "router_z_loss_mlp": 0.23999023, + "step": 2198, + "time_per_iteration": 2.5805225372314453 + }, + { + "auxiliary_loss_clip": 0.01061924, + "auxiliary_loss_mlp": 0.0100142, + "balance_loss_clip": 1.03299153, + "balance_loss_mlp": 0.99995345, + "epoch": 0.0638094132667866, + "flos": 74775246215040.0, + "grad_norm": 0.7099934869468766, + "language_loss": 0.49001834, + "learning_rate": 3.988024409929782e-06, + "loss": 0.51065177, + "num_input_tokens_seen": 62271555, + "router_z_loss_clip": 0.2890625, + "router_z_loss_mlp": 0.01464844, + "step": 2199, + "time_per_iteration": 3.1155667304992676 + }, + { + "auxiliary_loss_clip": 0.01184999, + "auxiliary_loss_mlp": 0.01063394, + "balance_loss_clip": 1.07397985, + "balance_loss_mlp": 1.04198432, + "epoch": 0.06383843073530265, + "flos": 12158994176640.0, + "grad_norm": 3.9847794600231263, + "language_loss": 1.00790489, + "learning_rate": 3.988003862739129e-06, + "loss": 1.03038883, + "num_input_tokens_seen": 62282560, + "router_z_loss_clip": 1.10986328, + "router_z_loss_mlp": 0.2142334, + "step": 2200, + "time_per_iteration": 2.4495761394500732 + }, + { + "auxiliary_loss_clip": 0.01196836, + "auxiliary_loss_mlp": 0.01073727, + "balance_loss_clip": 1.07906878, + "balance_loss_mlp": 1.05240011, + "epoch": 0.0638674482038187, + "flos": 70065989712000.0, + "grad_norm": 2.2206814713286898, + "language_loss": 0.91956341, + "learning_rate": 3.987983297989621e-06, + "loss": 0.94226897, + "num_input_tokens_seen": 62306780, + "router_z_loss_clip": 1.17675781, + "router_z_loss_mlp": 0.21313477, + "step": 2201, + "time_per_iteration": 2.880368947982788 + }, + { + "auxiliary_loss_clip": 0.01182909, + "auxiliary_loss_mlp": 0.01057923, + "balance_loss_clip": 1.07231522, + "balance_loss_mlp": 1.03769279, + "epoch": 0.06389646567233474, + "flos": 18690570351360.0, + "grad_norm": 2.961684832446519, + "language_loss": 0.77191257, + "learning_rate": 3.9879627156814415e-06, + "loss": 0.79432094, + "num_input_tokens_seen": 62320670, + "router_z_loss_clip": 1.10644531, + "router_z_loss_mlp": 0.20227051, + "step": 2202, + "time_per_iteration": 2.4591798782348633 + }, + { + "auxiliary_loss_clip": 0.01192604, + "auxiliary_loss_mlp": 0.01059431, + "balance_loss_clip": 1.07678938, + "balance_loss_mlp": 1.03817558, + "epoch": 0.0639254831408508, + "flos": 37303896504960.0, + "grad_norm": 1.9634157573991728, + "language_loss": 0.8294512, + "learning_rate": 3.98794211581477e-06, + "loss": 0.85197157, + "num_input_tokens_seen": 62345265, + "router_z_loss_clip": 1.15722656, + "router_z_loss_mlp": 0.21264648, + "step": 2203, + "time_per_iteration": 2.7114479541778564 + }, + { + "auxiliary_loss_clip": 0.01063389, + "auxiliary_loss_mlp": 0.0101425, + "balance_loss_clip": 1.03380227, + "balance_loss_mlp": 1.01295042, + "epoch": 0.06395450060936683, + "flos": 74781136045440.0, + "grad_norm": 0.6555816063135989, + "language_loss": 0.53091234, + "learning_rate": 3.9879214983897896e-06, + "loss": 0.55168873, + "num_input_tokens_seen": 62414170, + "router_z_loss_clip": 0.296875, + "router_z_loss_mlp": 0.01300049, + "step": 2204, + "time_per_iteration": 3.2843306064605713 + }, + { + "auxiliary_loss_clip": 0.01183569, + "auxiliary_loss_mlp": 0.01060342, + "balance_loss_clip": 1.07299185, + "balance_loss_mlp": 1.03982592, + "epoch": 0.06398351807788288, + "flos": 33213240028800.0, + "grad_norm": 2.289832994880593, + "language_loss": 0.84627473, + "learning_rate": 3.9879008634066815e-06, + "loss": 0.8687138, + "num_input_tokens_seen": 62431365, + "router_z_loss_clip": 1.10595703, + "router_z_loss_mlp": 0.20507812, + "step": 2205, + "time_per_iteration": 2.6017303466796875 + }, + { + "auxiliary_loss_clip": 0.01174016, + "auxiliary_loss_mlp": 0.01057178, + "balance_loss_clip": 1.07258034, + "balance_loss_mlp": 1.04010153, + "epoch": 0.06401253554639894, + "flos": 27665903399040.0, + "grad_norm": 2.250814089839509, + "language_loss": 0.698156, + "learning_rate": 3.987880210865629e-06, + "loss": 0.72046793, + "num_input_tokens_seen": 62446705, + "router_z_loss_clip": 1.01513672, + "router_z_loss_mlp": 0.1706543, + "step": 2206, + "time_per_iteration": 2.5870120525360107 + }, + { + "auxiliary_loss_clip": 0.01171623, + "auxiliary_loss_mlp": 0.01046679, + "balance_loss_clip": 1.0709964, + "balance_loss_mlp": 1.0286665, + "epoch": 0.06404155301491497, + "flos": 27922952102400.0, + "grad_norm": 2.228588932263961, + "language_loss": 0.97346759, + "learning_rate": 3.9878595407668144e-06, + "loss": 0.99565059, + "num_input_tokens_seen": 62461930, + "router_z_loss_clip": 1.00732422, + "router_z_loss_mlp": 0.18029785, + "step": 2207, + "time_per_iteration": 2.5841524600982666 + }, + { + "auxiliary_loss_clip": 0.01180898, + "auxiliary_loss_mlp": 0.01050711, + "balance_loss_clip": 1.07169366, + "balance_loss_mlp": 1.03011143, + "epoch": 0.06407057048343102, + "flos": 28835909107200.0, + "grad_norm": 3.099210177537756, + "language_loss": 0.89439529, + "learning_rate": 3.98783885311042e-06, + "loss": 0.91671145, + "num_input_tokens_seen": 62476810, + "router_z_loss_clip": 1.09179688, + "router_z_loss_mlp": 0.20599365, + "step": 2208, + "time_per_iteration": 7.3885462284088135 + }, + { + "auxiliary_loss_clip": 0.01181445, + "auxiliary_loss_mlp": 0.01056314, + "balance_loss_clip": 1.07328701, + "balance_loss_mlp": 1.03558373, + "epoch": 0.06409958795194708, + "flos": 18288873578880.0, + "grad_norm": 2.6253104724618934, + "language_loss": 0.9222818, + "learning_rate": 3.987818147896627e-06, + "loss": 0.94465941, + "num_input_tokens_seen": 62489925, + "router_z_loss_clip": 1.07910156, + "router_z_loss_mlp": 0.20727539, + "step": 2209, + "time_per_iteration": 4.777694940567017 + }, + { + "auxiliary_loss_clip": 0.01061744, + "auxiliary_loss_mlp": 0.01003483, + "balance_loss_clip": 1.0323894, + "balance_loss_mlp": 1.00214815, + "epoch": 0.06412860542046311, + "flos": 50873930605440.0, + "grad_norm": 0.7400223685246722, + "language_loss": 0.49615595, + "learning_rate": 3.987797425125621e-06, + "loss": 0.51680821, + "num_input_tokens_seen": 62548540, + "router_z_loss_clip": 0.29296875, + "router_z_loss_mlp": 0.0133667, + "step": 2210, + "time_per_iteration": 3.0828490257263184 + }, + { + "auxiliary_loss_clip": 0.0118814, + "auxiliary_loss_mlp": 0.01058843, + "balance_loss_clip": 1.07410955, + "balance_loss_mlp": 1.03614521, + "epoch": 0.06415762288897917, + "flos": 19966763640960.0, + "grad_norm": 2.4167922431134543, + "language_loss": 0.9016906, + "learning_rate": 3.987776684797583e-06, + "loss": 0.92416042, + "num_input_tokens_seen": 62561040, + "router_z_loss_clip": 1.13964844, + "router_z_loss_mlp": 0.22705078, + "step": 2211, + "time_per_iteration": 4.821083307266235 + }, + { + "auxiliary_loss_clip": 0.01061139, + "auxiliary_loss_mlp": 0.01005056, + "balance_loss_clip": 1.03177691, + "balance_loss_mlp": 1.00369668, + "epoch": 0.06418664035749522, + "flos": 71785036759680.0, + "grad_norm": 0.6641738663996758, + "language_loss": 0.5181067, + "learning_rate": 3.987755926912698e-06, + "loss": 0.53876865, + "num_input_tokens_seen": 62628715, + "router_z_loss_clip": 0.29296875, + "router_z_loss_mlp": 0.01361084, + "step": 2212, + "time_per_iteration": 3.172147512435913 + }, + { + "auxiliary_loss_clip": 0.01186954, + "auxiliary_loss_mlp": 0.01058635, + "balance_loss_clip": 1.07667243, + "balance_loss_mlp": 1.03845322, + "epoch": 0.06421565782601125, + "flos": 45981529632000.0, + "grad_norm": 3.94472666675732, + "language_loss": 0.90039015, + "learning_rate": 3.987735151471148e-06, + "loss": 0.92284608, + "num_input_tokens_seen": 62645180, + "router_z_loss_clip": 1.10253906, + "router_z_loss_mlp": 0.20178223, + "step": 2213, + "time_per_iteration": 2.629585027694702 + }, + { + "auxiliary_loss_clip": 0.01184527, + "auxiliary_loss_mlp": 0.01060876, + "balance_loss_clip": 1.07304478, + "balance_loss_mlp": 1.04127192, + "epoch": 0.0642446752945273, + "flos": 16029625893120.0, + "grad_norm": 2.2131966662623146, + "language_loss": 0.88383752, + "learning_rate": 3.987714358473116e-06, + "loss": 0.9062916, + "num_input_tokens_seen": 62658505, + "router_z_loss_clip": 1.11523438, + "router_z_loss_mlp": 0.19604492, + "step": 2214, + "time_per_iteration": 2.5364749431610107 + }, + { + "auxiliary_loss_clip": 0.01058637, + "auxiliary_loss_mlp": 0.01000484, + "balance_loss_clip": 1.02924871, + "balance_loss_mlp": 0.999107, + "epoch": 0.06427369276304336, + "flos": 62739857715840.0, + "grad_norm": 0.6629897597893866, + "language_loss": 0.51417208, + "learning_rate": 3.987693547918787e-06, + "loss": 0.53476322, + "num_input_tokens_seen": 62721085, + "router_z_loss_clip": 0.29296875, + "router_z_loss_mlp": 0.01379395, + "step": 2215, + "time_per_iteration": 3.0747292041778564 + }, + { + "auxiliary_loss_clip": 0.01183677, + "auxiliary_loss_mlp": 0.01050898, + "balance_loss_clip": 1.07113409, + "balance_loss_mlp": 1.0311147, + "epoch": 0.0643027102315594, + "flos": 15881422377600.0, + "grad_norm": 2.8237314912127904, + "language_loss": 0.91714525, + "learning_rate": 3.9876727198083445e-06, + "loss": 0.93949103, + "num_input_tokens_seen": 62733965, + "router_z_loss_clip": 1.12695312, + "router_z_loss_mlp": 0.19812012, + "step": 2216, + "time_per_iteration": 2.4467148780822754 + }, + { + "auxiliary_loss_clip": 0.01182018, + "auxiliary_loss_mlp": 0.01055158, + "balance_loss_clip": 1.07086194, + "balance_loss_mlp": 1.0356909, + "epoch": 0.06433172770007545, + "flos": 12345694093440.0, + "grad_norm": 1.7202758896629045, + "language_loss": 0.65966332, + "learning_rate": 3.987651874141972e-06, + "loss": 0.68203509, + "num_input_tokens_seen": 62752530, + "router_z_loss_clip": 1.11132812, + "router_z_loss_mlp": 0.19470215, + "step": 2217, + "time_per_iteration": 2.524109363555908 + }, + { + "auxiliary_loss_clip": 0.01056611, + "auxiliary_loss_mlp": 0.01003158, + "balance_loss_clip": 1.02751863, + "balance_loss_mlp": 1.00185895, + "epoch": 0.0643607451685915, + "flos": 74776395450240.0, + "grad_norm": 0.6884950726558617, + "language_loss": 0.5961414, + "learning_rate": 3.987631010919853e-06, + "loss": 0.61673909, + "num_input_tokens_seen": 62812975, + "router_z_loss_clip": 0.29101562, + "router_z_loss_mlp": 0.01300049, + "step": 2218, + "time_per_iteration": 3.131666660308838 + }, + { + "auxiliary_loss_clip": 0.01055514, + "auxiliary_loss_mlp": 0.01002057, + "balance_loss_clip": 1.02656114, + "balance_loss_mlp": 1.00066853, + "epoch": 0.06438976263710754, + "flos": 74794603645440.0, + "grad_norm": 0.7390697392256236, + "language_loss": 0.44544661, + "learning_rate": 3.9876101301421735e-06, + "loss": 0.46602237, + "num_input_tokens_seen": 62877450, + "router_z_loss_clip": 0.2890625, + "router_z_loss_mlp": 0.01391602, + "step": 2219, + "time_per_iteration": 3.2780511379241943 + }, + { + "auxiliary_loss_clip": 0.0118362, + "auxiliary_loss_mlp": 0.01058188, + "balance_loss_clip": 1.07045352, + "balance_loss_mlp": 1.03667116, + "epoch": 0.06441878010562359, + "flos": 27083935244160.0, + "grad_norm": 2.2239466556518646, + "language_loss": 0.85962486, + "learning_rate": 3.987589231809117e-06, + "loss": 0.88204294, + "num_input_tokens_seen": 62894975, + "router_z_loss_clip": 1.13085938, + "router_z_loss_mlp": 0.21508789, + "step": 2220, + "time_per_iteration": 2.557861566543579 + }, + { + "auxiliary_loss_clip": 0.01053737, + "auxiliary_loss_mlp": 0.0100298, + "balance_loss_clip": 1.02505636, + "balance_loss_mlp": 1.00163925, + "epoch": 0.06444779757413963, + "flos": 74779986810240.0, + "grad_norm": 0.6680962770324186, + "language_loss": 0.53834724, + "learning_rate": 3.987568315920868e-06, + "loss": 0.55891442, + "num_input_tokens_seen": 62964695, + "router_z_loss_clip": 0.28710938, + "router_z_loss_mlp": 0.01342773, + "step": 2221, + "time_per_iteration": 3.1802685260772705 + }, + { + "auxiliary_loss_clip": 0.01170695, + "auxiliary_loss_mlp": 0.01051196, + "balance_loss_clip": 1.0697217, + "balance_loss_mlp": 1.03363645, + "epoch": 0.06447681504265568, + "flos": 25112798547840.0, + "grad_norm": 2.373939793495944, + "language_loss": 0.98140246, + "learning_rate": 3.987547382477611e-06, + "loss": 1.00362134, + "num_input_tokens_seen": 62978280, + "router_z_loss_clip": 1.00830078, + "router_z_loss_mlp": 0.17553711, + "step": 2222, + "time_per_iteration": 2.4696528911590576 + }, + { + "auxiliary_loss_clip": 0.01183283, + "auxiliary_loss_mlp": 0.01067342, + "balance_loss_clip": 1.07063198, + "balance_loss_mlp": 1.04572988, + "epoch": 0.06450583251117173, + "flos": 26320654212480.0, + "grad_norm": 1.8869177184915715, + "language_loss": 0.83870137, + "learning_rate": 3.987526431479533e-06, + "loss": 0.8612076, + "num_input_tokens_seen": 62997420, + "router_z_loss_clip": 1.12695312, + "router_z_loss_mlp": 0.21630859, + "step": 2223, + "time_per_iteration": 2.562016248703003 + }, + { + "auxiliary_loss_clip": 0.01181999, + "auxiliary_loss_mlp": 0.01059641, + "balance_loss_clip": 1.07051659, + "balance_loss_mlp": 1.0378139, + "epoch": 0.06453484997968777, + "flos": 12962890512000.0, + "grad_norm": 2.5030830918556424, + "language_loss": 0.87073034, + "learning_rate": 3.987505462926815e-06, + "loss": 0.89314675, + "num_input_tokens_seen": 63009905, + "router_z_loss_clip": 1.11425781, + "router_z_loss_mlp": 0.21826172, + "step": 2224, + "time_per_iteration": 2.4651217460632324 + }, + { + "auxiliary_loss_clip": 0.01178552, + "auxiliary_loss_mlp": 0.01053282, + "balance_loss_clip": 1.07034266, + "balance_loss_mlp": 1.03311205, + "epoch": 0.06456386744820382, + "flos": 11868046012800.0, + "grad_norm": 2.237383693665359, + "language_loss": 0.80568695, + "learning_rate": 3.987484476819645e-06, + "loss": 0.82800519, + "num_input_tokens_seen": 63026240, + "router_z_loss_clip": 1.08300781, + "router_z_loss_mlp": 0.20178223, + "step": 2225, + "time_per_iteration": 2.4610068798065186 + }, + { + "auxiliary_loss_clip": 0.01052627, + "auxiliary_loss_mlp": 0.0101011, + "balance_loss_clip": 1.02408147, + "balance_loss_mlp": 1.00878072, + "epoch": 0.06459288491671987, + "flos": 74774204720640.0, + "grad_norm": 0.6428528146235357, + "language_loss": 0.48303682, + "learning_rate": 3.987463473158208e-06, + "loss": 0.5036642, + "num_input_tokens_seen": 63094350, + "router_z_loss_clip": 0.28515625, + "router_z_loss_mlp": 0.01330566, + "step": 2226, + "time_per_iteration": 3.3478047847747803 + }, + { + "auxiliary_loss_clip": 0.01053238, + "auxiliary_loss_mlp": 0.01012226, + "balance_loss_clip": 1.0246278, + "balance_loss_mlp": 1.01097453, + "epoch": 0.06462190238523591, + "flos": 74777831994240.0, + "grad_norm": 0.736465608780735, + "language_loss": 0.48054796, + "learning_rate": 3.98744245194269e-06, + "loss": 0.50120258, + "num_input_tokens_seen": 63152460, + "router_z_loss_clip": 0.28515625, + "router_z_loss_mlp": 0.01251221, + "step": 2227, + "time_per_iteration": 3.1088173389434814 + }, + { + "auxiliary_loss_clip": 0.0119132, + "auxiliary_loss_mlp": 0.01055338, + "balance_loss_clip": 1.0729903, + "balance_loss_mlp": 1.03487015, + "epoch": 0.06465091985375196, + "flos": 32562000495360.0, + "grad_norm": 2.6822908636475753, + "language_loss": 0.78433532, + "learning_rate": 3.9874214131732765e-06, + "loss": 0.80680192, + "num_input_tokens_seen": 63169900, + "router_z_loss_clip": 1.18261719, + "router_z_loss_mlp": 0.20446777, + "step": 2228, + "time_per_iteration": 2.5922489166259766 + }, + { + "auxiliary_loss_clip": 0.01186456, + "auxiliary_loss_mlp": 0.01056254, + "balance_loss_clip": 1.0699327, + "balance_loss_mlp": 1.03603625, + "epoch": 0.06467993732226801, + "flos": 25950450689280.0, + "grad_norm": 3.406722460482486, + "language_loss": 0.91765803, + "learning_rate": 3.987400356850152e-06, + "loss": 0.94008517, + "num_input_tokens_seen": 63183935, + "router_z_loss_clip": 1.16699219, + "router_z_loss_mlp": 0.20214844, + "step": 2229, + "time_per_iteration": 2.5567896366119385 + }, + { + "auxiliary_loss_clip": 0.01183375, + "auxiliary_loss_mlp": 0.01049443, + "balance_loss_clip": 1.07333493, + "balance_loss_mlp": 1.03028619, + "epoch": 0.06470895479078405, + "flos": 19057685304960.0, + "grad_norm": 3.45100824649942, + "language_loss": 0.74591649, + "learning_rate": 3.987379282973503e-06, + "loss": 0.76824468, + "num_input_tokens_seen": 63197010, + "router_z_loss_clip": 1.10058594, + "router_z_loss_mlp": 0.19165039, + "step": 2230, + "time_per_iteration": 2.4944891929626465 + }, + { + "auxiliary_loss_clip": 0.01175953, + "auxiliary_loss_mlp": 0.01044279, + "balance_loss_clip": 1.07046723, + "balance_loss_mlp": 1.02762532, + "epoch": 0.0647379722593001, + "flos": 40188421169280.0, + "grad_norm": 1.7753365743518268, + "language_loss": 0.73242188, + "learning_rate": 3.987358191543516e-06, + "loss": 0.75462413, + "num_input_tokens_seen": 63223310, + "router_z_loss_clip": 1.05517578, + "router_z_loss_mlp": 0.16662598, + "step": 2231, + "time_per_iteration": 2.688876152038574 + }, + { + "auxiliary_loss_clip": 0.01182383, + "auxiliary_loss_mlp": 0.01051979, + "balance_loss_clip": 1.07221699, + "balance_loss_mlp": 1.03338218, + "epoch": 0.06476698972781615, + "flos": 31072534202880.0, + "grad_norm": 1.6391995284924021, + "language_loss": 0.87212145, + "learning_rate": 3.987337082560378e-06, + "loss": 0.89446509, + "num_input_tokens_seen": 63248040, + "router_z_loss_clip": 1.1015625, + "router_z_loss_mlp": 0.18591309, + "step": 2232, + "time_per_iteration": 2.665264368057251 + }, + { + "auxiliary_loss_clip": 0.01179114, + "auxiliary_loss_mlp": 0.01056367, + "balance_loss_clip": 1.07261324, + "balance_loss_mlp": 1.03805006, + "epoch": 0.06479600719633219, + "flos": 30667856601600.0, + "grad_norm": 2.2545580120764863, + "language_loss": 0.70761585, + "learning_rate": 3.987315956024273e-06, + "loss": 0.72997069, + "num_input_tokens_seen": 63269710, + "router_z_loss_clip": 1.06542969, + "router_z_loss_mlp": 0.18328857, + "step": 2233, + "time_per_iteration": 2.585914134979248 + }, + { + "auxiliary_loss_clip": 0.01172154, + "auxiliary_loss_mlp": 0.01045317, + "balance_loss_clip": 1.0687592, + "balance_loss_mlp": 1.02691066, + "epoch": 0.06482502466484824, + "flos": 32409451434240.0, + "grad_norm": 2.078553711247239, + "language_loss": 0.78286386, + "learning_rate": 3.987294811935391e-06, + "loss": 0.80503857, + "num_input_tokens_seen": 63287420, + "router_z_loss_clip": 1.03320312, + "router_z_loss_mlp": 0.18414307, + "step": 2234, + "time_per_iteration": 2.6184847354888916 + }, + { + "auxiliary_loss_clip": 0.01172111, + "auxiliary_loss_mlp": 0.01057819, + "balance_loss_clip": 1.06910014, + "balance_loss_mlp": 1.04053926, + "epoch": 0.06485404213336428, + "flos": 24236829573120.0, + "grad_norm": 2.215262449448004, + "language_loss": 1.01481843, + "learning_rate": 3.987273650293917e-06, + "loss": 1.03711784, + "num_input_tokens_seen": 63305950, + "router_z_loss_clip": 1.02929688, + "router_z_loss_mlp": 0.17297363, + "step": 2235, + "time_per_iteration": 2.6506905555725098 + }, + { + "auxiliary_loss_clip": 0.0118118, + "auxiliary_loss_mlp": 0.01057146, + "balance_loss_clip": 1.06781387, + "balance_loss_mlp": 1.03711891, + "epoch": 0.06488305960188033, + "flos": 23543681846400.0, + "grad_norm": 2.190852021274531, + "language_loss": 0.89345306, + "learning_rate": 3.987252471100038e-06, + "loss": 0.91583639, + "num_input_tokens_seen": 63319245, + "router_z_loss_clip": 1.13183594, + "router_z_loss_mlp": 0.20043945, + "step": 2236, + "time_per_iteration": 2.5856895446777344 + }, + { + "auxiliary_loss_clip": 0.01178267, + "auxiliary_loss_mlp": 0.01071265, + "balance_loss_clip": 1.07032835, + "balance_loss_mlp": 1.05241776, + "epoch": 0.06491207707039638, + "flos": 25602477684480.0, + "grad_norm": 2.0790734390052097, + "language_loss": 0.73647249, + "learning_rate": 3.98723127435394e-06, + "loss": 0.75896782, + "num_input_tokens_seen": 63337360, + "router_z_loss_clip": 1.07910156, + "router_z_loss_mlp": 0.18835449, + "step": 2237, + "time_per_iteration": 2.565335750579834 + }, + { + "auxiliary_loss_clip": 0.01178633, + "auxiliary_loss_mlp": 0.01056048, + "balance_loss_clip": 1.07241106, + "balance_loss_mlp": 1.03916776, + "epoch": 0.06494109453891242, + "flos": 26935623987840.0, + "grad_norm": 2.7362967887221927, + "language_loss": 0.88285953, + "learning_rate": 3.987210060055812e-06, + "loss": 0.90520632, + "num_input_tokens_seen": 63349840, + "router_z_loss_clip": 1.06054688, + "router_z_loss_mlp": 0.16882324, + "step": 2238, + "time_per_iteration": 2.5547034740448 + }, + { + "auxiliary_loss_clip": 0.01180272, + "auxiliary_loss_mlp": 0.01058817, + "balance_loss_clip": 1.07413709, + "balance_loss_mlp": 1.03853405, + "epoch": 0.06497011200742847, + "flos": 43975739289600.0, + "grad_norm": 2.913272151841331, + "language_loss": 0.9103328, + "learning_rate": 3.98718882820584e-06, + "loss": 0.9327237, + "num_input_tokens_seen": 63365640, + "router_z_loss_clip": 1.06103516, + "router_z_loss_mlp": 0.20269775, + "step": 2239, + "time_per_iteration": 2.591001272201538 + }, + { + "auxiliary_loss_clip": 0.01179734, + "auxiliary_loss_mlp": 0.01068913, + "balance_loss_clip": 1.06931138, + "balance_loss_mlp": 1.04956496, + "epoch": 0.06499912947594452, + "flos": 66690531912960.0, + "grad_norm": 2.7738666448983476, + "language_loss": 0.83852553, + "learning_rate": 3.9871675788042125e-06, + "loss": 0.86101204, + "num_input_tokens_seen": 63391835, + "router_z_loss_clip": 1.10498047, + "router_z_loss_mlp": 0.19342041, + "step": 2240, + "time_per_iteration": 2.894853115081787 + }, + { + "auxiliary_loss_clip": 0.01184223, + "auxiliary_loss_mlp": 0.01046119, + "balance_loss_clip": 1.07188511, + "balance_loss_mlp": 1.02574563, + "epoch": 0.06502814694446056, + "flos": 59811662113920.0, + "grad_norm": 2.7433287788813727, + "language_loss": 0.81530166, + "learning_rate": 3.987146311851118e-06, + "loss": 0.83760512, + "num_input_tokens_seen": 63413250, + "router_z_loss_clip": 1.12451172, + "router_z_loss_mlp": 0.20367432, + "step": 2241, + "time_per_iteration": 2.760631799697876 + }, + { + "auxiliary_loss_clip": 0.01186736, + "auxiliary_loss_mlp": 0.01047242, + "balance_loss_clip": 1.07507181, + "balance_loss_mlp": 1.02691722, + "epoch": 0.06505716441297661, + "flos": 33099223282560.0, + "grad_norm": 2.9580647788710923, + "language_loss": 0.91967434, + "learning_rate": 3.987125027346741e-06, + "loss": 0.94201416, + "num_input_tokens_seen": 63425980, + "router_z_loss_clip": 1.11621094, + "router_z_loss_mlp": 0.20300293, + "step": 2242, + "time_per_iteration": 2.5785651206970215 + }, + { + "auxiliary_loss_clip": 0.01177062, + "auxiliary_loss_mlp": 0.01048975, + "balance_loss_clip": 1.06941307, + "balance_loss_mlp": 1.02969909, + "epoch": 0.06508618188149266, + "flos": 28431411073920.0, + "grad_norm": 2.143614580072257, + "language_loss": 0.8797586, + "learning_rate": 3.987103725291273e-06, + "loss": 0.9020189, + "num_input_tokens_seen": 63441355, + "router_z_loss_clip": 1.07714844, + "router_z_loss_mlp": 0.19262695, + "step": 2243, + "time_per_iteration": 2.570260524749756 + }, + { + "auxiliary_loss_clip": 0.01186975, + "auxiliary_loss_mlp": 0.0105586, + "balance_loss_clip": 1.07796669, + "balance_loss_mlp": 1.03696537, + "epoch": 0.0651151993500087, + "flos": 44122075297920.0, + "grad_norm": 2.167230971597651, + "language_loss": 0.82372588, + "learning_rate": 3.9870824056849e-06, + "loss": 0.84615427, + "num_input_tokens_seen": 63458600, + "router_z_loss_clip": 1.08984375, + "router_z_loss_mlp": 0.18896484, + "step": 2244, + "time_per_iteration": 2.6659164428710938 + }, + { + "auxiliary_loss_clip": 0.01179076, + "auxiliary_loss_mlp": 0.01059048, + "balance_loss_clip": 1.07298589, + "balance_loss_mlp": 1.03821623, + "epoch": 0.06514421681852475, + "flos": 27700305649920.0, + "grad_norm": 2.286366993739695, + "language_loss": 0.74127722, + "learning_rate": 3.987061068527812e-06, + "loss": 0.76365852, + "num_input_tokens_seen": 63472940, + "router_z_loss_clip": 1.06103516, + "router_z_loss_mlp": 0.20819092, + "step": 2245, + "time_per_iteration": 2.6660752296447754 + }, + { + "auxiliary_loss_clip": 0.01060476, + "auxiliary_loss_mlp": 0.01032891, + "balance_loss_clip": 1.03162861, + "balance_loss_mlp": 1.03141904, + "epoch": 0.0651732342870408, + "flos": 64451611324800.0, + "grad_norm": 0.7661344882259787, + "language_loss": 0.50627768, + "learning_rate": 3.987039713820196e-06, + "loss": 0.52721131, + "num_input_tokens_seen": 63534065, + "router_z_loss_clip": 0.2890625, + "router_z_loss_mlp": 0.01470947, + "step": 2246, + "time_per_iteration": 3.1567392349243164 + }, + { + "auxiliary_loss_clip": 0.01187138, + "auxiliary_loss_mlp": 0.01054216, + "balance_loss_clip": 1.07610965, + "balance_loss_mlp": 1.03399801, + "epoch": 0.06520225175555684, + "flos": 11357827274880.0, + "grad_norm": 2.9183811466972007, + "language_loss": 0.92786753, + "learning_rate": 3.9870183415622415e-06, + "loss": 0.95028096, + "num_input_tokens_seen": 63545540, + "router_z_loss_clip": 1.11132812, + "router_z_loss_mlp": 0.20227051, + "step": 2247, + "time_per_iteration": 2.504910945892334 + }, + { + "auxiliary_loss_clip": 0.01061113, + "auxiliary_loss_mlp": 0.01005468, + "balance_loss_clip": 1.0326426, + "balance_loss_mlp": 1.00390649, + "epoch": 0.0652312692240729, + "flos": 63013351870080.0, + "grad_norm": 0.7861809377627832, + "language_loss": 0.46556634, + "learning_rate": 3.986996951754136e-06, + "loss": 0.48623216, + "num_input_tokens_seen": 63598905, + "router_z_loss_clip": 0.28515625, + "router_z_loss_mlp": 0.015625, + "step": 2248, + "time_per_iteration": 3.080961227416992 + }, + { + "auxiliary_loss_clip": 0.01176035, + "auxiliary_loss_mlp": 0.01056502, + "balance_loss_clip": 1.07117069, + "balance_loss_mlp": 1.03639174, + "epoch": 0.06526028669258895, + "flos": 27777406193280.0, + "grad_norm": 2.396524947202804, + "language_loss": 0.88550627, + "learning_rate": 3.98697554439607e-06, + "loss": 0.90783167, + "num_input_tokens_seen": 63617915, + "router_z_loss_clip": 1.04980469, + "router_z_loss_mlp": 0.20117188, + "step": 2249, + "time_per_iteration": 2.646132469177246 + }, + { + "auxiliary_loss_clip": 0.01060307, + "auxiliary_loss_mlp": 0.00996993, + "balance_loss_clip": 1.03156948, + "balance_loss_mlp": 0.99540728, + "epoch": 0.06528930416110498, + "flos": 74767488877440.0, + "grad_norm": 0.7503431080584452, + "language_loss": 0.56557393, + "learning_rate": 3.9869541194882326e-06, + "loss": 0.58614695, + "num_input_tokens_seen": 63677335, + "router_z_loss_clip": 0.28710938, + "router_z_loss_mlp": 0.01586914, + "step": 2250, + "time_per_iteration": 3.0091700553894043 + }, + { + "auxiliary_loss_clip": 0.01177292, + "auxiliary_loss_mlp": 0.01068738, + "balance_loss_clip": 1.07204545, + "balance_loss_mlp": 1.04816866, + "epoch": 0.06531832162962103, + "flos": 24163643612160.0, + "grad_norm": 2.47617248953945, + "language_loss": 0.90598387, + "learning_rate": 3.986932677030812e-06, + "loss": 0.92844415, + "num_input_tokens_seen": 63692445, + "router_z_loss_clip": 1.05078125, + "router_z_loss_mlp": 0.20550537, + "step": 2251, + "time_per_iteration": 2.5650298595428467 + }, + { + "auxiliary_loss_clip": 0.01176329, + "auxiliary_loss_mlp": 0.01047756, + "balance_loss_clip": 1.07222748, + "balance_loss_mlp": 1.0290643, + "epoch": 0.06534733909813707, + "flos": 21791241106560.0, + "grad_norm": 2.6157177557136495, + "language_loss": 0.85856485, + "learning_rate": 3.9869112170239975e-06, + "loss": 0.88080573, + "num_input_tokens_seen": 63705245, + "router_z_loss_clip": 1.04199219, + "router_z_loss_mlp": 0.18701172, + "step": 2252, + "time_per_iteration": 2.514342784881592 + }, + { + "auxiliary_loss_clip": 0.01175943, + "auxiliary_loss_mlp": 0.01049601, + "balance_loss_clip": 1.07047427, + "balance_loss_mlp": 1.02974045, + "epoch": 0.06537635656665312, + "flos": 28178420607360.0, + "grad_norm": 1.828506312068361, + "language_loss": 0.81172574, + "learning_rate": 3.98688973946798e-06, + "loss": 0.83398116, + "num_input_tokens_seen": 63725945, + "router_z_loss_clip": 1.05566406, + "router_z_loss_mlp": 0.19873047, + "step": 2253, + "time_per_iteration": 2.6422674655914307 + }, + { + "auxiliary_loss_clip": 0.01183234, + "auxiliary_loss_mlp": 0.01069482, + "balance_loss_clip": 1.07534909, + "balance_loss_mlp": 1.04657054, + "epoch": 0.06540537403516918, + "flos": 11028346796160.0, + "grad_norm": 3.1428583675859434, + "language_loss": 0.79362011, + "learning_rate": 3.986868244362947e-06, + "loss": 0.81614733, + "num_input_tokens_seen": 63736825, + "router_z_loss_clip": 1.07861328, + "router_z_loss_mlp": 0.22924805, + "step": 2254, + "time_per_iteration": 2.5076510906219482 + }, + { + "auxiliary_loss_clip": 0.01198101, + "auxiliary_loss_mlp": 0.0106939, + "balance_loss_clip": 1.07825828, + "balance_loss_mlp": 1.0460428, + "epoch": 0.06543439150368521, + "flos": 34160347889280.0, + "grad_norm": 2.3828134474330125, + "language_loss": 1.0034827, + "learning_rate": 3.986846731709091e-06, + "loss": 1.02615762, + "num_input_tokens_seen": 63756660, + "router_z_loss_clip": 1.19824219, + "router_z_loss_mlp": 0.23352051, + "step": 2255, + "time_per_iteration": 2.6556496620178223 + }, + { + "auxiliary_loss_clip": 0.01169824, + "auxiliary_loss_mlp": 0.01058134, + "balance_loss_clip": 1.07148242, + "balance_loss_mlp": 1.04192138, + "epoch": 0.06546340897220126, + "flos": 41820524369280.0, + "grad_norm": 3.8821023052058874, + "language_loss": 0.84657466, + "learning_rate": 3.9868252015066e-06, + "loss": 0.86885428, + "num_input_tokens_seen": 63773305, + "router_z_loss_clip": 0.98193359, + "router_z_loss_mlp": 0.16223145, + "step": 2256, + "time_per_iteration": 2.700730085372925 + }, + { + "auxiliary_loss_clip": 0.01188293, + "auxiliary_loss_mlp": 0.01057794, + "balance_loss_clip": 1.07881224, + "balance_loss_mlp": 1.03818393, + "epoch": 0.06549242644071732, + "flos": 24309656398080.0, + "grad_norm": 2.4491529817056947, + "language_loss": 0.95558971, + "learning_rate": 3.9868036537556645e-06, + "loss": 0.97805059, + "num_input_tokens_seen": 63787955, + "router_z_loss_clip": 1.09472656, + "router_z_loss_mlp": 0.19592285, + "step": 2257, + "time_per_iteration": 2.515831708908081 + }, + { + "auxiliary_loss_clip": 0.01180796, + "auxiliary_loss_mlp": 0.01050875, + "balance_loss_clip": 1.07344103, + "balance_loss_mlp": 1.03190911, + "epoch": 0.06552144390923335, + "flos": 35511846042240.0, + "grad_norm": 3.2363302625580936, + "language_loss": 0.75245285, + "learning_rate": 3.986782088456476e-06, + "loss": 0.77476954, + "num_input_tokens_seen": 63804110, + "router_z_loss_clip": 1.07470703, + "router_z_loss_mlp": 0.18969727, + "step": 2258, + "time_per_iteration": 2.6189770698547363 + }, + { + "auxiliary_loss_clip": 0.01191738, + "auxiliary_loss_mlp": 0.01076253, + "balance_loss_clip": 1.07916403, + "balance_loss_mlp": 1.05400872, + "epoch": 0.0655504613777494, + "flos": 11174682804480.0, + "grad_norm": 3.895418701883983, + "language_loss": 0.90214688, + "learning_rate": 3.986760505609224e-06, + "loss": 0.9248268, + "num_input_tokens_seen": 63815595, + "router_z_loss_clip": 1.12792969, + "router_z_loss_mlp": 0.22265625, + "step": 2259, + "time_per_iteration": 2.5258774757385254 + }, + { + "auxiliary_loss_clip": 0.01187214, + "auxiliary_loss_mlp": 0.01061813, + "balance_loss_clip": 1.07535636, + "balance_loss_mlp": 1.03996754, + "epoch": 0.06557947884626546, + "flos": 21937900337280.0, + "grad_norm": 2.1559004943220206, + "language_loss": 0.95658827, + "learning_rate": 3.986738905214099e-06, + "loss": 0.97907853, + "num_input_tokens_seen": 63832200, + "router_z_loss_clip": 1.11767578, + "router_z_loss_mlp": 0.21844482, + "step": 2260, + "time_per_iteration": 2.5241410732269287 + }, + { + "auxiliary_loss_clip": 0.01058023, + "auxiliary_loss_mlp": 0.01020322, + "balance_loss_clip": 1.02960575, + "balance_loss_mlp": 1.01886141, + "epoch": 0.0656084963147815, + "flos": 62437848163200.0, + "grad_norm": 0.7277335286083877, + "language_loss": 0.5308066, + "learning_rate": 3.986717287271291e-06, + "loss": 0.55159009, + "num_input_tokens_seen": 63894035, + "router_z_loss_clip": 0.28320312, + "router_z_loss_mlp": 0.0145874, + "step": 2261, + "time_per_iteration": 3.134208917617798 + }, + { + "auxiliary_loss_clip": 0.01184132, + "auxiliary_loss_mlp": 0.01071581, + "balance_loss_clip": 1.07723093, + "balance_loss_mlp": 1.05206633, + "epoch": 0.06563751378329755, + "flos": 23324878149120.0, + "grad_norm": 3.23215007692184, + "language_loss": 1.01923907, + "learning_rate": 3.986695651780994e-06, + "loss": 1.04179621, + "num_input_tokens_seen": 63909680, + "router_z_loss_clip": 1.06933594, + "router_z_loss_mlp": 0.1953125, + "step": 2262, + "time_per_iteration": 2.544898271560669 + }, + { + "auxiliary_loss_clip": 0.01057001, + "auxiliary_loss_mlp": 0.01015496, + "balance_loss_clip": 1.02842259, + "balance_loss_mlp": 1.0140183, + "epoch": 0.0656665312518136, + "flos": 68258215048320.0, + "grad_norm": 0.6561524656902831, + "language_loss": 0.4971109, + "learning_rate": 3.986673998743396e-06, + "loss": 0.51783586, + "num_input_tokens_seen": 63971455, + "router_z_loss_clip": 0.28515625, + "router_z_loss_mlp": 0.01477051, + "step": 2263, + "time_per_iteration": 3.201497793197632 + }, + { + "auxiliary_loss_clip": 0.01176425, + "auxiliary_loss_mlp": 0.01050401, + "balance_loss_clip": 1.06950378, + "balance_loss_mlp": 1.03123212, + "epoch": 0.06569554872032964, + "flos": 28398624935040.0, + "grad_norm": 1.6172738101652626, + "language_loss": 0.69949722, + "learning_rate": 3.986652328158688e-06, + "loss": 0.72176546, + "num_input_tokens_seen": 63997475, + "router_z_loss_clip": 1.06835938, + "router_z_loss_mlp": 0.19177246, + "step": 2264, + "time_per_iteration": 2.861994981765747 + }, + { + "auxiliary_loss_clip": 0.01185912, + "auxiliary_loss_mlp": 0.01063015, + "balance_loss_clip": 1.0738529, + "balance_loss_mlp": 1.04140806, + "epoch": 0.06572456618884569, + "flos": 30036977101440.0, + "grad_norm": 2.5350325057957517, + "language_loss": 0.73659408, + "learning_rate": 3.986630640027065e-06, + "loss": 0.75908339, + "num_input_tokens_seen": 64011780, + "router_z_loss_clip": 1.12109375, + "router_z_loss_mlp": 0.21588135, + "step": 2265, + "time_per_iteration": 2.4974887371063232 + }, + { + "auxiliary_loss_clip": 0.01054744, + "auxiliary_loss_mlp": 0.01015085, + "balance_loss_clip": 1.0264262, + "balance_loss_mlp": 1.01354682, + "epoch": 0.06575358365736172, + "flos": 63176495443200.0, + "grad_norm": 0.7508485315883742, + "language_loss": 0.55221558, + "learning_rate": 3.9866089343487155e-06, + "loss": 0.57291389, + "num_input_tokens_seen": 64070275, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01538086, + "step": 2266, + "time_per_iteration": 3.020430326461792 + }, + { + "auxiliary_loss_clip": 0.01181078, + "auxiliary_loss_mlp": 0.01054329, + "balance_loss_clip": 1.07503343, + "balance_loss_mlp": 1.03440893, + "epoch": 0.06578260112587778, + "flos": 16647001879680.0, + "grad_norm": 3.6212528137905435, + "language_loss": 0.79595828, + "learning_rate": 3.986587211123832e-06, + "loss": 0.81831235, + "num_input_tokens_seen": 64081275, + "router_z_loss_clip": 1.06005859, + "router_z_loss_mlp": 0.19909668, + "step": 2267, + "time_per_iteration": 2.47898530960083 + }, + { + "auxiliary_loss_clip": 0.01186589, + "auxiliary_loss_mlp": 0.01061568, + "balance_loss_clip": 1.07231379, + "balance_loss_mlp": 1.03802967, + "epoch": 0.06581161859439383, + "flos": 33139874499840.0, + "grad_norm": 3.1089657471609597, + "language_loss": 0.88969648, + "learning_rate": 3.986565470352606e-06, + "loss": 0.91217804, + "num_input_tokens_seen": 64097745, + "router_z_loss_clip": 1.14160156, + "router_z_loss_mlp": 0.23529053, + "step": 2268, + "time_per_iteration": 2.6126813888549805 + }, + { + "auxiliary_loss_clip": 0.01184496, + "auxiliary_loss_mlp": 0.01052499, + "balance_loss_clip": 1.07381487, + "balance_loss_mlp": 1.03303182, + "epoch": 0.06584063606290987, + "flos": 23433220546560.0, + "grad_norm": 2.4749172397927146, + "language_loss": 0.92822242, + "learning_rate": 3.986543712035231e-06, + "loss": 0.95059234, + "num_input_tokens_seen": 64111170, + "router_z_loss_clip": 1.10546875, + "router_z_loss_mlp": 0.19470215, + "step": 2269, + "time_per_iteration": 2.5946967601776123 + }, + { + "auxiliary_loss_clip": 0.0118122, + "auxiliary_loss_mlp": 0.01055862, + "balance_loss_clip": 1.06923556, + "balance_loss_mlp": 1.03526258, + "epoch": 0.06586965353142592, + "flos": 24638346777600.0, + "grad_norm": 2.3996764409652327, + "language_loss": 0.90967488, + "learning_rate": 3.986521936171897e-06, + "loss": 0.9320457, + "num_input_tokens_seen": 64129810, + "router_z_loss_clip": 1.12060547, + "router_z_loss_mlp": 0.20593262, + "step": 2270, + "time_per_iteration": 2.565694570541382 + }, + { + "auxiliary_loss_clip": 0.01190949, + "auxiliary_loss_mlp": 0.0105375, + "balance_loss_clip": 1.0756284, + "balance_loss_mlp": 1.03081417, + "epoch": 0.06589867099994197, + "flos": 31760654025600.0, + "grad_norm": 1.8800910854456283, + "language_loss": 0.71903175, + "learning_rate": 3.986500142762797e-06, + "loss": 0.74147874, + "num_input_tokens_seen": 64149590, + "router_z_loss_clip": 1.15332031, + "router_z_loss_mlp": 0.22924805, + "step": 2271, + "time_per_iteration": 2.5764713287353516 + }, + { + "auxiliary_loss_clip": 0.01172425, + "auxiliary_loss_mlp": 0.0106038, + "balance_loss_clip": 1.07131314, + "balance_loss_mlp": 1.04046011, + "epoch": 0.065927688468458, + "flos": 17010848695680.0, + "grad_norm": 3.937379872652416, + "language_loss": 0.96839476, + "learning_rate": 3.986478331808125e-06, + "loss": 0.99072278, + "num_input_tokens_seen": 64160415, + "router_z_loss_clip": 1.01025391, + "router_z_loss_mlp": 0.19934082, + "step": 2272, + "time_per_iteration": 2.458378791809082 + }, + { + "auxiliary_loss_clip": 0.0118424, + "auxiliary_loss_mlp": 0.0105281, + "balance_loss_clip": 1.07503986, + "balance_loss_mlp": 1.03290224, + "epoch": 0.06595670593697406, + "flos": 13656325547520.0, + "grad_norm": 2.403604996138453, + "language_loss": 0.63562417, + "learning_rate": 3.986456503308072e-06, + "loss": 0.65799469, + "num_input_tokens_seen": 64172850, + "router_z_loss_clip": 1.09326172, + "router_z_loss_mlp": 0.19909668, + "step": 2273, + "time_per_iteration": 2.4656898975372314 + }, + { + "auxiliary_loss_clip": 0.01052015, + "auxiliary_loss_mlp": 0.01003401, + "balance_loss_clip": 1.02386665, + "balance_loss_mlp": 1.00201857, + "epoch": 0.06598572340549011, + "flos": 63287495447040.0, + "grad_norm": 0.6772104828214783, + "language_loss": 0.52462399, + "learning_rate": 3.98643465726283e-06, + "loss": 0.54517817, + "num_input_tokens_seen": 64235440, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01385498, + "step": 2274, + "time_per_iteration": 3.0872764587402344 + }, + { + "auxiliary_loss_clip": 0.01192618, + "auxiliary_loss_mlp": 0.01061535, + "balance_loss_clip": 1.07630849, + "balance_loss_mlp": 1.03980303, + "epoch": 0.06601474087400615, + "flos": 21682180437120.0, + "grad_norm": 2.254810608852077, + "language_loss": 0.9370414, + "learning_rate": 3.986412793672596e-06, + "loss": 0.95958292, + "num_input_tokens_seen": 64249415, + "router_z_loss_clip": 1.16308594, + "router_z_loss_mlp": 0.21765137, + "step": 2275, + "time_per_iteration": 2.5264854431152344 + }, + { + "auxiliary_loss_clip": 0.01178579, + "auxiliary_loss_mlp": 0.01059211, + "balance_loss_clip": 1.07117534, + "balance_loss_mlp": 1.04008365, + "epoch": 0.0660437583425222, + "flos": 16538515827840.0, + "grad_norm": 2.189558674058844, + "language_loss": 0.75635123, + "learning_rate": 3.986390912537558e-06, + "loss": 0.77872914, + "num_input_tokens_seen": 64262605, + "router_z_loss_clip": 1.07519531, + "router_z_loss_mlp": 0.19122314, + "step": 2276, + "time_per_iteration": 2.469045877456665 + }, + { + "auxiliary_loss_clip": 0.01185644, + "auxiliary_loss_mlp": 0.01057077, + "balance_loss_clip": 1.07119656, + "balance_loss_mlp": 1.03731179, + "epoch": 0.06607277581103825, + "flos": 34425369411840.0, + "grad_norm": 1.8368979872933597, + "language_loss": 0.85993671, + "learning_rate": 3.986369013857914e-06, + "loss": 0.88236392, + "num_input_tokens_seen": 64286460, + "router_z_loss_clip": 1.14355469, + "router_z_loss_mlp": 0.19763184, + "step": 2277, + "time_per_iteration": 2.607201099395752 + }, + { + "auxiliary_loss_clip": 0.01189032, + "auxiliary_loss_mlp": 0.01052499, + "balance_loss_clip": 1.08019066, + "balance_loss_mlp": 1.03141129, + "epoch": 0.06610179327955429, + "flos": 11174144100480.0, + "grad_norm": 2.845340984308743, + "language_loss": 0.76980442, + "learning_rate": 3.986347097633853e-06, + "loss": 0.79221976, + "num_input_tokens_seen": 64300125, + "router_z_loss_clip": 1.08789062, + "router_z_loss_mlp": 0.21069336, + "step": 2278, + "time_per_iteration": 2.449399471282959 + }, + { + "auxiliary_loss_clip": 0.01184109, + "auxiliary_loss_mlp": 0.01051483, + "balance_loss_clip": 1.07702732, + "balance_loss_mlp": 1.02991867, + "epoch": 0.06613081074807034, + "flos": 37626445658880.0, + "grad_norm": 3.346207307307456, + "language_loss": 0.98466963, + "learning_rate": 3.986325163865571e-06, + "loss": 1.0070256, + "num_input_tokens_seen": 64317400, + "router_z_loss_clip": 1.07177734, + "router_z_loss_mlp": 0.21557617, + "step": 2279, + "time_per_iteration": 5.073980808258057 + }, + { + "auxiliary_loss_clip": 0.01194627, + "auxiliary_loss_mlp": 0.01058266, + "balance_loss_clip": 1.07537889, + "balance_loss_mlp": 1.03409004, + "epoch": 0.06615982821658639, + "flos": 29094753490560.0, + "grad_norm": 2.2844795145417685, + "language_loss": 1.01549053, + "learning_rate": 3.986303212553262e-06, + "loss": 1.03801942, + "num_input_tokens_seen": 64336125, + "router_z_loss_clip": 1.19238281, + "router_z_loss_mlp": 0.24188232, + "step": 2280, + "time_per_iteration": 5.027520656585693 + }, + { + "auxiliary_loss_clip": 0.01175784, + "auxiliary_loss_mlp": 0.01052899, + "balance_loss_clip": 1.06923509, + "balance_loss_mlp": 1.03243017, + "epoch": 0.06618884568510243, + "flos": 43068169324800.0, + "grad_norm": 2.126914955466725, + "language_loss": 0.78978413, + "learning_rate": 3.986281243697119e-06, + "loss": 0.81207097, + "num_input_tokens_seen": 64354865, + "router_z_loss_clip": 1.06591797, + "router_z_loss_mlp": 0.20458984, + "step": 2281, + "time_per_iteration": 5.002503395080566 + }, + { + "auxiliary_loss_clip": 0.01174742, + "auxiliary_loss_mlp": 0.01049051, + "balance_loss_clip": 1.07105064, + "balance_loss_mlp": 1.0299654, + "epoch": 0.06621786315361848, + "flos": 19276093952640.0, + "grad_norm": 3.6283254027931933, + "language_loss": 0.82987142, + "learning_rate": 3.986259257297337e-06, + "loss": 0.85210931, + "num_input_tokens_seen": 64368545, + "router_z_loss_clip": 1.03710938, + "router_z_loss_mlp": 0.1907959, + "step": 2282, + "time_per_iteration": 2.4949750900268555 + }, + { + "auxiliary_loss_clip": 0.01056096, + "auxiliary_loss_mlp": 0.01000891, + "balance_loss_clip": 1.02713013, + "balance_loss_mlp": 0.99963921, + "epoch": 0.06624688062213452, + "flos": 74773235053440.0, + "grad_norm": 0.662568294948, + "language_loss": 0.45518368, + "learning_rate": 3.9862372533541085e-06, + "loss": 0.47575361, + "num_input_tokens_seen": 64430150, + "router_z_loss_clip": 0.2890625, + "router_z_loss_mlp": 0.01251221, + "step": 2283, + "time_per_iteration": 3.1460390090942383 + }, + { + "auxiliary_loss_clip": 0.0118091, + "auxiliary_loss_mlp": 0.01057521, + "balance_loss_clip": 1.07293987, + "balance_loss_mlp": 1.03843546, + "epoch": 0.06627589809065057, + "flos": 36092126257920.0, + "grad_norm": 2.381286835725292, + "language_loss": 0.89282167, + "learning_rate": 3.986215231867629e-06, + "loss": 0.91520602, + "num_input_tokens_seen": 64448820, + "router_z_loss_clip": 1.07910156, + "router_z_loss_mlp": 0.19104004, + "step": 2284, + "time_per_iteration": 2.642836570739746 + }, + { + "auxiliary_loss_clip": 0.01182542, + "auxiliary_loss_mlp": 0.01063249, + "balance_loss_clip": 1.07500267, + "balance_loss_mlp": 1.04329324, + "epoch": 0.06630491555916662, + "flos": 33176287912320.0, + "grad_norm": 2.765737988676244, + "language_loss": 0.95845246, + "learning_rate": 3.986193192838093e-06, + "loss": 0.9809103, + "num_input_tokens_seen": 64464680, + "router_z_loss_clip": 1.07617188, + "router_z_loss_mlp": 0.19970703, + "step": 2285, + "time_per_iteration": 2.5973572731018066 + }, + { + "auxiliary_loss_clip": 0.01182234, + "auxiliary_loss_mlp": 0.0105437, + "balance_loss_clip": 1.07017756, + "balance_loss_mlp": 1.03179181, + "epoch": 0.06633393302768266, + "flos": 22341536444160.0, + "grad_norm": 2.5928443079470385, + "language_loss": 0.99455488, + "learning_rate": 3.986171136265695e-06, + "loss": 1.01692092, + "num_input_tokens_seen": 64481345, + "router_z_loss_clip": 1.12109375, + "router_z_loss_mlp": 0.22558594, + "step": 2286, + "time_per_iteration": 2.4789438247680664 + }, + { + "auxiliary_loss_clip": 0.01178512, + "auxiliary_loss_mlp": 0.01059335, + "balance_loss_clip": 1.07213831, + "balance_loss_mlp": 1.04089355, + "epoch": 0.06636295049619871, + "flos": 21171135686400.0, + "grad_norm": 1.905500004904581, + "language_loss": 0.73391753, + "learning_rate": 3.98614906215063e-06, + "loss": 0.75629604, + "num_input_tokens_seen": 64495465, + "router_z_loss_clip": 1.06298828, + "router_z_loss_mlp": 0.18444824, + "step": 2287, + "time_per_iteration": 2.5133841037750244 + }, + { + "auxiliary_loss_clip": 0.01183688, + "auxiliary_loss_mlp": 0.01065589, + "balance_loss_clip": 1.07488871, + "balance_loss_mlp": 1.0462172, + "epoch": 0.06639196796471476, + "flos": 39047179276800.0, + "grad_norm": 2.5200639886787184, + "language_loss": 0.98592073, + "learning_rate": 3.986126970493092e-06, + "loss": 1.00841355, + "num_input_tokens_seen": 64510355, + "router_z_loss_clip": 1.08935547, + "router_z_loss_mlp": 0.19384766, + "step": 2288, + "time_per_iteration": 2.637213945388794 + }, + { + "auxiliary_loss_clip": 0.01056599, + "auxiliary_loss_mlp": 0.01003856, + "balance_loss_clip": 1.02780032, + "balance_loss_mlp": 1.00253284, + "epoch": 0.0664209854332308, + "flos": 59306905221120.0, + "grad_norm": 0.7110675166015339, + "language_loss": 0.5290243, + "learning_rate": 3.986104861293277e-06, + "loss": 0.54962885, + "num_input_tokens_seen": 64569395, + "router_z_loss_clip": 0.28710938, + "router_z_loss_mlp": 0.01324463, + "step": 2289, + "time_per_iteration": 3.0258896350860596 + }, + { + "auxiliary_loss_clip": 0.01184557, + "auxiliary_loss_mlp": 0.01059564, + "balance_loss_clip": 1.07674503, + "balance_loss_mlp": 1.03855944, + "epoch": 0.06645000290174685, + "flos": 40144106764800.0, + "grad_norm": 3.142667523653755, + "language_loss": 0.98574483, + "learning_rate": 3.986082734551381e-06, + "loss": 1.0081861, + "num_input_tokens_seen": 64585380, + "router_z_loss_clip": 1.07763672, + "router_z_loss_mlp": 0.20983887, + "step": 2290, + "time_per_iteration": 2.5962073802948 + }, + { + "auxiliary_loss_clip": 0.01056821, + "auxiliary_loss_mlp": 0.01007858, + "balance_loss_clip": 1.02815127, + "balance_loss_mlp": 1.00654066, + "epoch": 0.0664790203702629, + "flos": 62991088416000.0, + "grad_norm": 0.6388277203906214, + "language_loss": 0.47988778, + "learning_rate": 3.986060590267598e-06, + "loss": 0.50053459, + "num_input_tokens_seen": 64647895, + "router_z_loss_clip": 0.28710938, + "router_z_loss_mlp": 0.01318359, + "step": 2291, + "time_per_iteration": 3.2084169387817383 + }, + { + "auxiliary_loss_clip": 0.01185528, + "auxiliary_loss_mlp": 0.01066582, + "balance_loss_clip": 1.07274497, + "balance_loss_mlp": 1.04431343, + "epoch": 0.06650803783877894, + "flos": 28588844384640.0, + "grad_norm": 1.9326341301872025, + "language_loss": 1.08174372, + "learning_rate": 3.986038428442125e-06, + "loss": 1.10426486, + "num_input_tokens_seen": 64670735, + "router_z_loss_clip": 1.12744141, + "router_z_loss_mlp": 0.22265625, + "step": 2292, + "time_per_iteration": 2.6752800941467285 + }, + { + "auxiliary_loss_clip": 0.01057081, + "auxiliary_loss_mlp": 0.01007921, + "balance_loss_clip": 1.02818787, + "balance_loss_mlp": 1.00668073, + "epoch": 0.06653705530729499, + "flos": 74777401031040.0, + "grad_norm": 0.6740522602902492, + "language_loss": 0.4946475, + "learning_rate": 3.986016249075156e-06, + "loss": 0.51529753, + "num_input_tokens_seen": 64739680, + "router_z_loss_clip": 0.2890625, + "router_z_loss_mlp": 0.01239014, + "step": 2293, + "time_per_iteration": 3.324146032333374 + }, + { + "auxiliary_loss_clip": 0.01055164, + "auxiliary_loss_mlp": 0.01008154, + "balance_loss_clip": 1.02640557, + "balance_loss_mlp": 1.00686693, + "epoch": 0.06656607277581104, + "flos": 73097212498560.0, + "grad_norm": 0.653616529677956, + "language_loss": 0.49950078, + "learning_rate": 3.985994052166888e-06, + "loss": 0.52013397, + "num_input_tokens_seen": 64801605, + "router_z_loss_clip": 0.2890625, + "router_z_loss_mlp": 0.01287842, + "step": 2294, + "time_per_iteration": 3.087886095046997 + }, + { + "auxiliary_loss_clip": 0.01054384, + "auxiliary_loss_mlp": 0.01004738, + "balance_loss_clip": 1.02556372, + "balance_loss_mlp": 1.00348592, + "epoch": 0.06659509024432708, + "flos": 74761132170240.0, + "grad_norm": 0.6784059031187372, + "language_loss": 0.48863247, + "learning_rate": 3.985971837717517e-06, + "loss": 0.5092237, + "num_input_tokens_seen": 64862580, + "router_z_loss_clip": 0.2890625, + "router_z_loss_mlp": 0.01251221, + "step": 2295, + "time_per_iteration": 3.0413475036621094 + }, + { + "auxiliary_loss_clip": 0.01171704, + "auxiliary_loss_mlp": 0.01052873, + "balance_loss_clip": 1.07069921, + "balance_loss_mlp": 1.03455043, + "epoch": 0.06662410771284313, + "flos": 16499193413760.0, + "grad_norm": 3.9667649844946355, + "language_loss": 0.74055845, + "learning_rate": 3.985949605727239e-06, + "loss": 0.76280415, + "num_input_tokens_seen": 64874330, + "router_z_loss_clip": 1.00976562, + "router_z_loss_mlp": 0.18328857, + "step": 2296, + "time_per_iteration": 2.4613819122314453 + }, + { + "auxiliary_loss_clip": 0.01182772, + "auxiliary_loss_mlp": 0.01050867, + "balance_loss_clip": 1.07324576, + "balance_loss_mlp": 1.03128052, + "epoch": 0.06665312518135917, + "flos": 74737429194240.0, + "grad_norm": 1.8919898604084284, + "language_loss": 0.73267412, + "learning_rate": 3.9859273561962516e-06, + "loss": 0.75501049, + "num_input_tokens_seen": 64898580, + "router_z_loss_clip": 1.09472656, + "router_z_loss_mlp": 0.19592285, + "step": 2297, + "time_per_iteration": 2.88862943649292 + }, + { + "auxiliary_loss_clip": 0.01172311, + "auxiliary_loss_mlp": 0.01056799, + "balance_loss_clip": 1.07059562, + "balance_loss_mlp": 1.03987086, + "epoch": 0.06668214264987522, + "flos": 31532081829120.0, + "grad_norm": 2.4594219520388836, + "language_loss": 0.7786178, + "learning_rate": 3.985905089124749e-06, + "loss": 0.80090886, + "num_input_tokens_seen": 64912900, + "router_z_loss_clip": 1.01757812, + "router_z_loss_mlp": 0.16918945, + "step": 2298, + "time_per_iteration": 2.5407869815826416 + }, + { + "auxiliary_loss_clip": 0.01052159, + "auxiliary_loss_mlp": 0.01001663, + "balance_loss_clip": 1.023597, + "balance_loss_mlp": 1.00045896, + "epoch": 0.06671116011839127, + "flos": 60689752968960.0, + "grad_norm": 0.7502857013953578, + "language_loss": 0.50286877, + "learning_rate": 3.9858828045129285e-06, + "loss": 0.52340698, + "num_input_tokens_seen": 64970595, + "router_z_loss_clip": 0.28515625, + "router_z_loss_mlp": 0.01202393, + "step": 2299, + "time_per_iteration": 2.9467697143554688 + }, + { + "auxiliary_loss_clip": 0.01181506, + "auxiliary_loss_mlp": 0.01059552, + "balance_loss_clip": 1.07323325, + "balance_loss_mlp": 1.03917301, + "epoch": 0.06674017758690731, + "flos": 21537101404800.0, + "grad_norm": 2.107987635428309, + "language_loss": 0.98613143, + "learning_rate": 3.985860502360988e-06, + "loss": 1.00854206, + "num_input_tokens_seen": 64985405, + "router_z_loss_clip": 1.08300781, + "router_z_loss_mlp": 0.20355225, + "step": 2300, + "time_per_iteration": 2.5701940059661865 + }, + { + "auxiliary_loss_clip": 0.01186167, + "auxiliary_loss_mlp": 0.01062971, + "balance_loss_clip": 1.07121921, + "balance_loss_mlp": 1.04065514, + "epoch": 0.06676919505542336, + "flos": 28835945020800.0, + "grad_norm": 2.1777299868621856, + "language_loss": 1.08228004, + "learning_rate": 3.9858381826691245e-06, + "loss": 1.10477138, + "num_input_tokens_seen": 65002615, + "router_z_loss_clip": 1.14941406, + "router_z_loss_mlp": 0.22302246, + "step": 2301, + "time_per_iteration": 2.5519602298736572 + }, + { + "auxiliary_loss_clip": 0.01184532, + "auxiliary_loss_mlp": 0.01058428, + "balance_loss_clip": 1.07016575, + "balance_loss_mlp": 1.03530717, + "epoch": 0.06679821252393942, + "flos": 74737177799040.0, + "grad_norm": 1.9220640199065497, + "language_loss": 0.8715409, + "learning_rate": 3.985815845437535e-06, + "loss": 0.89397049, + "num_input_tokens_seen": 65028225, + "router_z_loss_clip": 1.14355469, + "router_z_loss_mlp": 0.23138428, + "step": 2302, + "time_per_iteration": 2.9108989238739014 + }, + { + "auxiliary_loss_clip": 0.01183153, + "auxiliary_loss_mlp": 0.01049565, + "balance_loss_clip": 1.07450354, + "balance_loss_mlp": 1.02969289, + "epoch": 0.06682722999245545, + "flos": 30110701766400.0, + "grad_norm": 2.457395228651539, + "language_loss": 0.85790867, + "learning_rate": 3.985793490666415e-06, + "loss": 0.88023579, + "num_input_tokens_seen": 65042385, + "router_z_loss_clip": 1.0859375, + "router_z_loss_mlp": 0.1987915, + "step": 2303, + "time_per_iteration": 2.550428867340088 + }, + { + "auxiliary_loss_clip": 0.01179652, + "auxiliary_loss_mlp": 0.01064877, + "balance_loss_clip": 1.06728935, + "balance_loss_mlp": 1.0426569, + "epoch": 0.0668562474609715, + "flos": 35072407054080.0, + "grad_norm": 1.8525100138441835, + "language_loss": 0.79519641, + "learning_rate": 3.9857711183559636e-06, + "loss": 0.81764174, + "num_input_tokens_seen": 65059690, + "router_z_loss_clip": 1.12402344, + "router_z_loss_mlp": 0.22229004, + "step": 2304, + "time_per_iteration": 2.6130125522613525 + }, + { + "auxiliary_loss_clip": 0.01179771, + "auxiliary_loss_mlp": 0.0105639, + "balance_loss_clip": 1.06909394, + "balance_loss_mlp": 1.03671992, + "epoch": 0.06688526492948756, + "flos": 19384472263680.0, + "grad_norm": 2.5614429835671104, + "language_loss": 0.95274538, + "learning_rate": 3.985748728506379e-06, + "loss": 0.97510695, + "num_input_tokens_seen": 65074565, + "router_z_loss_clip": 1.10595703, + "router_z_loss_mlp": 0.19677734, + "step": 2305, + "time_per_iteration": 2.4974937438964844 + }, + { + "auxiliary_loss_clip": 0.01055088, + "auxiliary_loss_mlp": 0.01011992, + "balance_loss_clip": 1.02722001, + "balance_loss_mlp": 1.01088333, + "epoch": 0.0669142823980036, + "flos": 74770505619840.0, + "grad_norm": 0.7144687916005723, + "language_loss": 0.49875388, + "learning_rate": 3.985726321117857e-06, + "loss": 0.51942468, + "num_input_tokens_seen": 65130105, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.0111084, + "step": 2306, + "time_per_iteration": 3.068343162536621 + }, + { + "auxiliary_loss_clip": 0.01186709, + "auxiliary_loss_mlp": 0.01058831, + "balance_loss_clip": 1.07182264, + "balance_loss_mlp": 1.0373975, + "epoch": 0.06694329986651965, + "flos": 31166762555520.0, + "grad_norm": 2.726529980347502, + "language_loss": 0.99079359, + "learning_rate": 3.985703896190597e-06, + "loss": 1.01324904, + "num_input_tokens_seen": 65144705, + "router_z_loss_clip": 1.1484375, + "router_z_loss_mlp": 0.21429443, + "step": 2307, + "time_per_iteration": 2.5879693031311035 + }, + { + "auxiliary_loss_clip": 0.01162638, + "auxiliary_loss_mlp": 0.01048444, + "balance_loss_clip": 1.06557465, + "balance_loss_mlp": 1.03095019, + "epoch": 0.0669723173350357, + "flos": 39050267846400.0, + "grad_norm": 3.052103612067467, + "language_loss": 0.78789866, + "learning_rate": 3.985681453724797e-06, + "loss": 0.81000948, + "num_input_tokens_seen": 65160610, + "router_z_loss_clip": 0.96972656, + "router_z_loss_mlp": 0.17492676, + "step": 2308, + "time_per_iteration": 2.636331081390381 + }, + { + "auxiliary_loss_clip": 0.01179089, + "auxiliary_loss_mlp": 0.01057085, + "balance_loss_clip": 1.06893384, + "balance_loss_mlp": 1.03662884, + "epoch": 0.06700133480355173, + "flos": 26024426749440.0, + "grad_norm": 2.229080338785681, + "language_loss": 0.71460885, + "learning_rate": 3.985658993720655e-06, + "loss": 0.73697066, + "num_input_tokens_seen": 65177205, + "router_z_loss_clip": 1.09960938, + "router_z_loss_mlp": 0.20446777, + "step": 2309, + "time_per_iteration": 2.5304441452026367 + }, + { + "auxiliary_loss_clip": 0.01177639, + "auxiliary_loss_mlp": 0.01054638, + "balance_loss_clip": 1.06855583, + "balance_loss_mlp": 1.03426492, + "epoch": 0.06703035227206779, + "flos": 35583595459200.0, + "grad_norm": 2.1595193741446175, + "language_loss": 0.84818637, + "learning_rate": 3.9856365161783685e-06, + "loss": 0.87050915, + "num_input_tokens_seen": 65193585, + "router_z_loss_clip": 1.08984375, + "router_z_loss_mlp": 0.20373535, + "step": 2310, + "time_per_iteration": 2.5993213653564453 + }, + { + "auxiliary_loss_clip": 0.01054155, + "auxiliary_loss_mlp": 0.01000019, + "balance_loss_clip": 1.02624488, + "balance_loss_mlp": 0.9988212, + "epoch": 0.06705936974058384, + "flos": 68251642859520.0, + "grad_norm": 0.6358584699288541, + "language_loss": 0.49873191, + "learning_rate": 3.985614021098138e-06, + "loss": 0.51927364, + "num_input_tokens_seen": 65257105, + "router_z_loss_clip": 0.27929688, + "router_z_loss_mlp": 0.01196289, + "step": 2311, + "time_per_iteration": 3.1311161518096924 + }, + { + "auxiliary_loss_clip": 0.01178195, + "auxiliary_loss_mlp": 0.0105302, + "balance_loss_clip": 1.07077181, + "balance_loss_mlp": 1.03282595, + "epoch": 0.06708838720909988, + "flos": 56855352119040.0, + "grad_norm": 1.77320195982654, + "language_loss": 0.84497678, + "learning_rate": 3.98559150848016e-06, + "loss": 0.86728901, + "num_input_tokens_seen": 65276950, + "router_z_loss_clip": 1.07421875, + "router_z_loss_mlp": 0.2019043, + "step": 2312, + "time_per_iteration": 2.897887706756592 + }, + { + "auxiliary_loss_clip": 0.01162823, + "auxiliary_loss_mlp": 0.01043433, + "balance_loss_clip": 1.06402075, + "balance_loss_mlp": 1.02496719, + "epoch": 0.06711740467761593, + "flos": 18803401948800.0, + "grad_norm": 1.9030766832798243, + "language_loss": 0.71716273, + "learning_rate": 3.985568978324634e-06, + "loss": 0.73922521, + "num_input_tokens_seen": 65292740, + "router_z_loss_clip": 0.98730469, + "router_z_loss_mlp": 0.18457031, + "step": 2313, + "time_per_iteration": 2.4955642223358154 + }, + { + "auxiliary_loss_clip": 0.01193066, + "auxiliary_loss_mlp": 0.01065094, + "balance_loss_clip": 1.07338309, + "balance_loss_mlp": 1.04312348, + "epoch": 0.06714642214613196, + "flos": 33435635086080.0, + "grad_norm": 2.692172939976985, + "language_loss": 0.96955121, + "learning_rate": 3.98554643063176e-06, + "loss": 0.99213278, + "num_input_tokens_seen": 65309145, + "router_z_loss_clip": 1.19726562, + "router_z_loss_mlp": 0.21960449, + "step": 2314, + "time_per_iteration": 2.631340265274048 + }, + { + "auxiliary_loss_clip": 0.011747, + "auxiliary_loss_mlp": 0.01048833, + "balance_loss_clip": 1.06882787, + "balance_loss_mlp": 1.02873397, + "epoch": 0.06717543961464802, + "flos": 18144979695360.0, + "grad_norm": 2.666397422835796, + "language_loss": 0.82773244, + "learning_rate": 3.985523865401736e-06, + "loss": 0.84996784, + "num_input_tokens_seen": 65324540, + "router_z_loss_clip": 1.06054688, + "router_z_loss_mlp": 0.2010498, + "step": 2315, + "time_per_iteration": 2.521608591079712 + }, + { + "auxiliary_loss_clip": 0.01185256, + "auxiliary_loss_mlp": 0.01057598, + "balance_loss_clip": 1.07428563, + "balance_loss_mlp": 1.0363431, + "epoch": 0.06720445708316407, + "flos": 11613008471040.0, + "grad_norm": 2.722795838344708, + "language_loss": 0.85940176, + "learning_rate": 3.985501282634762e-06, + "loss": 0.88183033, + "num_input_tokens_seen": 65336085, + "router_z_loss_clip": 1.109375, + "router_z_loss_mlp": 0.21264648, + "step": 2316, + "time_per_iteration": 2.4902584552764893 + }, + { + "auxiliary_loss_clip": 0.0116988, + "auxiliary_loss_mlp": 0.01061984, + "balance_loss_clip": 1.06979084, + "balance_loss_mlp": 1.04322052, + "epoch": 0.0672334745516801, + "flos": 13509594489600.0, + "grad_norm": 2.070788711360717, + "language_loss": 0.73138785, + "learning_rate": 3.985478682331037e-06, + "loss": 0.75370657, + "num_input_tokens_seen": 65348915, + "router_z_loss_clip": 1.0, + "router_z_loss_mlp": 0.18768311, + "step": 2317, + "time_per_iteration": 2.4776697158813477 + }, + { + "auxiliary_loss_clip": 0.01169897, + "auxiliary_loss_mlp": 0.01048926, + "balance_loss_clip": 1.067981, + "balance_loss_mlp": 1.03028727, + "epoch": 0.06726249202019616, + "flos": 48901785350400.0, + "grad_norm": 1.763810762897475, + "language_loss": 0.79146045, + "learning_rate": 3.985456064490761e-06, + "loss": 0.8136487, + "num_input_tokens_seen": 65369650, + "router_z_loss_clip": 1.02001953, + "router_z_loss_mlp": 0.18652344, + "step": 2318, + "time_per_iteration": 2.776613235473633 + }, + { + "auxiliary_loss_clip": 0.01173605, + "auxiliary_loss_mlp": 0.01054379, + "balance_loss_clip": 1.06747043, + "balance_loss_mlp": 1.03578866, + "epoch": 0.06729150948871221, + "flos": 42661875611520.0, + "grad_norm": 2.4067284701957528, + "language_loss": 0.94418538, + "learning_rate": 3.9854334291141335e-06, + "loss": 0.96646518, + "num_input_tokens_seen": 65388550, + "router_z_loss_clip": 1.06152344, + "router_z_loss_mlp": 0.18597412, + "step": 2319, + "time_per_iteration": 2.6497535705566406 + }, + { + "auxiliary_loss_clip": 0.01183814, + "auxiliary_loss_mlp": 0.01052966, + "balance_loss_clip": 1.07356596, + "balance_loss_mlp": 1.03225946, + "epoch": 0.06732052695722825, + "flos": 25111397917440.0, + "grad_norm": 2.71673330702151, + "language_loss": 0.87324673, + "learning_rate": 3.985410776201355e-06, + "loss": 0.8956145, + "num_input_tokens_seen": 65401865, + "router_z_loss_clip": 1.10009766, + "router_z_loss_mlp": 0.20715332, + "step": 2320, + "time_per_iteration": 2.614063024520874 + }, + { + "auxiliary_loss_clip": 0.01054903, + "auxiliary_loss_mlp": 0.01005936, + "balance_loss_clip": 1.02688587, + "balance_loss_mlp": 1.00474977, + "epoch": 0.0673495444257443, + "flos": 74781962058240.0, + "grad_norm": 0.6788991704758844, + "language_loss": 0.48929474, + "learning_rate": 3.985388105752625e-06, + "loss": 0.50990307, + "num_input_tokens_seen": 65463570, + "router_z_loss_clip": 0.27929688, + "router_z_loss_mlp": 0.01184082, + "step": 2321, + "time_per_iteration": 3.1756930351257324 + }, + { + "auxiliary_loss_clip": 0.01172815, + "auxiliary_loss_mlp": 0.01062065, + "balance_loss_clip": 1.06948233, + "balance_loss_mlp": 1.04207325, + "epoch": 0.06737856189426035, + "flos": 42185484506880.0, + "grad_norm": 3.334482049737631, + "language_loss": 0.81138951, + "learning_rate": 3.985365417768144e-06, + "loss": 0.83373833, + "num_input_tokens_seen": 65479125, + "router_z_loss_clip": 1.03466797, + "router_z_loss_mlp": 0.19995117, + "step": 2322, + "time_per_iteration": 2.7219324111938477 + }, + { + "auxiliary_loss_clip": 0.01052878, + "auxiliary_loss_mlp": 0.01007614, + "balance_loss_clip": 1.02502036, + "balance_loss_mlp": 1.00648165, + "epoch": 0.06740757936277639, + "flos": 72251443883520.0, + "grad_norm": 0.6744760782045662, + "language_loss": 0.47358671, + "learning_rate": 3.985342712248112e-06, + "loss": 0.49419165, + "num_input_tokens_seen": 65541450, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01135254, + "step": 2323, + "time_per_iteration": 3.128401279449463 + }, + { + "auxiliary_loss_clip": 0.01174728, + "auxiliary_loss_mlp": 0.01062391, + "balance_loss_clip": 1.0685463, + "balance_loss_mlp": 1.04267406, + "epoch": 0.06743659683129244, + "flos": 16136926796160.0, + "grad_norm": 2.4772954906297144, + "language_loss": 0.779513, + "learning_rate": 3.985319989192729e-06, + "loss": 0.80188423, + "num_input_tokens_seen": 65555405, + "router_z_loss_clip": 1.0625, + "router_z_loss_mlp": 0.19714355, + "step": 2324, + "time_per_iteration": 2.4709160327911377 + }, + { + "auxiliary_loss_clip": 0.01179746, + "auxiliary_loss_mlp": 0.01044664, + "balance_loss_clip": 1.07281876, + "balance_loss_mlp": 1.0266161, + "epoch": 0.06746561429980849, + "flos": 17852990037120.0, + "grad_norm": 2.618025521417534, + "language_loss": 0.82880902, + "learning_rate": 3.985297248602197e-06, + "loss": 0.85105318, + "num_input_tokens_seen": 65569855, + "router_z_loss_clip": 1.06982422, + "router_z_loss_mlp": 0.18041992, + "step": 2325, + "time_per_iteration": 2.45053768157959 + }, + { + "auxiliary_loss_clip": 0.0105152, + "auxiliary_loss_mlp": 0.01010109, + "balance_loss_clip": 1.02344608, + "balance_loss_mlp": 1.00886917, + "epoch": 0.06749463176832453, + "flos": 74744974028160.0, + "grad_norm": 0.7024926479740509, + "language_loss": 0.4786638, + "learning_rate": 3.985274490476717e-06, + "loss": 0.4992801, + "num_input_tokens_seen": 65631380, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01239014, + "step": 2326, + "time_per_iteration": 3.142590045928955 + }, + { + "auxiliary_loss_clip": 0.01173897, + "auxiliary_loss_mlp": 0.01053111, + "balance_loss_clip": 1.06853819, + "balance_loss_mlp": 1.03285766, + "epoch": 0.06752364923684058, + "flos": 15738031284480.0, + "grad_norm": 2.297980512928848, + "language_loss": 0.87487298, + "learning_rate": 3.985251714816489e-06, + "loss": 0.89714301, + "num_input_tokens_seen": 65648075, + "router_z_loss_clip": 1.05419922, + "router_z_loss_mlp": 0.20227051, + "step": 2327, + "time_per_iteration": 2.482877016067505 + }, + { + "auxiliary_loss_clip": 0.0118256, + "auxiliary_loss_mlp": 0.01054309, + "balance_loss_clip": 1.07189131, + "balance_loss_mlp": 1.03254151, + "epoch": 0.06755266670535663, + "flos": 20076901718400.0, + "grad_norm": 2.303933538394581, + "language_loss": 0.9172045, + "learning_rate": 3.985228921621714e-06, + "loss": 0.93957317, + "num_input_tokens_seen": 65663485, + "router_z_loss_clip": 1.10742188, + "router_z_loss_mlp": 0.21801758, + "step": 2328, + "time_per_iteration": 2.476120710372925 + }, + { + "auxiliary_loss_clip": 0.01188376, + "auxiliary_loss_mlp": 0.01061431, + "balance_loss_clip": 1.07810903, + "balance_loss_mlp": 1.03940153, + "epoch": 0.06758168417387267, + "flos": 33104179359360.0, + "grad_norm": 3.472321059667518, + "language_loss": 1.04054379, + "learning_rate": 3.985206110892594e-06, + "loss": 1.06304181, + "num_input_tokens_seen": 65682665, + "router_z_loss_clip": 1.10253906, + "router_z_loss_mlp": 0.22058105, + "step": 2329, + "time_per_iteration": 2.573331594467163 + }, + { + "auxiliary_loss_clip": 0.01049745, + "auxiliary_loss_mlp": 0.01003836, + "balance_loss_clip": 1.02170885, + "balance_loss_mlp": 1.00241745, + "epoch": 0.06761070164238872, + "flos": 60361529466240.0, + "grad_norm": 0.7420847781887063, + "language_loss": 0.53389925, + "learning_rate": 3.985183282629331e-06, + "loss": 0.55443501, + "num_input_tokens_seen": 65737030, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01416016, + "step": 2330, + "time_per_iteration": 2.955322742462158 + }, + { + "auxiliary_loss_clip": 0.01170446, + "auxiliary_loss_mlp": 0.01047728, + "balance_loss_clip": 1.06637394, + "balance_loss_mlp": 1.02821374, + "epoch": 0.06763971911090476, + "flos": 19604173800960.0, + "grad_norm": 3.0043561398921694, + "language_loss": 0.86537898, + "learning_rate": 3.985160436832126e-06, + "loss": 0.88756061, + "num_input_tokens_seen": 65751585, + "router_z_loss_clip": 1.04101562, + "router_z_loss_mlp": 0.19519043, + "step": 2331, + "time_per_iteration": 2.5052196979522705 + }, + { + "auxiliary_loss_clip": 0.01049714, + "auxiliary_loss_mlp": 0.00997415, + "balance_loss_clip": 1.02165842, + "balance_loss_mlp": 0.99603224, + "epoch": 0.06766873657942081, + "flos": 74778370698240.0, + "grad_norm": 0.6066593973332208, + "language_loss": 0.49384564, + "learning_rate": 3.985137573501179e-06, + "loss": 0.51431692, + "num_input_tokens_seen": 65818270, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01385498, + "step": 2332, + "time_per_iteration": 3.3085291385650635 + }, + { + "auxiliary_loss_clip": 0.01049786, + "auxiliary_loss_mlp": 0.01002027, + "balance_loss_clip": 1.02169561, + "balance_loss_mlp": 1.00064981, + "epoch": 0.06769775404793686, + "flos": 74777436944640.0, + "grad_norm": 0.6859698925251204, + "language_loss": 0.52423108, + "learning_rate": 3.985114692636695e-06, + "loss": 0.5447492, + "num_input_tokens_seen": 65877890, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01379395, + "step": 2333, + "time_per_iteration": 3.118434429168701 + }, + { + "auxiliary_loss_clip": 0.01186259, + "auxiliary_loss_mlp": 0.01059749, + "balance_loss_clip": 1.07152915, + "balance_loss_mlp": 1.03916204, + "epoch": 0.0677267715164529, + "flos": 36425916368640.0, + "grad_norm": 2.151927827858173, + "language_loss": 0.93456864, + "learning_rate": 3.985091794238875e-06, + "loss": 0.95702875, + "num_input_tokens_seen": 65899115, + "router_z_loss_clip": 1.14746094, + "router_z_loss_mlp": 0.20574951, + "step": 2334, + "time_per_iteration": 2.6454625129699707 + }, + { + "auxiliary_loss_clip": 0.01175682, + "auxiliary_loss_mlp": 0.01047011, + "balance_loss_clip": 1.07049799, + "balance_loss_mlp": 1.02657247, + "epoch": 0.06775578898496895, + "flos": 32955904016640.0, + "grad_norm": 1.842658654736469, + "language_loss": 0.79305309, + "learning_rate": 3.98506887830792e-06, + "loss": 0.81528002, + "num_input_tokens_seen": 65919125, + "router_z_loss_clip": 1.05126953, + "router_z_loss_mlp": 0.20452881, + "step": 2335, + "time_per_iteration": 2.6304895877838135 + }, + { + "auxiliary_loss_clip": 0.01049438, + "auxiliary_loss_mlp": 0.01000216, + "balance_loss_clip": 1.02116501, + "balance_loss_mlp": 0.99894029, + "epoch": 0.067784806453485, + "flos": 67433560652160.0, + "grad_norm": 0.7390692648143847, + "language_loss": 0.5309996, + "learning_rate": 3.985045944844034e-06, + "loss": 0.55149609, + "num_input_tokens_seen": 65974245, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01275635, + "step": 2336, + "time_per_iteration": 3.0051329135894775 + }, + { + "auxiliary_loss_clip": 0.01049752, + "auxiliary_loss_mlp": 0.01000549, + "balance_loss_clip": 1.02141857, + "balance_loss_mlp": 0.99919552, + "epoch": 0.06781382392200104, + "flos": 66999796012800.0, + "grad_norm": 0.5788771325660023, + "language_loss": 0.46587074, + "learning_rate": 3.985022993847419e-06, + "loss": 0.48637372, + "num_input_tokens_seen": 66041205, + "router_z_loss_clip": 0.28320312, + "router_z_loss_mlp": 0.0135498, + "step": 2337, + "time_per_iteration": 3.2509353160858154 + }, + { + "auxiliary_loss_clip": 0.01170728, + "auxiliary_loss_mlp": 0.01047975, + "balance_loss_clip": 1.06776571, + "balance_loss_mlp": 1.03037977, + "epoch": 0.06784284139051709, + "flos": 20985225868800.0, + "grad_norm": 3.5254880450862065, + "language_loss": 0.98406637, + "learning_rate": 3.985000025318277e-06, + "loss": 1.00625336, + "num_input_tokens_seen": 66052845, + "router_z_loss_clip": 1.02929688, + "router_z_loss_mlp": 0.17596436, + "step": 2338, + "time_per_iteration": 2.5032691955566406 + }, + { + "auxiliary_loss_clip": 0.01179294, + "auxiliary_loss_mlp": 0.0104983, + "balance_loss_clip": 1.0685029, + "balance_loss_mlp": 1.0310185, + "epoch": 0.06787185885903314, + "flos": 33320325450240.0, + "grad_norm": 2.191369053416373, + "language_loss": 0.75360489, + "learning_rate": 3.984977039256812e-06, + "loss": 0.77589607, + "num_input_tokens_seen": 66067730, + "router_z_loss_clip": 1.10742188, + "router_z_loss_mlp": 0.18823242, + "step": 2339, + "time_per_iteration": 2.6717820167541504 + }, + { + "auxiliary_loss_clip": 0.01050811, + "auxiliary_loss_mlp": 0.01002547, + "balance_loss_clip": 1.02257514, + "balance_loss_mlp": 1.00125933, + "epoch": 0.06790087632754918, + "flos": 60291324334080.0, + "grad_norm": 0.6731035568568325, + "language_loss": 0.50814486, + "learning_rate": 3.984954035663227e-06, + "loss": 0.52867842, + "num_input_tokens_seen": 66131640, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01287842, + "step": 2340, + "time_per_iteration": 3.0590946674346924 + }, + { + "auxiliary_loss_clip": 0.01177062, + "auxiliary_loss_mlp": 0.01058892, + "balance_loss_clip": 1.07042241, + "balance_loss_mlp": 1.0387094, + "epoch": 0.06792989379606523, + "flos": 22924725661440.0, + "grad_norm": 2.7507559469254734, + "language_loss": 0.86706585, + "learning_rate": 3.984931014537724e-06, + "loss": 0.88942534, + "num_input_tokens_seen": 66145470, + "router_z_loss_clip": 1.06494141, + "router_z_loss_mlp": 0.20178223, + "step": 2341, + "time_per_iteration": 2.5497448444366455 + }, + { + "auxiliary_loss_clip": 0.01168991, + "auxiliary_loss_mlp": 0.01054621, + "balance_loss_clip": 1.06601202, + "balance_loss_mlp": 1.03575635, + "epoch": 0.06795891126458128, + "flos": 25733765894400.0, + "grad_norm": 2.2918993460154984, + "language_loss": 0.8952623, + "learning_rate": 3.984907975880508e-06, + "loss": 0.91749841, + "num_input_tokens_seen": 66163045, + "router_z_loss_clip": 1.03027344, + "router_z_loss_mlp": 0.18859863, + "step": 2342, + "time_per_iteration": 2.5367956161499023 + }, + { + "auxiliary_loss_clip": 0.0105031, + "auxiliary_loss_mlp": 0.01005686, + "balance_loss_clip": 1.02216983, + "balance_loss_mlp": 1.00451815, + "epoch": 0.06798792873309732, + "flos": 61684263803520.0, + "grad_norm": 0.6498082216274841, + "language_loss": 0.4601776, + "learning_rate": 3.984884919691781e-06, + "loss": 0.48073757, + "num_input_tokens_seen": 66222835, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01165771, + "step": 2343, + "time_per_iteration": 3.116891860961914 + }, + { + "auxiliary_loss_clip": 0.01187574, + "auxiliary_loss_mlp": 0.01058461, + "balance_loss_clip": 1.07165015, + "balance_loss_mlp": 1.03655064, + "epoch": 0.06801694620161337, + "flos": 17595366716160.0, + "grad_norm": 3.7387678420528134, + "language_loss": 0.91788703, + "learning_rate": 3.984861845971747e-06, + "loss": 0.94034737, + "num_input_tokens_seen": 66236400, + "router_z_loss_clip": 1.15966797, + "router_z_loss_mlp": 0.21911621, + "step": 2344, + "time_per_iteration": 2.5031485557556152 + }, + { + "auxiliary_loss_clip": 0.0118068, + "auxiliary_loss_mlp": 0.01054877, + "balance_loss_clip": 1.0735569, + "balance_loss_mlp": 1.03411055, + "epoch": 0.06804596367012941, + "flos": 51982347467520.0, + "grad_norm": 2.1141960304531446, + "language_loss": 1.15563178, + "learning_rate": 3.98483875472061e-06, + "loss": 1.17798734, + "num_input_tokens_seen": 66261695, + "router_z_loss_clip": 1.07128906, + "router_z_loss_mlp": 0.20751953, + "step": 2345, + "time_per_iteration": 2.8949456214904785 + }, + { + "auxiliary_loss_clip": 0.01050675, + "auxiliary_loss_mlp": 0.01000634, + "balance_loss_clip": 1.022807, + "balance_loss_mlp": 0.99947733, + "epoch": 0.06807498113864546, + "flos": 69267662962560.0, + "grad_norm": 0.7465228949322226, + "language_loss": 0.52725708, + "learning_rate": 3.984815645938575e-06, + "loss": 0.54777026, + "num_input_tokens_seen": 66317225, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01153564, + "step": 2346, + "time_per_iteration": 3.0404250621795654 + }, + { + "auxiliary_loss_clip": 0.01167595, + "auxiliary_loss_mlp": 0.01056437, + "balance_loss_clip": 1.0684166, + "balance_loss_mlp": 1.03590918, + "epoch": 0.06810399860716151, + "flos": 26793597611520.0, + "grad_norm": 2.160733370612517, + "language_loss": 0.77796841, + "learning_rate": 3.984792519625844e-06, + "loss": 0.80020869, + "num_input_tokens_seen": 66337570, + "router_z_loss_clip": 0.99267578, + "router_z_loss_mlp": 0.2052002, + "step": 2347, + "time_per_iteration": 2.6266303062438965 + }, + { + "auxiliary_loss_clip": 0.01174049, + "auxiliary_loss_mlp": 0.01049741, + "balance_loss_clip": 1.06658196, + "balance_loss_mlp": 1.03053665, + "epoch": 0.06813301607567755, + "flos": 21063475647360.0, + "grad_norm": 2.331122281457913, + "language_loss": 0.74033582, + "learning_rate": 3.984769375782622e-06, + "loss": 0.76257372, + "num_input_tokens_seen": 66352685, + "router_z_loss_clip": 1.07421875, + "router_z_loss_mlp": 0.19189453, + "step": 2348, + "time_per_iteration": 2.460031509399414 + }, + { + "auxiliary_loss_clip": 0.01171248, + "auxiliary_loss_mlp": 0.01048358, + "balance_loss_clip": 1.06818986, + "balance_loss_mlp": 1.03125715, + "epoch": 0.0681620335441936, + "flos": 21538430208000.0, + "grad_norm": 2.463092656455816, + "language_loss": 0.85987324, + "learning_rate": 3.984746214409114e-06, + "loss": 0.88206929, + "num_input_tokens_seen": 66368580, + "router_z_loss_clip": 1.03027344, + "router_z_loss_mlp": 0.17120361, + "step": 2349, + "time_per_iteration": 2.502596855163574 + }, + { + "auxiliary_loss_clip": 0.01172666, + "auxiliary_loss_mlp": 0.01045877, + "balance_loss_clip": 1.0685215, + "balance_loss_mlp": 1.02902138, + "epoch": 0.06819105101270966, + "flos": 27082283218560.0, + "grad_norm": 3.4398491139087013, + "language_loss": 1.06372356, + "learning_rate": 3.9847230355055245e-06, + "loss": 1.08590901, + "num_input_tokens_seen": 66382370, + "router_z_loss_clip": 1.04296875, + "router_z_loss_mlp": 0.16864014, + "step": 2350, + "time_per_iteration": 7.318811655044556 + }, + { + "auxiliary_loss_clip": 0.0116832, + "auxiliary_loss_mlp": 0.01060715, + "balance_loss_clip": 1.07056236, + "balance_loss_mlp": 1.0435791, + "epoch": 0.06822006848122569, + "flos": 28395320883840.0, + "grad_norm": 3.7677404334997675, + "language_loss": 0.89227092, + "learning_rate": 3.984699839072058e-06, + "loss": 0.91456127, + "num_input_tokens_seen": 66396740, + "router_z_loss_clip": 0.97900391, + "router_z_loss_mlp": 0.17138672, + "step": 2351, + "time_per_iteration": 4.812373161315918 + }, + { + "auxiliary_loss_clip": 0.01163248, + "auxiliary_loss_mlp": 0.01041717, + "balance_loss_clip": 1.0652709, + "balance_loss_mlp": 1.02458656, + "epoch": 0.06824908594974174, + "flos": 16317485487360.0, + "grad_norm": 2.5489058368096322, + "language_loss": 0.85041851, + "learning_rate": 3.9846766251089195e-06, + "loss": 0.87246811, + "num_input_tokens_seen": 66409440, + "router_z_loss_clip": 0.97998047, + "router_z_loss_mlp": 0.17138672, + "step": 2352, + "time_per_iteration": 4.659198760986328 + }, + { + "auxiliary_loss_clip": 0.01172838, + "auxiliary_loss_mlp": 0.01049103, + "balance_loss_clip": 1.07009792, + "balance_loss_mlp": 1.02930284, + "epoch": 0.0682781034182578, + "flos": 28177738248960.0, + "grad_norm": 3.3699104404227254, + "language_loss": 0.9310624, + "learning_rate": 3.984653393616313e-06, + "loss": 0.95328182, + "num_input_tokens_seen": 66425485, + "router_z_loss_clip": 1.02832031, + "router_z_loss_mlp": 0.19799805, + "step": 2353, + "time_per_iteration": 2.561885118484497 + }, + { + "auxiliary_loss_clip": 0.0118072, + "auxiliary_loss_mlp": 0.01051758, + "balance_loss_clip": 1.07217038, + "balance_loss_mlp": 1.03331637, + "epoch": 0.06830712088677383, + "flos": 12378659800320.0, + "grad_norm": 4.0059761036423245, + "language_loss": 0.88061845, + "learning_rate": 3.984630144594446e-06, + "loss": 0.90294325, + "num_input_tokens_seen": 66436315, + "router_z_loss_clip": 1.08349609, + "router_z_loss_mlp": 0.18432617, + "step": 2354, + "time_per_iteration": 2.5141329765319824 + }, + { + "auxiliary_loss_clip": 0.01176416, + "auxiliary_loss_mlp": 0.01049733, + "balance_loss_clip": 1.07060766, + "balance_loss_mlp": 1.02839422, + "epoch": 0.06833613835528989, + "flos": 18836044433280.0, + "grad_norm": 3.1061299211073394, + "language_loss": 0.89453471, + "learning_rate": 3.984606878043522e-06, + "loss": 0.91679621, + "num_input_tokens_seen": 66448050, + "router_z_loss_clip": 1.05615234, + "router_z_loss_mlp": 0.21343994, + "step": 2355, + "time_per_iteration": 2.4796524047851562 + }, + { + "auxiliary_loss_clip": 0.01168117, + "auxiliary_loss_mlp": 0.01049395, + "balance_loss_clip": 1.06715989, + "balance_loss_mlp": 1.03106117, + "epoch": 0.06836515582380594, + "flos": 27383825894400.0, + "grad_norm": 2.085786358969434, + "language_loss": 1.08070588, + "learning_rate": 3.984583593963747e-06, + "loss": 1.10288095, + "num_input_tokens_seen": 66466985, + "router_z_loss_clip": 1.00976562, + "router_z_loss_mlp": 0.18328857, + "step": 2356, + "time_per_iteration": 2.612807512283325 + }, + { + "auxiliary_loss_clip": 0.01179091, + "auxiliary_loss_mlp": 0.01062424, + "balance_loss_clip": 1.07060087, + "balance_loss_mlp": 1.04036987, + "epoch": 0.06839417329232197, + "flos": 25734268684800.0, + "grad_norm": 1.9810008332792495, + "language_loss": 0.85169601, + "learning_rate": 3.984560292355326e-06, + "loss": 0.87411118, + "num_input_tokens_seen": 66484575, + "router_z_loss_clip": 1.08544922, + "router_z_loss_mlp": 0.22070312, + "step": 2357, + "time_per_iteration": 2.5762124061584473 + }, + { + "auxiliary_loss_clip": 0.01186883, + "auxiliary_loss_mlp": 0.01061485, + "balance_loss_clip": 1.0704062, + "balance_loss_mlp": 1.04098094, + "epoch": 0.06842319076083803, + "flos": 47479471534080.0, + "grad_norm": 2.9221797249357757, + "language_loss": 0.98177254, + "learning_rate": 3.984536973218466e-06, + "loss": 1.00425625, + "num_input_tokens_seen": 66503110, + "router_z_loss_clip": 1.16503906, + "router_z_loss_mlp": 0.2052002, + "step": 2358, + "time_per_iteration": 2.7258870601654053 + }, + { + "auxiliary_loss_clip": 0.01181921, + "auxiliary_loss_mlp": 0.01046395, + "balance_loss_clip": 1.07572627, + "balance_loss_mlp": 1.02891934, + "epoch": 0.06845220822935408, + "flos": 11175832039680.0, + "grad_norm": 2.853858706178328, + "language_loss": 0.789451, + "learning_rate": 3.984513636553372e-06, + "loss": 0.8117342, + "num_input_tokens_seen": 66514520, + "router_z_loss_clip": 1.06152344, + "router_z_loss_mlp": 0.17468262, + "step": 2359, + "time_per_iteration": 2.4977500438690186 + }, + { + "auxiliary_loss_clip": 0.01185668, + "auxiliary_loss_mlp": 0.01057404, + "balance_loss_clip": 1.07333291, + "balance_loss_mlp": 1.0360657, + "epoch": 0.06848122569787012, + "flos": 30399566941440.0, + "grad_norm": 2.4069139185699435, + "language_loss": 0.77879333, + "learning_rate": 3.984490282360251e-06, + "loss": 0.80122405, + "num_input_tokens_seen": 66527920, + "router_z_loss_clip": 1.12255859, + "router_z_loss_mlp": 0.21350098, + "step": 2360, + "time_per_iteration": 2.6007444858551025 + }, + { + "auxiliary_loss_clip": 0.01178293, + "auxiliary_loss_mlp": 0.01049479, + "balance_loss_clip": 1.07456684, + "balance_loss_mlp": 1.02917147, + "epoch": 0.06851024316638617, + "flos": 58754667571200.0, + "grad_norm": 1.9739498604908845, + "language_loss": 0.85210371, + "learning_rate": 3.984466910639309e-06, + "loss": 0.87438142, + "num_input_tokens_seen": 66552745, + "router_z_loss_clip": 1.03662109, + "router_z_loss_mlp": 0.203125, + "step": 2361, + "time_per_iteration": 2.8242664337158203 + }, + { + "auxiliary_loss_clip": 0.01172876, + "auxiliary_loss_mlp": 0.01049774, + "balance_loss_clip": 1.07224822, + "balance_loss_mlp": 1.03285825, + "epoch": 0.0685392606349022, + "flos": 16137106364160.0, + "grad_norm": 4.034296846595869, + "language_loss": 0.68632138, + "learning_rate": 3.984443521390752e-06, + "loss": 0.70854795, + "num_input_tokens_seen": 66565650, + "router_z_loss_clip": 1.00585938, + "router_z_loss_mlp": 0.16918945, + "step": 2362, + "time_per_iteration": 2.480363607406616 + }, + { + "auxiliary_loss_clip": 0.01178312, + "auxiliary_loss_mlp": 0.01051572, + "balance_loss_clip": 1.07040191, + "balance_loss_mlp": 1.03198636, + "epoch": 0.06856827810341826, + "flos": 29789158193280.0, + "grad_norm": 1.8247742978272166, + "language_loss": 0.84907287, + "learning_rate": 3.984420114614786e-06, + "loss": 0.87137175, + "num_input_tokens_seen": 66586585, + "router_z_loss_clip": 1.08056641, + "router_z_loss_mlp": 0.19592285, + "step": 2363, + "time_per_iteration": 2.6253182888031006 + }, + { + "auxiliary_loss_clip": 0.01173055, + "auxiliary_loss_mlp": 0.01047634, + "balance_loss_clip": 1.07183433, + "balance_loss_mlp": 1.03096855, + "epoch": 0.06859729557193431, + "flos": 16180774323840.0, + "grad_norm": 2.593426716321418, + "language_loss": 0.76331758, + "learning_rate": 3.984396690311619e-06, + "loss": 0.78552449, + "num_input_tokens_seen": 66600515, + "router_z_loss_clip": 1.01269531, + "router_z_loss_mlp": 0.16687012, + "step": 2364, + "time_per_iteration": 2.4759581089019775 + }, + { + "auxiliary_loss_clip": 0.01057641, + "auxiliary_loss_mlp": 0.01005664, + "balance_loss_clip": 1.02954078, + "balance_loss_mlp": 1.00436485, + "epoch": 0.06862631304045035, + "flos": 61274414643840.0, + "grad_norm": 0.7575627802375694, + "language_loss": 0.52284265, + "learning_rate": 3.9843732484814585e-06, + "loss": 0.54347575, + "num_input_tokens_seen": 66658005, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01300049, + "step": 2365, + "time_per_iteration": 3.0943610668182373 + }, + { + "auxiliary_loss_clip": 0.01058502, + "auxiliary_loss_mlp": 0.0100396, + "balance_loss_clip": 1.03022671, + "balance_loss_mlp": 1.002738, + "epoch": 0.0686553305089664, + "flos": 74782285280640.0, + "grad_norm": 0.6657398049725342, + "language_loss": 0.48253712, + "learning_rate": 3.984349789124509e-06, + "loss": 0.50316179, + "num_input_tokens_seen": 66726810, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01220703, + "step": 2366, + "time_per_iteration": 3.2258331775665283 + }, + { + "auxiliary_loss_clip": 0.01176916, + "auxiliary_loss_mlp": 0.01052237, + "balance_loss_clip": 1.06896496, + "balance_loss_mlp": 1.03292561, + "epoch": 0.06868434797748245, + "flos": 25842539255040.0, + "grad_norm": 2.193912295751311, + "language_loss": 0.77366781, + "learning_rate": 3.98432631224098e-06, + "loss": 0.79595935, + "num_input_tokens_seen": 66743380, + "router_z_loss_clip": 1.07958984, + "router_z_loss_mlp": 0.1930542, + "step": 2367, + "time_per_iteration": 2.5576887130737305 + }, + { + "auxiliary_loss_clip": 0.01174644, + "auxiliary_loss_mlp": 0.01058532, + "balance_loss_clip": 1.07113504, + "balance_loss_mlp": 1.03938651, + "epoch": 0.06871336544599849, + "flos": 34087449237120.0, + "grad_norm": 2.68930893424419, + "language_loss": 0.99448127, + "learning_rate": 3.984302817831078e-06, + "loss": 1.01681304, + "num_input_tokens_seen": 66761635, + "router_z_loss_clip": 1.03369141, + "router_z_loss_mlp": 0.19110107, + "step": 2368, + "time_per_iteration": 2.636423349380493 + }, + { + "auxiliary_loss_clip": 0.01056097, + "auxiliary_loss_mlp": 0.01004289, + "balance_loss_clip": 1.02794099, + "balance_loss_mlp": 1.00301957, + "epoch": 0.06874238291451454, + "flos": 74773342794240.0, + "grad_norm": 0.689242707619083, + "language_loss": 0.48310035, + "learning_rate": 3.98427930589501e-06, + "loss": 0.50370425, + "num_input_tokens_seen": 66824315, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01269531, + "step": 2369, + "time_per_iteration": 3.093912124633789 + }, + { + "auxiliary_loss_clip": 0.01171543, + "auxiliary_loss_mlp": 0.01048348, + "balance_loss_clip": 1.06733668, + "balance_loss_mlp": 1.02862549, + "epoch": 0.06877140038303059, + "flos": 47346674952960.0, + "grad_norm": 2.8133343633837966, + "language_loss": 0.77874565, + "learning_rate": 3.984255776432984e-06, + "loss": 0.80094457, + "num_input_tokens_seen": 66843070, + "router_z_loss_clip": 1.04248047, + "router_z_loss_mlp": 0.19720459, + "step": 2370, + "time_per_iteration": 2.7714202404022217 + }, + { + "auxiliary_loss_clip": 0.01189316, + "auxiliary_loss_mlp": 0.01059434, + "balance_loss_clip": 1.07435465, + "balance_loss_mlp": 1.03828669, + "epoch": 0.06880041785154663, + "flos": 14790276979200.0, + "grad_norm": 2.2524335078995126, + "language_loss": 0.82234246, + "learning_rate": 3.984232229445209e-06, + "loss": 0.84482992, + "num_input_tokens_seen": 66855350, + "router_z_loss_clip": 1.14990234, + "router_z_loss_mlp": 0.21130371, + "step": 2371, + "time_per_iteration": 2.497096061706543 + }, + { + "auxiliary_loss_clip": 0.01054152, + "auxiliary_loss_mlp": 0.00999808, + "balance_loss_clip": 1.02586436, + "balance_loss_mlp": 0.99844915, + "epoch": 0.06882943532006268, + "flos": 74776610931840.0, + "grad_norm": 0.735938750797132, + "language_loss": 0.51600003, + "learning_rate": 3.984208664931891e-06, + "loss": 0.53653955, + "num_input_tokens_seen": 66918380, + "router_z_loss_clip": 0.28320312, + "router_z_loss_mlp": 0.01361084, + "step": 2372, + "time_per_iteration": 3.116684675216675 + }, + { + "auxiliary_loss_clip": 0.0105386, + "auxiliary_loss_mlp": 0.01003199, + "balance_loss_clip": 1.02587068, + "balance_loss_mlp": 1.00191176, + "epoch": 0.06885845278857873, + "flos": 69967239223680.0, + "grad_norm": 0.6686132187273819, + "language_loss": 0.54143798, + "learning_rate": 3.984185082893241e-06, + "loss": 0.5620085, + "num_input_tokens_seen": 66986760, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01287842, + "step": 2373, + "time_per_iteration": 3.201925754547119 + }, + { + "auxiliary_loss_clip": 0.01053885, + "auxiliary_loss_mlp": 0.01006842, + "balance_loss_clip": 1.02563143, + "balance_loss_mlp": 1.00553036, + "epoch": 0.06888747025709477, + "flos": 66052724065920.0, + "grad_norm": 0.7581576672073606, + "language_loss": 0.51389372, + "learning_rate": 3.984161483329464e-06, + "loss": 0.53450096, + "num_input_tokens_seen": 67047025, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01312256, + "step": 2374, + "time_per_iteration": 3.082893133163452 + }, + { + "auxiliary_loss_clip": 0.01179606, + "auxiliary_loss_mlp": 0.01063927, + "balance_loss_clip": 1.07394922, + "balance_loss_mlp": 1.04592597, + "epoch": 0.06891648772561082, + "flos": 11977537645440.0, + "grad_norm": 3.181635450097956, + "language_loss": 0.90306026, + "learning_rate": 3.9841378662407696e-06, + "loss": 0.92549562, + "num_input_tokens_seen": 67059635, + "router_z_loss_clip": 1.05712891, + "router_z_loss_mlp": 0.17987061, + "step": 2375, + "time_per_iteration": 2.4934871196746826 + }, + { + "auxiliary_loss_clip": 0.01175814, + "auxiliary_loss_mlp": 0.01049835, + "balance_loss_clip": 1.07252431, + "balance_loss_mlp": 1.03082156, + "epoch": 0.06894550519412686, + "flos": 51570343491840.0, + "grad_norm": 2.1225000807883, + "language_loss": 0.91225928, + "learning_rate": 3.984114231627367e-06, + "loss": 0.93451577, + "num_input_tokens_seen": 67079745, + "router_z_loss_clip": 1.03369141, + "router_z_loss_mlp": 0.19030762, + "step": 2376, + "time_per_iteration": 2.825746774673462 + }, + { + "auxiliary_loss_clip": 0.01160087, + "auxiliary_loss_mlp": 0.01045862, + "balance_loss_clip": 1.06719613, + "balance_loss_mlp": 1.03018594, + "epoch": 0.06897452266264291, + "flos": 19056643810560.0, + "grad_norm": 2.365468652776217, + "language_loss": 0.78929579, + "learning_rate": 3.984090579489465e-06, + "loss": 0.81135535, + "num_input_tokens_seen": 67094325, + "router_z_loss_clip": 0.9296875, + "router_z_loss_mlp": 0.15679932, + "step": 2377, + "time_per_iteration": 2.5312540531158447 + }, + { + "auxiliary_loss_clip": 0.0117755, + "auxiliary_loss_mlp": 0.01045745, + "balance_loss_clip": 1.06652796, + "balance_loss_mlp": 1.02739835, + "epoch": 0.06900354013115896, + "flos": 17783036300160.0, + "grad_norm": 1.937441804867808, + "language_loss": 0.81449234, + "learning_rate": 3.9840669098272715e-06, + "loss": 0.83672523, + "num_input_tokens_seen": 67110825, + "router_z_loss_clip": 1.11035156, + "router_z_loss_mlp": 0.18347168, + "step": 2378, + "time_per_iteration": 2.5065603256225586 + }, + { + "auxiliary_loss_clip": 0.01174193, + "auxiliary_loss_mlp": 0.01052581, + "balance_loss_clip": 1.0700866, + "balance_loss_mlp": 1.034688, + "epoch": 0.069032557599675, + "flos": 25476753104640.0, + "grad_norm": 5.8171073314371355, + "language_loss": 0.97415864, + "learning_rate": 3.984043222640997e-06, + "loss": 0.99642634, + "num_input_tokens_seen": 67126200, + "router_z_loss_clip": 1.04150391, + "router_z_loss_mlp": 0.17907715, + "step": 2379, + "time_per_iteration": 2.5818378925323486 + }, + { + "auxiliary_loss_clip": 0.01175982, + "auxiliary_loss_mlp": 0.01064123, + "balance_loss_clip": 1.0691216, + "balance_loss_mlp": 1.04483545, + "epoch": 0.06906157506819105, + "flos": 12851998248960.0, + "grad_norm": 3.3143631483646705, + "language_loss": 0.84670156, + "learning_rate": 3.984019517930849e-06, + "loss": 0.8691026, + "num_input_tokens_seen": 67137415, + "router_z_loss_clip": 1.06933594, + "router_z_loss_mlp": 0.19287109, + "step": 2380, + "time_per_iteration": 2.512028455734253 + }, + { + "auxiliary_loss_clip": 0.01164266, + "auxiliary_loss_mlp": 0.01045029, + "balance_loss_clip": 1.06723118, + "balance_loss_mlp": 1.03002667, + "epoch": 0.0690905925367071, + "flos": 16210400065920.0, + "grad_norm": 2.2944215412324778, + "language_loss": 0.71779537, + "learning_rate": 3.983995795697038e-06, + "loss": 0.73988831, + "num_input_tokens_seen": 67152110, + "router_z_loss_clip": 0.96875, + "router_z_loss_mlp": 0.15008545, + "step": 2381, + "time_per_iteration": 2.4814088344573975 + }, + { + "auxiliary_loss_clip": 0.01178418, + "auxiliary_loss_mlp": 0.01053556, + "balance_loss_clip": 1.07061541, + "balance_loss_mlp": 1.03433943, + "epoch": 0.06911961000522314, + "flos": 16319209340160.0, + "grad_norm": 2.5746845864036167, + "language_loss": 0.78372216, + "learning_rate": 3.983972055939774e-06, + "loss": 0.80604196, + "num_input_tokens_seen": 67164450, + "router_z_loss_clip": 1.07714844, + "router_z_loss_mlp": 0.19226074, + "step": 2382, + "time_per_iteration": 2.4822793006896973 + }, + { + "auxiliary_loss_clip": 0.01181549, + "auxiliary_loss_mlp": 0.01050205, + "balance_loss_clip": 1.07312691, + "balance_loss_mlp": 1.03115582, + "epoch": 0.06914862747373919, + "flos": 27445878639360.0, + "grad_norm": 3.066209277733726, + "language_loss": 1.00322974, + "learning_rate": 3.983948298659266e-06, + "loss": 1.02554727, + "num_input_tokens_seen": 67177580, + "router_z_loss_clip": 1.08496094, + "router_z_loss_mlp": 0.19042969, + "step": 2383, + "time_per_iteration": 2.5707051753997803 + }, + { + "auxiliary_loss_clip": 0.01176867, + "auxiliary_loss_mlp": 0.0105777, + "balance_loss_clip": 1.0670501, + "balance_loss_mlp": 1.03750455, + "epoch": 0.06917764494225524, + "flos": 41601397449600.0, + "grad_norm": 2.623649975348723, + "language_loss": 1.06224501, + "learning_rate": 3.983924523855723e-06, + "loss": 1.08459139, + "num_input_tokens_seen": 67192595, + "router_z_loss_clip": 1.09814453, + "router_z_loss_mlp": 0.20275879, + "step": 2384, + "time_per_iteration": 2.689267635345459 + }, + { + "auxiliary_loss_clip": 0.01169232, + "auxiliary_loss_mlp": 0.01058005, + "balance_loss_clip": 1.06745315, + "balance_loss_mlp": 1.03857374, + "epoch": 0.06920666241077128, + "flos": 19966907295360.0, + "grad_norm": 3.3798518835930804, + "language_loss": 0.74215341, + "learning_rate": 3.983900731529356e-06, + "loss": 0.76442575, + "num_input_tokens_seen": 67204425, + "router_z_loss_clip": 1.01806641, + "router_z_loss_mlp": 0.19445801, + "step": 2385, + "time_per_iteration": 2.524813175201416 + }, + { + "auxiliary_loss_clip": 0.01172475, + "auxiliary_loss_mlp": 0.01041906, + "balance_loss_clip": 1.06503534, + "balance_loss_mlp": 1.02149141, + "epoch": 0.06923567987928733, + "flos": 18217662865920.0, + "grad_norm": 2.6364275510909216, + "language_loss": 0.78721106, + "learning_rate": 3.983876921680375e-06, + "loss": 0.80935484, + "num_input_tokens_seen": 67220650, + "router_z_loss_clip": 1.07421875, + "router_z_loss_mlp": 0.20397949, + "step": 2386, + "time_per_iteration": 2.530398368835449 + }, + { + "auxiliary_loss_clip": 0.01174452, + "auxiliary_loss_mlp": 0.01052574, + "balance_loss_clip": 1.06954372, + "balance_loss_mlp": 1.03220105, + "epoch": 0.06926469734780338, + "flos": 16427946787200.0, + "grad_norm": 2.677863058181403, + "language_loss": 0.86086309, + "learning_rate": 3.98385309430899e-06, + "loss": 0.88313335, + "num_input_tokens_seen": 67233180, + "router_z_loss_clip": 1.04882812, + "router_z_loss_mlp": 0.20385742, + "step": 2387, + "time_per_iteration": 2.4850096702575684 + }, + { + "auxiliary_loss_clip": 0.0118054, + "auxiliary_loss_mlp": 0.01051397, + "balance_loss_clip": 1.0712781, + "balance_loss_mlp": 1.03115523, + "epoch": 0.06929371481631942, + "flos": 29235271495680.0, + "grad_norm": 2.074375464406192, + "language_loss": 1.00543904, + "learning_rate": 3.9838292494154125e-06, + "loss": 1.02775836, + "num_input_tokens_seen": 67250545, + "router_z_loss_clip": 1.09179688, + "router_z_loss_mlp": 0.20251465, + "step": 2388, + "time_per_iteration": 2.5531704425811768 + }, + { + "auxiliary_loss_clip": 0.01059861, + "auxiliary_loss_mlp": 0.00999503, + "balance_loss_clip": 1.03143477, + "balance_loss_mlp": 0.99812049, + "epoch": 0.06932273228483547, + "flos": 74795321917440.0, + "grad_norm": 0.7831101691399782, + "language_loss": 0.53295881, + "learning_rate": 3.983805386999851e-06, + "loss": 0.55355245, + "num_input_tokens_seen": 67316275, + "router_z_loss_clip": 0.28515625, + "router_z_loss_mlp": 0.01385498, + "step": 2389, + "time_per_iteration": 3.322878360748291 + }, + { + "auxiliary_loss_clip": 0.01166414, + "auxiliary_loss_mlp": 0.01050943, + "balance_loss_clip": 1.06727386, + "balance_loss_mlp": 1.03390825, + "epoch": 0.06935174975335152, + "flos": 18106626948480.0, + "grad_norm": 2.9704370503448514, + "language_loss": 0.75673509, + "learning_rate": 3.9837815070625185e-06, + "loss": 0.77890861, + "num_input_tokens_seen": 67329310, + "router_z_loss_clip": 0.99267578, + "router_z_loss_mlp": 0.17028809, + "step": 2390, + "time_per_iteration": 2.434044361114502 + }, + { + "auxiliary_loss_clip": 0.01179714, + "auxiliary_loss_mlp": 0.01049688, + "balance_loss_clip": 1.06915176, + "balance_loss_mlp": 1.03042412, + "epoch": 0.06938076722186756, + "flos": 32308758633600.0, + "grad_norm": 1.822247679458434, + "language_loss": 0.78838861, + "learning_rate": 3.983757609603625e-06, + "loss": 0.81068265, + "num_input_tokens_seen": 67352275, + "router_z_loss_clip": 1.10400391, + "router_z_loss_mlp": 0.19262695, + "step": 2391, + "time_per_iteration": 2.5906949043273926 + }, + { + "auxiliary_loss_clip": 0.01157562, + "auxiliary_loss_mlp": 0.01040217, + "balance_loss_clip": 1.06665874, + "balance_loss_mlp": 1.02558947, + "epoch": 0.06940978469038361, + "flos": 28868551591680.0, + "grad_norm": 1.9924153474928845, + "language_loss": 0.63830066, + "learning_rate": 3.983733694623382e-06, + "loss": 0.66027844, + "num_input_tokens_seen": 67366460, + "router_z_loss_clip": 0.90771484, + "router_z_loss_mlp": 0.14624023, + "step": 2392, + "time_per_iteration": 2.5625112056732178 + }, + { + "auxiliary_loss_clip": 0.01057098, + "auxiliary_loss_mlp": 0.01000921, + "balance_loss_clip": 1.02904952, + "balance_loss_mlp": 0.99954396, + "epoch": 0.06943880215889965, + "flos": 59590131960960.0, + "grad_norm": 0.7439501848790643, + "language_loss": 0.47759792, + "learning_rate": 3.983709762121999e-06, + "loss": 0.49817812, + "num_input_tokens_seen": 67419300, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01379395, + "step": 2393, + "time_per_iteration": 2.939671754837036 + }, + { + "auxiliary_loss_clip": 0.01175368, + "auxiliary_loss_mlp": 0.0105543, + "balance_loss_clip": 1.06953788, + "balance_loss_mlp": 1.03613055, + "epoch": 0.0694678196274157, + "flos": 10991502420480.0, + "grad_norm": 2.8797178346147834, + "language_loss": 0.8405475, + "learning_rate": 3.983685812099689e-06, + "loss": 0.86285543, + "num_input_tokens_seen": 67429205, + "router_z_loss_clip": 1.05566406, + "router_z_loss_mlp": 0.19311523, + "step": 2394, + "time_per_iteration": 2.552687406539917 + }, + { + "auxiliary_loss_clip": 0.01175079, + "auxiliary_loss_mlp": 0.01055641, + "balance_loss_clip": 1.06879473, + "balance_loss_mlp": 1.03414714, + "epoch": 0.06949683709593175, + "flos": 54631332696960.0, + "grad_norm": 1.9741645169772581, + "language_loss": 0.92475665, + "learning_rate": 3.983661844556664e-06, + "loss": 0.9470638, + "num_input_tokens_seen": 67448635, + "router_z_loss_clip": 1.06396484, + "router_z_loss_mlp": 0.21496582, + "step": 2395, + "time_per_iteration": 2.82738995552063 + }, + { + "auxiliary_loss_clip": 0.01054236, + "auxiliary_loss_mlp": 0.01001018, + "balance_loss_clip": 1.02622724, + "balance_loss_mlp": 0.99965918, + "epoch": 0.06952585456444779, + "flos": 74776682759040.0, + "grad_norm": 0.7033226578494944, + "language_loss": 0.52461022, + "learning_rate": 3.983637859493134e-06, + "loss": 0.54516274, + "num_input_tokens_seen": 67510405, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01361084, + "step": 2396, + "time_per_iteration": 3.187086820602417 + }, + { + "auxiliary_loss_clip": 0.01175773, + "auxiliary_loss_mlp": 0.01044003, + "balance_loss_clip": 1.06782532, + "balance_loss_mlp": 1.02532291, + "epoch": 0.06955487203296384, + "flos": 21794581071360.0, + "grad_norm": 2.934534766157612, + "language_loss": 0.84078741, + "learning_rate": 3.9836138569093125e-06, + "loss": 0.86298513, + "num_input_tokens_seen": 67525045, + "router_z_loss_clip": 1.07763672, + "router_z_loss_mlp": 0.18664551, + "step": 2397, + "time_per_iteration": 2.4717280864715576 + }, + { + "auxiliary_loss_clip": 0.01172851, + "auxiliary_loss_mlp": 0.01056606, + "balance_loss_clip": 1.06659579, + "balance_loss_mlp": 1.03708577, + "epoch": 0.0695838895014799, + "flos": 37627307585280.0, + "grad_norm": 2.2433132009607926, + "language_loss": 0.85710686, + "learning_rate": 3.98358983680541e-06, + "loss": 0.87940145, + "num_input_tokens_seen": 67540305, + "router_z_loss_clip": 1.06201172, + "router_z_loss_mlp": 0.19543457, + "step": 2398, + "time_per_iteration": 2.6369683742523193 + }, + { + "auxiliary_loss_clip": 0.01173139, + "auxiliary_loss_mlp": 0.01055671, + "balance_loss_clip": 1.06562662, + "balance_loss_mlp": 1.03686023, + "epoch": 0.06961290696999593, + "flos": 23836174295040.0, + "grad_norm": 1.9342575267906512, + "language_loss": 0.81027889, + "learning_rate": 3.98356579918164e-06, + "loss": 0.83256698, + "num_input_tokens_seen": 67558600, + "router_z_loss_clip": 1.07470703, + "router_z_loss_mlp": 0.18811035, + "step": 2399, + "time_per_iteration": 2.5091540813446045 + }, + { + "auxiliary_loss_clip": 0.01172656, + "auxiliary_loss_mlp": 0.01053025, + "balance_loss_clip": 1.06482959, + "balance_loss_mlp": 1.03432059, + "epoch": 0.06964192443851198, + "flos": 30702905297280.0, + "grad_norm": 1.797136668086608, + "language_loss": 0.84404248, + "learning_rate": 3.983541744038214e-06, + "loss": 0.86629927, + "num_input_tokens_seen": 67579145, + "router_z_loss_clip": 1.078125, + "router_z_loss_mlp": 0.18701172, + "step": 2400, + "time_per_iteration": 2.583832263946533 + }, + { + "auxiliary_loss_clip": 0.01176352, + "auxiliary_loss_mlp": 0.01049147, + "balance_loss_clip": 1.07099867, + "balance_loss_mlp": 1.02988291, + "epoch": 0.06967094190702804, + "flos": 21537496454400.0, + "grad_norm": 2.198077633725202, + "language_loss": 0.77586371, + "learning_rate": 3.983517671375344e-06, + "loss": 0.79811871, + "num_input_tokens_seen": 67593515, + "router_z_loss_clip": 1.05615234, + "router_z_loss_mlp": 0.19256592, + "step": 2401, + "time_per_iteration": 2.4633495807647705 + }, + { + "auxiliary_loss_clip": 0.01175866, + "auxiliary_loss_mlp": 0.01050533, + "balance_loss_clip": 1.0678798, + "balance_loss_mlp": 1.02973044, + "epoch": 0.06969995937554407, + "flos": 21281956122240.0, + "grad_norm": 3.0007219609851483, + "language_loss": 1.02517104, + "learning_rate": 3.983493581193243e-06, + "loss": 1.04743505, + "num_input_tokens_seen": 67605750, + "router_z_loss_clip": 1.07910156, + "router_z_loss_mlp": 0.20794678, + "step": 2402, + "time_per_iteration": 2.4953243732452393 + }, + { + "auxiliary_loss_clip": 0.01168919, + "auxiliary_loss_mlp": 0.01045483, + "balance_loss_clip": 1.06747782, + "balance_loss_mlp": 1.02756619, + "epoch": 0.06972897684406013, + "flos": 11983247907840.0, + "grad_norm": 2.84274949276529, + "language_loss": 0.74077332, + "learning_rate": 3.983469473492126e-06, + "loss": 0.76291734, + "num_input_tokens_seen": 67619705, + "router_z_loss_clip": 1.01611328, + "router_z_loss_mlp": 0.17919922, + "step": 2403, + "time_per_iteration": 2.4493093490600586 + }, + { + "auxiliary_loss_clip": 0.01179866, + "auxiliary_loss_mlp": 0.01063227, + "balance_loss_clip": 1.07037652, + "balance_loss_mlp": 1.04167342, + "epoch": 0.06975799431257618, + "flos": 45653449783680.0, + "grad_norm": 2.133516071768301, + "language_loss": 0.90299577, + "learning_rate": 3.983445348272203e-06, + "loss": 0.92542672, + "num_input_tokens_seen": 67639530, + "router_z_loss_clip": 1.09472656, + "router_z_loss_mlp": 0.21557617, + "step": 2404, + "time_per_iteration": 2.787989377975464 + }, + { + "auxiliary_loss_clip": 0.01178845, + "auxiliary_loss_mlp": 0.01056041, + "balance_loss_clip": 1.07003832, + "balance_loss_mlp": 1.03503621, + "epoch": 0.06978701178109221, + "flos": 17924919022080.0, + "grad_norm": 2.1705410765783433, + "language_loss": 0.75288701, + "learning_rate": 3.983421205533688e-06, + "loss": 0.77523589, + "num_input_tokens_seen": 67654340, + "router_z_loss_clip": 1.08789062, + "router_z_loss_mlp": 0.20996094, + "step": 2405, + "time_per_iteration": 2.460893392562866 + }, + { + "auxiliary_loss_clip": 0.01164564, + "auxiliary_loss_mlp": 0.01051899, + "balance_loss_clip": 1.06525326, + "balance_loss_mlp": 1.03379726, + "epoch": 0.06981602924960827, + "flos": 60976891313280.0, + "grad_norm": 1.7728313852829864, + "language_loss": 0.87545037, + "learning_rate": 3.9833970452767935e-06, + "loss": 0.89761496, + "num_input_tokens_seen": 67682505, + "router_z_loss_clip": 0.99316406, + "router_z_loss_mlp": 0.18115234, + "step": 2406, + "time_per_iteration": 2.8288888931274414 + }, + { + "auxiliary_loss_clip": 0.01052053, + "auxiliary_loss_mlp": 0.01012016, + "balance_loss_clip": 1.0238167, + "balance_loss_mlp": 1.01064491, + "epoch": 0.0698450467181243, + "flos": 60828726689280.0, + "grad_norm": 0.718579677005347, + "language_loss": 0.51000559, + "learning_rate": 3.9833728675017355e-06, + "loss": 0.5306462, + "num_input_tokens_seen": 67735405, + "router_z_loss_clip": 0.28320312, + "router_z_loss_mlp": 0.01373291, + "step": 2407, + "time_per_iteration": 2.9327051639556885 + }, + { + "auxiliary_loss_clip": 0.0116908, + "auxiliary_loss_mlp": 0.01058038, + "balance_loss_clip": 1.06558359, + "balance_loss_mlp": 1.03944123, + "epoch": 0.06987406418664036, + "flos": 16357562087040.0, + "grad_norm": 2.2072995229141417, + "language_loss": 0.73088914, + "learning_rate": 3.983348672208724e-06, + "loss": 0.75316036, + "num_input_tokens_seen": 67750465, + "router_z_loss_clip": 1.03369141, + "router_z_loss_mlp": 0.18603516, + "step": 2408, + "time_per_iteration": 2.5236353874206543 + }, + { + "auxiliary_loss_clip": 0.01173993, + "auxiliary_loss_mlp": 0.01055857, + "balance_loss_clip": 1.06719542, + "balance_loss_mlp": 1.03580546, + "epoch": 0.06990308165515641, + "flos": 74732437203840.0, + "grad_norm": 1.8838176760113194, + "language_loss": 0.80942404, + "learning_rate": 3.983324459397975e-06, + "loss": 0.8317225, + "num_input_tokens_seen": 67775430, + "router_z_loss_clip": 1.06884766, + "router_z_loss_mlp": 0.20050049, + "step": 2409, + "time_per_iteration": 2.9532480239868164 + }, + { + "auxiliary_loss_clip": 0.01052023, + "auxiliary_loss_mlp": 0.01002078, + "balance_loss_clip": 1.02339506, + "balance_loss_mlp": 1.00071335, + "epoch": 0.06993209912367244, + "flos": 67189692240000.0, + "grad_norm": 0.7380609537987697, + "language_loss": 0.5180732, + "learning_rate": 3.983300229069703e-06, + "loss": 0.53861415, + "num_input_tokens_seen": 67835045, + "router_z_loss_clip": 0.28515625, + "router_z_loss_mlp": 0.01367188, + "step": 2410, + "time_per_iteration": 3.1111767292022705 + }, + { + "auxiliary_loss_clip": 0.01169146, + "auxiliary_loss_mlp": 0.01046113, + "balance_loss_clip": 1.06849647, + "balance_loss_mlp": 1.02730203, + "epoch": 0.0699611165921885, + "flos": 16975512691200.0, + "grad_norm": 2.2535597165353702, + "language_loss": 0.64719599, + "learning_rate": 3.9832759812241195e-06, + "loss": 0.66934854, + "num_input_tokens_seen": 67850030, + "router_z_loss_clip": 1.00634766, + "router_z_loss_mlp": 0.18798828, + "step": 2411, + "time_per_iteration": 2.5457470417022705 + }, + { + "auxiliary_loss_clip": 0.01173704, + "auxiliary_loss_mlp": 0.01047747, + "balance_loss_clip": 1.06830883, + "balance_loss_mlp": 1.02975273, + "epoch": 0.06999013406070455, + "flos": 12196269515520.0, + "grad_norm": 2.625739903881718, + "language_loss": 0.83333939, + "learning_rate": 3.98325171586144e-06, + "loss": 0.85555393, + "num_input_tokens_seen": 67861730, + "router_z_loss_clip": 1.05517578, + "router_z_loss_mlp": 0.17980957, + "step": 2412, + "time_per_iteration": 2.436823844909668 + }, + { + "auxiliary_loss_clip": 0.01170055, + "auxiliary_loss_mlp": 0.01047459, + "balance_loss_clip": 1.06843603, + "balance_loss_mlp": 1.03009069, + "epoch": 0.07001915152922059, + "flos": 16866703416960.0, + "grad_norm": 4.378142692276706, + "language_loss": 0.81234008, + "learning_rate": 3.983227432981879e-06, + "loss": 0.83451521, + "num_input_tokens_seen": 67877200, + "router_z_loss_clip": 1.01513672, + "router_z_loss_mlp": 0.17370605, + "step": 2413, + "time_per_iteration": 2.4706802368164062 + }, + { + "auxiliary_loss_clip": 0.01170427, + "auxiliary_loss_mlp": 0.0104683, + "balance_loss_clip": 1.06805825, + "balance_loss_mlp": 1.02694595, + "epoch": 0.07004816899773664, + "flos": 20554154749440.0, + "grad_norm": 2.1035825107802655, + "language_loss": 0.73190379, + "learning_rate": 3.9832031325856515e-06, + "loss": 0.75407636, + "num_input_tokens_seen": 67893830, + "router_z_loss_clip": 1.0234375, + "router_z_loss_mlp": 0.1987915, + "step": 2414, + "time_per_iteration": 2.5354208946228027 + }, + { + "auxiliary_loss_clip": 0.01176654, + "auxiliary_loss_mlp": 0.01057307, + "balance_loss_clip": 1.07199121, + "balance_loss_mlp": 1.03859758, + "epoch": 0.07007718646625269, + "flos": 39821701265280.0, + "grad_norm": 1.9784687788370126, + "language_loss": 1.16582537, + "learning_rate": 3.98317881467297e-06, + "loss": 1.18816495, + "num_input_tokens_seen": 67917965, + "router_z_loss_clip": 1.046875, + "router_z_loss_mlp": 0.18701172, + "step": 2415, + "time_per_iteration": 2.6987032890319824 + }, + { + "auxiliary_loss_clip": 0.01169923, + "auxiliary_loss_mlp": 0.01052285, + "balance_loss_clip": 1.06651235, + "balance_loss_mlp": 1.03409362, + "epoch": 0.07010620393476873, + "flos": 24714621308160.0, + "grad_norm": 2.606297482823207, + "language_loss": 0.94206524, + "learning_rate": 3.983154479244051e-06, + "loss": 0.96428728, + "num_input_tokens_seen": 67933780, + "router_z_loss_clip": 1.03369141, + "router_z_loss_mlp": 0.18188477, + "step": 2416, + "time_per_iteration": 2.53574275970459 + }, + { + "auxiliary_loss_clip": 0.01171671, + "auxiliary_loss_mlp": 0.01055765, + "balance_loss_clip": 1.06903791, + "balance_loss_mlp": 1.03673875, + "epoch": 0.07013522140328478, + "flos": 15557436679680.0, + "grad_norm": 2.7233908439575987, + "language_loss": 0.94342542, + "learning_rate": 3.9831301262991105e-06, + "loss": 0.96569973, + "num_input_tokens_seen": 67946980, + "router_z_loss_clip": 1.02636719, + "router_z_loss_mlp": 0.19042969, + "step": 2417, + "time_per_iteration": 2.4664931297302246 + }, + { + "auxiliary_loss_clip": 0.01056081, + "auxiliary_loss_mlp": 0.01010283, + "balance_loss_clip": 1.02749372, + "balance_loss_mlp": 1.0088886, + "epoch": 0.07016423887180083, + "flos": 61096262163840.0, + "grad_norm": 0.7285454574294177, + "language_loss": 0.49295059, + "learning_rate": 3.983105755838361e-06, + "loss": 0.51361418, + "num_input_tokens_seen": 68007435, + "router_z_loss_clip": 0.28515625, + "router_z_loss_mlp": 0.01397705, + "step": 2418, + "time_per_iteration": 3.062796115875244 + }, + { + "auxiliary_loss_clip": 0.01056161, + "auxiliary_loss_mlp": 0.01004897, + "balance_loss_clip": 1.02763629, + "balance_loss_mlp": 1.00354433, + "epoch": 0.07019325634031687, + "flos": 67000729766400.0, + "grad_norm": 0.701669050567708, + "language_loss": 0.52531469, + "learning_rate": 3.983081367862019e-06, + "loss": 0.54592526, + "num_input_tokens_seen": 68071090, + "router_z_loss_clip": 0.28515625, + "router_z_loss_mlp": 0.0135498, + "step": 2419, + "time_per_iteration": 3.1202430725097656 + }, + { + "auxiliary_loss_clip": 0.01054987, + "auxiliary_loss_mlp": 0.01003752, + "balance_loss_clip": 1.02666879, + "balance_loss_mlp": 1.00235772, + "epoch": 0.07022227380883292, + "flos": 74784152787840.0, + "grad_norm": 0.6295156224040762, + "language_loss": 0.51793253, + "learning_rate": 3.983056962370301e-06, + "loss": 0.53851998, + "num_input_tokens_seen": 68142680, + "router_z_loss_clip": 0.28320312, + "router_z_loss_mlp": 0.01397705, + "step": 2420, + "time_per_iteration": 3.223851203918457 + }, + { + "auxiliary_loss_clip": 0.01182121, + "auxiliary_loss_mlp": 0.01050382, + "balance_loss_clip": 1.07253861, + "balance_loss_mlp": 1.02930021, + "epoch": 0.07025129127734897, + "flos": 27594405377280.0, + "grad_norm": 3.9498107519564654, + "language_loss": 1.11832654, + "learning_rate": 3.9830325393634205e-06, + "loss": 1.14065146, + "num_input_tokens_seen": 68159435, + "router_z_loss_clip": 1.09814453, + "router_z_loss_mlp": 0.21075439, + "step": 2421, + "time_per_iteration": 5.0270819664001465 + }, + { + "auxiliary_loss_clip": 0.01055153, + "auxiliary_loss_mlp": 0.00998621, + "balance_loss_clip": 1.02692759, + "balance_loss_mlp": 0.99736369, + "epoch": 0.07028030874586501, + "flos": 66212024883840.0, + "grad_norm": 0.7140450541317741, + "language_loss": 0.520118, + "learning_rate": 3.983008098841594e-06, + "loss": 0.54065573, + "num_input_tokens_seen": 68226570, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01257324, + "step": 2422, + "time_per_iteration": 8.021208047866821 + }, + { + "auxiliary_loss_clip": 0.01054898, + "auxiliary_loss_mlp": 0.01002196, + "balance_loss_clip": 1.02664995, + "balance_loss_mlp": 1.00089669, + "epoch": 0.07030932621438106, + "flos": 63718566566400.0, + "grad_norm": 0.7179589099139652, + "language_loss": 0.51719177, + "learning_rate": 3.9829836408050385e-06, + "loss": 0.53776264, + "num_input_tokens_seen": 68286765, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01300049, + "step": 2423, + "time_per_iteration": 5.265848398208618 + }, + { + "auxiliary_loss_clip": 0.01166601, + "auxiliary_loss_mlp": 0.01043389, + "balance_loss_clip": 1.0707432, + "balance_loss_mlp": 1.02579367, + "epoch": 0.0703383436828971, + "flos": 15880416796800.0, + "grad_norm": 2.488974484823663, + "language_loss": 0.66501379, + "learning_rate": 3.982959165253967e-06, + "loss": 0.6871137, + "num_input_tokens_seen": 68297915, + "router_z_loss_clip": 0.95800781, + "router_z_loss_mlp": 0.17590332, + "step": 2424, + "time_per_iteration": 2.4645020961761475 + }, + { + "auxiliary_loss_clip": 0.01169945, + "auxiliary_loss_mlp": 0.01057764, + "balance_loss_clip": 1.06828833, + "balance_loss_mlp": 1.03873777, + "epoch": 0.07036736115141315, + "flos": 11647267067520.0, + "grad_norm": 3.3906376940761374, + "language_loss": 0.84282887, + "learning_rate": 3.9829346721886e-06, + "loss": 0.86510599, + "num_input_tokens_seen": 68308665, + "router_z_loss_clip": 1.01660156, + "router_z_loss_mlp": 0.19042969, + "step": 2425, + "time_per_iteration": 2.448245048522949 + }, + { + "auxiliary_loss_clip": 0.01052213, + "auxiliary_loss_mlp": 0.01010257, + "balance_loss_clip": 1.02431321, + "balance_loss_mlp": 1.00899911, + "epoch": 0.0703963786199292, + "flos": 70907918549760.0, + "grad_norm": 0.790810710426032, + "language_loss": 0.58118021, + "learning_rate": 3.982910161609151e-06, + "loss": 0.60180485, + "num_input_tokens_seen": 68365385, + "router_z_loss_clip": 0.27929688, + "router_z_loss_mlp": 0.01257324, + "step": 2426, + "time_per_iteration": 3.0344412326812744 + }, + { + "auxiliary_loss_clip": 0.01174658, + "auxiliary_loss_mlp": 0.0104688, + "balance_loss_clip": 1.06851935, + "balance_loss_mlp": 1.02883184, + "epoch": 0.07042539608844524, + "flos": 16158403128960.0, + "grad_norm": 3.638844903325996, + "language_loss": 0.83407062, + "learning_rate": 3.982885633515837e-06, + "loss": 0.85628599, + "num_input_tokens_seen": 68382125, + "router_z_loss_clip": 1.06152344, + "router_z_loss_mlp": 0.18054199, + "step": 2427, + "time_per_iteration": 2.504620313644409 + }, + { + "auxiliary_loss_clip": 0.01165726, + "auxiliary_loss_mlp": 0.01050537, + "balance_loss_clip": 1.06985891, + "balance_loss_mlp": 1.03349054, + "epoch": 0.07045441355696129, + "flos": 36898141495680.0, + "grad_norm": 2.1650652636319, + "language_loss": 0.70557821, + "learning_rate": 3.982861087908874e-06, + "loss": 0.72774088, + "num_input_tokens_seen": 68398300, + "router_z_loss_clip": 0.95947266, + "router_z_loss_mlp": 0.1706543, + "step": 2428, + "time_per_iteration": 2.642442226409912 + }, + { + "auxiliary_loss_clip": 0.01189241, + "auxiliary_loss_mlp": 0.01061369, + "balance_loss_clip": 1.07553816, + "balance_loss_mlp": 1.03936934, + "epoch": 0.07048343102547734, + "flos": 30146396906880.0, + "grad_norm": 2.1584801267898497, + "language_loss": 0.91173995, + "learning_rate": 3.98283652478848e-06, + "loss": 0.93424606, + "num_input_tokens_seen": 68416270, + "router_z_loss_clip": 1.13769531, + "router_z_loss_mlp": 0.2199707, + "step": 2429, + "time_per_iteration": 2.567405939102173 + }, + { + "auxiliary_loss_clip": 0.01176744, + "auxiliary_loss_mlp": 0.01060068, + "balance_loss_clip": 1.06571984, + "balance_loss_mlp": 1.03783536, + "epoch": 0.07051244849399338, + "flos": 18980189712000.0, + "grad_norm": 3.0238887594483064, + "language_loss": 0.86658335, + "learning_rate": 3.982811944154872e-06, + "loss": 0.88895148, + "num_input_tokens_seen": 68427270, + "router_z_loss_clip": 1.10986328, + "router_z_loss_mlp": 0.22253418, + "step": 2430, + "time_per_iteration": 2.557703733444214 + }, + { + "auxiliary_loss_clip": 0.01052586, + "auxiliary_loss_mlp": 0.0100641, + "balance_loss_clip": 1.02462363, + "balance_loss_mlp": 1.00511682, + "epoch": 0.07054146596250943, + "flos": 74780525514240.0, + "grad_norm": 0.7011293693652804, + "language_loss": 0.58471078, + "learning_rate": 3.982787346008265e-06, + "loss": 0.60530078, + "num_input_tokens_seen": 68487990, + "router_z_loss_clip": 0.27929688, + "router_z_loss_mlp": 0.01293945, + "step": 2431, + "time_per_iteration": 3.1609127521514893 + }, + { + "auxiliary_loss_clip": 0.01176904, + "auxiliary_loss_mlp": 0.01051088, + "balance_loss_clip": 1.06816804, + "balance_loss_mlp": 1.03088248, + "epoch": 0.07057048343102548, + "flos": 21063403820160.0, + "grad_norm": 2.349489131718107, + "language_loss": 0.81940186, + "learning_rate": 3.98276273034888e-06, + "loss": 0.84168178, + "num_input_tokens_seen": 68502085, + "router_z_loss_clip": 1.08642578, + "router_z_loss_mlp": 0.2019043, + "step": 2432, + "time_per_iteration": 2.4622724056243896 + }, + { + "auxiliary_loss_clip": 0.01166749, + "auxiliary_loss_mlp": 0.01054686, + "balance_loss_clip": 1.06889784, + "balance_loss_mlp": 1.03892028, + "epoch": 0.07059950089954152, + "flos": 28612221160320.0, + "grad_norm": 2.9989028857780946, + "language_loss": 0.88804811, + "learning_rate": 3.98273809717693e-06, + "loss": 0.91026241, + "num_input_tokens_seen": 68515850, + "router_z_loss_clip": 0.97900391, + "router_z_loss_mlp": 0.15771484, + "step": 2433, + "time_per_iteration": 2.575028657913208 + }, + { + "auxiliary_loss_clip": 0.01050878, + "auxiliary_loss_mlp": 0.01009231, + "balance_loss_clip": 1.02292919, + "balance_loss_mlp": 1.00802732, + "epoch": 0.07062851836805757, + "flos": 57115348715520.0, + "grad_norm": 0.6584284419280285, + "language_loss": 0.52910775, + "learning_rate": 3.982713446492637e-06, + "loss": 0.54970884, + "num_input_tokens_seen": 68573430, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01202393, + "step": 2434, + "time_per_iteration": 2.9822676181793213 + }, + { + "auxiliary_loss_clip": 0.01179351, + "auxiliary_loss_mlp": 0.01057295, + "balance_loss_clip": 1.06830204, + "balance_loss_mlp": 1.03671932, + "epoch": 0.07065753583657362, + "flos": 25330345269120.0, + "grad_norm": 2.245997944502763, + "language_loss": 0.99929273, + "learning_rate": 3.982688778296215e-06, + "loss": 1.02165914, + "num_input_tokens_seen": 68588735, + "router_z_loss_clip": 1.11035156, + "router_z_loss_mlp": 0.20581055, + "step": 2435, + "time_per_iteration": 2.575079917907715 + }, + { + "auxiliary_loss_clip": 0.01168678, + "auxiliary_loss_mlp": 0.01051293, + "balance_loss_clip": 1.06781983, + "balance_loss_mlp": 1.03180194, + "epoch": 0.07068655330508966, + "flos": 29709256389120.0, + "grad_norm": 2.541884218248202, + "language_loss": 0.70764709, + "learning_rate": 3.982664092587884e-06, + "loss": 0.72984678, + "num_input_tokens_seen": 68603985, + "router_z_loss_clip": 1.00878906, + "router_z_loss_mlp": 0.19470215, + "step": 2436, + "time_per_iteration": 2.5057175159454346 + }, + { + "auxiliary_loss_clip": 0.01171096, + "auxiliary_loss_mlp": 0.01043407, + "balance_loss_clip": 1.06586361, + "balance_loss_mlp": 1.02492321, + "epoch": 0.07071557077360571, + "flos": 16865697836160.0, + "grad_norm": 3.6891848793179096, + "language_loss": 0.93114638, + "learning_rate": 3.98263938936786e-06, + "loss": 0.95329142, + "num_input_tokens_seen": 68619415, + "router_z_loss_clip": 1.05175781, + "router_z_loss_mlp": 0.18475342, + "step": 2437, + "time_per_iteration": 2.5705440044403076 + }, + { + "auxiliary_loss_clip": 0.01172166, + "auxiliary_loss_mlp": 0.01043563, + "balance_loss_clip": 1.06811047, + "balance_loss_mlp": 1.0268681, + "epoch": 0.07074458824212176, + "flos": 39303401967360.0, + "grad_norm": 2.6391192336837026, + "language_loss": 0.96230656, + "learning_rate": 3.982614668636365e-06, + "loss": 0.98446381, + "num_input_tokens_seen": 68638860, + "router_z_loss_clip": 1.04101562, + "router_z_loss_mlp": 0.16699219, + "step": 2438, + "time_per_iteration": 2.6666362285614014 + }, + { + "auxiliary_loss_clip": 0.01052858, + "auxiliary_loss_mlp": 0.01002304, + "balance_loss_clip": 1.02493453, + "balance_loss_mlp": 1.00104046, + "epoch": 0.0707736057106378, + "flos": 67106989175040.0, + "grad_norm": 0.7832241261097317, + "language_loss": 0.5308603, + "learning_rate": 3.982589930393613e-06, + "loss": 0.55141193, + "num_input_tokens_seen": 68694985, + "router_z_loss_clip": 0.27929688, + "router_z_loss_mlp": 0.01263428, + "step": 2439, + "time_per_iteration": 2.979496955871582 + }, + { + "auxiliary_loss_clip": 0.01054177, + "auxiliary_loss_mlp": 0.01002668, + "balance_loss_clip": 1.02610183, + "balance_loss_mlp": 1.00127923, + "epoch": 0.07080262317915385, + "flos": 59178846257280.0, + "grad_norm": 0.7275003578263773, + "language_loss": 0.45464805, + "learning_rate": 3.982565174639825e-06, + "loss": 0.47521648, + "num_input_tokens_seen": 68740160, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01391602, + "step": 2440, + "time_per_iteration": 2.7675576210021973 + }, + { + "auxiliary_loss_clip": 0.01178572, + "auxiliary_loss_mlp": 0.01052142, + "balance_loss_clip": 1.07242227, + "balance_loss_mlp": 1.03585196, + "epoch": 0.07083164064766989, + "flos": 14932626577920.0, + "grad_norm": 2.079979873372311, + "language_loss": 0.70108652, + "learning_rate": 3.982540401375219e-06, + "loss": 0.72339362, + "num_input_tokens_seen": 68753730, + "router_z_loss_clip": 1.06054688, + "router_z_loss_mlp": 0.16308594, + "step": 2441, + "time_per_iteration": 2.5525660514831543 + }, + { + "auxiliary_loss_clip": 0.01176934, + "auxiliary_loss_mlp": 0.01047945, + "balance_loss_clip": 1.07039201, + "balance_loss_mlp": 1.02726245, + "epoch": 0.07086065811618594, + "flos": 39816421966080.0, + "grad_norm": 2.7837026109203653, + "language_loss": 0.64496422, + "learning_rate": 3.982515610600015e-06, + "loss": 0.66721302, + "num_input_tokens_seen": 68769855, + "router_z_loss_clip": 1.06640625, + "router_z_loss_mlp": 0.20715332, + "step": 2442, + "time_per_iteration": 2.598414421081543 + }, + { + "auxiliary_loss_clip": 0.01178288, + "auxiliary_loss_mlp": 0.01051432, + "balance_loss_clip": 1.06968594, + "balance_loss_mlp": 1.03034949, + "epoch": 0.070889675584702, + "flos": 36092305825920.0, + "grad_norm": 4.062971662616356, + "language_loss": 0.99518061, + "learning_rate": 3.98249080231443e-06, + "loss": 1.01747775, + "num_input_tokens_seen": 68785715, + "router_z_loss_clip": 1.08496094, + "router_z_loss_mlp": 0.21075439, + "step": 2443, + "time_per_iteration": 2.611112117767334 + }, + { + "auxiliary_loss_clip": 0.01052116, + "auxiliary_loss_mlp": 0.0100386, + "balance_loss_clip": 1.02385163, + "balance_loss_mlp": 1.00250661, + "epoch": 0.07091869305321803, + "flos": 74784296442240.0, + "grad_norm": 0.6826512819265704, + "language_loss": 0.58131099, + "learning_rate": 3.982465976518685e-06, + "loss": 0.60187078, + "num_input_tokens_seen": 68854945, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.0135498, + "step": 2444, + "time_per_iteration": 3.2262356281280518 + }, + { + "auxiliary_loss_clip": 0.01051044, + "auxiliary_loss_mlp": 0.01003676, + "balance_loss_clip": 1.02277088, + "balance_loss_mlp": 1.00234056, + "epoch": 0.07094771052173408, + "flos": 74772265386240.0, + "grad_norm": 0.6843290823867045, + "language_loss": 0.52649647, + "learning_rate": 3.982441133212997e-06, + "loss": 0.54704368, + "num_input_tokens_seen": 68918040, + "router_z_loss_clip": 0.28320312, + "router_z_loss_mlp": 0.0133667, + "step": 2445, + "time_per_iteration": 3.112826108932495 + }, + { + "auxiliary_loss_clip": 0.01184954, + "auxiliary_loss_mlp": 0.01053902, + "balance_loss_clip": 1.07374668, + "balance_loss_mlp": 1.03345776, + "epoch": 0.07097672799025014, + "flos": 16793373801600.0, + "grad_norm": 2.91380662784409, + "language_loss": 0.73635077, + "learning_rate": 3.982416272397587e-06, + "loss": 0.75873935, + "num_input_tokens_seen": 68931285, + "router_z_loss_clip": 1.11035156, + "router_z_loss_mlp": 0.20452881, + "step": 2446, + "time_per_iteration": 2.4608333110809326 + }, + { + "auxiliary_loss_clip": 0.010492, + "auxiliary_loss_mlp": 0.01002832, + "balance_loss_clip": 1.02073622, + "balance_loss_mlp": 1.00146091, + "epoch": 0.07100574545876617, + "flos": 57332356732800.0, + "grad_norm": 0.6474350417329094, + "language_loss": 0.49308121, + "learning_rate": 3.982391394072675e-06, + "loss": 0.51360154, + "num_input_tokens_seen": 68991230, + "router_z_loss_clip": 0.28515625, + "router_z_loss_mlp": 0.01373291, + "step": 2447, + "time_per_iteration": 3.0398247241973877 + }, + { + "auxiliary_loss_clip": 0.0117345, + "auxiliary_loss_mlp": 0.01059073, + "balance_loss_clip": 1.06817687, + "balance_loss_mlp": 1.04044652, + "epoch": 0.07103476292728222, + "flos": 26715096437760.0, + "grad_norm": 2.708352956337526, + "language_loss": 0.90436786, + "learning_rate": 3.982366498238478e-06, + "loss": 0.92669308, + "num_input_tokens_seen": 69004245, + "router_z_loss_clip": 1.05175781, + "router_z_loss_mlp": 0.18634033, + "step": 2448, + "time_per_iteration": 2.548820734024048 + }, + { + "auxiliary_loss_clip": 0.01175008, + "auxiliary_loss_mlp": 0.01057198, + "balance_loss_clip": 1.07535017, + "balance_loss_mlp": 1.03839839, + "epoch": 0.07106378039579828, + "flos": 28324038343680.0, + "grad_norm": 3.0366156604480627, + "language_loss": 0.77342182, + "learning_rate": 3.98234158489522e-06, + "loss": 0.79574382, + "num_input_tokens_seen": 69020540, + "router_z_loss_clip": 0.99707031, + "router_z_loss_mlp": 0.18774414, + "step": 2449, + "time_per_iteration": 2.561095952987671 + }, + { + "auxiliary_loss_clip": 0.0105207, + "auxiliary_loss_mlp": 0.01003316, + "balance_loss_clip": 1.02341795, + "balance_loss_mlp": 1.00196934, + "epoch": 0.07109279786431431, + "flos": 69555522556800.0, + "grad_norm": 0.6737066403638926, + "language_loss": 0.50074524, + "learning_rate": 3.982316654043118e-06, + "loss": 0.52129912, + "num_input_tokens_seen": 69080925, + "router_z_loss_clip": 0.28710938, + "router_z_loss_mlp": 0.01348877, + "step": 2450, + "time_per_iteration": 3.094940662384033 + }, + { + "auxiliary_loss_clip": 0.01180454, + "auxiliary_loss_mlp": 0.01050493, + "balance_loss_clip": 1.0727551, + "balance_loss_mlp": 1.03013229, + "epoch": 0.07112181533283037, + "flos": 39304335720960.0, + "grad_norm": 2.0968735690324487, + "language_loss": 0.70555311, + "learning_rate": 3.982291705682393e-06, + "loss": 0.7278626, + "num_input_tokens_seen": 69096295, + "router_z_loss_clip": 1.07714844, + "router_z_loss_mlp": 0.20361328, + "step": 2451, + "time_per_iteration": 2.6636862754821777 + }, + { + "auxiliary_loss_clip": 0.01178572, + "auxiliary_loss_mlp": 0.01055207, + "balance_loss_clip": 1.0715816, + "balance_loss_mlp": 1.03566909, + "epoch": 0.07115083280134642, + "flos": 43076429452800.0, + "grad_norm": 2.2987943665322956, + "language_loss": 0.87675244, + "learning_rate": 3.9822667398132665e-06, + "loss": 0.89909017, + "num_input_tokens_seen": 69121620, + "router_z_loss_clip": 1.06884766, + "router_z_loss_mlp": 0.19543457, + "step": 2452, + "time_per_iteration": 2.6942882537841797 + }, + { + "auxiliary_loss_clip": 0.01050569, + "auxiliary_loss_mlp": 0.01003246, + "balance_loss_clip": 1.02180862, + "balance_loss_mlp": 1.00198269, + "epoch": 0.07117985026986245, + "flos": 70981212251520.0, + "grad_norm": 0.7419098134085231, + "language_loss": 0.48418099, + "learning_rate": 3.982241756435958e-06, + "loss": 0.50471914, + "num_input_tokens_seen": 69185190, + "router_z_loss_clip": 0.2890625, + "router_z_loss_mlp": 0.01263428, + "step": 2453, + "time_per_iteration": 3.2551469802856445 + }, + { + "auxiliary_loss_clip": 0.01050543, + "auxiliary_loss_mlp": 0.01003775, + "balance_loss_clip": 1.02149081, + "balance_loss_mlp": 1.00254154, + "epoch": 0.0712088677383785, + "flos": 71970587441280.0, + "grad_norm": 0.6957579778908127, + "language_loss": 0.5596211, + "learning_rate": 3.982216755550687e-06, + "loss": 0.58016431, + "num_input_tokens_seen": 69249070, + "router_z_loss_clip": 0.2890625, + "router_z_loss_mlp": 0.0123291, + "step": 2454, + "time_per_iteration": 3.1216073036193848 + }, + { + "auxiliary_loss_clip": 0.01049846, + "auxiliary_loss_mlp": 0.01006407, + "balance_loss_clip": 1.0205791, + "balance_loss_mlp": 1.00517297, + "epoch": 0.07123788520689454, + "flos": 54370587870720.0, + "grad_norm": 0.7164635563820616, + "language_loss": 0.55595994, + "learning_rate": 3.982191737157677e-06, + "loss": 0.57652247, + "num_input_tokens_seen": 69300100, + "router_z_loss_clip": 0.29296875, + "router_z_loss_mlp": 0.0123291, + "step": 2455, + "time_per_iteration": 2.8753745555877686 + }, + { + "auxiliary_loss_clip": 0.01184635, + "auxiliary_loss_mlp": 0.01055184, + "balance_loss_clip": 1.07065868, + "balance_loss_mlp": 1.03516865, + "epoch": 0.0712669026754106, + "flos": 30957835098240.0, + "grad_norm": 1.9931950290082372, + "language_loss": 0.86588609, + "learning_rate": 3.982166701257146e-06, + "loss": 0.88828433, + "num_input_tokens_seen": 69322415, + "router_z_loss_clip": 1.14013672, + "router_z_loss_mlp": 0.20019531, + "step": 2456, + "time_per_iteration": 2.6271238327026367 + }, + { + "auxiliary_loss_clip": 0.01170062, + "auxiliary_loss_mlp": 0.01038079, + "balance_loss_clip": 1.06620121, + "balance_loss_mlp": 1.02087665, + "epoch": 0.07129592014392665, + "flos": 28981059966720.0, + "grad_norm": 2.2572995676165477, + "language_loss": 0.70432574, + "learning_rate": 3.982141647849318e-06, + "loss": 0.72640717, + "num_input_tokens_seen": 69339720, + "router_z_loss_clip": 1.03662109, + "router_z_loss_mlp": 0.17211914, + "step": 2457, + "time_per_iteration": 2.6184868812561035 + }, + { + "auxiliary_loss_clip": 0.01169618, + "auxiliary_loss_mlp": 0.01059445, + "balance_loss_clip": 1.06777644, + "balance_loss_mlp": 1.03990042, + "epoch": 0.07132493761244268, + "flos": 34415385431040.0, + "grad_norm": 2.7108041559954907, + "language_loss": 1.07505226, + "learning_rate": 3.9821165769344115e-06, + "loss": 1.09734297, + "num_input_tokens_seen": 69355595, + "router_z_loss_clip": 1.01757812, + "router_z_loss_mlp": 0.19537354, + "step": 2458, + "time_per_iteration": 2.674534559249878 + }, + { + "auxiliary_loss_clip": 0.01176334, + "auxiliary_loss_mlp": 0.01048993, + "balance_loss_clip": 1.06923997, + "balance_loss_mlp": 1.02770221, + "epoch": 0.07135395508095874, + "flos": 30768944451840.0, + "grad_norm": 3.215360494365664, + "language_loss": 0.94529688, + "learning_rate": 3.98209148851265e-06, + "loss": 0.96755016, + "num_input_tokens_seen": 69372860, + "router_z_loss_clip": 1.07128906, + "router_z_loss_mlp": 0.2130127, + "step": 2459, + "time_per_iteration": 2.532536029815674 + }, + { + "auxiliary_loss_clip": 0.01171788, + "auxiliary_loss_mlp": 0.01044884, + "balance_loss_clip": 1.07183027, + "balance_loss_mlp": 1.02797377, + "epoch": 0.07138297254947479, + "flos": 16791757689600.0, + "grad_norm": 2.5489984431157304, + "language_loss": 0.92466027, + "learning_rate": 3.982066382584254e-06, + "loss": 0.94682705, + "num_input_tokens_seen": 69385140, + "router_z_loss_clip": 1.00048828, + "router_z_loss_mlp": 0.16912842, + "step": 2460, + "time_per_iteration": 2.5045418739318848 + }, + { + "auxiliary_loss_clip": 0.01175176, + "auxiliary_loss_mlp": 0.01053755, + "balance_loss_clip": 1.06679285, + "balance_loss_mlp": 1.03385854, + "epoch": 0.07141199001799083, + "flos": 30877538244480.0, + "grad_norm": 2.402136858920361, + "language_loss": 1.17985988, + "learning_rate": 3.982041259149446e-06, + "loss": 1.20214915, + "num_input_tokens_seen": 69402850, + "router_z_loss_clip": 1.08349609, + "router_z_loss_mlp": 0.19885254, + "step": 2461, + "time_per_iteration": 2.613905429840088 + }, + { + "auxiliary_loss_clip": 0.01163843, + "auxiliary_loss_mlp": 0.01041609, + "balance_loss_clip": 1.06536913, + "balance_loss_mlp": 1.02527142, + "epoch": 0.07144100748650688, + "flos": 15917728049280.0, + "grad_norm": 2.6233380511726123, + "language_loss": 0.78359056, + "learning_rate": 3.982016118208448e-06, + "loss": 0.80564505, + "num_input_tokens_seen": 69415360, + "router_z_loss_clip": 0.984375, + "router_z_loss_mlp": 0.16339111, + "step": 2462, + "time_per_iteration": 2.4659552574157715 + }, + { + "auxiliary_loss_clip": 0.01055675, + "auxiliary_loss_mlp": 0.01001831, + "balance_loss_clip": 1.02615118, + "balance_loss_mlp": 1.00052547, + "epoch": 0.07147002495502293, + "flos": 61419314108160.0, + "grad_norm": 0.7219667672615003, + "language_loss": 0.51580364, + "learning_rate": 3.98199095976148e-06, + "loss": 0.53637868, + "num_input_tokens_seen": 69474755, + "router_z_loss_clip": 0.296875, + "router_z_loss_mlp": 0.01306152, + "step": 2463, + "time_per_iteration": 2.9934468269348145 + }, + { + "auxiliary_loss_clip": 0.0117734, + "auxiliary_loss_mlp": 0.01042856, + "balance_loss_clip": 1.07031465, + "balance_loss_mlp": 1.02284062, + "epoch": 0.07149904242353897, + "flos": 39231688464000.0, + "grad_norm": 2.383959253697052, + "language_loss": 0.9247545, + "learning_rate": 3.981965783808768e-06, + "loss": 0.94695646, + "num_input_tokens_seen": 69491895, + "router_z_loss_clip": 1.07177734, + "router_z_loss_mlp": 0.20007324, + "step": 2464, + "time_per_iteration": 2.6714444160461426 + }, + { + "auxiliary_loss_clip": 0.01178116, + "auxiliary_loss_mlp": 0.01054813, + "balance_loss_clip": 1.0710969, + "balance_loss_mlp": 1.03567362, + "epoch": 0.07152805989205502, + "flos": 27044828311680.0, + "grad_norm": 2.77887141729854, + "language_loss": 0.81921136, + "learning_rate": 3.981940590350531e-06, + "loss": 0.84154058, + "num_input_tokens_seen": 69507155, + "router_z_loss_clip": 1.07128906, + "router_z_loss_mlp": 0.19146729, + "step": 2465, + "time_per_iteration": 2.4710466861724854 + }, + { + "auxiliary_loss_clip": 0.01053052, + "auxiliary_loss_mlp": 0.0100588, + "balance_loss_clip": 1.0236249, + "balance_loss_mlp": 1.00449753, + "epoch": 0.07155707736057107, + "flos": 55102519307520.0, + "grad_norm": 0.719425723620848, + "language_loss": 0.52165836, + "learning_rate": 3.981915379386992e-06, + "loss": 0.54224771, + "num_input_tokens_seen": 69558720, + "router_z_loss_clip": 0.29296875, + "router_z_loss_mlp": 0.01385498, + "step": 2466, + "time_per_iteration": 2.8551156520843506 + }, + { + "auxiliary_loss_clip": 0.01174906, + "auxiliary_loss_mlp": 0.01047939, + "balance_loss_clip": 1.07162476, + "balance_loss_mlp": 1.02964067, + "epoch": 0.07158609482908711, + "flos": 15880560451200.0, + "grad_norm": 2.713737216791729, + "language_loss": 0.96456563, + "learning_rate": 3.981890150918376e-06, + "loss": 0.98679411, + "num_input_tokens_seen": 69570470, + "router_z_loss_clip": 1.03271484, + "router_z_loss_mlp": 0.18286133, + "step": 2467, + "time_per_iteration": 2.472895383834839 + }, + { + "auxiliary_loss_clip": 0.01182446, + "auxiliary_loss_mlp": 0.01049755, + "balance_loss_clip": 1.07196975, + "balance_loss_mlp": 1.02963257, + "epoch": 0.07161511229760316, + "flos": 30438099256320.0, + "grad_norm": 3.6234671638458855, + "language_loss": 0.81982672, + "learning_rate": 3.981864904944903e-06, + "loss": 0.84214884, + "num_input_tokens_seen": 69585825, + "router_z_loss_clip": 1.10400391, + "router_z_loss_mlp": 0.2010498, + "step": 2468, + "time_per_iteration": 2.576599597930908 + }, + { + "auxiliary_loss_clip": 0.0105333, + "auxiliary_loss_mlp": 0.00998111, + "balance_loss_clip": 1.02389693, + "balance_loss_mlp": 0.9967761, + "epoch": 0.07164412976611921, + "flos": 74771654855040.0, + "grad_norm": 0.6402093406683681, + "language_loss": 0.49156606, + "learning_rate": 3.981839641466798e-06, + "loss": 0.51208043, + "num_input_tokens_seen": 69648430, + "router_z_loss_clip": 0.29296875, + "router_z_loss_mlp": 0.0133667, + "step": 2469, + "time_per_iteration": 3.1269757747650146 + }, + { + "auxiliary_loss_clip": 0.01178912, + "auxiliary_loss_mlp": 0.01058803, + "balance_loss_clip": 1.0710175, + "balance_loss_mlp": 1.03763115, + "epoch": 0.07167314723463525, + "flos": 32299241529600.0, + "grad_norm": 3.5249705157517504, + "language_loss": 0.93159795, + "learning_rate": 3.981814360484283e-06, + "loss": 0.95397514, + "num_input_tokens_seen": 69664895, + "router_z_loss_clip": 1.07958984, + "router_z_loss_mlp": 0.21179199, + "step": 2470, + "time_per_iteration": 2.6312365531921387 + }, + { + "auxiliary_loss_clip": 0.01176518, + "auxiliary_loss_mlp": 0.01036865, + "balance_loss_clip": 1.07051492, + "balance_loss_mlp": 1.01929998, + "epoch": 0.0717021647031513, + "flos": 22885295506560.0, + "grad_norm": 1.9671596662031903, + "language_loss": 0.84182703, + "learning_rate": 3.981789061997581e-06, + "loss": 0.86396092, + "num_input_tokens_seen": 69681975, + "router_z_loss_clip": 1.06103516, + "router_z_loss_mlp": 0.17565918, + "step": 2471, + "time_per_iteration": 2.450438976287842 + }, + { + "auxiliary_loss_clip": 0.01179406, + "auxiliary_loss_mlp": 0.0104868, + "balance_loss_clip": 1.070997, + "balance_loss_mlp": 1.02923656, + "epoch": 0.07173118217166734, + "flos": 23433507855360.0, + "grad_norm": 2.2573873404996756, + "language_loss": 0.92553592, + "learning_rate": 3.981763746006916e-06, + "loss": 0.94781679, + "num_input_tokens_seen": 69698630, + "router_z_loss_clip": 1.08447266, + "router_z_loss_mlp": 0.19458008, + "step": 2472, + "time_per_iteration": 2.5294954776763916 + }, + { + "auxiliary_loss_clip": 0.01173214, + "auxiliary_loss_mlp": 0.01056504, + "balance_loss_clip": 1.06843948, + "balance_loss_mlp": 1.0383364, + "epoch": 0.07176019964018339, + "flos": 16794020246400.0, + "grad_norm": 2.0594057281328104, + "language_loss": 0.80517244, + "learning_rate": 3.981738412512513e-06, + "loss": 0.82746965, + "num_input_tokens_seen": 69713210, + "router_z_loss_clip": 1.04785156, + "router_z_loss_mlp": 0.18157959, + "step": 2473, + "time_per_iteration": 2.4883103370666504 + }, + { + "auxiliary_loss_clip": 0.01174199, + "auxiliary_loss_mlp": 0.01044361, + "balance_loss_clip": 1.06688535, + "balance_loss_mlp": 1.02535367, + "epoch": 0.07178921710869944, + "flos": 27482615274240.0, + "grad_norm": 2.3716437298194304, + "language_loss": 0.89375317, + "learning_rate": 3.981713061514593e-06, + "loss": 0.91593879, + "num_input_tokens_seen": 69732020, + "router_z_loss_clip": 1.07421875, + "router_z_loss_mlp": 0.19006348, + "step": 2474, + "time_per_iteration": 2.7086544036865234 + }, + { + "auxiliary_loss_clip": 0.01055886, + "auxiliary_loss_mlp": 0.01001822, + "balance_loss_clip": 1.02642846, + "balance_loss_mlp": 1.00050473, + "epoch": 0.07181823457721548, + "flos": 69796844081280.0, + "grad_norm": 0.6555207931764028, + "language_loss": 0.50426304, + "learning_rate": 3.981687693013383e-06, + "loss": 0.52484012, + "num_input_tokens_seen": 69799915, + "router_z_loss_clip": 0.29296875, + "router_z_loss_mlp": 0.01318359, + "step": 2475, + "time_per_iteration": 3.3038594722747803 + }, + { + "auxiliary_loss_clip": 0.01158308, + "auxiliary_loss_mlp": 0.01044857, + "balance_loss_clip": 1.06618571, + "balance_loss_mlp": 1.02882934, + "epoch": 0.07184725204573153, + "flos": 42881397580800.0, + "grad_norm": 2.8996676335085585, + "language_loss": 0.68967307, + "learning_rate": 3.981662307009104e-06, + "loss": 0.71170467, + "num_input_tokens_seen": 69819485, + "router_z_loss_clip": 0.92285156, + "router_z_loss_mlp": 0.16033936, + "step": 2476, + "time_per_iteration": 2.696378231048584 + }, + { + "auxiliary_loss_clip": 0.01161107, + "auxiliary_loss_mlp": 0.01043059, + "balance_loss_clip": 1.06467319, + "balance_loss_mlp": 1.02536249, + "epoch": 0.07187626951424758, + "flos": 30074683403520.0, + "grad_norm": 2.154739447384949, + "language_loss": 0.76902378, + "learning_rate": 3.981636903501982e-06, + "loss": 0.79106545, + "num_input_tokens_seen": 69834970, + "router_z_loss_clip": 0.96386719, + "router_z_loss_mlp": 0.17706299, + "step": 2477, + "time_per_iteration": 2.6141037940979004 + }, + { + "auxiliary_loss_clip": 0.01184663, + "auxiliary_loss_mlp": 0.01054044, + "balance_loss_clip": 1.07434797, + "balance_loss_mlp": 1.03282499, + "epoch": 0.07190528698276362, + "flos": 20623462041600.0, + "grad_norm": 2.648960466560044, + "language_loss": 0.92943251, + "learning_rate": 3.9816114824922406e-06, + "loss": 0.9518196, + "num_input_tokens_seen": 69846675, + "router_z_loss_clip": 1.10351562, + "router_z_loss_mlp": 0.21221924, + "step": 2478, + "time_per_iteration": 2.5534136295318604 + }, + { + "auxiliary_loss_clip": 0.01169439, + "auxiliary_loss_mlp": 0.01050663, + "balance_loss_clip": 1.06623971, + "balance_loss_mlp": 1.03119588, + "epoch": 0.07193430445127967, + "flos": 27229588894080.0, + "grad_norm": 1.99597020204994, + "language_loss": 0.85902059, + "learning_rate": 3.981586043980106e-06, + "loss": 0.88122153, + "num_input_tokens_seen": 69864930, + "router_z_loss_clip": 1.03222656, + "router_z_loss_mlp": 0.19470215, + "step": 2479, + "time_per_iteration": 2.609652519226074 + }, + { + "auxiliary_loss_clip": 0.0117169, + "auxiliary_loss_mlp": 0.01051928, + "balance_loss_clip": 1.07180715, + "balance_loss_mlp": 1.03475034, + "epoch": 0.07196332191979572, + "flos": 31681937370240.0, + "grad_norm": 1.996362594356961, + "language_loss": 0.82754099, + "learning_rate": 3.9815605879658e-06, + "loss": 0.84977716, + "num_input_tokens_seen": 69884395, + "router_z_loss_clip": 0.99853516, + "router_z_loss_mlp": 0.17163086, + "step": 2480, + "time_per_iteration": 2.5783777236938477 + }, + { + "auxiliary_loss_clip": 0.01056832, + "auxiliary_loss_mlp": 0.01003654, + "balance_loss_clip": 1.02722311, + "balance_loss_mlp": 1.00245619, + "epoch": 0.07199233938831176, + "flos": 60064619644800.0, + "grad_norm": 0.704468187731858, + "language_loss": 0.52455461, + "learning_rate": 3.98153511444955e-06, + "loss": 0.54515946, + "num_input_tokens_seen": 69941095, + "router_z_loss_clip": 0.296875, + "router_z_loss_mlp": 0.01196289, + "step": 2481, + "time_per_iteration": 2.9788858890533447 + }, + { + "auxiliary_loss_clip": 0.01171562, + "auxiliary_loss_mlp": 0.01045861, + "balance_loss_clip": 1.06779885, + "balance_loss_mlp": 1.02896953, + "epoch": 0.07202135685682781, + "flos": 38977333280640.0, + "grad_norm": 2.267453951610927, + "language_loss": 0.98221296, + "learning_rate": 3.981509623431579e-06, + "loss": 1.00438714, + "num_input_tokens_seen": 69964405, + "router_z_loss_clip": 1.03759766, + "router_z_loss_mlp": 0.16882324, + "step": 2482, + "time_per_iteration": 2.7938618659973145 + }, + { + "auxiliary_loss_clip": 0.0105719, + "auxiliary_loss_mlp": 0.01002757, + "balance_loss_clip": 1.02749276, + "balance_loss_mlp": 1.00154102, + "epoch": 0.07205037432534386, + "flos": 72434121477120.0, + "grad_norm": 0.7156978918132939, + "language_loss": 0.4749454, + "learning_rate": 3.981484114912114e-06, + "loss": 0.49554488, + "num_input_tokens_seen": 70024980, + "router_z_loss_clip": 0.296875, + "router_z_loss_mlp": 0.012146, + "step": 2483, + "time_per_iteration": 3.102461814880371 + }, + { + "auxiliary_loss_clip": 0.01171655, + "auxiliary_loss_mlp": 0.0104248, + "balance_loss_clip": 1.06891513, + "balance_loss_mlp": 1.02359176, + "epoch": 0.0720793917938599, + "flos": 24236434523520.0, + "grad_norm": 2.9726336617527815, + "language_loss": 0.93778855, + "learning_rate": 3.98145858889138e-06, + "loss": 0.95992988, + "num_input_tokens_seen": 70038345, + "router_z_loss_clip": 1.02734375, + "router_z_loss_mlp": 0.18902588, + "step": 2484, + "time_per_iteration": 2.5100231170654297 + }, + { + "auxiliary_loss_clip": 0.0117128, + "auxiliary_loss_mlp": 0.01044577, + "balance_loss_clip": 1.06443429, + "balance_loss_mlp": 1.02518725, + "epoch": 0.07210840926237595, + "flos": 20313303079680.0, + "grad_norm": 2.278649312926952, + "language_loss": 0.98481107, + "learning_rate": 3.981433045369601e-06, + "loss": 1.00696957, + "num_input_tokens_seen": 70055065, + "router_z_loss_clip": 1.06933594, + "router_z_loss_mlp": 0.19390869, + "step": 2485, + "time_per_iteration": 2.497924327850342 + }, + { + "auxiliary_loss_clip": 0.01178292, + "auxiliary_loss_mlp": 0.01045249, + "balance_loss_clip": 1.06605232, + "balance_loss_mlp": 1.02468503, + "epoch": 0.07213742673089199, + "flos": 18984176121600.0, + "grad_norm": 2.2650883325926334, + "language_loss": 0.95643473, + "learning_rate": 3.981407484347004e-06, + "loss": 0.97867018, + "num_input_tokens_seen": 70071940, + "router_z_loss_clip": 1.12304688, + "router_z_loss_mlp": 0.20556641, + "step": 2486, + "time_per_iteration": 2.579498529434204 + }, + { + "auxiliary_loss_clip": 0.01166601, + "auxiliary_loss_mlp": 0.01040551, + "balance_loss_clip": 1.06657386, + "balance_loss_mlp": 1.02238405, + "epoch": 0.07216644419940804, + "flos": 18334409045760.0, + "grad_norm": 2.4572261870219574, + "language_loss": 0.83535022, + "learning_rate": 3.981381905823814e-06, + "loss": 0.85742176, + "num_input_tokens_seen": 70087875, + "router_z_loss_clip": 0.99902344, + "router_z_loss_mlp": 0.1817627, + "step": 2487, + "time_per_iteration": 2.465522527694702 + }, + { + "auxiliary_loss_clip": 0.01171346, + "auxiliary_loss_mlp": 0.01064189, + "balance_loss_clip": 1.06548238, + "balance_loss_mlp": 1.04435241, + "epoch": 0.0721954616679241, + "flos": 35766021657600.0, + "grad_norm": 2.012365483335632, + "language_loss": 0.8614862, + "learning_rate": 3.981356309800257e-06, + "loss": 0.88384151, + "num_input_tokens_seen": 70106815, + "router_z_loss_clip": 1.05761719, + "router_z_loss_mlp": 0.19848633, + "step": 2488, + "time_per_iteration": 2.597210645675659 + }, + { + "auxiliary_loss_clip": 0.01169671, + "auxiliary_loss_mlp": 0.01048155, + "balance_loss_clip": 1.06957865, + "balance_loss_mlp": 1.03002334, + "epoch": 0.07222447913644013, + "flos": 21244788524160.0, + "grad_norm": 2.341028546543971, + "language_loss": 0.83026844, + "learning_rate": 3.98133069627656e-06, + "loss": 0.85244668, + "num_input_tokens_seen": 70120375, + "router_z_loss_clip": 1.0, + "router_z_loss_mlp": 0.18127441, + "step": 2489, + "time_per_iteration": 2.4782354831695557 + }, + { + "auxiliary_loss_clip": 0.01164469, + "auxiliary_loss_mlp": 0.01047638, + "balance_loss_clip": 1.06728411, + "balance_loss_mlp": 1.03167009, + "epoch": 0.07225349660495618, + "flos": 10775140848000.0, + "grad_norm": 2.8873610453732996, + "language_loss": 0.82424545, + "learning_rate": 3.981305065252948e-06, + "loss": 0.84636652, + "num_input_tokens_seen": 70132445, + "router_z_loss_clip": 0.97216797, + "router_z_loss_mlp": 0.15960693, + "step": 2490, + "time_per_iteration": 2.443741798400879 + }, + { + "auxiliary_loss_clip": 0.01173313, + "auxiliary_loss_mlp": 0.01045433, + "balance_loss_clip": 1.0694164, + "balance_loss_mlp": 1.02566814, + "epoch": 0.07228251407347223, + "flos": 13107969544320.0, + "grad_norm": 3.225029958779138, + "language_loss": 0.95049191, + "learning_rate": 3.981279416729649e-06, + "loss": 0.97267938, + "num_input_tokens_seen": 70144855, + "router_z_loss_clip": 1.03808594, + "router_z_loss_mlp": 0.19787598, + "step": 2491, + "time_per_iteration": 2.465467929840088 + }, + { + "auxiliary_loss_clip": 0.01060306, + "auxiliary_loss_mlp": 0.01003759, + "balance_loss_clip": 1.03104413, + "balance_loss_mlp": 1.0026921, + "epoch": 0.07231153154198827, + "flos": 54740539998720.0, + "grad_norm": 0.7738953924399555, + "language_loss": 0.56308305, + "learning_rate": 3.981253750706887e-06, + "loss": 0.58372366, + "num_input_tokens_seen": 70199510, + "router_z_loss_clip": 0.29296875, + "router_z_loss_mlp": 0.01068115, + "step": 2492, + "time_per_iteration": 5.402881860733032 + }, + { + "auxiliary_loss_clip": 0.01168181, + "auxiliary_loss_mlp": 0.01044999, + "balance_loss_clip": 1.06591022, + "balance_loss_mlp": 1.02552044, + "epoch": 0.07234054901050432, + "flos": 18034410654720.0, + "grad_norm": 2.501061242805749, + "language_loss": 0.71199459, + "learning_rate": 3.981228067184891e-06, + "loss": 0.73412645, + "num_input_tokens_seen": 70215605, + "router_z_loss_clip": 1.02246094, + "router_z_loss_mlp": 0.19458008, + "step": 2493, + "time_per_iteration": 4.759463310241699 + }, + { + "auxiliary_loss_clip": 0.01058092, + "auxiliary_loss_mlp": 0.0099973, + "balance_loss_clip": 1.02895379, + "balance_loss_mlp": 0.9985795, + "epoch": 0.07236956647902038, + "flos": 67926076963200.0, + "grad_norm": 0.7550047637270512, + "language_loss": 0.53709066, + "learning_rate": 3.981202366163886e-06, + "loss": 0.55766892, + "num_input_tokens_seen": 70281495, + "router_z_loss_clip": 0.29296875, + "router_z_loss_mlp": 0.01147461, + "step": 2494, + "time_per_iteration": 5.547833204269409 + }, + { + "auxiliary_loss_clip": 0.01171258, + "auxiliary_loss_mlp": 0.01052962, + "balance_loss_clip": 1.0658989, + "balance_loss_mlp": 1.03378677, + "epoch": 0.07239858394753641, + "flos": 62983974545280.0, + "grad_norm": 2.8988247917129297, + "language_loss": 0.86951518, + "learning_rate": 3.981176647644101e-06, + "loss": 0.89175737, + "num_input_tokens_seen": 70300985, + "router_z_loss_clip": 1.05322266, + "router_z_loss_mlp": 0.19189453, + "step": 2495, + "time_per_iteration": 2.8107047080993652 + }, + { + "auxiliary_loss_clip": 0.01177511, + "auxiliary_loss_mlp": 0.01050026, + "balance_loss_clip": 1.06787801, + "balance_loss_mlp": 1.03067875, + "epoch": 0.07242760141605246, + "flos": 48134661563520.0, + "grad_norm": 2.1636584701425665, + "language_loss": 0.90717775, + "learning_rate": 3.981150911625762e-06, + "loss": 0.92945313, + "num_input_tokens_seen": 70321255, + "router_z_loss_clip": 1.09667969, + "router_z_loss_mlp": 0.19354248, + "step": 2496, + "time_per_iteration": 2.71307373046875 + }, + { + "auxiliary_loss_clip": 0.01185277, + "auxiliary_loss_mlp": 0.01074629, + "balance_loss_clip": 1.0740428, + "balance_loss_mlp": 1.05062068, + "epoch": 0.07245661888456852, + "flos": 62913733499520.0, + "grad_norm": 1.9442617758735095, + "language_loss": 0.97581774, + "learning_rate": 3.981125158109096e-06, + "loss": 0.99841684, + "num_input_tokens_seen": 70345955, + "router_z_loss_clip": 1.11328125, + "router_z_loss_mlp": 0.24017334, + "step": 2497, + "time_per_iteration": 2.875450849533081 + }, + { + "auxiliary_loss_clip": 0.01172216, + "auxiliary_loss_mlp": 0.01048221, + "balance_loss_clip": 1.06575561, + "balance_loss_mlp": 1.02890897, + "epoch": 0.07248563635308455, + "flos": 39266557591680.0, + "grad_norm": 3.3318835747274953, + "language_loss": 0.81599963, + "learning_rate": 3.981099387094332e-06, + "loss": 0.83820397, + "num_input_tokens_seen": 70360510, + "router_z_loss_clip": 1.06494141, + "router_z_loss_mlp": 0.19299316, + "step": 2498, + "time_per_iteration": 2.5521016120910645 + }, + { + "auxiliary_loss_clip": 0.01172167, + "auxiliary_loss_mlp": 0.01052831, + "balance_loss_clip": 1.06765223, + "balance_loss_mlp": 1.03276169, + "epoch": 0.0725146538216006, + "flos": 60209156995200.0, + "grad_norm": 2.2090490056872882, + "language_loss": 0.7654618, + "learning_rate": 3.981073598581696e-06, + "loss": 0.78771186, + "num_input_tokens_seen": 70381760, + "router_z_loss_clip": 1.04541016, + "router_z_loss_mlp": 0.20068359, + "step": 2499, + "time_per_iteration": 2.872499942779541 + }, + { + "auxiliary_loss_clip": 0.01056244, + "auxiliary_loss_mlp": 0.0102298, + "balance_loss_clip": 1.02734113, + "balance_loss_mlp": 1.02188897, + "epoch": 0.07254367129011666, + "flos": 59730542225280.0, + "grad_norm": 0.7066406583283238, + "language_loss": 0.50899023, + "learning_rate": 3.9810477925714154e-06, + "loss": 0.52978247, + "num_input_tokens_seen": 70434125, + "router_z_loss_clip": 0.2890625, + "router_z_loss_mlp": 0.01092529, + "step": 2500, + "time_per_iteration": 2.8775839805603027 + }, + { + "auxiliary_loss_clip": 0.01056752, + "auxiliary_loss_mlp": 0.01013564, + "balance_loss_clip": 1.02832901, + "balance_loss_mlp": 1.01246154, + "epoch": 0.0725726887586327, + "flos": 67256377839360.0, + "grad_norm": 0.6829114344593863, + "language_loss": 0.52262002, + "learning_rate": 3.98102196906372e-06, + "loss": 0.54332316, + "num_input_tokens_seen": 70498705, + "router_z_loss_clip": 0.28515625, + "router_z_loss_mlp": 0.01104736, + "step": 2501, + "time_per_iteration": 3.176166534423828 + }, + { + "auxiliary_loss_clip": 0.01169703, + "auxiliary_loss_mlp": 0.01044414, + "balance_loss_clip": 1.06638145, + "balance_loss_mlp": 1.02601433, + "epoch": 0.07260170622714875, + "flos": 74735777168640.0, + "grad_norm": 1.973654867136005, + "language_loss": 0.71182686, + "learning_rate": 3.980996128058837e-06, + "loss": 0.73396802, + "num_input_tokens_seen": 70523090, + "router_z_loss_clip": 1.03173828, + "router_z_loss_mlp": 0.18408203, + "step": 2502, + "time_per_iteration": 2.9187450408935547 + }, + { + "auxiliary_loss_clip": 0.01169839, + "auxiliary_loss_mlp": 0.01039438, + "balance_loss_clip": 1.06722999, + "balance_loss_mlp": 1.02169931, + "epoch": 0.07263072369566478, + "flos": 27337643982720.0, + "grad_norm": 2.682425216188927, + "language_loss": 0.80310798, + "learning_rate": 3.980970269556994e-06, + "loss": 0.8252008, + "num_input_tokens_seen": 70536330, + "router_z_loss_clip": 1.02587891, + "router_z_loss_mlp": 0.17730713, + "step": 2503, + "time_per_iteration": 2.51778244972229 + }, + { + "auxiliary_loss_clip": 0.01169022, + "auxiliary_loss_mlp": 0.01041851, + "balance_loss_clip": 1.0712719, + "balance_loss_mlp": 1.02537596, + "epoch": 0.07265974116418084, + "flos": 41714480442240.0, + "grad_norm": 3.164631874297662, + "language_loss": 0.56462926, + "learning_rate": 3.98094439355842e-06, + "loss": 0.58673799, + "num_input_tokens_seen": 70555270, + "router_z_loss_clip": 0.97900391, + "router_z_loss_mlp": 0.16479492, + "step": 2504, + "time_per_iteration": 2.6525752544403076 + }, + { + "auxiliary_loss_clip": 0.01165438, + "auxiliary_loss_mlp": 0.01044373, + "balance_loss_clip": 1.06236529, + "balance_loss_mlp": 1.02657533, + "epoch": 0.07268875863269689, + "flos": 20264140339200.0, + "grad_norm": 2.2370776298950843, + "language_loss": 0.77923441, + "learning_rate": 3.980918500063344e-06, + "loss": 0.80133259, + "num_input_tokens_seen": 70575110, + "router_z_loss_clip": 1.03125, + "router_z_loss_mlp": 0.17797852, + "step": 2505, + "time_per_iteration": 2.526477813720703 + }, + { + "auxiliary_loss_clip": 0.01178814, + "auxiliary_loss_mlp": 0.0105274, + "balance_loss_clip": 1.06859756, + "balance_loss_mlp": 1.0322603, + "epoch": 0.07271777610121292, + "flos": 38063837571840.0, + "grad_norm": 3.3990683241098, + "language_loss": 0.89764124, + "learning_rate": 3.980892589071993e-06, + "loss": 0.9199568, + "num_input_tokens_seen": 70590185, + "router_z_loss_clip": 1.1015625, + "router_z_loss_mlp": 0.20483398, + "step": 2506, + "time_per_iteration": 2.6620495319366455 + }, + { + "auxiliary_loss_clip": 0.01057005, + "auxiliary_loss_mlp": 0.01009647, + "balance_loss_clip": 1.02893496, + "balance_loss_mlp": 1.00850892, + "epoch": 0.07274679356972898, + "flos": 64420333557120.0, + "grad_norm": 0.8365867743365591, + "language_loss": 0.48605952, + "learning_rate": 3.9808666605845985e-06, + "loss": 0.50672609, + "num_input_tokens_seen": 70652365, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01141357, + "step": 2507, + "time_per_iteration": 3.1550631523132324 + }, + { + "auxiliary_loss_clip": 0.01179043, + "auxiliary_loss_mlp": 0.01058655, + "balance_loss_clip": 1.0753715, + "balance_loss_mlp": 1.03978384, + "epoch": 0.07277581103824503, + "flos": 16500881352960.0, + "grad_norm": 4.3534041812066215, + "language_loss": 0.87891972, + "learning_rate": 3.980840714601388e-06, + "loss": 0.90129668, + "num_input_tokens_seen": 70664340, + "router_z_loss_clip": 1.03662109, + "router_z_loss_mlp": 0.1887207, + "step": 2508, + "time_per_iteration": 2.4872279167175293 + }, + { + "auxiliary_loss_clip": 0.01056242, + "auxiliary_loss_mlp": 0.01005652, + "balance_loss_clip": 1.02802384, + "balance_loss_mlp": 1.00443637, + "epoch": 0.07280482850676107, + "flos": 74772983658240.0, + "grad_norm": 0.6439638902166204, + "language_loss": 0.47239023, + "learning_rate": 3.98081475112259e-06, + "loss": 0.49300915, + "num_input_tokens_seen": 70723695, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.012146, + "step": 2509, + "time_per_iteration": 3.111920118331909 + }, + { + "auxiliary_loss_clip": 0.01054503, + "auxiliary_loss_mlp": 0.0099856, + "balance_loss_clip": 1.02641845, + "balance_loss_mlp": 0.99740392, + "epoch": 0.07283384597527712, + "flos": 65073476511360.0, + "grad_norm": 0.6670760222544049, + "language_loss": 0.53185105, + "learning_rate": 3.980788770148435e-06, + "loss": 0.55238169, + "num_input_tokens_seen": 70788860, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01153564, + "step": 2510, + "time_per_iteration": 3.177136182785034 + }, + { + "auxiliary_loss_clip": 0.01167479, + "auxiliary_loss_mlp": 0.01043193, + "balance_loss_clip": 1.06463504, + "balance_loss_mlp": 1.02501357, + "epoch": 0.07286286344379317, + "flos": 25440016469760.0, + "grad_norm": 2.181509963207211, + "language_loss": 0.84624511, + "learning_rate": 3.980762771679152e-06, + "loss": 0.86835182, + "num_input_tokens_seen": 70805550, + "router_z_loss_clip": 1.02880859, + "router_z_loss_mlp": 0.18164062, + "step": 2511, + "time_per_iteration": 2.5822291374206543 + }, + { + "auxiliary_loss_clip": 0.01163516, + "auxiliary_loss_mlp": 0.01054135, + "balance_loss_clip": 1.06332517, + "balance_loss_mlp": 1.03608084, + "epoch": 0.0728918809123092, + "flos": 31680249431040.0, + "grad_norm": 2.931375084018335, + "language_loss": 0.961797, + "learning_rate": 3.980736755714971e-06, + "loss": 0.98397356, + "num_input_tokens_seen": 70821520, + "router_z_loss_clip": 1.0, + "router_z_loss_mlp": 0.18054199, + "step": 2512, + "time_per_iteration": 2.6062259674072266 + }, + { + "auxiliary_loss_clip": 0.0105419, + "auxiliary_loss_mlp": 0.0101859, + "balance_loss_clip": 1.02623796, + "balance_loss_mlp": 1.0173502, + "epoch": 0.07292089838082526, + "flos": 69488765130240.0, + "grad_norm": 0.6408076799106863, + "language_loss": 0.53227955, + "learning_rate": 3.98071072225612e-06, + "loss": 0.55300736, + "num_input_tokens_seen": 70888150, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01239014, + "step": 2513, + "time_per_iteration": 3.1212196350097656 + }, + { + "auxiliary_loss_clip": 0.01179983, + "auxiliary_loss_mlp": 0.01068039, + "balance_loss_clip": 1.06953096, + "balance_loss_mlp": 1.04716563, + "epoch": 0.07294991584934131, + "flos": 39233340489600.0, + "grad_norm": 2.628919140134863, + "language_loss": 0.96166986, + "learning_rate": 3.980684671302832e-06, + "loss": 0.98415005, + "num_input_tokens_seen": 70904640, + "router_z_loss_clip": 1.10253906, + "router_z_loss_mlp": 0.2088623, + "step": 2514, + "time_per_iteration": 2.722796678543091 + }, + { + "auxiliary_loss_clip": 0.01171256, + "auxiliary_loss_mlp": 0.01057443, + "balance_loss_clip": 1.06682432, + "balance_loss_mlp": 1.03821468, + "epoch": 0.07297893331785735, + "flos": 10299468015360.0, + "grad_norm": 4.484033544275769, + "language_loss": 0.85910285, + "learning_rate": 3.980658602855335e-06, + "loss": 0.88138986, + "num_input_tokens_seen": 70915955, + "router_z_loss_clip": 1.04345703, + "router_z_loss_mlp": 0.19226074, + "step": 2515, + "time_per_iteration": 2.5269505977630615 + }, + { + "auxiliary_loss_clip": 0.01174629, + "auxiliary_loss_mlp": 0.01045044, + "balance_loss_clip": 1.06814575, + "balance_loss_mlp": 1.0259583, + "epoch": 0.0730079507863734, + "flos": 29637758367360.0, + "grad_norm": 2.3228057676836, + "language_loss": 0.8558532, + "learning_rate": 3.98063251691386e-06, + "loss": 0.87804997, + "num_input_tokens_seen": 70932235, + "router_z_loss_clip": 1.06396484, + "router_z_loss_mlp": 0.19091797, + "step": 2516, + "time_per_iteration": 2.5349180698394775 + }, + { + "auxiliary_loss_clip": 0.01158043, + "auxiliary_loss_mlp": 0.01045832, + "balance_loss_clip": 1.06629491, + "balance_loss_mlp": 1.03002453, + "epoch": 0.07303696825488944, + "flos": 11039587752960.0, + "grad_norm": 3.139116606996356, + "language_loss": 0.92401564, + "learning_rate": 3.980606413478637e-06, + "loss": 0.94605446, + "num_input_tokens_seen": 70945635, + "router_z_loss_clip": 0.91699219, + "router_z_loss_mlp": 0.15802002, + "step": 2517, + "time_per_iteration": 2.5465190410614014 + }, + { + "auxiliary_loss_clip": 0.01167651, + "auxiliary_loss_mlp": 0.01048022, + "balance_loss_clip": 1.06641877, + "balance_loss_mlp": 1.03084433, + "epoch": 0.07306598572340549, + "flos": 37591432876800.0, + "grad_norm": 2.4842641057780757, + "language_loss": 0.82210362, + "learning_rate": 3.980580292549896e-06, + "loss": 0.84426039, + "num_input_tokens_seen": 70961890, + "router_z_loss_clip": 1.01318359, + "router_z_loss_mlp": 0.17193604, + "step": 2518, + "time_per_iteration": 2.6817314624786377 + }, + { + "auxiliary_loss_clip": 0.01166138, + "auxiliary_loss_mlp": 0.01043434, + "balance_loss_clip": 1.06731975, + "balance_loss_mlp": 1.02638662, + "epoch": 0.07309500319192154, + "flos": 25949552849280.0, + "grad_norm": 2.4342100890494196, + "language_loss": 0.95982414, + "learning_rate": 3.980554154127869e-06, + "loss": 0.98191983, + "num_input_tokens_seen": 70975425, + "router_z_loss_clip": 0.98828125, + "router_z_loss_mlp": 0.17028809, + "step": 2519, + "time_per_iteration": 2.530183792114258 + }, + { + "auxiliary_loss_clip": 0.01183194, + "auxiliary_loss_mlp": 0.0105491, + "balance_loss_clip": 1.07360196, + "balance_loss_mlp": 1.03461432, + "epoch": 0.07312402066043758, + "flos": 33101916802560.0, + "grad_norm": 2.140924232683483, + "language_loss": 0.9088496, + "learning_rate": 3.980527998212786e-06, + "loss": 0.93123072, + "num_input_tokens_seen": 70998235, + "router_z_loss_clip": 1.09765625, + "router_z_loss_mlp": 0.20306396, + "step": 2520, + "time_per_iteration": 2.637835741043091 + }, + { + "auxiliary_loss_clip": 0.01056084, + "auxiliary_loss_mlp": 0.01006436, + "balance_loss_clip": 1.02827477, + "balance_loss_mlp": 1.0052495, + "epoch": 0.07315303812895363, + "flos": 56389163454720.0, + "grad_norm": 0.6785828652722413, + "language_loss": 0.50736064, + "learning_rate": 3.980501824804879e-06, + "loss": 0.52798581, + "num_input_tokens_seen": 71059830, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01184082, + "step": 2521, + "time_per_iteration": 3.066305160522461 + }, + { + "auxiliary_loss_clip": 0.01053861, + "auxiliary_loss_mlp": 0.0100624, + "balance_loss_clip": 1.02629089, + "balance_loss_mlp": 1.00504768, + "epoch": 0.07318205559746968, + "flos": 68764195981440.0, + "grad_norm": 0.6082348330251375, + "language_loss": 0.50621408, + "learning_rate": 3.980475633904378e-06, + "loss": 0.52681512, + "num_input_tokens_seen": 71128680, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01190186, + "step": 2522, + "time_per_iteration": 3.1754729747772217 + }, + { + "auxiliary_loss_clip": 0.01174001, + "auxiliary_loss_mlp": 0.0105275, + "balance_loss_clip": 1.070436, + "balance_loss_mlp": 1.03365242, + "epoch": 0.07321107306598572, + "flos": 68275196225280.0, + "grad_norm": 1.941409622216073, + "language_loss": 0.70901906, + "learning_rate": 3.980449425511515e-06, + "loss": 0.73128653, + "num_input_tokens_seen": 71150085, + "router_z_loss_clip": 1.03662109, + "router_z_loss_mlp": 0.190979, + "step": 2523, + "time_per_iteration": 2.8523941040039062 + }, + { + "auxiliary_loss_clip": 0.01165503, + "auxiliary_loss_mlp": 0.01039984, + "balance_loss_clip": 1.06737328, + "balance_loss_mlp": 1.02214992, + "epoch": 0.07324009053450177, + "flos": 18147780956160.0, + "grad_norm": 2.9306222899620984, + "language_loss": 0.76609081, + "learning_rate": 3.980423199626521e-06, + "loss": 0.78814566, + "num_input_tokens_seen": 71163065, + "router_z_loss_clip": 0.98242188, + "router_z_loss_mlp": 0.17834473, + "step": 2524, + "time_per_iteration": 2.482661247253418 + }, + { + "auxiliary_loss_clip": 0.01183311, + "auxiliary_loss_mlp": 0.01057741, + "balance_loss_clip": 1.07374883, + "balance_loss_mlp": 1.03648591, + "epoch": 0.07326910800301782, + "flos": 10407128054400.0, + "grad_norm": 3.292126329029783, + "language_loss": 0.92870677, + "learning_rate": 3.980396956249628e-06, + "loss": 0.95111728, + "num_input_tokens_seen": 71173325, + "router_z_loss_clip": 1.09667969, + "router_z_loss_mlp": 0.21276855, + "step": 2525, + "time_per_iteration": 2.4704930782318115 + }, + { + "auxiliary_loss_clip": 0.01183996, + "auxiliary_loss_mlp": 0.01056751, + "balance_loss_clip": 1.07443333, + "balance_loss_mlp": 1.0357821, + "epoch": 0.07329812547153386, + "flos": 13738920871680.0, + "grad_norm": 2.5852677535167636, + "language_loss": 0.9598912, + "learning_rate": 3.980370695381067e-06, + "loss": 0.98229879, + "num_input_tokens_seen": 71187550, + "router_z_loss_clip": 1.09667969, + "router_z_loss_mlp": 0.2097168, + "step": 2526, + "time_per_iteration": 2.5564382076263428 + }, + { + "auxiliary_loss_clip": 0.01176952, + "auxiliary_loss_mlp": 0.01048433, + "balance_loss_clip": 1.07264614, + "balance_loss_mlp": 1.03013432, + "epoch": 0.07332714294004991, + "flos": 25553566339200.0, + "grad_norm": 2.3525071222762346, + "language_loss": 0.78050649, + "learning_rate": 3.980344417021071e-06, + "loss": 0.80276036, + "num_input_tokens_seen": 71206815, + "router_z_loss_clip": 1.04296875, + "router_z_loss_mlp": 0.1829834, + "step": 2527, + "time_per_iteration": 2.587451457977295 + }, + { + "auxiliary_loss_clip": 0.01051592, + "auxiliary_loss_mlp": 0.01012149, + "balance_loss_clip": 1.02389205, + "balance_loss_mlp": 1.01092708, + "epoch": 0.07335616040856596, + "flos": 74775749005440.0, + "grad_norm": 0.7098252911104684, + "language_loss": 0.46970719, + "learning_rate": 3.980318121169872e-06, + "loss": 0.49034458, + "num_input_tokens_seen": 71269880, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01220703, + "step": 2528, + "time_per_iteration": 3.1238248348236084 + }, + { + "auxiliary_loss_clip": 0.0105167, + "auxiliary_loss_mlp": 0.01013254, + "balance_loss_clip": 1.02378964, + "balance_loss_mlp": 1.01199663, + "epoch": 0.073385177877082, + "flos": 62593270312320.0, + "grad_norm": 0.7026210752085227, + "language_loss": 0.5420922, + "learning_rate": 3.980291807827702e-06, + "loss": 0.56274146, + "num_input_tokens_seen": 71332500, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01257324, + "step": 2529, + "time_per_iteration": 3.113800287246704 + }, + { + "auxiliary_loss_clip": 0.01173643, + "auxiliary_loss_mlp": 0.01049057, + "balance_loss_clip": 1.07179368, + "balance_loss_mlp": 1.03012645, + "epoch": 0.07341419534559805, + "flos": 11101209534720.0, + "grad_norm": 3.9053215187333845, + "language_loss": 0.86734819, + "learning_rate": 3.980265476994794e-06, + "loss": 0.88957524, + "num_input_tokens_seen": 71344260, + "router_z_loss_clip": 1.01855469, + "router_z_loss_mlp": 0.18933105, + "step": 2530, + "time_per_iteration": 2.4542856216430664 + }, + { + "auxiliary_loss_clip": 0.01178576, + "auxiliary_loss_mlp": 0.01064423, + "balance_loss_clip": 1.06899977, + "balance_loss_mlp": 1.04353714, + "epoch": 0.0734432128141141, + "flos": 20224746097920.0, + "grad_norm": 2.3171248677852905, + "language_loss": 0.85529578, + "learning_rate": 3.9802391286713796e-06, + "loss": 0.87772578, + "num_input_tokens_seen": 71362700, + "router_z_loss_clip": 1.09570312, + "router_z_loss_mlp": 0.20874023, + "step": 2531, + "time_per_iteration": 2.5722365379333496 + }, + { + "auxiliary_loss_clip": 0.01168123, + "auxiliary_loss_mlp": 0.01046716, + "balance_loss_clip": 1.06936955, + "balance_loss_mlp": 1.02869725, + "epoch": 0.07347223028263014, + "flos": 38757093039360.0, + "grad_norm": 2.3446504497354668, + "language_loss": 0.92417115, + "learning_rate": 3.980212762857691e-06, + "loss": 0.94631964, + "num_input_tokens_seen": 71379280, + "router_z_loss_clip": 0.98632812, + "router_z_loss_mlp": 0.18017578, + "step": 2532, + "time_per_iteration": 2.675616979598999 + }, + { + "auxiliary_loss_clip": 0.01172645, + "auxiliary_loss_mlp": 0.01059879, + "balance_loss_clip": 1.0666256, + "balance_loss_mlp": 1.04073405, + "epoch": 0.07350124775114619, + "flos": 32118646924800.0, + "grad_norm": 2.697661926712523, + "language_loss": 1.00351489, + "learning_rate": 3.980186379553963e-06, + "loss": 1.02584028, + "num_input_tokens_seen": 71393745, + "router_z_loss_clip": 1.05908203, + "router_z_loss_mlp": 0.19134521, + "step": 2533, + "time_per_iteration": 2.6631288528442383 + }, + { + "auxiliary_loss_clip": 0.01173386, + "auxiliary_loss_mlp": 0.01051182, + "balance_loss_clip": 1.06861663, + "balance_loss_mlp": 1.03365231, + "epoch": 0.07353026521966223, + "flos": 25587250318080.0, + "grad_norm": 2.8049750155411965, + "language_loss": 0.76647401, + "learning_rate": 3.980159978760427e-06, + "loss": 0.78871965, + "num_input_tokens_seen": 71409120, + "router_z_loss_clip": 1.04882812, + "router_z_loss_mlp": 0.17541504, + "step": 2534, + "time_per_iteration": 2.5971498489379883 + }, + { + "auxiliary_loss_clip": 0.01166249, + "auxiliary_loss_mlp": 0.01051149, + "balance_loss_clip": 1.06380749, + "balance_loss_mlp": 1.03273153, + "epoch": 0.07355928268817828, + "flos": 22487261921280.0, + "grad_norm": 3.775670955156282, + "language_loss": 0.95079827, + "learning_rate": 3.9801335604773175e-06, + "loss": 0.97297215, + "num_input_tokens_seen": 71423795, + "router_z_loss_clip": 1.02392578, + "router_z_loss_mlp": 0.18408203, + "step": 2535, + "time_per_iteration": 2.4978294372558594 + }, + { + "auxiliary_loss_clip": 0.01166664, + "auxiliary_loss_mlp": 0.01052908, + "balance_loss_clip": 1.06813633, + "balance_loss_mlp": 1.03468084, + "epoch": 0.07358830015669433, + "flos": 11221295679360.0, + "grad_norm": 2.964718814821082, + "language_loss": 0.84341377, + "learning_rate": 3.980107124704866e-06, + "loss": 0.86560947, + "num_input_tokens_seen": 71435585, + "router_z_loss_clip": 0.98681641, + "router_z_loss_mlp": 0.18231201, + "step": 2536, + "time_per_iteration": 2.516660213470459 + }, + { + "auxiliary_loss_clip": 0.01173343, + "auxiliary_loss_mlp": 0.01057128, + "balance_loss_clip": 1.06745958, + "balance_loss_mlp": 1.03642178, + "epoch": 0.07361731762521037, + "flos": 17815175994240.0, + "grad_norm": 2.498453675844928, + "language_loss": 0.79580462, + "learning_rate": 3.980080671443308e-06, + "loss": 0.81810933, + "num_input_tokens_seen": 71452840, + "router_z_loss_clip": 1.05859375, + "router_z_loss_mlp": 0.20709229, + "step": 2537, + "time_per_iteration": 2.5488665103912354 + }, + { + "auxiliary_loss_clip": 0.01164249, + "auxiliary_loss_mlp": 0.0105048, + "balance_loss_clip": 1.06503832, + "balance_loss_mlp": 1.03294456, + "epoch": 0.07364633509372642, + "flos": 17702595792000.0, + "grad_norm": 2.355432683441265, + "language_loss": 0.79755497, + "learning_rate": 3.980054200692876e-06, + "loss": 0.81970227, + "num_input_tokens_seen": 71463575, + "router_z_loss_clip": 0.99121094, + "router_z_loss_mlp": 0.17547607, + "step": 2538, + "time_per_iteration": 2.4749069213867188 + }, + { + "auxiliary_loss_clip": 0.01174723, + "auxiliary_loss_mlp": 0.01063543, + "balance_loss_clip": 1.06750178, + "balance_loss_mlp": 1.0432421, + "epoch": 0.07367535256224247, + "flos": 12048604704000.0, + "grad_norm": 3.0910942192217816, + "language_loss": 1.00470197, + "learning_rate": 3.9800277124538036e-06, + "loss": 1.02708471, + "num_input_tokens_seen": 71476530, + "router_z_loss_clip": 1.07275391, + "router_z_loss_mlp": 0.20324707, + "step": 2539, + "time_per_iteration": 2.5138752460479736 + }, + { + "auxiliary_loss_clip": 0.01161532, + "auxiliary_loss_mlp": 0.01063806, + "balance_loss_clip": 1.06464148, + "balance_loss_mlp": 1.04665792, + "epoch": 0.07370437003075851, + "flos": 34233928899840.0, + "grad_norm": 2.452227510002382, + "language_loss": 0.92763704, + "learning_rate": 3.980001206726326e-06, + "loss": 0.94989043, + "num_input_tokens_seen": 71493550, + "router_z_loss_clip": 0.96923828, + "router_z_loss_mlp": 0.17156982, + "step": 2540, + "time_per_iteration": 2.6558730602264404 + }, + { + "auxiliary_loss_clip": 0.01169707, + "auxiliary_loss_mlp": 0.01066992, + "balance_loss_clip": 1.06652474, + "balance_loss_mlp": 1.04760313, + "epoch": 0.07373338749927456, + "flos": 38429982858240.0, + "grad_norm": 1.9080431411127423, + "language_loss": 0.80105168, + "learning_rate": 3.979974683510676e-06, + "loss": 0.82341862, + "num_input_tokens_seen": 71513375, + "router_z_loss_clip": 1.03320312, + "router_z_loss_mlp": 0.1940918, + "step": 2541, + "time_per_iteration": 2.7112417221069336 + }, + { + "auxiliary_loss_clip": 0.01168358, + "auxiliary_loss_mlp": 0.01057349, + "balance_loss_clip": 1.06710052, + "balance_loss_mlp": 1.03961027, + "epoch": 0.07376240496779062, + "flos": 32409164125440.0, + "grad_norm": 2.345098926586201, + "language_loss": 0.62204558, + "learning_rate": 3.979948142807089e-06, + "loss": 0.64430267, + "num_input_tokens_seen": 71528245, + "router_z_loss_clip": 1.01269531, + "router_z_loss_mlp": 0.17736816, + "step": 2542, + "time_per_iteration": 2.523747682571411 + }, + { + "auxiliary_loss_clip": 0.01059549, + "auxiliary_loss_mlp": 0.01038211, + "balance_loss_clip": 1.0309391, + "balance_loss_mlp": 1.03701878, + "epoch": 0.07379142243630665, + "flos": 57479698321920.0, + "grad_norm": 0.6954718853324329, + "language_loss": 0.48219675, + "learning_rate": 3.979921584615798e-06, + "loss": 0.50317436, + "num_input_tokens_seen": 71588245, + "router_z_loss_clip": 0.28515625, + "router_z_loss_mlp": 0.01190186, + "step": 2543, + "time_per_iteration": 3.0939507484436035 + }, + { + "auxiliary_loss_clip": 0.01180239, + "auxiliary_loss_mlp": 0.01052725, + "balance_loss_clip": 1.07033181, + "balance_loss_mlp": 1.03234029, + "epoch": 0.0738204399048227, + "flos": 44011506257280.0, + "grad_norm": 2.321569079364186, + "language_loss": 0.82520199, + "learning_rate": 3.979895008937039e-06, + "loss": 0.84753156, + "num_input_tokens_seen": 71606775, + "router_z_loss_clip": 1.09765625, + "router_z_loss_mlp": 0.20385742, + "step": 2544, + "time_per_iteration": 2.6696994304656982 + }, + { + "auxiliary_loss_clip": 0.01174822, + "auxiliary_loss_mlp": 0.01059471, + "balance_loss_clip": 1.07132161, + "balance_loss_mlp": 1.04033732, + "epoch": 0.07384945737333876, + "flos": 27118229754240.0, + "grad_norm": 2.5739968018015116, + "language_loss": 0.95016181, + "learning_rate": 3.979868415771046e-06, + "loss": 0.97250479, + "num_input_tokens_seen": 71621510, + "router_z_loss_clip": 1.03564453, + "router_z_loss_mlp": 0.19128418, + "step": 2545, + "time_per_iteration": 2.638251543045044 + }, + { + "auxiliary_loss_clip": 0.01171443, + "auxiliary_loss_mlp": 0.01046285, + "balance_loss_clip": 1.06845546, + "balance_loss_mlp": 1.02689004, + "epoch": 0.0738784748418548, + "flos": 24785975675520.0, + "grad_norm": 2.158695367629314, + "language_loss": 0.84772211, + "learning_rate": 3.979841805118054e-06, + "loss": 0.86989939, + "num_input_tokens_seen": 71635925, + "router_z_loss_clip": 1.03125, + "router_z_loss_mlp": 0.19396973, + "step": 2546, + "time_per_iteration": 2.532796621322632 + }, + { + "auxiliary_loss_clip": 0.01052085, + "auxiliary_loss_mlp": 0.01000245, + "balance_loss_clip": 1.02450931, + "balance_loss_mlp": 0.99904138, + "epoch": 0.07390749231037085, + "flos": 63162668707200.0, + "grad_norm": 0.6770303174304054, + "language_loss": 0.45987356, + "learning_rate": 3.979815176978298e-06, + "loss": 0.48039687, + "num_input_tokens_seen": 71685165, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.01202393, + "step": 2547, + "time_per_iteration": 2.8986682891845703 + }, + { + "auxiliary_loss_clip": 0.01174886, + "auxiliary_loss_mlp": 0.01048953, + "balance_loss_clip": 1.06909776, + "balance_loss_mlp": 1.02966523, + "epoch": 0.07393650977888688, + "flos": 17524407398400.0, + "grad_norm": 3.9915554868847374, + "language_loss": 0.92232323, + "learning_rate": 3.979788531352013e-06, + "loss": 0.9445616, + "num_input_tokens_seen": 71698315, + "router_z_loss_clip": 1.05712891, + "router_z_loss_mlp": 0.19274902, + "step": 2548, + "time_per_iteration": 2.496797800064087 + }, + { + "auxiliary_loss_clip": 0.01174755, + "auxiliary_loss_mlp": 0.01057151, + "balance_loss_clip": 1.07114792, + "balance_loss_mlp": 1.03733253, + "epoch": 0.07396552724740293, + "flos": 31861167258240.0, + "grad_norm": 2.3029887032937846, + "language_loss": 0.86197686, + "learning_rate": 3.979761868239434e-06, + "loss": 0.884296, + "num_input_tokens_seen": 71714640, + "router_z_loss_clip": 1.03564453, + "router_z_loss_mlp": 0.19818115, + "step": 2549, + "time_per_iteration": 2.611830711364746 + }, + { + "auxiliary_loss_clip": 0.01051988, + "auxiliary_loss_mlp": 0.01008559, + "balance_loss_clip": 1.02407074, + "balance_loss_mlp": 1.00730753, + "epoch": 0.07399454471591899, + "flos": 74765944592640.0, + "grad_norm": 0.7148216925452694, + "language_loss": 0.53495741, + "learning_rate": 3.979735187640798e-06, + "loss": 0.55556285, + "num_input_tokens_seen": 71769100, + "router_z_loss_clip": 0.27929688, + "router_z_loss_mlp": 0.01251221, + "step": 2550, + "time_per_iteration": 3.0461599826812744 + }, + { + "auxiliary_loss_clip": 0.01051012, + "auxiliary_loss_mlp": 0.01010825, + "balance_loss_clip": 1.02329862, + "balance_loss_mlp": 1.00966871, + "epoch": 0.07402356218443502, + "flos": 63971485205760.0, + "grad_norm": 0.6538832758957528, + "language_loss": 0.46071514, + "learning_rate": 3.97970848955634e-06, + "loss": 0.48133352, + "num_input_tokens_seen": 71831430, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01153564, + "step": 2551, + "time_per_iteration": 3.018155574798584 + }, + { + "auxiliary_loss_clip": 0.0118207, + "auxiliary_loss_mlp": 0.01053491, + "balance_loss_clip": 1.07276416, + "balance_loss_mlp": 1.03411329, + "epoch": 0.07405257965295108, + "flos": 20186106042240.0, + "grad_norm": 2.6535309144722494, + "language_loss": 1.02579546, + "learning_rate": 3.9796817739862945e-06, + "loss": 1.04815114, + "num_input_tokens_seen": 71845075, + "router_z_loss_clip": 1.09277344, + "router_z_loss_mlp": 0.19396973, + "step": 2552, + "time_per_iteration": 2.615811586380005 + }, + { + "auxiliary_loss_clip": 0.01177916, + "auxiliary_loss_mlp": 0.01052491, + "balance_loss_clip": 1.067729, + "balance_loss_mlp": 1.03220129, + "epoch": 0.07408159712146713, + "flos": 41566456494720.0, + "grad_norm": 2.582537849624267, + "language_loss": 0.99280393, + "learning_rate": 3.979655040930898e-06, + "loss": 1.01510811, + "num_input_tokens_seen": 71864950, + "router_z_loss_clip": 1.10205078, + "router_z_loss_mlp": 0.20300293, + "step": 2553, + "time_per_iteration": 2.6925835609436035 + }, + { + "auxiliary_loss_clip": 0.01176973, + "auxiliary_loss_mlp": 0.01050273, + "balance_loss_clip": 1.06645894, + "balance_loss_mlp": 1.02987671, + "epoch": 0.07411061458998316, + "flos": 21828121395840.0, + "grad_norm": 2.5939884425100566, + "language_loss": 0.98871738, + "learning_rate": 3.979628290390389e-06, + "loss": 1.0109899, + "num_input_tokens_seen": 71879415, + "router_z_loss_clip": 1.10449219, + "router_z_loss_mlp": 0.20397949, + "step": 2554, + "time_per_iteration": 2.531306505203247 + }, + { + "auxiliary_loss_clip": 0.01169422, + "auxiliary_loss_mlp": 0.0104946, + "balance_loss_clip": 1.06610763, + "balance_loss_mlp": 1.03057122, + "epoch": 0.07413963205849922, + "flos": 12560726862720.0, + "grad_norm": 3.71879415527313, + "language_loss": 1.13161969, + "learning_rate": 3.979601522365e-06, + "loss": 1.15380847, + "num_input_tokens_seen": 71890295, + "router_z_loss_clip": 1.03417969, + "router_z_loss_mlp": 0.18902588, + "step": 2555, + "time_per_iteration": 2.481100082397461 + }, + { + "auxiliary_loss_clip": 0.01050562, + "auxiliary_loss_mlp": 0.01022947, + "balance_loss_clip": 1.02324367, + "balance_loss_mlp": 1.02177262, + "epoch": 0.07416864952701527, + "flos": 69950395745280.0, + "grad_norm": 0.6490146879629891, + "language_loss": 0.50439548, + "learning_rate": 3.979574736854971e-06, + "loss": 0.52513051, + "num_input_tokens_seen": 71947230, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.01171875, + "step": 2556, + "time_per_iteration": 3.0238239765167236 + }, + { + "auxiliary_loss_clip": 0.01168667, + "auxiliary_loss_mlp": 0.0104486, + "balance_loss_clip": 1.07059717, + "balance_loss_mlp": 1.02797973, + "epoch": 0.0741976669955313, + "flos": 29856957114240.0, + "grad_norm": 1.6606082037544987, + "language_loss": 0.83638489, + "learning_rate": 3.979547933860535e-06, + "loss": 0.85852015, + "num_input_tokens_seen": 71968905, + "router_z_loss_clip": 0.97949219, + "router_z_loss_mlp": 0.16876221, + "step": 2557, + "time_per_iteration": 2.688110589981079 + }, + { + "auxiliary_loss_clip": 0.01176568, + "auxiliary_loss_mlp": 0.01048668, + "balance_loss_clip": 1.06976509, + "balance_loss_mlp": 1.02924931, + "epoch": 0.07422668446404736, + "flos": 13984154000640.0, + "grad_norm": 3.3608735255459274, + "language_loss": 0.84608555, + "learning_rate": 3.979521113381932e-06, + "loss": 0.86833799, + "num_input_tokens_seen": 71981365, + "router_z_loss_clip": 1.06835938, + "router_z_loss_mlp": 0.19396973, + "step": 2558, + "time_per_iteration": 2.5363454818725586 + }, + { + "auxiliary_loss_clip": 0.01175307, + "auxiliary_loss_mlp": 0.01053048, + "balance_loss_clip": 1.06999278, + "balance_loss_mlp": 1.03300881, + "epoch": 0.07425570193256341, + "flos": 28065373528320.0, + "grad_norm": 3.104610409602041, + "language_loss": 0.82477307, + "learning_rate": 3.979494275419398e-06, + "loss": 0.84705663, + "num_input_tokens_seen": 71994990, + "router_z_loss_clip": 1.05273438, + "router_z_loss_mlp": 0.20031738, + "step": 2559, + "time_per_iteration": 2.5770468711853027 + }, + { + "auxiliary_loss_clip": 0.01163975, + "auxiliary_loss_mlp": 0.01051972, + "balance_loss_clip": 1.06549621, + "balance_loss_mlp": 1.03069329, + "epoch": 0.07428471940107945, + "flos": 66780702028800.0, + "grad_norm": 2.468089903881021, + "language_loss": 0.72843039, + "learning_rate": 3.979467419973168e-06, + "loss": 0.75058991, + "num_input_tokens_seen": 72018165, + "router_z_loss_clip": 0.98535156, + "router_z_loss_mlp": 0.2130127, + "step": 2560, + "time_per_iteration": 2.859034299850464 + }, + { + "auxiliary_loss_clip": 0.01172898, + "auxiliary_loss_mlp": 0.01057487, + "balance_loss_clip": 1.06833911, + "balance_loss_mlp": 1.03887892, + "epoch": 0.0743137368695955, + "flos": 43098908388480.0, + "grad_norm": 2.378977387721811, + "language_loss": 0.83906108, + "learning_rate": 3.979440547043482e-06, + "loss": 0.8613649, + "num_input_tokens_seen": 72038680, + "router_z_loss_clip": 1.04541016, + "router_z_loss_mlp": 0.18603516, + "step": 2561, + "time_per_iteration": 2.7904956340789795 + }, + { + "auxiliary_loss_clip": 0.0105321, + "auxiliary_loss_mlp": 0.01028513, + "balance_loss_clip": 1.02581763, + "balance_loss_mlp": 1.02728558, + "epoch": 0.07434275433811155, + "flos": 74778191130240.0, + "grad_norm": 0.7074707685003973, + "language_loss": 0.49793059, + "learning_rate": 3.979413656630575e-06, + "loss": 0.51874781, + "num_input_tokens_seen": 72104160, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.01226807, + "step": 2562, + "time_per_iteration": 3.177717685699463 + }, + { + "auxiliary_loss_clip": 0.01175105, + "auxiliary_loss_mlp": 0.01051558, + "balance_loss_clip": 1.07307279, + "balance_loss_mlp": 1.03244913, + "epoch": 0.07437177180662759, + "flos": 25477219981440.0, + "grad_norm": 2.721173879979392, + "language_loss": 0.85148346, + "learning_rate": 3.979386748734686e-06, + "loss": 0.87375009, + "num_input_tokens_seen": 72118480, + "router_z_loss_clip": 1.02148438, + "router_z_loss_mlp": 0.19104004, + "step": 2563, + "time_per_iteration": 7.5423829555511475 + }, + { + "auxiliary_loss_clip": 0.0118035, + "auxiliary_loss_mlp": 0.01045568, + "balance_loss_clip": 1.0692997, + "balance_loss_mlp": 1.02639914, + "epoch": 0.07440078927514364, + "flos": 16064459107200.0, + "grad_norm": 3.657539882255734, + "language_loss": 0.89035887, + "learning_rate": 3.979359823356053e-06, + "loss": 0.91261804, + "num_input_tokens_seen": 72133135, + "router_z_loss_clip": 1.11181641, + "router_z_loss_mlp": 0.19152832, + "step": 2564, + "time_per_iteration": 4.800133943557739 + }, + { + "auxiliary_loss_clip": 0.01174138, + "auxiliary_loss_mlp": 0.01050045, + "balance_loss_clip": 1.07123244, + "balance_loss_mlp": 1.03137648, + "epoch": 0.07442980674365968, + "flos": 17412904604160.0, + "grad_norm": 2.7937708932486016, + "language_loss": 0.76595676, + "learning_rate": 3.979332880494912e-06, + "loss": 0.78819859, + "num_input_tokens_seen": 72144875, + "router_z_loss_clip": 1.02880859, + "router_z_loss_mlp": 0.18682861, + "step": 2565, + "time_per_iteration": 4.95654034614563 + }, + { + "auxiliary_loss_clip": 0.01178963, + "auxiliary_loss_mlp": 0.01056147, + "balance_loss_clip": 1.073282, + "balance_loss_mlp": 1.03365231, + "epoch": 0.07445882421217573, + "flos": 42953649788160.0, + "grad_norm": 1.7821533500172262, + "language_loss": 0.82460213, + "learning_rate": 3.9793059201515025e-06, + "loss": 0.84695327, + "num_input_tokens_seen": 72179375, + "router_z_loss_clip": 1.05761719, + "router_z_loss_mlp": 0.22503662, + "step": 2566, + "time_per_iteration": 3.222581386566162 + }, + { + "auxiliary_loss_clip": 0.01182357, + "auxiliary_loss_mlp": 0.01054356, + "balance_loss_clip": 1.07135201, + "balance_loss_mlp": 1.03451991, + "epoch": 0.07448784168069178, + "flos": 16902542211840.0, + "grad_norm": 3.2899221084789967, + "language_loss": 0.91623461, + "learning_rate": 3.979278942326062e-06, + "loss": 0.93860179, + "num_input_tokens_seen": 72192120, + "router_z_loss_clip": 1.11132812, + "router_z_loss_mlp": 0.19836426, + "step": 2567, + "time_per_iteration": 2.5562922954559326 + }, + { + "auxiliary_loss_clip": 0.01055643, + "auxiliary_loss_mlp": 0.01003879, + "balance_loss_clip": 1.02858686, + "balance_loss_mlp": 1.00259733, + "epoch": 0.07451685914920782, + "flos": 74773917411840.0, + "grad_norm": 0.6727021033529793, + "language_loss": 0.50779319, + "learning_rate": 3.979251947018829e-06, + "loss": 0.52838838, + "num_input_tokens_seen": 72253810, + "router_z_loss_clip": 0.26953125, + "router_z_loss_mlp": 0.01281738, + "step": 2568, + "time_per_iteration": 3.150907278060913 + }, + { + "auxiliary_loss_clip": 0.01180887, + "auxiliary_loss_mlp": 0.01055585, + "balance_loss_clip": 1.07590592, + "balance_loss_mlp": 1.03561759, + "epoch": 0.07454587661772387, + "flos": 39270400346880.0, + "grad_norm": 2.094854549485409, + "language_loss": 0.64481664, + "learning_rate": 3.979224934230043e-06, + "loss": 0.66718137, + "num_input_tokens_seen": 72271910, + "router_z_loss_clip": 1.04882812, + "router_z_loss_mlp": 0.19976807, + "step": 2569, + "time_per_iteration": 2.629777431488037 + }, + { + "auxiliary_loss_clip": 0.01056244, + "auxiliary_loss_mlp": 0.01000896, + "balance_loss_clip": 1.02905703, + "balance_loss_mlp": 0.99966806, + "epoch": 0.07457489408623992, + "flos": 63755877818880.0, + "grad_norm": 0.7646716271465361, + "language_loss": 0.54000628, + "learning_rate": 3.9791979039599395e-06, + "loss": 0.56057763, + "num_input_tokens_seen": 72332235, + "router_z_loss_clip": 0.27148438, + "router_z_loss_mlp": 0.01226807, + "step": 2570, + "time_per_iteration": 3.0428426265716553 + }, + { + "auxiliary_loss_clip": 0.01163777, + "auxiliary_loss_mlp": 0.01043222, + "balance_loss_clip": 1.06763244, + "balance_loss_mlp": 1.02524483, + "epoch": 0.07460391155475596, + "flos": 39817607114880.0, + "grad_norm": 2.2466760497166227, + "language_loss": 0.84202665, + "learning_rate": 3.979170856208761e-06, + "loss": 0.86409664, + "num_input_tokens_seen": 72350020, + "router_z_loss_clip": 0.9609375, + "router_z_loss_mlp": 0.17980957, + "step": 2571, + "time_per_iteration": 2.6723880767822266 + }, + { + "auxiliary_loss_clip": 0.01178939, + "auxiliary_loss_mlp": 0.01049054, + "balance_loss_clip": 1.07479179, + "balance_loss_mlp": 1.02987313, + "epoch": 0.07463292902327201, + "flos": 10920183966720.0, + "grad_norm": 4.130299085974133, + "language_loss": 0.82421291, + "learning_rate": 3.979143790976744e-06, + "loss": 0.84649289, + "num_input_tokens_seen": 72361700, + "router_z_loss_clip": 1.04199219, + "router_z_loss_mlp": 0.19189453, + "step": 2572, + "time_per_iteration": 2.47824764251709 + }, + { + "auxiliary_loss_clip": 0.01175521, + "auxiliary_loss_mlp": 0.01053154, + "balance_loss_clip": 1.07091784, + "balance_loss_mlp": 1.03286481, + "epoch": 0.07466194649178806, + "flos": 31898263029120.0, + "grad_norm": 2.137384816770213, + "language_loss": 0.7777133, + "learning_rate": 3.9791167082641275e-06, + "loss": 0.80000007, + "num_input_tokens_seen": 72378065, + "router_z_loss_clip": 1.04443359, + "router_z_loss_mlp": 0.20275879, + "step": 2573, + "time_per_iteration": 2.598762273788452 + }, + { + "auxiliary_loss_clip": 0.01172623, + "auxiliary_loss_mlp": 0.01048382, + "balance_loss_clip": 1.0687077, + "balance_loss_mlp": 1.02867043, + "epoch": 0.0746909639603041, + "flos": 28031833203840.0, + "grad_norm": 2.6229943102252684, + "language_loss": 0.91047597, + "learning_rate": 3.979089608071152e-06, + "loss": 0.93268609, + "num_input_tokens_seen": 72392355, + "router_z_loss_clip": 1.03662109, + "router_z_loss_mlp": 0.19732666, + "step": 2574, + "time_per_iteration": 2.590792179107666 + }, + { + "auxiliary_loss_clip": 0.01056543, + "auxiliary_loss_mlp": 0.01005434, + "balance_loss_clip": 1.0295949, + "balance_loss_mlp": 1.0041585, + "epoch": 0.07471998142882015, + "flos": 59415427186560.0, + "grad_norm": 0.6786470732614627, + "language_loss": 0.48302457, + "learning_rate": 3.979062490398056e-06, + "loss": 0.50364435, + "num_input_tokens_seen": 72452110, + "router_z_loss_clip": 0.26953125, + "router_z_loss_mlp": 0.01275635, + "step": 2575, + "time_per_iteration": 3.0054585933685303 + }, + { + "auxiliary_loss_clip": 0.01185195, + "auxiliary_loss_mlp": 0.01061904, + "balance_loss_clip": 1.07832396, + "balance_loss_mlp": 1.04123354, + "epoch": 0.0747489988973362, + "flos": 34199634389760.0, + "grad_norm": 1.8946479902161386, + "language_loss": 0.87830973, + "learning_rate": 3.979035355245079e-06, + "loss": 0.90078068, + "num_input_tokens_seen": 72473345, + "router_z_loss_clip": 1.06884766, + "router_z_loss_mlp": 0.20654297, + "step": 2576, + "time_per_iteration": 2.6706879138946533 + }, + { + "auxiliary_loss_clip": 0.01055008, + "auxiliary_loss_mlp": 0.01010544, + "balance_loss_clip": 1.02796173, + "balance_loss_mlp": 1.00922096, + "epoch": 0.07477801636585224, + "flos": 74781818403840.0, + "grad_norm": 0.6546659405282436, + "language_loss": 0.53114074, + "learning_rate": 3.979008202612461e-06, + "loss": 0.5517962, + "num_input_tokens_seen": 72539500, + "router_z_loss_clip": 0.26953125, + "router_z_loss_mlp": 0.01324463, + "step": 2577, + "time_per_iteration": 3.167642116546631 + }, + { + "auxiliary_loss_clip": 0.01176475, + "auxiliary_loss_mlp": 0.01049034, + "balance_loss_clip": 1.07367706, + "balance_loss_mlp": 1.02893579, + "epoch": 0.07480703383436829, + "flos": 27373195468800.0, + "grad_norm": 2.470013009982439, + "language_loss": 0.91895437, + "learning_rate": 3.9789810325004425e-06, + "loss": 0.94120944, + "num_input_tokens_seen": 72554415, + "router_z_loss_clip": 1.02636719, + "router_z_loss_mlp": 0.20117188, + "step": 2578, + "time_per_iteration": 2.5997612476348877 + }, + { + "auxiliary_loss_clip": 0.01176766, + "auxiliary_loss_mlp": 0.0105863, + "balance_loss_clip": 1.0768261, + "balance_loss_mlp": 1.04055834, + "epoch": 0.07483605130288434, + "flos": 24168025071360.0, + "grad_norm": 3.116866554836578, + "language_loss": 0.81020266, + "learning_rate": 3.978953844909262e-06, + "loss": 0.83255661, + "num_input_tokens_seen": 72571150, + "router_z_loss_clip": 1.00097656, + "router_z_loss_mlp": 0.18078613, + "step": 2579, + "time_per_iteration": 2.5946428775787354 + }, + { + "auxiliary_loss_clip": 0.01176216, + "auxiliary_loss_mlp": 0.01045694, + "balance_loss_clip": 1.07031918, + "balance_loss_mlp": 1.02714562, + "epoch": 0.07486506877140038, + "flos": 18945643806720.0, + "grad_norm": 2.3163255230684228, + "language_loss": 0.88113731, + "learning_rate": 3.9789266398391605e-06, + "loss": 0.90335643, + "num_input_tokens_seen": 72585230, + "router_z_loss_clip": 1.06005859, + "router_z_loss_mlp": 0.18548584, + "step": 2580, + "time_per_iteration": 2.5232443809509277 + }, + { + "auxiliary_loss_clip": 0.0105025, + "auxiliary_loss_mlp": 0.01008503, + "balance_loss_clip": 1.02320206, + "balance_loss_mlp": 1.00731683, + "epoch": 0.07489408623991643, + "flos": 56783821161600.0, + "grad_norm": 0.6234489790462152, + "language_loss": 0.50086987, + "learning_rate": 3.978899417290378e-06, + "loss": 0.52145737, + "num_input_tokens_seen": 72647280, + "router_z_loss_clip": 0.26953125, + "router_z_loss_mlp": 0.01184082, + "step": 2581, + "time_per_iteration": 3.1125566959381104 + }, + { + "auxiliary_loss_clip": 0.01179622, + "auxiliary_loss_mlp": 0.01045445, + "balance_loss_clip": 1.07257652, + "balance_loss_mlp": 1.0266155, + "epoch": 0.07492310370843247, + "flos": 42807924311040.0, + "grad_norm": 2.583649031355927, + "language_loss": 0.83015251, + "learning_rate": 3.978872177263156e-06, + "loss": 0.85240316, + "num_input_tokens_seen": 72665970, + "router_z_loss_clip": 1.0703125, + "router_z_loss_mlp": 0.18835449, + "step": 2582, + "time_per_iteration": 2.642024040222168 + }, + { + "auxiliary_loss_clip": 0.01171061, + "auxiliary_loss_mlp": 0.01049532, + "balance_loss_clip": 1.0704689, + "balance_loss_mlp": 1.03171587, + "epoch": 0.07495212117694852, + "flos": 29872615443840.0, + "grad_norm": 1.761214473764191, + "language_loss": 0.92456001, + "learning_rate": 3.978844919757733e-06, + "loss": 0.94676596, + "num_input_tokens_seen": 72691320, + "router_z_loss_clip": 1.00585938, + "router_z_loss_mlp": 0.17822266, + "step": 2583, + "time_per_iteration": 2.686025857925415 + }, + { + "auxiliary_loss_clip": 0.01177992, + "auxiliary_loss_mlp": 0.01049977, + "balance_loss_clip": 1.07179832, + "balance_loss_mlp": 1.03116572, + "epoch": 0.07498113864546457, + "flos": 26316416407680.0, + "grad_norm": 3.4104886776881953, + "language_loss": 0.75241089, + "learning_rate": 3.9788176447743516e-06, + "loss": 0.77469063, + "num_input_tokens_seen": 72707485, + "router_z_loss_clip": 1.0625, + "router_z_loss_mlp": 0.18817139, + "step": 2584, + "time_per_iteration": 2.5614001750946045 + }, + { + "auxiliary_loss_clip": 0.01166948, + "auxiliary_loss_mlp": 0.01050094, + "balance_loss_clip": 1.06675744, + "balance_loss_mlp": 1.03242755, + "epoch": 0.07501015611398061, + "flos": 40734658270080.0, + "grad_norm": 2.2946765109887437, + "language_loss": 0.87248778, + "learning_rate": 3.978790352313251e-06, + "loss": 0.89465821, + "num_input_tokens_seen": 72729350, + "router_z_loss_clip": 1.00244141, + "router_z_loss_mlp": 0.17687988, + "step": 2585, + "time_per_iteration": 2.698011636734009 + }, + { + "auxiliary_loss_clip": 0.01175843, + "auxiliary_loss_mlp": 0.01054929, + "balance_loss_clip": 1.07348931, + "balance_loss_mlp": 1.03446114, + "epoch": 0.07503917358249666, + "flos": 26718615970560.0, + "grad_norm": 2.1273729015588643, + "language_loss": 0.79331684, + "learning_rate": 3.978763042374674e-06, + "loss": 0.81562459, + "num_input_tokens_seen": 72742020, + "router_z_loss_clip": 1.02294922, + "router_z_loss_mlp": 0.20483398, + "step": 2586, + "time_per_iteration": 2.5435962677001953 + }, + { + "auxiliary_loss_clip": 0.01165762, + "auxiliary_loss_mlp": 0.01047879, + "balance_loss_clip": 1.06633055, + "balance_loss_mlp": 1.02918744, + "epoch": 0.07506819105101271, + "flos": 15953064053760.0, + "grad_norm": 3.0947170214785826, + "language_loss": 0.99593353, + "learning_rate": 3.978735714958861e-06, + "loss": 1.01806998, + "num_input_tokens_seen": 72754120, + "router_z_loss_clip": 0.99511719, + "router_z_loss_mlp": 0.18688965, + "step": 2587, + "time_per_iteration": 2.471381187438965 + }, + { + "auxiliary_loss_clip": 0.0116615, + "auxiliary_loss_mlp": 0.01047891, + "balance_loss_clip": 1.06787503, + "balance_loss_mlp": 1.02925265, + "epoch": 0.07509720851952875, + "flos": 16427587651200.0, + "grad_norm": 2.836769941371693, + "language_loss": 0.94313782, + "learning_rate": 3.9787083700660535e-06, + "loss": 0.96527821, + "num_input_tokens_seen": 72766050, + "router_z_loss_clip": 0.98242188, + "router_z_loss_mlp": 0.18658447, + "step": 2588, + "time_per_iteration": 2.453033208847046 + }, + { + "auxiliary_loss_clip": 0.01179023, + "auxiliary_loss_mlp": 0.0105483, + "balance_loss_clip": 1.07154095, + "balance_loss_mlp": 1.03619766, + "epoch": 0.0751262259880448, + "flos": 31936328467200.0, + "grad_norm": 2.0632192563476695, + "language_loss": 0.9422999, + "learning_rate": 3.978681007696493e-06, + "loss": 0.96463841, + "num_input_tokens_seen": 72788195, + "router_z_loss_clip": 1.07373047, + "router_z_loss_mlp": 0.1862793, + "step": 2589, + "time_per_iteration": 2.7617523670196533 + }, + { + "auxiliary_loss_clip": 0.011756, + "auxiliary_loss_mlp": 0.01053767, + "balance_loss_clip": 1.06943011, + "balance_loss_mlp": 1.03332281, + "epoch": 0.07515524345656086, + "flos": 11575086687360.0, + "grad_norm": 2.5748697997863847, + "language_loss": 0.7865786, + "learning_rate": 3.978653627850422e-06, + "loss": 0.80887228, + "num_input_tokens_seen": 72799595, + "router_z_loss_clip": 1.0625, + "router_z_loss_mlp": 0.20446777, + "step": 2590, + "time_per_iteration": 2.4374561309814453 + }, + { + "auxiliary_loss_clip": 0.01178564, + "auxiliary_loss_mlp": 0.01062451, + "balance_loss_clip": 1.0700165, + "balance_loss_mlp": 1.04373479, + "epoch": 0.07518426092507689, + "flos": 30913484780160.0, + "grad_norm": 2.2854538961487214, + "language_loss": 0.91939676, + "learning_rate": 3.97862623052808e-06, + "loss": 0.94180691, + "num_input_tokens_seen": 72820365, + "router_z_loss_clip": 1.08740234, + "router_z_loss_mlp": 0.18737793, + "step": 2591, + "time_per_iteration": 2.612025737762451 + }, + { + "auxiliary_loss_clip": 0.01174748, + "auxiliary_loss_mlp": 0.01051163, + "balance_loss_clip": 1.0712471, + "balance_loss_mlp": 1.03225684, + "epoch": 0.07521327839359294, + "flos": 49922510135040.0, + "grad_norm": 2.5103081472671813, + "language_loss": 0.90583104, + "learning_rate": 3.978598815729711e-06, + "loss": 0.92809016, + "num_input_tokens_seen": 72837700, + "router_z_loss_clip": 1.03564453, + "router_z_loss_mlp": 0.18920898, + "step": 2592, + "time_per_iteration": 2.7440903186798096 + }, + { + "auxiliary_loss_clip": 0.01174045, + "auxiliary_loss_mlp": 0.01051436, + "balance_loss_clip": 1.0736897, + "balance_loss_mlp": 1.0321002, + "epoch": 0.075242295862109, + "flos": 26317565642880.0, + "grad_norm": 2.110891348073229, + "language_loss": 0.79598761, + "learning_rate": 3.978571383455557e-06, + "loss": 0.81824243, + "num_input_tokens_seen": 72854155, + "router_z_loss_clip": 1.00390625, + "router_z_loss_mlp": 0.19317627, + "step": 2593, + "time_per_iteration": 2.5700595378875732 + }, + { + "auxiliary_loss_clip": 0.01170594, + "auxiliary_loss_mlp": 0.0105571, + "balance_loss_clip": 1.0711453, + "balance_loss_mlp": 1.03844833, + "epoch": 0.07527131333062503, + "flos": 33100695740160.0, + "grad_norm": 1.902318175625478, + "language_loss": 0.78944808, + "learning_rate": 3.978543933705859e-06, + "loss": 0.81171113, + "num_input_tokens_seen": 72868965, + "router_z_loss_clip": 0.99462891, + "router_z_loss_mlp": 0.17242432, + "step": 2594, + "time_per_iteration": 2.67895770072937 + }, + { + "auxiliary_loss_clip": 0.01164632, + "auxiliary_loss_mlp": 0.01037851, + "balance_loss_clip": 1.06969702, + "balance_loss_mlp": 1.02100074, + "epoch": 0.07530033079914109, + "flos": 17267717831040.0, + "grad_norm": 2.3061123462001056, + "language_loss": 0.74173135, + "learning_rate": 3.978516466480862e-06, + "loss": 0.76375616, + "num_input_tokens_seen": 72881580, + "router_z_loss_clip": 0.94873047, + "router_z_loss_mlp": 0.16833496, + "step": 2595, + "time_per_iteration": 2.560631275177002 + }, + { + "auxiliary_loss_clip": 0.01178193, + "auxiliary_loss_mlp": 0.0104912, + "balance_loss_clip": 1.07362247, + "balance_loss_mlp": 1.03091693, + "epoch": 0.07532934826765712, + "flos": 22047930673920.0, + "grad_norm": 2.3909099243726177, + "language_loss": 0.84457338, + "learning_rate": 3.978488981780805e-06, + "loss": 0.86684656, + "num_input_tokens_seen": 72895465, + "router_z_loss_clip": 1.04589844, + "router_z_loss_mlp": 0.1819458, + "step": 2596, + "time_per_iteration": 2.5195698738098145 + }, + { + "auxiliary_loss_clip": 0.01175713, + "auxiliary_loss_mlp": 0.01058966, + "balance_loss_clip": 1.07392943, + "balance_loss_mlp": 1.03996444, + "epoch": 0.07535836573617317, + "flos": 20624323968000.0, + "grad_norm": 2.5342378490182513, + "language_loss": 0.82642508, + "learning_rate": 3.978461479605933e-06, + "loss": 0.84877181, + "num_input_tokens_seen": 72907820, + "router_z_loss_clip": 1.01708984, + "router_z_loss_mlp": 0.19000244, + "step": 2597, + "time_per_iteration": 2.6584208011627197 + }, + { + "auxiliary_loss_clip": 0.01065128, + "auxiliary_loss_mlp": 0.01000471, + "balance_loss_clip": 1.03744769, + "balance_loss_mlp": 0.9992016, + "epoch": 0.07538738320468923, + "flos": 71496638461440.0, + "grad_norm": 0.6705000654295896, + "language_loss": 0.48646361, + "learning_rate": 3.97843395995649e-06, + "loss": 0.5071196, + "num_input_tokens_seen": 72972135, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01269531, + "step": 2598, + "time_per_iteration": 3.163515567779541 + }, + { + "auxiliary_loss_clip": 0.0117996, + "auxiliary_loss_mlp": 0.01050618, + "balance_loss_clip": 1.07504475, + "balance_loss_mlp": 1.03025675, + "epoch": 0.07541640067320526, + "flos": 12023037198720.0, + "grad_norm": 3.1981953457166528, + "language_loss": 0.86210775, + "learning_rate": 3.978406422832717e-06, + "loss": 0.88441348, + "num_input_tokens_seen": 72984960, + "router_z_loss_clip": 1.04736328, + "router_z_loss_mlp": 0.20336914, + "step": 2599, + "time_per_iteration": 2.523827075958252 + }, + { + "auxiliary_loss_clip": 0.01181193, + "auxiliary_loss_mlp": 0.01059143, + "balance_loss_clip": 1.0752573, + "balance_loss_mlp": 1.0390507, + "epoch": 0.07544541814172132, + "flos": 19566718894080.0, + "grad_norm": 3.3593667553263784, + "language_loss": 0.85585606, + "learning_rate": 3.978378868234858e-06, + "loss": 0.87825942, + "num_input_tokens_seen": 72997870, + "router_z_loss_clip": 1.05859375, + "router_z_loss_mlp": 0.2008667, + "step": 2600, + "time_per_iteration": 2.514578104019165 + }, + { + "auxiliary_loss_clip": 0.01191802, + "auxiliary_loss_mlp": 0.01058854, + "balance_loss_clip": 1.07736969, + "balance_loss_mlp": 1.03445208, + "epoch": 0.07547443561023737, + "flos": 35111262591360.0, + "grad_norm": 2.42451959604169, + "language_loss": 1.1217922, + "learning_rate": 3.978351296163156e-06, + "loss": 1.14429879, + "num_input_tokens_seen": 73019975, + "router_z_loss_clip": 1.14257812, + "router_z_loss_mlp": 0.24401855, + "step": 2601, + "time_per_iteration": 2.6590514183044434 + }, + { + "auxiliary_loss_clip": 0.01188744, + "auxiliary_loss_mlp": 0.01062414, + "balance_loss_clip": 1.07837439, + "balance_loss_mlp": 1.04076529, + "epoch": 0.0755034530787534, + "flos": 53285293411200.0, + "grad_norm": 2.713327144302891, + "language_loss": 0.95861572, + "learning_rate": 3.978323706617855e-06, + "loss": 0.98112738, + "num_input_tokens_seen": 73040055, + "router_z_loss_clip": 1.10498047, + "router_z_loss_mlp": 0.2166748, + "step": 2602, + "time_per_iteration": 2.843740224838257 + }, + { + "auxiliary_loss_clip": 0.01065175, + "auxiliary_loss_mlp": 0.01002251, + "balance_loss_clip": 1.03780746, + "balance_loss_mlp": 1.00091541, + "epoch": 0.07553247054726946, + "flos": 73759118371200.0, + "grad_norm": 0.6299240086045056, + "language_loss": 0.51453751, + "learning_rate": 3.978296099599198e-06, + "loss": 0.5352118, + "num_input_tokens_seen": 73107940, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.0133667, + "step": 2603, + "time_per_iteration": 3.1961326599121094 + }, + { + "auxiliary_loss_clip": 0.01171573, + "auxiliary_loss_mlp": 0.01047223, + "balance_loss_clip": 1.07617474, + "balance_loss_mlp": 1.03073049, + "epoch": 0.07556148801578551, + "flos": 34342630433280.0, + "grad_norm": 2.199094456560371, + "language_loss": 0.88170981, + "learning_rate": 3.97826847510743e-06, + "loss": 0.90389776, + "num_input_tokens_seen": 73123475, + "router_z_loss_clip": 0.95458984, + "router_z_loss_mlp": 0.16497803, + "step": 2604, + "time_per_iteration": 2.6229679584503174 + }, + { + "auxiliary_loss_clip": 0.01060388, + "auxiliary_loss_mlp": 0.01001786, + "balance_loss_clip": 1.03311014, + "balance_loss_mlp": 1.00033712, + "epoch": 0.07559050548430155, + "flos": 71602395079680.0, + "grad_norm": 0.693417495184596, + "language_loss": 0.52524567, + "learning_rate": 3.978240833142794e-06, + "loss": 0.54586744, + "num_input_tokens_seen": 73184880, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.01446533, + "step": 2605, + "time_per_iteration": 3.1369597911834717 + }, + { + "auxiliary_loss_clip": 0.01178766, + "auxiliary_loss_mlp": 0.01056637, + "balance_loss_clip": 1.07327032, + "balance_loss_mlp": 1.03686035, + "epoch": 0.0756195229528176, + "flos": 30477960374400.0, + "grad_norm": 2.449886010473987, + "language_loss": 0.91204596, + "learning_rate": 3.978213173705534e-06, + "loss": 0.93439996, + "num_input_tokens_seen": 73202505, + "router_z_loss_clip": 1.05371094, + "router_z_loss_mlp": 0.19787598, + "step": 2606, + "time_per_iteration": 2.626380205154419 + }, + { + "auxiliary_loss_clip": 0.01179543, + "auxiliary_loss_mlp": 0.01059673, + "balance_loss_clip": 1.07668257, + "balance_loss_mlp": 1.04051614, + "epoch": 0.07564854042133365, + "flos": 16033289080320.0, + "grad_norm": 1.9157752661086052, + "language_loss": 0.7670663, + "learning_rate": 3.978185496795896e-06, + "loss": 0.78945851, + "num_input_tokens_seen": 73216290, + "router_z_loss_clip": 1.03027344, + "router_z_loss_mlp": 0.19152832, + "step": 2607, + "time_per_iteration": 2.5214555263519287 + }, + { + "auxiliary_loss_clip": 0.01185844, + "auxiliary_loss_mlp": 0.01055982, + "balance_loss_clip": 1.07557988, + "balance_loss_mlp": 1.0357995, + "epoch": 0.07567755788984969, + "flos": 33868501885440.0, + "grad_norm": 2.3983056624025005, + "language_loss": 0.77079618, + "learning_rate": 3.978157802414122e-06, + "loss": 0.79321444, + "num_input_tokens_seen": 73231450, + "router_z_loss_clip": 1.10351562, + "router_z_loss_mlp": 0.20178223, + "step": 2608, + "time_per_iteration": 2.644287347793579 + }, + { + "auxiliary_loss_clip": 0.01053898, + "auxiliary_loss_mlp": 0.0100255, + "balance_loss_clip": 1.02672195, + "balance_loss_mlp": 1.00115538, + "epoch": 0.07570657535836574, + "flos": 74235617216640.0, + "grad_norm": 0.7461140102776506, + "language_loss": 0.53319311, + "learning_rate": 3.978130090560458e-06, + "loss": 0.55375767, + "num_input_tokens_seen": 73294030, + "router_z_loss_clip": 0.27148438, + "router_z_loss_mlp": 0.01397705, + "step": 2609, + "time_per_iteration": 3.1628854274749756 + }, + { + "auxiliary_loss_clip": 0.01176048, + "auxiliary_loss_mlp": 0.01063382, + "balance_loss_clip": 1.07446718, + "balance_loss_mlp": 1.04249597, + "epoch": 0.07573559282688179, + "flos": 23944408951680.0, + "grad_norm": 2.723880632842705, + "language_loss": 0.90895081, + "learning_rate": 3.978102361235149e-06, + "loss": 0.93134511, + "num_input_tokens_seen": 73310480, + "router_z_loss_clip": 1.01708984, + "router_z_loss_mlp": 0.20910645, + "step": 2610, + "time_per_iteration": 2.5548596382141113 + }, + { + "auxiliary_loss_clip": 0.01185849, + "auxiliary_loss_mlp": 0.01052509, + "balance_loss_clip": 1.07480109, + "balance_loss_mlp": 1.03181434, + "epoch": 0.07576461029539783, + "flos": 24749131299840.0, + "grad_norm": 2.084376682890447, + "language_loss": 0.96577996, + "learning_rate": 3.978074614438439e-06, + "loss": 0.98816359, + "num_input_tokens_seen": 73325480, + "router_z_loss_clip": 1.11035156, + "router_z_loss_mlp": 0.20703125, + "step": 2611, + "time_per_iteration": 2.5346670150756836 + }, + { + "auxiliary_loss_clip": 0.01194741, + "auxiliary_loss_mlp": 0.01062905, + "balance_loss_clip": 1.07716835, + "balance_loss_mlp": 1.03974307, + "epoch": 0.07579362776391388, + "flos": 15989657034240.0, + "grad_norm": 2.3865915553502295, + "language_loss": 0.84147805, + "learning_rate": 3.978046850170574e-06, + "loss": 0.86405456, + "num_input_tokens_seen": 73338820, + "router_z_loss_clip": 1.17578125, + "router_z_loss_mlp": 0.23168945, + "step": 2612, + "time_per_iteration": 2.480438470840454 + }, + { + "auxiliary_loss_clip": 0.01054705, + "auxiliary_loss_mlp": 0.01005087, + "balance_loss_clip": 1.02734292, + "balance_loss_mlp": 1.00364494, + "epoch": 0.07582264523242992, + "flos": 61535306102400.0, + "grad_norm": 0.6380287283175382, + "language_loss": 0.51161003, + "learning_rate": 3.978019068431799e-06, + "loss": 0.53220797, + "num_input_tokens_seen": 73403590, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.0144043, + "step": 2613, + "time_per_iteration": 3.103782892227173 + }, + { + "auxiliary_loss_clip": 0.01169298, + "auxiliary_loss_mlp": 0.01054113, + "balance_loss_clip": 1.07005954, + "balance_loss_mlp": 1.0362432, + "epoch": 0.07585166270094597, + "flos": 18544701219840.0, + "grad_norm": 3.009804152658557, + "language_loss": 0.87122607, + "learning_rate": 3.977991269222358e-06, + "loss": 0.89346015, + "num_input_tokens_seen": 73415705, + "router_z_loss_clip": 0.99316406, + "router_z_loss_mlp": 0.17858887, + "step": 2614, + "time_per_iteration": 2.477879285812378 + }, + { + "auxiliary_loss_clip": 0.01178426, + "auxiliary_loss_mlp": 0.01042197, + "balance_loss_clip": 1.07049036, + "balance_loss_mlp": 1.02176476, + "epoch": 0.07588068016946202, + "flos": 34016920882560.0, + "grad_norm": 1.7090451670070743, + "language_loss": 0.85933781, + "learning_rate": 3.977963452542499e-06, + "loss": 0.88154411, + "num_input_tokens_seen": 73437920, + "router_z_loss_clip": 1.07958984, + "router_z_loss_mlp": 0.2043457, + "step": 2615, + "time_per_iteration": 2.6773629188537598 + }, + { + "auxiliary_loss_clip": 0.01176686, + "auxiliary_loss_mlp": 0.01057952, + "balance_loss_clip": 1.07345104, + "balance_loss_mlp": 1.04059482, + "epoch": 0.07590969763797806, + "flos": 19272251197440.0, + "grad_norm": 3.5159336833742882, + "language_loss": 0.76910418, + "learning_rate": 3.977935618392466e-06, + "loss": 0.79145056, + "num_input_tokens_seen": 73448955, + "router_z_loss_clip": 1.03173828, + "router_z_loss_mlp": 0.17352295, + "step": 2616, + "time_per_iteration": 2.5464019775390625 + }, + { + "auxiliary_loss_clip": 0.01168554, + "auxiliary_loss_mlp": 0.01044506, + "balance_loss_clip": 1.06955814, + "balance_loss_mlp": 1.02719665, + "epoch": 0.07593871510649411, + "flos": 29964617153280.0, + "grad_norm": 2.755983627822095, + "language_loss": 0.80101609, + "learning_rate": 3.977907766772505e-06, + "loss": 0.8231467, + "num_input_tokens_seen": 73464185, + "router_z_loss_clip": 0.99072266, + "router_z_loss_mlp": 0.1730957, + "step": 2617, + "time_per_iteration": 2.6080896854400635 + }, + { + "auxiliary_loss_clip": 0.01176528, + "auxiliary_loss_mlp": 0.01046983, + "balance_loss_clip": 1.07199585, + "balance_loss_mlp": 1.02820802, + "epoch": 0.07596773257501016, + "flos": 61342641550080.0, + "grad_norm": 2.3620888166878333, + "language_loss": 1.06416678, + "learning_rate": 3.977879897682862e-06, + "loss": 1.08640194, + "num_input_tokens_seen": 73485365, + "router_z_loss_clip": 1.04541016, + "router_z_loss_mlp": 0.18786621, + "step": 2618, + "time_per_iteration": 2.8849990367889404 + }, + { + "auxiliary_loss_clip": 0.01174583, + "auxiliary_loss_mlp": 0.01047983, + "balance_loss_clip": 1.07146561, + "balance_loss_mlp": 1.02970815, + "epoch": 0.0759967500435262, + "flos": 12671188162560.0, + "grad_norm": 3.225074132094528, + "language_loss": 0.83525389, + "learning_rate": 3.977852011123784e-06, + "loss": 0.85747957, + "num_input_tokens_seen": 73498175, + "router_z_loss_clip": 1.03125, + "router_z_loss_mlp": 0.18273926, + "step": 2619, + "time_per_iteration": 2.563971757888794 + }, + { + "auxiliary_loss_clip": 0.01160761, + "auxiliary_loss_mlp": 0.01042414, + "balance_loss_clip": 1.06713343, + "balance_loss_mlp": 1.02651763, + "epoch": 0.07602576751204225, + "flos": 41134020658560.0, + "grad_norm": 2.391745984523749, + "language_loss": 0.71037209, + "learning_rate": 3.977824107095516e-06, + "loss": 0.73240381, + "num_input_tokens_seen": 73516305, + "router_z_loss_clip": 0.93603516, + "router_z_loss_mlp": 0.15905762, + "step": 2620, + "time_per_iteration": 2.7638168334960938 + }, + { + "auxiliary_loss_clip": 0.01166631, + "auxiliary_loss_mlp": 0.01043854, + "balance_loss_clip": 1.0663209, + "balance_loss_mlp": 1.02355289, + "epoch": 0.0760547849805583, + "flos": 23286669056640.0, + "grad_norm": 2.143107139838692, + "language_loss": 0.69966674, + "learning_rate": 3.977796185598306e-06, + "loss": 0.7217716, + "num_input_tokens_seen": 73530400, + "router_z_loss_clip": 1.00341797, + "router_z_loss_mlp": 0.203125, + "step": 2621, + "time_per_iteration": 2.5534090995788574 + }, + { + "auxiliary_loss_clip": 0.01050158, + "auxiliary_loss_mlp": 0.01017105, + "balance_loss_clip": 1.0228616, + "balance_loss_mlp": 1.01573408, + "epoch": 0.07608380244907434, + "flos": 69772925623680.0, + "grad_norm": 0.7463690096174596, + "language_loss": 0.49669856, + "learning_rate": 3.977768246632399e-06, + "loss": 0.51737118, + "num_input_tokens_seen": 73588970, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.01373291, + "step": 2622, + "time_per_iteration": 3.025071620941162 + }, + { + "auxiliary_loss_clip": 0.01049987, + "auxiliary_loss_mlp": 0.01004803, + "balance_loss_clip": 1.02262783, + "balance_loss_mlp": 1.00347376, + "epoch": 0.07611281991759039, + "flos": 67255839135360.0, + "grad_norm": 0.7165717989418448, + "language_loss": 0.55616492, + "learning_rate": 3.977740290198043e-06, + "loss": 0.57671279, + "num_input_tokens_seen": 73650380, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.01330566, + "step": 2623, + "time_per_iteration": 3.040200710296631 + }, + { + "auxiliary_loss_clip": 0.0118518, + "auxiliary_loss_mlp": 0.01053725, + "balance_loss_clip": 1.07231903, + "balance_loss_mlp": 1.03251803, + "epoch": 0.07614183738610644, + "flos": 31862603802240.0, + "grad_norm": 2.074184330292082, + "language_loss": 0.95294428, + "learning_rate": 3.977712316295484e-06, + "loss": 0.97533333, + "num_input_tokens_seen": 73669550, + "router_z_loss_clip": 1.12841797, + "router_z_loss_mlp": 0.2121582, + "step": 2624, + "time_per_iteration": 2.6238324642181396 + }, + { + "auxiliary_loss_clip": 0.01174061, + "auxiliary_loss_mlp": 0.01048241, + "balance_loss_clip": 1.06703198, + "balance_loss_mlp": 1.02910829, + "epoch": 0.07617085485462248, + "flos": 11357863188480.0, + "grad_norm": 3.0439078796898, + "language_loss": 0.80965608, + "learning_rate": 3.97768432492497e-06, + "loss": 0.83187908, + "num_input_tokens_seen": 73681775, + "router_z_loss_clip": 1.07177734, + "router_z_loss_mlp": 0.19122314, + "step": 2625, + "time_per_iteration": 2.502629041671753 + }, + { + "auxiliary_loss_clip": 0.01048268, + "auxiliary_loss_mlp": 0.00999364, + "balance_loss_clip": 1.02086616, + "balance_loss_mlp": 0.99796957, + "epoch": 0.07619987232313853, + "flos": 74765082666240.0, + "grad_norm": 0.6471643804689358, + "language_loss": 0.51217663, + "learning_rate": 3.977656316086748e-06, + "loss": 0.53265297, + "num_input_tokens_seen": 73740250, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.01397705, + "step": 2626, + "time_per_iteration": 3.066662073135376 + }, + { + "auxiliary_loss_clip": 0.01177247, + "auxiliary_loss_mlp": 0.01050091, + "balance_loss_clip": 1.07108581, + "balance_loss_mlp": 1.03098726, + "epoch": 0.07622888979165457, + "flos": 23835384195840.0, + "grad_norm": 1.907739206782287, + "language_loss": 0.76822132, + "learning_rate": 3.977628289781064e-06, + "loss": 0.79049474, + "num_input_tokens_seen": 73761195, + "router_z_loss_clip": 1.0625, + "router_z_loss_mlp": 0.19116211, + "step": 2627, + "time_per_iteration": 2.7257585525512695 + }, + { + "auxiliary_loss_clip": 0.01174652, + "auxiliary_loss_mlp": 0.0105766, + "balance_loss_clip": 1.06664181, + "balance_loss_mlp": 1.03614235, + "epoch": 0.07625790726017062, + "flos": 23727113625600.0, + "grad_norm": 2.54494432333068, + "language_loss": 0.95096552, + "learning_rate": 3.977600246008167e-06, + "loss": 0.9732886, + "num_input_tokens_seen": 73780540, + "router_z_loss_clip": 1.08007812, + "router_z_loss_mlp": 0.21496582, + "step": 2628, + "time_per_iteration": 2.5827255249023438 + }, + { + "auxiliary_loss_clip": 0.01048504, + "auxiliary_loss_mlp": 0.0100079, + "balance_loss_clip": 1.02070522, + "balance_loss_mlp": 0.99947262, + "epoch": 0.07628692472868667, + "flos": 60325834325760.0, + "grad_norm": 0.6819174524185931, + "language_loss": 0.47678563, + "learning_rate": 3.977572184768305e-06, + "loss": 0.49727857, + "num_input_tokens_seen": 73837700, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01318359, + "step": 2629, + "time_per_iteration": 3.0441222190856934 + }, + { + "auxiliary_loss_clip": 0.01163199, + "auxiliary_loss_mlp": 0.01038912, + "balance_loss_clip": 1.06709266, + "balance_loss_mlp": 1.02159119, + "epoch": 0.07631594219720271, + "flos": 17850547912320.0, + "grad_norm": 2.5899949611591997, + "language_loss": 0.81148273, + "learning_rate": 3.977544106061725e-06, + "loss": 0.83350384, + "num_input_tokens_seen": 73849375, + "router_z_loss_clip": 0.9609375, + "router_z_loss_mlp": 0.1730957, + "step": 2630, + "time_per_iteration": 2.455867052078247 + }, + { + "auxiliary_loss_clip": 0.01179941, + "auxiliary_loss_mlp": 0.01063102, + "balance_loss_clip": 1.07126904, + "balance_loss_mlp": 1.04381442, + "epoch": 0.07634495966571876, + "flos": 16210040929920.0, + "grad_norm": 2.2654953497573356, + "language_loss": 0.84463072, + "learning_rate": 3.977516009888675e-06, + "loss": 0.86706126, + "num_input_tokens_seen": 73863710, + "router_z_loss_clip": 1.08642578, + "router_z_loss_mlp": 0.19274902, + "step": 2631, + "time_per_iteration": 2.4872634410858154 + }, + { + "auxiliary_loss_clip": 0.01158846, + "auxiliary_loss_mlp": 0.01049731, + "balance_loss_clip": 1.06549013, + "balance_loss_mlp": 1.03353691, + "epoch": 0.07637397713423481, + "flos": 19057649391360.0, + "grad_norm": 2.1264024217401913, + "language_loss": 0.8665632, + "learning_rate": 3.977487896249404e-06, + "loss": 0.88864899, + "num_input_tokens_seen": 73878780, + "router_z_loss_clip": 0.93505859, + "router_z_loss_mlp": 0.16192627, + "step": 2632, + "time_per_iteration": 2.5111210346221924 + }, + { + "auxiliary_loss_clip": 0.01169121, + "auxiliary_loss_mlp": 0.01051424, + "balance_loss_clip": 1.0667994, + "balance_loss_mlp": 1.03387618, + "epoch": 0.07640299460275085, + "flos": 53206325360640.0, + "grad_norm": 2.665284664583716, + "language_loss": 0.60935032, + "learning_rate": 3.977459765144159e-06, + "loss": 0.63155568, + "num_input_tokens_seen": 73907175, + "router_z_loss_clip": 1.02294922, + "router_z_loss_mlp": 0.17553711, + "step": 2633, + "time_per_iteration": 2.8034839630126953 + }, + { + "auxiliary_loss_clip": 0.01168865, + "auxiliary_loss_mlp": 0.01044824, + "balance_loss_clip": 1.06943774, + "balance_loss_mlp": 1.02572668, + "epoch": 0.0764320120712669, + "flos": 35436900314880.0, + "grad_norm": 6.8948285036748675, + "language_loss": 1.12219727, + "learning_rate": 3.9774316165731895e-06, + "loss": 1.1443342, + "num_input_tokens_seen": 73922135, + "router_z_loss_clip": 0.99365234, + "router_z_loss_mlp": 0.190979, + "step": 2634, + "time_per_iteration": 4.856901168823242 + }, + { + "auxiliary_loss_clip": 0.01048486, + "auxiliary_loss_mlp": 0.01002319, + "balance_loss_clip": 1.02101803, + "balance_loss_mlp": 1.00093579, + "epoch": 0.07646102953978295, + "flos": 67398152820480.0, + "grad_norm": 0.6902987665939668, + "language_loss": 0.52589655, + "learning_rate": 3.977403450536744e-06, + "loss": 0.5464046, + "num_input_tokens_seen": 73983365, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.01385498, + "step": 2635, + "time_per_iteration": 7.771939039230347 + }, + { + "auxiliary_loss_clip": 0.01165462, + "auxiliary_loss_mlp": 0.01041617, + "balance_loss_clip": 1.06771278, + "balance_loss_mlp": 1.02194786, + "epoch": 0.07649004700829899, + "flos": 24344345957760.0, + "grad_norm": 2.7625290310877553, + "language_loss": 0.823282, + "learning_rate": 3.977375267035071e-06, + "loss": 0.84535277, + "num_input_tokens_seen": 73997605, + "router_z_loss_clip": 0.97753906, + "router_z_loss_mlp": 0.19665527, + "step": 2636, + "time_per_iteration": 2.609200954437256 + }, + { + "auxiliary_loss_clip": 0.01047778, + "auxiliary_loss_mlp": 0.01006692, + "balance_loss_clip": 1.02033341, + "balance_loss_mlp": 1.00536919, + "epoch": 0.07651906447681504, + "flos": 62082476956800.0, + "grad_norm": 0.6571035386256792, + "language_loss": 0.48554575, + "learning_rate": 3.977347066068419e-06, + "loss": 0.50609052, + "num_input_tokens_seen": 74060975, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.01324463, + "step": 2637, + "time_per_iteration": 5.396805047988892 + }, + { + "auxiliary_loss_clip": 0.01172309, + "auxiliary_loss_mlp": 0.01046744, + "balance_loss_clip": 1.07123411, + "balance_loss_mlp": 1.02908969, + "epoch": 0.0765480819453311, + "flos": 13948135637760.0, + "grad_norm": 3.2007139797155553, + "language_loss": 0.92448777, + "learning_rate": 3.977318847637038e-06, + "loss": 0.94667828, + "num_input_tokens_seen": 74072655, + "router_z_loss_clip": 1.01074219, + "router_z_loss_mlp": 0.17663574, + "step": 2638, + "time_per_iteration": 2.487766742706299 + }, + { + "auxiliary_loss_clip": 0.01047037, + "auxiliary_loss_mlp": 0.01003233, + "balance_loss_clip": 1.0198195, + "balance_loss_mlp": 1.00193346, + "epoch": 0.07657709941384713, + "flos": 69248054136960.0, + "grad_norm": 0.6877486977861533, + "language_loss": 0.47504765, + "learning_rate": 3.977290611741177e-06, + "loss": 0.49555033, + "num_input_tokens_seen": 74125000, + "router_z_loss_clip": 0.27148438, + "router_z_loss_mlp": 0.01300049, + "step": 2639, + "time_per_iteration": 2.9308090209960938 + }, + { + "auxiliary_loss_clip": 0.01046447, + "auxiliary_loss_mlp": 0.01006319, + "balance_loss_clip": 1.01912546, + "balance_loss_mlp": 1.00508535, + "epoch": 0.07660611688236318, + "flos": 61510633459200.0, + "grad_norm": 0.7910672783544157, + "language_loss": 0.50349784, + "learning_rate": 3.977262358381084e-06, + "loss": 0.5240255, + "num_input_tokens_seen": 74176480, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.0123291, + "step": 2640, + "time_per_iteration": 3.1654365062713623 + }, + { + "auxiliary_loss_clip": 0.01045833, + "auxiliary_loss_mlp": 0.01002404, + "balance_loss_clip": 1.01849079, + "balance_loss_mlp": 1.00109315, + "epoch": 0.07663513435087924, + "flos": 60131987602560.0, + "grad_norm": 0.6654799957214015, + "language_loss": 0.47954416, + "learning_rate": 3.977234087557011e-06, + "loss": 0.50002658, + "num_input_tokens_seen": 74232010, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.01312256, + "step": 2641, + "time_per_iteration": 3.008322238922119 + }, + { + "auxiliary_loss_clip": 0.01045659, + "auxiliary_loss_mlp": 0.01004308, + "balance_loss_clip": 1.01853561, + "balance_loss_mlp": 1.00297248, + "epoch": 0.07666415181939527, + "flos": 74774922992640.0, + "grad_norm": 0.6641167559588983, + "language_loss": 0.47747257, + "learning_rate": 3.977205799269206e-06, + "loss": 0.49797225, + "num_input_tokens_seen": 74298780, + "router_z_loss_clip": 0.26953125, + "router_z_loss_mlp": 0.0133667, + "step": 2642, + "time_per_iteration": 3.183079719543457 + }, + { + "auxiliary_loss_clip": 0.01165828, + "auxiliary_loss_mlp": 0.01051848, + "balance_loss_clip": 1.06820941, + "balance_loss_mlp": 1.0344013, + "epoch": 0.07669316928791133, + "flos": 74729815511040.0, + "grad_norm": 2.6643142370232265, + "language_loss": 0.94708031, + "learning_rate": 3.977177493517919e-06, + "loss": 0.96925706, + "num_input_tokens_seen": 74321455, + "router_z_loss_clip": 0.97607422, + "router_z_loss_mlp": 0.17437744, + "step": 2643, + "time_per_iteration": 2.925802230834961 + }, + { + "auxiliary_loss_clip": 0.01168161, + "auxiliary_loss_mlp": 0.01044766, + "balance_loss_clip": 1.06667352, + "balance_loss_mlp": 1.02709913, + "epoch": 0.07672218675642736, + "flos": 10553715457920.0, + "grad_norm": 3.4680338391257104, + "language_loss": 0.80890381, + "learning_rate": 3.977149170303401e-06, + "loss": 0.83103311, + "num_input_tokens_seen": 74331995, + "router_z_loss_clip": 1.01416016, + "router_z_loss_mlp": 0.17663574, + "step": 2644, + "time_per_iteration": 2.463358163833618 + }, + { + "auxiliary_loss_clip": 0.01045745, + "auxiliary_loss_mlp": 0.00999762, + "balance_loss_clip": 1.01864767, + "balance_loss_mlp": 0.99856359, + "epoch": 0.07675120422494341, + "flos": 62188413143040.0, + "grad_norm": 0.6877120943108374, + "language_loss": 0.51846206, + "learning_rate": 3.977120829625901e-06, + "loss": 0.53891706, + "num_input_tokens_seen": 74392960, + "router_z_loss_clip": 0.26953125, + "router_z_loss_mlp": 0.01196289, + "step": 2645, + "time_per_iteration": 3.051929473876953 + }, + { + "auxiliary_loss_clip": 0.0104597, + "auxiliary_loss_mlp": 0.00999039, + "balance_loss_clip": 1.01901436, + "balance_loss_mlp": 0.99793667, + "epoch": 0.07678022169345947, + "flos": 74775928573440.0, + "grad_norm": 0.6337096382482061, + "language_loss": 0.5271709, + "learning_rate": 3.97709247148567e-06, + "loss": 0.54762101, + "num_input_tokens_seen": 74457960, + "router_z_loss_clip": 0.26953125, + "router_z_loss_mlp": 0.01104736, + "step": 2646, + "time_per_iteration": 3.315993070602417 + }, + { + "auxiliary_loss_clip": 0.01150389, + "auxiliary_loss_mlp": 0.01048139, + "balance_loss_clip": 1.0634129, + "balance_loss_mlp": 1.03416729, + "epoch": 0.0768092391619755, + "flos": 24091211836800.0, + "grad_norm": 2.5879528376014034, + "language_loss": 0.74907231, + "learning_rate": 3.977064095882958e-06, + "loss": 0.77105761, + "num_input_tokens_seen": 74472125, + "router_z_loss_clip": 0.86914062, + "router_z_loss_mlp": 0.13977051, + "step": 2647, + "time_per_iteration": 2.5246059894561768 + }, + { + "auxiliary_loss_clip": 0.01171445, + "auxiliary_loss_mlp": 0.01046072, + "balance_loss_clip": 1.07002604, + "balance_loss_mlp": 1.02701616, + "epoch": 0.07683825663049156, + "flos": 46164243139200.0, + "grad_norm": 2.4847214402131095, + "language_loss": 0.8195017, + "learning_rate": 3.977035702818016e-06, + "loss": 0.84167689, + "num_input_tokens_seen": 74490740, + "router_z_loss_clip": 1.01318359, + "router_z_loss_mlp": 0.1907959, + "step": 2648, + "time_per_iteration": 2.6171715259552 + }, + { + "auxiliary_loss_clip": 0.0116033, + "auxiliary_loss_mlp": 0.01052245, + "balance_loss_clip": 1.06491542, + "balance_loss_mlp": 1.0343399, + "epoch": 0.07686727409900761, + "flos": 18269806435200.0, + "grad_norm": 2.735480622450641, + "language_loss": 0.85898042, + "learning_rate": 3.977007292291094e-06, + "loss": 0.8811062, + "num_input_tokens_seen": 74509400, + "router_z_loss_clip": 0.95361328, + "router_z_loss_mlp": 0.17895508, + "step": 2649, + "time_per_iteration": 2.533233404159546 + }, + { + "auxiliary_loss_clip": 0.01174914, + "auxiliary_loss_mlp": 0.01058995, + "balance_loss_clip": 1.06914306, + "balance_loss_mlp": 1.04112005, + "epoch": 0.07689629156752364, + "flos": 26571848999040.0, + "grad_norm": 2.138283228010025, + "language_loss": 0.89960074, + "learning_rate": 3.976978864302445e-06, + "loss": 0.92193985, + "num_input_tokens_seen": 74526865, + "router_z_loss_clip": 1.05664062, + "router_z_loss_mlp": 0.17883301, + "step": 2650, + "time_per_iteration": 2.566412925720215 + }, + { + "auxiliary_loss_clip": 0.01044763, + "auxiliary_loss_mlp": 0.0099937, + "balance_loss_clip": 1.0179522, + "balance_loss_mlp": 0.99815971, + "epoch": 0.0769253090360397, + "flos": 74777365117440.0, + "grad_norm": 0.6757336431539226, + "language_loss": 0.44962764, + "learning_rate": 3.976950418852317e-06, + "loss": 0.47006893, + "num_input_tokens_seen": 74592660, + "router_z_loss_clip": 0.26757812, + "router_z_loss_mlp": 0.01208496, + "step": 2651, + "time_per_iteration": 3.148305892944336 + }, + { + "auxiliary_loss_clip": 0.01044392, + "auxiliary_loss_mlp": 0.01000253, + "balance_loss_clip": 1.01756275, + "balance_loss_mlp": 0.999138, + "epoch": 0.07695432650455575, + "flos": 69555809865600.0, + "grad_norm": 0.6684642657799619, + "language_loss": 0.53790945, + "learning_rate": 3.976921955940964e-06, + "loss": 0.55835593, + "num_input_tokens_seen": 74654495, + "router_z_loss_clip": 0.26953125, + "router_z_loss_mlp": 0.01116943, + "step": 2652, + "time_per_iteration": 3.075209617614746 + }, + { + "auxiliary_loss_clip": 0.01175756, + "auxiliary_loss_mlp": 0.01051225, + "balance_loss_clip": 1.07189059, + "balance_loss_mlp": 1.0325985, + "epoch": 0.07698334397307179, + "flos": 28100278569600.0, + "grad_norm": 2.7530127366460673, + "language_loss": 0.9922179, + "learning_rate": 3.976893475568636e-06, + "loss": 1.01448774, + "num_input_tokens_seen": 74666805, + "router_z_loss_clip": 1.0390625, + "router_z_loss_mlp": 0.18615723, + "step": 2653, + "time_per_iteration": 2.588700771331787 + }, + { + "auxiliary_loss_clip": 0.01166371, + "auxiliary_loss_mlp": 0.0105775, + "balance_loss_clip": 1.0683949, + "balance_loss_mlp": 1.03869998, + "epoch": 0.07701236144158784, + "flos": 13511282428800.0, + "grad_norm": 3.0088869576660073, + "language_loss": 1.01672244, + "learning_rate": 3.976864977735585e-06, + "loss": 1.03896368, + "num_input_tokens_seen": 74678975, + "router_z_loss_clip": 0.97949219, + "router_z_loss_mlp": 0.19055176, + "step": 2654, + "time_per_iteration": 2.450751304626465 + }, + { + "auxiliary_loss_clip": 0.01185137, + "auxiliary_loss_mlp": 0.01051925, + "balance_loss_clip": 1.07252955, + "balance_loss_mlp": 1.03047967, + "epoch": 0.07704137891010389, + "flos": 26032579136640.0, + "grad_norm": 6.167634190400982, + "language_loss": 0.81323463, + "learning_rate": 3.976836462442062e-06, + "loss": 0.83560526, + "num_input_tokens_seen": 74695205, + "router_z_loss_clip": 1.12646484, + "router_z_loss_mlp": 0.21459961, + "step": 2655, + "time_per_iteration": 2.593416452407837 + }, + { + "auxiliary_loss_clip": 0.01172279, + "auxiliary_loss_mlp": 0.01046428, + "balance_loss_clip": 1.07115197, + "balance_loss_mlp": 1.02784944, + "epoch": 0.07707039637861993, + "flos": 24528603749760.0, + "grad_norm": 2.542410506660972, + "language_loss": 0.91954255, + "learning_rate": 3.976807929688321e-06, + "loss": 0.94172955, + "num_input_tokens_seen": 74711340, + "router_z_loss_clip": 1.01171875, + "router_z_loss_mlp": 0.18560791, + "step": 2656, + "time_per_iteration": 2.5356571674346924 + }, + { + "auxiliary_loss_clip": 0.01169531, + "auxiliary_loss_mlp": 0.0104699, + "balance_loss_clip": 1.06819415, + "balance_loss_mlp": 1.02766657, + "epoch": 0.07709941384713598, + "flos": 48798973647360.0, + "grad_norm": 2.0537628253433895, + "language_loss": 0.87586915, + "learning_rate": 3.976779379474611e-06, + "loss": 0.89803433, + "num_input_tokens_seen": 74733100, + "router_z_loss_clip": 1.01367188, + "router_z_loss_mlp": 0.1932373, + "step": 2657, + "time_per_iteration": 2.854072332382202 + }, + { + "auxiliary_loss_clip": 0.01176559, + "auxiliary_loss_mlp": 0.01059538, + "balance_loss_clip": 1.07087719, + "balance_loss_mlp": 1.03778231, + "epoch": 0.07712843131565202, + "flos": 34273754104320.0, + "grad_norm": 2.31351744560546, + "language_loss": 0.87230599, + "learning_rate": 3.976750811801186e-06, + "loss": 0.89466697, + "num_input_tokens_seen": 74755560, + "router_z_loss_clip": 1.05712891, + "router_z_loss_mlp": 0.2175293, + "step": 2658, + "time_per_iteration": 2.61929988861084 + }, + { + "auxiliary_loss_clip": 0.01047236, + "auxiliary_loss_mlp": 0.01014207, + "balance_loss_clip": 1.020859, + "balance_loss_mlp": 1.01311016, + "epoch": 0.07715744878416807, + "flos": 74775605351040.0, + "grad_norm": 0.6531804746776005, + "language_loss": 0.49243718, + "learning_rate": 3.9767222266682975e-06, + "loss": 0.51305169, + "num_input_tokens_seen": 74822545, + "router_z_loss_clip": 0.26367188, + "router_z_loss_mlp": 0.01098633, + "step": 2659, + "time_per_iteration": 3.313877582550049 + }, + { + "auxiliary_loss_clip": 0.01176937, + "auxiliary_loss_mlp": 0.01048899, + "balance_loss_clip": 1.07262468, + "balance_loss_mlp": 1.03034413, + "epoch": 0.07718646625268412, + "flos": 44742108890880.0, + "grad_norm": 2.2491502039237377, + "language_loss": 0.80539882, + "learning_rate": 3.976693624076199e-06, + "loss": 0.8276571, + "num_input_tokens_seen": 74841230, + "router_z_loss_clip": 1.04248047, + "router_z_loss_mlp": 0.18566895, + "step": 2660, + "time_per_iteration": 2.711353063583374 + }, + { + "auxiliary_loss_clip": 0.01172391, + "auxiliary_loss_mlp": 0.01055092, + "balance_loss_clip": 1.06894612, + "balance_loss_mlp": 1.03812826, + "epoch": 0.07721548372120016, + "flos": 13253012663040.0, + "grad_norm": 3.9772480748226346, + "language_loss": 0.82810724, + "learning_rate": 3.9766650040251426e-06, + "loss": 0.85038197, + "num_input_tokens_seen": 74854345, + "router_z_loss_clip": 1.03417969, + "router_z_loss_mlp": 0.16955566, + "step": 2661, + "time_per_iteration": 2.483405828475952 + }, + { + "auxiliary_loss_clip": 0.01048578, + "auxiliary_loss_mlp": 0.01006437, + "balance_loss_clip": 1.02180386, + "balance_loss_mlp": 1.00526834, + "epoch": 0.07724450118971621, + "flos": 71534883467520.0, + "grad_norm": 0.6146351559459508, + "language_loss": 0.50395799, + "learning_rate": 3.976636366515381e-06, + "loss": 0.52450812, + "num_input_tokens_seen": 74924010, + "router_z_loss_clip": 0.26757812, + "router_z_loss_mlp": 0.01165771, + "step": 2662, + "time_per_iteration": 3.218858003616333 + }, + { + "auxiliary_loss_clip": 0.01172932, + "auxiliary_loss_mlp": 0.01061095, + "balance_loss_clip": 1.07124686, + "balance_loss_mlp": 1.04021502, + "epoch": 0.07727351865823226, + "flos": 25259565519360.0, + "grad_norm": 2.372835153888448, + "language_loss": 0.85419202, + "learning_rate": 3.976607711547166e-06, + "loss": 0.87653232, + "num_input_tokens_seen": 74939925, + "router_z_loss_clip": 1.01708984, + "router_z_loss_mlp": 0.2086792, + "step": 2663, + "time_per_iteration": 2.5006604194641113 + }, + { + "auxiliary_loss_clip": 0.01173672, + "auxiliary_loss_mlp": 0.010574, + "balance_loss_clip": 1.06782699, + "balance_loss_mlp": 1.03619266, + "epoch": 0.0773025361267483, + "flos": 19640479472640.0, + "grad_norm": 2.7009626534563926, + "language_loss": 0.94914138, + "learning_rate": 3.976579039120753e-06, + "loss": 0.97145212, + "num_input_tokens_seen": 74955055, + "router_z_loss_clip": 1.05859375, + "router_z_loss_mlp": 0.2121582, + "step": 2664, + "time_per_iteration": 2.522608518600464 + }, + { + "auxiliary_loss_clip": 0.01163012, + "auxiliary_loss_mlp": 0.0104434, + "balance_loss_clip": 1.0652014, + "balance_loss_mlp": 1.02589822, + "epoch": 0.07733155359526435, + "flos": 32190683650560.0, + "grad_norm": 2.924842963413923, + "language_loss": 0.75592494, + "learning_rate": 3.976550349236394e-06, + "loss": 0.77799851, + "num_input_tokens_seen": 74971665, + "router_z_loss_clip": 0.97802734, + "router_z_loss_mlp": 0.18457031, + "step": 2665, + "time_per_iteration": 2.6093509197235107 + }, + { + "auxiliary_loss_clip": 0.01174565, + "auxiliary_loss_mlp": 0.01054067, + "balance_loss_clip": 1.07062542, + "balance_loss_mlp": 1.03564334, + "epoch": 0.0773605710637804, + "flos": 11428571111040.0, + "grad_norm": 2.959392934328981, + "language_loss": 0.82280856, + "learning_rate": 3.976521641894342e-06, + "loss": 0.84509492, + "num_input_tokens_seen": 74982995, + "router_z_loss_clip": 1.03955078, + "router_z_loss_mlp": 0.18426514, + "step": 2666, + "time_per_iteration": 2.4657888412475586 + }, + { + "auxiliary_loss_clip": 0.0117682, + "auxiliary_loss_mlp": 0.0104921, + "balance_loss_clip": 1.07557726, + "balance_loss_mlp": 1.03067279, + "epoch": 0.07738958853229644, + "flos": 29126210826240.0, + "grad_norm": 2.873479193050233, + "language_loss": 0.82628649, + "learning_rate": 3.976492917094851e-06, + "loss": 0.84854686, + "num_input_tokens_seen": 75000725, + "router_z_loss_clip": 1.01367188, + "router_z_loss_mlp": 0.1854248, + "step": 2667, + "time_per_iteration": 2.5673391819000244 + }, + { + "auxiliary_loss_clip": 0.01051823, + "auxiliary_loss_mlp": 0.01005269, + "balance_loss_clip": 1.02473235, + "balance_loss_mlp": 1.00395155, + "epoch": 0.07741860600081249, + "flos": 74776862327040.0, + "grad_norm": 0.6934753045466869, + "language_loss": 0.49396223, + "learning_rate": 3.976464174838175e-06, + "loss": 0.5145331, + "num_input_tokens_seen": 75061805, + "router_z_loss_clip": 0.26953125, + "router_z_loss_mlp": 0.01318359, + "step": 2668, + "time_per_iteration": 3.2099270820617676 + }, + { + "auxiliary_loss_clip": 0.01052023, + "auxiliary_loss_mlp": 0.01003445, + "balance_loss_clip": 1.0248841, + "balance_loss_mlp": 1.00223529, + "epoch": 0.07744762346932854, + "flos": 74599248551040.0, + "grad_norm": 0.6595598651981011, + "language_loss": 0.50613898, + "learning_rate": 3.976435415124568e-06, + "loss": 0.52669364, + "num_input_tokens_seen": 75131735, + "router_z_loss_clip": 0.26953125, + "router_z_loss_mlp": 0.01208496, + "step": 2669, + "time_per_iteration": 3.2363338470458984 + }, + { + "auxiliary_loss_clip": 0.01163749, + "auxiliary_loss_mlp": 0.01044208, + "balance_loss_clip": 1.06671667, + "balance_loss_mlp": 1.02727985, + "epoch": 0.07747664093784458, + "flos": 11171809716480.0, + "grad_norm": 3.3596786454906287, + "language_loss": 1.04746008, + "learning_rate": 3.976406637954283e-06, + "loss": 1.06953955, + "num_input_tokens_seen": 75141625, + "router_z_loss_clip": 0.97021484, + "router_z_loss_mlp": 0.16943359, + "step": 2670, + "time_per_iteration": 2.4319992065429688 + }, + { + "auxiliary_loss_clip": 0.01180389, + "auxiliary_loss_mlp": 0.01056412, + "balance_loss_clip": 1.07490277, + "balance_loss_mlp": 1.0363729, + "epoch": 0.07750565840636063, + "flos": 16355730493440.0, + "grad_norm": 3.742221137970311, + "language_loss": 0.90001899, + "learning_rate": 3.9763778433275755e-06, + "loss": 0.922387, + "num_input_tokens_seen": 75155325, + "router_z_loss_clip": 1.05517578, + "router_z_loss_mlp": 0.20031738, + "step": 2671, + "time_per_iteration": 2.4872355461120605 + }, + { + "auxiliary_loss_clip": 0.01050295, + "auxiliary_loss_mlp": 0.01004151, + "balance_loss_clip": 1.02337432, + "balance_loss_mlp": 1.00285745, + "epoch": 0.07753467587487668, + "flos": 55077634160640.0, + "grad_norm": 0.7891044642757827, + "language_loss": 0.49075124, + "learning_rate": 3.976349031244699e-06, + "loss": 0.51129568, + "num_input_tokens_seen": 75213155, + "router_z_loss_clip": 0.26953125, + "router_z_loss_mlp": 0.01293945, + "step": 2672, + "time_per_iteration": 3.0253961086273193 + }, + { + "auxiliary_loss_clip": 0.01182831, + "auxiliary_loss_mlp": 0.01054527, + "balance_loss_clip": 1.07507837, + "balance_loss_mlp": 1.03306317, + "epoch": 0.07756369334339272, + "flos": 40328544124800.0, + "grad_norm": 1.6655626515326476, + "language_loss": 0.75560963, + "learning_rate": 3.976320201705908e-06, + "loss": 0.77798319, + "num_input_tokens_seen": 75233480, + "router_z_loss_clip": 1.07763672, + "router_z_loss_mlp": 0.21466064, + "step": 2673, + "time_per_iteration": 2.6995580196380615 + }, + { + "auxiliary_loss_clip": 0.01050584, + "auxiliary_loss_mlp": 0.01016374, + "balance_loss_clip": 1.02361608, + "balance_loss_mlp": 1.01506901, + "epoch": 0.07759271081190877, + "flos": 73171368126720.0, + "grad_norm": 0.6824105908847776, + "language_loss": 0.54018039, + "learning_rate": 3.976291354711457e-06, + "loss": 0.56084996, + "num_input_tokens_seen": 75295235, + "router_z_loss_clip": 0.26953125, + "router_z_loss_mlp": 0.01306152, + "step": 2674, + "time_per_iteration": 3.1214354038238525 + }, + { + "auxiliary_loss_clip": 0.0118675, + "auxiliary_loss_mlp": 0.01058353, + "balance_loss_clip": 1.08246112, + "balance_loss_mlp": 1.03883851, + "epoch": 0.07762172828042481, + "flos": 11721386782080.0, + "grad_norm": 2.813191878155912, + "language_loss": 0.87157881, + "learning_rate": 3.9762624902616015e-06, + "loss": 0.8940298, + "num_input_tokens_seen": 75306415, + "router_z_loss_clip": 1.04199219, + "router_z_loss_mlp": 0.19494629, + "step": 2675, + "time_per_iteration": 2.4896903038024902 + }, + { + "auxiliary_loss_clip": 0.01174673, + "auxiliary_loss_mlp": 0.01045677, + "balance_loss_clip": 1.07398772, + "balance_loss_mlp": 1.02752757, + "epoch": 0.07765074574894086, + "flos": 34286611173120.0, + "grad_norm": 2.0064497321950623, + "language_loss": 0.87895548, + "learning_rate": 3.976233608356595e-06, + "loss": 0.90115893, + "num_input_tokens_seen": 75326270, + "router_z_loss_clip": 1.00830078, + "router_z_loss_mlp": 0.18133545, + "step": 2676, + "time_per_iteration": 2.6267645359039307 + }, + { + "auxiliary_loss_clip": 0.01173469, + "auxiliary_loss_mlp": 0.01047866, + "balance_loss_clip": 1.07521415, + "balance_loss_mlp": 1.02986574, + "epoch": 0.07767976321745691, + "flos": 37810487969280.0, + "grad_norm": 4.2490681744318906, + "language_loss": 0.82067037, + "learning_rate": 3.976204708996694e-06, + "loss": 0.84288377, + "num_input_tokens_seen": 75343840, + "router_z_loss_clip": 0.98291016, + "router_z_loss_mlp": 0.18011475, + "step": 2677, + "time_per_iteration": 2.656559705734253 + }, + { + "auxiliary_loss_clip": 0.01167218, + "auxiliary_loss_mlp": 0.01046884, + "balance_loss_clip": 1.07026899, + "balance_loss_mlp": 1.02943206, + "epoch": 0.07770878068597295, + "flos": 28581625751040.0, + "grad_norm": 1.9857971906657435, + "language_loss": 0.81395894, + "learning_rate": 3.976175792182154e-06, + "loss": 0.83609998, + "num_input_tokens_seen": 75361930, + "router_z_loss_clip": 0.96923828, + "router_z_loss_mlp": 0.17462158, + "step": 2678, + "time_per_iteration": 2.5759682655334473 + }, + { + "auxiliary_loss_clip": 0.0116612, + "auxiliary_loss_mlp": 0.01046698, + "balance_loss_clip": 1.06960106, + "balance_loss_mlp": 1.02975214, + "epoch": 0.077737798154489, + "flos": 11647518462720.0, + "grad_norm": 2.896079605514391, + "language_loss": 1.03014791, + "learning_rate": 3.976146857913229e-06, + "loss": 1.05227613, + "num_input_tokens_seen": 75374260, + "router_z_loss_clip": 0.96484375, + "router_z_loss_mlp": 0.16949463, + "step": 2679, + "time_per_iteration": 2.488046884536743 + }, + { + "auxiliary_loss_clip": 0.01051902, + "auxiliary_loss_mlp": 0.01007139, + "balance_loss_clip": 1.02521634, + "balance_loss_mlp": 1.00581598, + "epoch": 0.07776681562300505, + "flos": 65852628376320.0, + "grad_norm": 0.7266811222944767, + "language_loss": 0.49466652, + "learning_rate": 3.976117906190176e-06, + "loss": 0.51525694, + "num_input_tokens_seen": 75428390, + "router_z_loss_clip": 0.26757812, + "router_z_loss_mlp": 0.01324463, + "step": 2680, + "time_per_iteration": 3.0011348724365234 + }, + { + "auxiliary_loss_clip": 0.01176662, + "auxiliary_loss_mlp": 0.01044166, + "balance_loss_clip": 1.07623577, + "balance_loss_mlp": 1.02693963, + "epoch": 0.07779583309152109, + "flos": 16063453526400.0, + "grad_norm": 2.8824416665859998, + "language_loss": 0.71024406, + "learning_rate": 3.97608893701325e-06, + "loss": 0.73245239, + "num_input_tokens_seen": 75441375, + "router_z_loss_clip": 1.00439453, + "router_z_loss_mlp": 0.17230225, + "step": 2681, + "time_per_iteration": 2.5418665409088135 + }, + { + "auxiliary_loss_clip": 0.01052705, + "auxiliary_loss_mlp": 0.0100718, + "balance_loss_clip": 1.02613163, + "balance_loss_mlp": 1.0059042, + "epoch": 0.07782485056003714, + "flos": 74782895811840.0, + "grad_norm": 0.6519112200719878, + "language_loss": 0.47826213, + "learning_rate": 3.976059950382706e-06, + "loss": 0.49886096, + "num_input_tokens_seen": 75506280, + "router_z_loss_clip": 0.265625, + "router_z_loss_mlp": 0.01275635, + "step": 2682, + "time_per_iteration": 3.230821132659912 + }, + { + "auxiliary_loss_clip": 0.01173139, + "auxiliary_loss_mlp": 0.01054653, + "balance_loss_clip": 1.07315755, + "balance_loss_mlp": 1.03573442, + "epoch": 0.0778538680285532, + "flos": 19360628611200.0, + "grad_norm": 3.535900804339009, + "language_loss": 0.82855088, + "learning_rate": 3.976030946298802e-06, + "loss": 0.85082877, + "num_input_tokens_seen": 75522010, + "router_z_loss_clip": 0.99951172, + "router_z_loss_mlp": 0.18933105, + "step": 2683, + "time_per_iteration": 2.5887317657470703 + }, + { + "auxiliary_loss_clip": 0.01049643, + "auxiliary_loss_mlp": 0.01005415, + "balance_loss_clip": 1.02311015, + "balance_loss_mlp": 1.00408554, + "epoch": 0.07788288549706923, + "flos": 74781387440640.0, + "grad_norm": 0.6400436311035315, + "language_loss": 0.48817056, + "learning_rate": 3.976001924761791e-06, + "loss": 0.50872111, + "num_input_tokens_seen": 75592465, + "router_z_loss_clip": 0.265625, + "router_z_loss_mlp": 0.01330566, + "step": 2684, + "time_per_iteration": 3.198906183242798 + }, + { + "auxiliary_loss_clip": 0.01163879, + "auxiliary_loss_mlp": 0.01039985, + "balance_loss_clip": 1.06589913, + "balance_loss_mlp": 1.02263999, + "epoch": 0.07791190296558528, + "flos": 25550513683200.0, + "grad_norm": 2.9528006874049977, + "language_loss": 0.8445949, + "learning_rate": 3.975972885771934e-06, + "loss": 0.86663353, + "num_input_tokens_seen": 75606620, + "router_z_loss_clip": 0.97949219, + "router_z_loss_mlp": 0.17358398, + "step": 2685, + "time_per_iteration": 2.5281429290771484 + }, + { + "auxiliary_loss_clip": 0.01168529, + "auxiliary_loss_mlp": 0.01048901, + "balance_loss_clip": 1.07005048, + "balance_loss_mlp": 1.02952909, + "epoch": 0.07794092043410134, + "flos": 28432452568320.0, + "grad_norm": 2.205348492269838, + "language_loss": 0.80261755, + "learning_rate": 3.975943829329483e-06, + "loss": 0.82479179, + "num_input_tokens_seen": 75622240, + "router_z_loss_clip": 0.98535156, + "router_z_loss_mlp": 0.19366455, + "step": 2686, + "time_per_iteration": 2.557507038116455 + }, + { + "auxiliary_loss_clip": 0.01177356, + "auxiliary_loss_mlp": 0.01055709, + "balance_loss_clip": 1.07119346, + "balance_loss_mlp": 1.03517532, + "epoch": 0.07796993790261737, + "flos": 44816372259840.0, + "grad_norm": 2.1664166328404444, + "language_loss": 0.75225055, + "learning_rate": 3.975914755434697e-06, + "loss": 0.77458119, + "num_input_tokens_seen": 75644675, + "router_z_loss_clip": 1.06298828, + "router_z_loss_mlp": 0.20562744, + "step": 2687, + "time_per_iteration": 2.693267583847046 + }, + { + "auxiliary_loss_clip": 0.01048161, + "auxiliary_loss_mlp": 0.01004526, + "balance_loss_clip": 1.02156115, + "balance_loss_mlp": 1.00320268, + "epoch": 0.07799895537113342, + "flos": 64408410241920.0, + "grad_norm": 0.6460065194997203, + "language_loss": 0.50156581, + "learning_rate": 3.975885664087833e-06, + "loss": 0.5220927, + "num_input_tokens_seen": 75704125, + "router_z_loss_clip": 0.265625, + "router_z_loss_mlp": 0.01324463, + "step": 2688, + "time_per_iteration": 3.0667455196380615 + }, + { + "auxiliary_loss_clip": 0.01165164, + "auxiliary_loss_mlp": 0.01050768, + "balance_loss_clip": 1.06683636, + "balance_loss_mlp": 1.03217113, + "epoch": 0.07802797283964946, + "flos": 15632813370240.0, + "grad_norm": 2.2734787243080317, + "language_loss": 0.66097367, + "learning_rate": 3.975856555289146e-06, + "loss": 0.68313301, + "num_input_tokens_seen": 75720065, + "router_z_loss_clip": 0.98339844, + "router_z_loss_mlp": 0.18615723, + "step": 2689, + "time_per_iteration": 2.544890880584717 + }, + { + "auxiliary_loss_clip": 0.01188427, + "auxiliary_loss_mlp": 0.01058937, + "balance_loss_clip": 1.07731867, + "balance_loss_mlp": 1.03890967, + "epoch": 0.07805699030816551, + "flos": 26059762753920.0, + "grad_norm": 2.523799654639191, + "language_loss": 0.71744633, + "learning_rate": 3.975827429038895e-06, + "loss": 0.7399199, + "num_input_tokens_seen": 75733235, + "router_z_loss_clip": 1.11230469, + "router_z_loss_mlp": 0.20031738, + "step": 2690, + "time_per_iteration": 2.480640411376953 + }, + { + "auxiliary_loss_clip": 0.01048359, + "auxiliary_loss_mlp": 0.01004833, + "balance_loss_clip": 1.02170146, + "balance_loss_mlp": 1.00340271, + "epoch": 0.07808600777668157, + "flos": 70443630328320.0, + "grad_norm": 0.6569058554301647, + "language_loss": 0.58065701, + "learning_rate": 3.975798285337337e-06, + "loss": 0.6011889, + "num_input_tokens_seen": 75798305, + "router_z_loss_clip": 0.265625, + "router_z_loss_mlp": 0.01428223, + "step": 2691, + "time_per_iteration": 3.168069362640381 + }, + { + "auxiliary_loss_clip": 0.01169802, + "auxiliary_loss_mlp": 0.01048504, + "balance_loss_clip": 1.07030129, + "balance_loss_mlp": 1.02851284, + "epoch": 0.0781150252451976, + "flos": 25296122586240.0, + "grad_norm": 2.880474022306748, + "language_loss": 0.64547288, + "learning_rate": 3.975769124184729e-06, + "loss": 0.66765594, + "num_input_tokens_seen": 75816115, + "router_z_loss_clip": 0.99365234, + "router_z_loss_mlp": 0.19995117, + "step": 2692, + "time_per_iteration": 2.565795660018921 + }, + { + "auxiliary_loss_clip": 0.01166342, + "auxiliary_loss_mlp": 0.01041639, + "balance_loss_clip": 1.06992602, + "balance_loss_mlp": 1.02518833, + "epoch": 0.07814404271371365, + "flos": 22886911618560.0, + "grad_norm": 2.201036502558227, + "language_loss": 0.79901981, + "learning_rate": 3.975739945581328e-06, + "loss": 0.82109952, + "num_input_tokens_seen": 75830595, + "router_z_loss_clip": 0.96337891, + "router_z_loss_mlp": 0.16442871, + "step": 2693, + "time_per_iteration": 2.504183292388916 + }, + { + "auxiliary_loss_clip": 0.01176632, + "auxiliary_loss_mlp": 0.0106186, + "balance_loss_clip": 1.07205963, + "balance_loss_mlp": 1.04270256, + "epoch": 0.0781730601822297, + "flos": 17559240612480.0, + "grad_norm": 2.5316744126782087, + "language_loss": 0.80206686, + "learning_rate": 3.975710749527393e-06, + "loss": 0.82445174, + "num_input_tokens_seen": 75843780, + "router_z_loss_clip": 1.04589844, + "router_z_loss_mlp": 0.19177246, + "step": 2694, + "time_per_iteration": 2.5373809337615967 + }, + { + "auxiliary_loss_clip": 0.01178337, + "auxiliary_loss_mlp": 0.01046657, + "balance_loss_clip": 1.07265234, + "balance_loss_mlp": 1.02733314, + "epoch": 0.07820207765074574, + "flos": 15083559527040.0, + "grad_norm": 2.0797023838890127, + "language_loss": 0.74768388, + "learning_rate": 3.9756815360231814e-06, + "loss": 0.76993382, + "num_input_tokens_seen": 75858440, + "router_z_loss_clip": 1.05664062, + "router_z_loss_mlp": 0.19311523, + "step": 2695, + "time_per_iteration": 2.532409906387329 + }, + { + "auxiliary_loss_clip": 0.01174857, + "auxiliary_loss_mlp": 0.01050977, + "balance_loss_clip": 1.07307422, + "balance_loss_mlp": 1.03302968, + "epoch": 0.0782310951192618, + "flos": 74736172218240.0, + "grad_norm": 2.134948564899093, + "language_loss": 0.96024877, + "learning_rate": 3.97565230506895e-06, + "loss": 0.98250711, + "num_input_tokens_seen": 75881125, + "router_z_loss_clip": 1.01904297, + "router_z_loss_mlp": 0.17956543, + "step": 2696, + "time_per_iteration": 2.9467995166778564 + }, + { + "auxiliary_loss_clip": 0.0105078, + "auxiliary_loss_mlp": 0.01002406, + "balance_loss_clip": 1.02461839, + "balance_loss_mlp": 1.00092173, + "epoch": 0.07826011258777785, + "flos": 59995563747840.0, + "grad_norm": 0.6767131337224075, + "language_loss": 0.49710411, + "learning_rate": 3.9756230566649584e-06, + "loss": 0.51763606, + "num_input_tokens_seen": 75947155, + "router_z_loss_clip": 0.26171875, + "router_z_loss_mlp": 0.01483154, + "step": 2697, + "time_per_iteration": 3.163784980773926 + }, + { + "auxiliary_loss_clip": 0.01166952, + "auxiliary_loss_mlp": 0.01049361, + "balance_loss_clip": 1.07001376, + "balance_loss_mlp": 1.03237343, + "epoch": 0.07828913005629388, + "flos": 18689995733760.0, + "grad_norm": 2.2364930914524974, + "language_loss": 0.8968209, + "learning_rate": 3.9755937908114646e-06, + "loss": 0.91898406, + "num_input_tokens_seen": 75960355, + "router_z_loss_clip": 0.96728516, + "router_z_loss_mlp": 0.1697998, + "step": 2698, + "time_per_iteration": 2.4974639415740967 + }, + { + "auxiliary_loss_clip": 0.01049967, + "auxiliary_loss_mlp": 0.01002861, + "balance_loss_clip": 1.0236063, + "balance_loss_mlp": 1.00141239, + "epoch": 0.07831814752480994, + "flos": 57223439717760.0, + "grad_norm": 0.6709445881566822, + "language_loss": 0.47253758, + "learning_rate": 3.975564507508727e-06, + "loss": 0.49306583, + "num_input_tokens_seen": 76021415, + "router_z_loss_clip": 0.26171875, + "router_z_loss_mlp": 0.01446533, + "step": 2699, + "time_per_iteration": 3.014780282974243 + }, + { + "auxiliary_loss_clip": 0.01170208, + "auxiliary_loss_mlp": 0.01040917, + "balance_loss_clip": 1.06708276, + "balance_loss_mlp": 1.02370906, + "epoch": 0.07834716499332599, + "flos": 12013520094720.0, + "grad_norm": 3.062622804600172, + "language_loss": 0.75304919, + "learning_rate": 3.975535206757004e-06, + "loss": 0.77516043, + "num_input_tokens_seen": 76035080, + "router_z_loss_clip": 1.03173828, + "router_z_loss_mlp": 0.17181396, + "step": 2700, + "time_per_iteration": 2.487541437149048 + }, + { + "auxiliary_loss_clip": 0.0117059, + "auxiliary_loss_mlp": 0.01058055, + "balance_loss_clip": 1.07077551, + "balance_loss_mlp": 1.0393157, + "epoch": 0.07837618246184203, + "flos": 13910680730880.0, + "grad_norm": 2.8016556801111907, + "language_loss": 0.80239868, + "learning_rate": 3.9755058885565545e-06, + "loss": 0.8246851, + "num_input_tokens_seen": 76046950, + "router_z_loss_clip": 0.99804688, + "router_z_loss_mlp": 0.18756104, + "step": 2701, + "time_per_iteration": 2.4729526042938232 + }, + { + "auxiliary_loss_clip": 0.0117635, + "auxiliary_loss_mlp": 0.01052611, + "balance_loss_clip": 1.07125711, + "balance_loss_mlp": 1.03379929, + "epoch": 0.07840519993035808, + "flos": 54007923225600.0, + "grad_norm": 2.2115118410781807, + "language_loss": 0.77041113, + "learning_rate": 3.975476552907638e-06, + "loss": 0.79270077, + "num_input_tokens_seen": 76070880, + "router_z_loss_clip": 1.05126953, + "router_z_loss_mlp": 0.18823242, + "step": 2702, + "time_per_iteration": 2.861204147338867 + }, + { + "auxiliary_loss_clip": 0.01173077, + "auxiliary_loss_mlp": 0.01050325, + "balance_loss_clip": 1.06748295, + "balance_loss_mlp": 1.03100085, + "epoch": 0.07843421739887413, + "flos": 11247725111040.0, + "grad_norm": 3.432186518692089, + "language_loss": 0.90406567, + "learning_rate": 3.975447199810513e-06, + "loss": 0.92629969, + "num_input_tokens_seen": 76083240, + "router_z_loss_clip": 1.05664062, + "router_z_loss_mlp": 0.19317627, + "step": 2703, + "time_per_iteration": 2.5033748149871826 + }, + { + "auxiliary_loss_clip": 0.01164018, + "auxiliary_loss_mlp": 0.01042801, + "balance_loss_clip": 1.06701159, + "balance_loss_mlp": 1.02445471, + "epoch": 0.07846323486739017, + "flos": 20148687048960.0, + "grad_norm": 2.3103835293904806, + "language_loss": 0.79518521, + "learning_rate": 3.975417829265439e-06, + "loss": 0.81725347, + "num_input_tokens_seen": 76098920, + "router_z_loss_clip": 0.97070312, + "router_z_loss_mlp": 0.18334961, + "step": 2704, + "time_per_iteration": 2.4937427043914795 + }, + { + "auxiliary_loss_clip": 0.01174704, + "auxiliary_loss_mlp": 0.01047391, + "balance_loss_clip": 1.0698812, + "balance_loss_mlp": 1.02908015, + "epoch": 0.07849225233590622, + "flos": 25951025306880.0, + "grad_norm": 2.1253299270686257, + "language_loss": 0.9346174, + "learning_rate": 3.975388441272676e-06, + "loss": 0.95683837, + "num_input_tokens_seen": 76117625, + "router_z_loss_clip": 1.04931641, + "router_z_loss_mlp": 0.1829834, + "step": 2705, + "time_per_iteration": 7.353763818740845 + }, + { + "auxiliary_loss_clip": 0.01167327, + "auxiliary_loss_mlp": 0.01048585, + "balance_loss_clip": 1.06513977, + "balance_loss_mlp": 1.03063154, + "epoch": 0.07852126980442226, + "flos": 39413288649600.0, + "grad_norm": 4.055485752383478, + "language_loss": 0.97954965, + "learning_rate": 3.975359035832482e-06, + "loss": 1.00170875, + "num_input_tokens_seen": 76133145, + "router_z_loss_clip": 1.02246094, + "router_z_loss_mlp": 0.17956543, + "step": 2706, + "time_per_iteration": 4.807420015335083 + }, + { + "auxiliary_loss_clip": 0.0104839, + "auxiliary_loss_mlp": 0.01006581, + "balance_loss_clip": 1.02231157, + "balance_loss_mlp": 1.00514495, + "epoch": 0.07855028727293831, + "flos": 74775174387840.0, + "grad_norm": 0.7307117816897188, + "language_loss": 0.49467507, + "learning_rate": 3.975329612945118e-06, + "loss": 0.51522481, + "num_input_tokens_seen": 76196575, + "router_z_loss_clip": 0.26171875, + "router_z_loss_mlp": 0.01434326, + "step": 2707, + "time_per_iteration": 3.2124462127685547 + }, + { + "auxiliary_loss_clip": 0.01167453, + "auxiliary_loss_mlp": 0.01041408, + "balance_loss_clip": 1.06830502, + "balance_loss_mlp": 1.02508175, + "epoch": 0.07857930474145436, + "flos": 17636305242240.0, + "grad_norm": 2.4486537002694266, + "language_loss": 0.73034811, + "learning_rate": 3.975300172610844e-06, + "loss": 0.75243664, + "num_input_tokens_seen": 76211780, + "router_z_loss_clip": 0.9921875, + "router_z_loss_mlp": 0.16326904, + "step": 2708, + "time_per_iteration": 4.7365007400512695 + }, + { + "auxiliary_loss_clip": 0.01157455, + "auxiliary_loss_mlp": 0.01047402, + "balance_loss_clip": 1.06713164, + "balance_loss_mlp": 1.03160644, + "epoch": 0.0786083222099704, + "flos": 34597667975040.0, + "grad_norm": 2.0016376270295546, + "language_loss": 0.67419428, + "learning_rate": 3.975270714829919e-06, + "loss": 0.69624281, + "num_input_tokens_seen": 76228080, + "router_z_loss_clip": 0.90283203, + "router_z_loss_mlp": 0.15802002, + "step": 2709, + "time_per_iteration": 2.658548593521118 + }, + { + "auxiliary_loss_clip": 0.01183784, + "auxiliary_loss_mlp": 0.01051158, + "balance_loss_clip": 1.07276201, + "balance_loss_mlp": 1.03072512, + "epoch": 0.07863733967848645, + "flos": 10770077030400.0, + "grad_norm": 2.9371751415471135, + "language_loss": 0.99456918, + "learning_rate": 3.975241239602604e-06, + "loss": 1.01691854, + "num_input_tokens_seen": 76237385, + "router_z_loss_clip": 1.11083984, + "router_z_loss_mlp": 0.20422363, + "step": 2710, + "time_per_iteration": 2.485184907913208 + }, + { + "auxiliary_loss_clip": 0.01049011, + "auxiliary_loss_mlp": 0.01000406, + "balance_loss_clip": 1.02284455, + "balance_loss_mlp": 0.99914247, + "epoch": 0.0786663571470025, + "flos": 60519178258560.0, + "grad_norm": 0.7014495992989159, + "language_loss": 0.5512234, + "learning_rate": 3.975211746929158e-06, + "loss": 0.5717175, + "num_input_tokens_seen": 76303305, + "router_z_loss_clip": 0.26171875, + "router_z_loss_mlp": 0.01263428, + "step": 2711, + "time_per_iteration": 3.2189650535583496 + }, + { + "auxiliary_loss_clip": 0.01178343, + "auxiliary_loss_mlp": 0.01046255, + "balance_loss_clip": 1.07243729, + "balance_loss_mlp": 1.02721739, + "epoch": 0.07869537461551854, + "flos": 23907277267200.0, + "grad_norm": 2.251106900673283, + "language_loss": 0.81829679, + "learning_rate": 3.975182236809844e-06, + "loss": 0.84054273, + "num_input_tokens_seen": 76317690, + "router_z_loss_clip": 1.05908203, + "router_z_loss_mlp": 0.19030762, + "step": 2712, + "time_per_iteration": 2.513737201690674 + }, + { + "auxiliary_loss_clip": 0.01167145, + "auxiliary_loss_mlp": 0.01053643, + "balance_loss_clip": 1.06900597, + "balance_loss_mlp": 1.03499889, + "epoch": 0.07872439208403459, + "flos": 39524719616640.0, + "grad_norm": 2.252321257701644, + "language_loss": 0.65033484, + "learning_rate": 3.97515270924492e-06, + "loss": 0.67254275, + "num_input_tokens_seen": 76335030, + "router_z_loss_clip": 0.97900391, + "router_z_loss_mlp": 0.18664551, + "step": 2713, + "time_per_iteration": 2.647026300430298 + }, + { + "auxiliary_loss_clip": 0.01158532, + "auxiliary_loss_mlp": 0.01047781, + "balance_loss_clip": 1.06839299, + "balance_loss_mlp": 1.02978063, + "epoch": 0.07875340955255064, + "flos": 42479844462720.0, + "grad_norm": 3.389212909947724, + "language_loss": 0.90404797, + "learning_rate": 3.9751231642346485e-06, + "loss": 0.9261111, + "num_input_tokens_seen": 76351790, + "router_z_loss_clip": 0.90185547, + "router_z_loss_mlp": 0.17987061, + "step": 2714, + "time_per_iteration": 2.722287178039551 + }, + { + "auxiliary_loss_clip": 0.01176128, + "auxiliary_loss_mlp": 0.01047777, + "balance_loss_clip": 1.07270432, + "balance_loss_mlp": 1.03142166, + "epoch": 0.07878242702106668, + "flos": 32664668544000.0, + "grad_norm": 2.5426639099198285, + "language_loss": 0.92106736, + "learning_rate": 3.97509360177929e-06, + "loss": 0.94330645, + "num_input_tokens_seen": 76366260, + "router_z_loss_clip": 1.03417969, + "router_z_loss_mlp": 0.16345215, + "step": 2715, + "time_per_iteration": 2.687513828277588 + }, + { + "auxiliary_loss_clip": 0.0104849, + "auxiliary_loss_mlp": 0.00999516, + "balance_loss_clip": 1.02218843, + "balance_loss_mlp": 0.99815065, + "epoch": 0.07881144448958273, + "flos": 59690214230400.0, + "grad_norm": 0.7438442617724276, + "language_loss": 0.51029658, + "learning_rate": 3.975064021879106e-06, + "loss": 0.53077662, + "num_input_tokens_seen": 76419485, + "router_z_loss_clip": 0.26171875, + "router_z_loss_mlp": 0.01367188, + "step": 2716, + "time_per_iteration": 2.960756778717041 + }, + { + "auxiliary_loss_clip": 0.01168123, + "auxiliary_loss_mlp": 0.01048671, + "balance_loss_clip": 1.06863189, + "balance_loss_mlp": 1.02982402, + "epoch": 0.07884046195809878, + "flos": 30257360997120.0, + "grad_norm": 2.2445572612084246, + "language_loss": 1.01512742, + "learning_rate": 3.975034424534358e-06, + "loss": 1.03729534, + "num_input_tokens_seen": 76436070, + "router_z_loss_clip": 0.99462891, + "router_z_loss_mlp": 0.18841553, + "step": 2717, + "time_per_iteration": 2.5986592769622803 + }, + { + "auxiliary_loss_clip": 0.01171724, + "auxiliary_loss_mlp": 0.01050902, + "balance_loss_clip": 1.06999111, + "balance_loss_mlp": 1.03176928, + "epoch": 0.07886947942661482, + "flos": 43725370515840.0, + "grad_norm": 2.20475811902191, + "language_loss": 0.82340109, + "learning_rate": 3.975004809745305e-06, + "loss": 0.84562743, + "num_input_tokens_seen": 76458335, + "router_z_loss_clip": 1.01660156, + "router_z_loss_mlp": 0.19152832, + "step": 2718, + "time_per_iteration": 2.6950182914733887 + }, + { + "auxiliary_loss_clip": 0.01048728, + "auxiliary_loss_mlp": 0.01001806, + "balance_loss_clip": 1.02266252, + "balance_loss_mlp": 1.00041091, + "epoch": 0.07889849689513087, + "flos": 74775210301440.0, + "grad_norm": 0.7261066898981372, + "language_loss": 0.53312516, + "learning_rate": 3.974975177512212e-06, + "loss": 0.55363047, + "num_input_tokens_seen": 76521570, + "router_z_loss_clip": 0.25976562, + "router_z_loss_mlp": 0.01397705, + "step": 2719, + "time_per_iteration": 3.173992156982422 + }, + { + "auxiliary_loss_clip": 0.01177759, + "auxiliary_loss_mlp": 0.01055903, + "balance_loss_clip": 1.07164383, + "balance_loss_mlp": 1.03636432, + "epoch": 0.07892751436364692, + "flos": 29495444682240.0, + "grad_norm": 2.2029302354661167, + "language_loss": 1.01552224, + "learning_rate": 3.9749455278353375e-06, + "loss": 1.03785896, + "num_input_tokens_seen": 76544700, + "router_z_loss_clip": 1.06054688, + "router_z_loss_mlp": 0.19537354, + "step": 2720, + "time_per_iteration": 2.676403760910034 + }, + { + "auxiliary_loss_clip": 0.01163608, + "auxiliary_loss_mlp": 0.01038626, + "balance_loss_clip": 1.06820273, + "balance_loss_mlp": 1.02119112, + "epoch": 0.07895653183216296, + "flos": 32919490604160.0, + "grad_norm": 2.109948416962157, + "language_loss": 0.80800784, + "learning_rate": 3.974915860714946e-06, + "loss": 0.8300302, + "num_input_tokens_seen": 76561395, + "router_z_loss_clip": 0.95410156, + "router_z_loss_mlp": 0.17443848, + "step": 2721, + "time_per_iteration": 2.6813275814056396 + }, + { + "auxiliary_loss_clip": 0.01050663, + "auxiliary_loss_mlp": 0.01004757, + "balance_loss_clip": 1.02438307, + "balance_loss_mlp": 1.0033797, + "epoch": 0.07898554930067901, + "flos": 61881052464000.0, + "grad_norm": 0.6998818921328182, + "language_loss": 0.51851153, + "learning_rate": 3.9748861761512975e-06, + "loss": 0.53906572, + "num_input_tokens_seen": 76618945, + "router_z_loss_clip": 0.26171875, + "router_z_loss_mlp": 0.01379395, + "step": 2722, + "time_per_iteration": 2.9987378120422363 + }, + { + "auxiliary_loss_clip": 0.01048963, + "auxiliary_loss_mlp": 0.01005562, + "balance_loss_clip": 1.02272964, + "balance_loss_mlp": 1.00420332, + "epoch": 0.07901456676919505, + "flos": 70543784424960.0, + "grad_norm": 0.630394015838598, + "language_loss": 0.49456733, + "learning_rate": 3.9748564741446556e-06, + "loss": 0.51511264, + "num_input_tokens_seen": 76685010, + "router_z_loss_clip": 0.26171875, + "router_z_loss_mlp": 0.01361084, + "step": 2723, + "time_per_iteration": 3.206238031387329 + }, + { + "auxiliary_loss_clip": 0.01182925, + "auxiliary_loss_mlp": 0.01058539, + "balance_loss_clip": 1.07285953, + "balance_loss_mlp": 1.03585315, + "epoch": 0.0790435842377111, + "flos": 11757979762560.0, + "grad_norm": 3.0098347636709426, + "language_loss": 0.92879397, + "learning_rate": 3.974826754695283e-06, + "loss": 0.95120859, + "num_input_tokens_seen": 76697200, + "router_z_loss_clip": 1.10058594, + "router_z_loss_mlp": 0.22668457, + "step": 2724, + "time_per_iteration": 2.4709346294403076 + }, + { + "auxiliary_loss_clip": 0.01174357, + "auxiliary_loss_mlp": 0.0104825, + "balance_loss_clip": 1.07094169, + "balance_loss_mlp": 1.02860415, + "epoch": 0.07907260170622715, + "flos": 10918460113920.0, + "grad_norm": 3.063541680393536, + "language_loss": 0.9360103, + "learning_rate": 3.97479701780344e-06, + "loss": 0.95823634, + "num_input_tokens_seen": 76706635, + "router_z_loss_clip": 1.03417969, + "router_z_loss_mlp": 0.19641113, + "step": 2725, + "time_per_iteration": 2.4754648208618164 + }, + { + "auxiliary_loss_clip": 0.01174234, + "auxiliary_loss_mlp": 0.01050549, + "balance_loss_clip": 1.07144594, + "balance_loss_mlp": 1.03135586, + "epoch": 0.07910161917474319, + "flos": 15665671336320.0, + "grad_norm": 2.4975676825063786, + "language_loss": 0.83706594, + "learning_rate": 3.974767263469391e-06, + "loss": 0.85931385, + "num_input_tokens_seen": 76720415, + "router_z_loss_clip": 1.02539062, + "router_z_loss_mlp": 0.19177246, + "step": 2726, + "time_per_iteration": 2.5148510932922363 + }, + { + "auxiliary_loss_clip": 0.0104903, + "auxiliary_loss_mlp": 0.01003413, + "balance_loss_clip": 1.02285099, + "balance_loss_mlp": 1.00194097, + "epoch": 0.07913063664325924, + "flos": 60039519016320.0, + "grad_norm": 0.6761047369778989, + "language_loss": 0.51674062, + "learning_rate": 3.974737491693399e-06, + "loss": 0.537265, + "num_input_tokens_seen": 76785720, + "router_z_loss_clip": 0.26171875, + "router_z_loss_mlp": 0.01470947, + "step": 2727, + "time_per_iteration": 3.086843490600586 + }, + { + "auxiliary_loss_clip": 0.01174955, + "auxiliary_loss_mlp": 0.01056061, + "balance_loss_clip": 1.07258654, + "balance_loss_mlp": 1.03753626, + "epoch": 0.0791596541117753, + "flos": 28541225928960.0, + "grad_norm": 2.3783032081364195, + "language_loss": 0.87288839, + "learning_rate": 3.9747077024757255e-06, + "loss": 0.89519858, + "num_input_tokens_seen": 76799525, + "router_z_loss_clip": 1.02441406, + "router_z_loss_mlp": 0.18530273, + "step": 2728, + "time_per_iteration": 2.553131341934204 + }, + { + "auxiliary_loss_clip": 0.01173865, + "auxiliary_loss_mlp": 0.01041765, + "balance_loss_clip": 1.07179499, + "balance_loss_mlp": 1.02250075, + "epoch": 0.07918867158029133, + "flos": 13436803578240.0, + "grad_norm": 2.6830812268047493, + "language_loss": 0.65903306, + "learning_rate": 3.974677895816636e-06, + "loss": 0.6811893, + "num_input_tokens_seen": 76811155, + "router_z_loss_clip": 1.02050781, + "router_z_loss_mlp": 0.19274902, + "step": 2729, + "time_per_iteration": 2.467829465866089 + }, + { + "auxiliary_loss_clip": 0.01051836, + "auxiliary_loss_mlp": 0.01003695, + "balance_loss_clip": 1.0256629, + "balance_loss_mlp": 1.0023365, + "epoch": 0.07921768904880738, + "flos": 74767309309440.0, + "grad_norm": 0.6845683567730914, + "language_loss": 0.53557193, + "learning_rate": 3.974648071716391e-06, + "loss": 0.55612731, + "num_input_tokens_seen": 76870800, + "router_z_loss_clip": 0.26171875, + "router_z_loss_mlp": 0.01361084, + "step": 2730, + "time_per_iteration": 3.195890188217163 + }, + { + "auxiliary_loss_clip": 0.01050384, + "auxiliary_loss_mlp": 0.0100328, + "balance_loss_clip": 1.02402616, + "balance_loss_mlp": 1.00190282, + "epoch": 0.07924670651732343, + "flos": 72022443701760.0, + "grad_norm": 0.6748463526606437, + "language_loss": 0.46058798, + "learning_rate": 3.974618230175255e-06, + "loss": 0.48112461, + "num_input_tokens_seen": 76936485, + "router_z_loss_clip": 0.26367188, + "router_z_loss_mlp": 0.01379395, + "step": 2731, + "time_per_iteration": 3.308412551879883 + }, + { + "auxiliary_loss_clip": 0.01170372, + "auxiliary_loss_mlp": 0.01049985, + "balance_loss_clip": 1.06936371, + "balance_loss_mlp": 1.03179979, + "epoch": 0.07927572398583947, + "flos": 33512699715840.0, + "grad_norm": 2.0724433226568166, + "language_loss": 0.78280276, + "learning_rate": 3.974588371193492e-06, + "loss": 0.80500638, + "num_input_tokens_seen": 76958055, + "router_z_loss_clip": 1.00878906, + "router_z_loss_mlp": 0.18182373, + "step": 2732, + "time_per_iteration": 2.644538402557373 + }, + { + "auxiliary_loss_clip": 0.01183143, + "auxiliary_loss_mlp": 0.01049084, + "balance_loss_clip": 1.07482243, + "balance_loss_mlp": 1.02669692, + "epoch": 0.07930474145435552, + "flos": 32665243161600.0, + "grad_norm": 2.0435386862359914, + "language_loss": 1.06119752, + "learning_rate": 3.974558494771366e-06, + "loss": 1.08351982, + "num_input_tokens_seen": 76976190, + "router_z_loss_clip": 1.08398438, + "router_z_loss_mlp": 0.22387695, + "step": 2733, + "time_per_iteration": 2.6639046669006348 + }, + { + "auxiliary_loss_clip": 0.01174406, + "auxiliary_loss_mlp": 0.01046767, + "balance_loss_clip": 1.07504618, + "balance_loss_mlp": 1.02778912, + "epoch": 0.07933375892287158, + "flos": 31578048259200.0, + "grad_norm": 2.0058882182149698, + "language_loss": 0.68632138, + "learning_rate": 3.97452860090914e-06, + "loss": 0.70853311, + "num_input_tokens_seen": 76998640, + "router_z_loss_clip": 0.99365234, + "router_z_loss_mlp": 0.18981934, + "step": 2734, + "time_per_iteration": 2.6181890964508057 + }, + { + "auxiliary_loss_clip": 0.01175585, + "auxiliary_loss_mlp": 0.01049806, + "balance_loss_clip": 1.07325172, + "balance_loss_mlp": 1.03051782, + "epoch": 0.07936277639138761, + "flos": 40617876176640.0, + "grad_norm": 3.1189593149957595, + "language_loss": 1.01108503, + "learning_rate": 3.974498689607078e-06, + "loss": 1.03333879, + "num_input_tokens_seen": 77015445, + "router_z_loss_clip": 1.02294922, + "router_z_loss_mlp": 0.19256592, + "step": 2735, + "time_per_iteration": 2.679558753967285 + }, + { + "auxiliary_loss_clip": 0.01173512, + "auxiliary_loss_mlp": 0.01042067, + "balance_loss_clip": 1.07224846, + "balance_loss_mlp": 1.02479339, + "epoch": 0.07939179385990366, + "flos": 16249399257600.0, + "grad_norm": 2.3711755932777376, + "language_loss": 0.76252139, + "learning_rate": 3.974468760865446e-06, + "loss": 0.78467715, + "num_input_tokens_seen": 77028385, + "router_z_loss_clip": 1.01269531, + "router_z_loss_mlp": 0.17272949, + "step": 2736, + "time_per_iteration": 2.5372042655944824 + }, + { + "auxiliary_loss_clip": 0.01173116, + "auxiliary_loss_mlp": 0.01047907, + "balance_loss_clip": 1.07274842, + "balance_loss_mlp": 1.02937007, + "epoch": 0.0794208113284197, + "flos": 69879756672000.0, + "grad_norm": 2.0156358579132596, + "language_loss": 0.80905282, + "learning_rate": 3.974438814684506e-06, + "loss": 0.83126307, + "num_input_tokens_seen": 77050635, + "router_z_loss_clip": 1.00341797, + "router_z_loss_mlp": 0.18518066, + "step": 2737, + "time_per_iteration": 2.9434831142425537 + }, + { + "auxiliary_loss_clip": 0.01051897, + "auxiliary_loss_mlp": 0.0100255, + "balance_loss_clip": 1.02582645, + "balance_loss_mlp": 1.00114346, + "epoch": 0.07944982879693575, + "flos": 67377469564800.0, + "grad_norm": 0.6673103219705189, + "language_loss": 0.48481381, + "learning_rate": 3.974408851064523e-06, + "loss": 0.50535828, + "num_input_tokens_seen": 77117345, + "router_z_loss_clip": 0.26171875, + "router_z_loss_mlp": 0.01403809, + "step": 2738, + "time_per_iteration": 3.236741781234741 + }, + { + "auxiliary_loss_clip": 0.01178699, + "auxiliary_loss_mlp": 0.01051585, + "balance_loss_clip": 1.07315922, + "balance_loss_mlp": 1.03098607, + "epoch": 0.0794788462654518, + "flos": 14787870768000.0, + "grad_norm": 3.019048296698285, + "language_loss": 0.9055115, + "learning_rate": 3.974378870005762e-06, + "loss": 0.92781436, + "num_input_tokens_seen": 77130245, + "router_z_loss_clip": 1.05517578, + "router_z_loss_mlp": 0.20605469, + "step": 2739, + "time_per_iteration": 2.50032377243042 + }, + { + "auxiliary_loss_clip": 0.01051667, + "auxiliary_loss_mlp": 0.01001976, + "balance_loss_clip": 1.02577281, + "balance_loss_mlp": 1.00068271, + "epoch": 0.07950786373396784, + "flos": 70442552920320.0, + "grad_norm": 0.7258790135770117, + "language_loss": 0.52850294, + "learning_rate": 3.9743488715084884e-06, + "loss": 0.54903936, + "num_input_tokens_seen": 77198770, + "router_z_loss_clip": 0.2578125, + "router_z_loss_mlp": 0.01293945, + "step": 2740, + "time_per_iteration": 3.240680456161499 + }, + { + "auxiliary_loss_clip": 0.01176554, + "auxiliary_loss_mlp": 0.01056581, + "balance_loss_clip": 1.07455945, + "balance_loss_mlp": 1.03699505, + "epoch": 0.0795368812024839, + "flos": 28944000109440.0, + "grad_norm": 2.0618091903862164, + "language_loss": 0.8855139, + "learning_rate": 3.974318855572967e-06, + "loss": 0.9078452, + "num_input_tokens_seen": 77216390, + "router_z_loss_clip": 1.01953125, + "router_z_loss_mlp": 0.19592285, + "step": 2741, + "time_per_iteration": 2.575139284133911 + }, + { + "auxiliary_loss_clip": 0.01170158, + "auxiliary_loss_mlp": 0.01044757, + "balance_loss_clip": 1.06942677, + "balance_loss_mlp": 1.02596378, + "epoch": 0.07956589867099995, + "flos": 17740553489280.0, + "grad_norm": 2.408880813935227, + "language_loss": 0.86961651, + "learning_rate": 3.9742888221994616e-06, + "loss": 0.89176571, + "num_input_tokens_seen": 77228980, + "router_z_loss_clip": 1.00732422, + "router_z_loss_mlp": 0.18792725, + "step": 2742, + "time_per_iteration": 2.4923818111419678 + }, + { + "auxiliary_loss_clip": 0.01175109, + "auxiliary_loss_mlp": 0.01049905, + "balance_loss_clip": 1.07548022, + "balance_loss_mlp": 1.03143394, + "epoch": 0.07959491613951598, + "flos": 55576716704640.0, + "grad_norm": 2.433641852951559, + "language_loss": 0.7014243, + "learning_rate": 3.974258771388239e-06, + "loss": 0.72367442, + "num_input_tokens_seen": 77247560, + "router_z_loss_clip": 0.99658203, + "router_z_loss_mlp": 0.18475342, + "step": 2743, + "time_per_iteration": 2.7690317630767822 + }, + { + "auxiliary_loss_clip": 0.01174339, + "auxiliary_loss_mlp": 0.01051069, + "balance_loss_clip": 1.07419431, + "balance_loss_mlp": 1.03274691, + "epoch": 0.07962393360803204, + "flos": 18907398800640.0, + "grad_norm": 2.9078677699557156, + "language_loss": 0.99553025, + "learning_rate": 3.974228703139564e-06, + "loss": 1.01778436, + "num_input_tokens_seen": 77260075, + "router_z_loss_clip": 1.00048828, + "router_z_loss_mlp": 0.18322754, + "step": 2744, + "time_per_iteration": 2.4983503818511963 + }, + { + "auxiliary_loss_clip": 0.01173384, + "auxiliary_loss_mlp": 0.01051869, + "balance_loss_clip": 1.07229197, + "balance_loss_mlp": 1.0328908, + "epoch": 0.07965295107654809, + "flos": 38244108954240.0, + "grad_norm": 3.2217060034677365, + "language_loss": 0.95396781, + "learning_rate": 3.9741986174537026e-06, + "loss": 0.97622037, + "num_input_tokens_seen": 77274705, + "router_z_loss_clip": 1.01171875, + "router_z_loss_mlp": 0.18981934, + "step": 2745, + "time_per_iteration": 2.6347548961639404 + }, + { + "auxiliary_loss_clip": 0.01187987, + "auxiliary_loss_mlp": 0.01056393, + "balance_loss_clip": 1.07634187, + "balance_loss_mlp": 1.03484011, + "epoch": 0.07968196854506412, + "flos": 10589231030400.0, + "grad_norm": 2.3482192999074782, + "language_loss": 0.75301623, + "learning_rate": 3.97416851433092e-06, + "loss": 0.77546, + "num_input_tokens_seen": 77287630, + "router_z_loss_clip": 1.11425781, + "router_z_loss_mlp": 0.21533203, + "step": 2746, + "time_per_iteration": 2.468384027481079 + }, + { + "auxiliary_loss_clip": 0.01181331, + "auxiliary_loss_mlp": 0.01055064, + "balance_loss_clip": 1.07547414, + "balance_loss_mlp": 1.03446472, + "epoch": 0.07971098601358018, + "flos": 32593421917440.0, + "grad_norm": 1.9817402363791687, + "language_loss": 0.88104218, + "learning_rate": 3.974138393771481e-06, + "loss": 0.90340614, + "num_input_tokens_seen": 77307445, + "router_z_loss_clip": 1.06005859, + "router_z_loss_mlp": 0.20605469, + "step": 2747, + "time_per_iteration": 2.632481813430786 + }, + { + "auxiliary_loss_clip": 0.01175217, + "auxiliary_loss_mlp": 0.01049117, + "balance_loss_clip": 1.0746659, + "balance_loss_mlp": 1.03035367, + "epoch": 0.07974000348209623, + "flos": 21608886735360.0, + "grad_norm": 2.415763411948825, + "language_loss": 0.88226223, + "learning_rate": 3.974108255775654e-06, + "loss": 0.90450561, + "num_input_tokens_seen": 77322030, + "router_z_loss_clip": 1.00585938, + "router_z_loss_mlp": 0.18762207, + "step": 2748, + "time_per_iteration": 2.5380916595458984 + }, + { + "auxiliary_loss_clip": 0.01175208, + "auxiliary_loss_mlp": 0.01044202, + "balance_loss_clip": 1.07437801, + "balance_loss_mlp": 1.02522957, + "epoch": 0.07976902095061227, + "flos": 29453105525760.0, + "grad_norm": 2.3358545270937103, + "language_loss": 0.79673088, + "learning_rate": 3.9740781003437035e-06, + "loss": 0.81892496, + "num_input_tokens_seen": 77337660, + "router_z_loss_clip": 1.00683594, + "router_z_loss_mlp": 0.18969727, + "step": 2749, + "time_per_iteration": 2.54492449760437 + }, + { + "auxiliary_loss_clip": 0.01053116, + "auxiliary_loss_mlp": 0.01004463, + "balance_loss_clip": 1.02703667, + "balance_loss_mlp": 1.00314617, + "epoch": 0.07979803841912832, + "flos": 63315002286720.0, + "grad_norm": 0.7244343838004134, + "language_loss": 0.49303606, + "learning_rate": 3.974047927475897e-06, + "loss": 0.51361179, + "num_input_tokens_seen": 77392040, + "router_z_loss_clip": 0.26171875, + "router_z_loss_mlp": 0.01318359, + "step": 2750, + "time_per_iteration": 3.023045539855957 + }, + { + "auxiliary_loss_clip": 0.01171831, + "auxiliary_loss_mlp": 0.01064948, + "balance_loss_clip": 1.07038033, + "balance_loss_mlp": 1.04397917, + "epoch": 0.07982705588764437, + "flos": 54228127553280.0, + "grad_norm": 2.0711915077667293, + "language_loss": 0.94970715, + "learning_rate": 3.9740177371725e-06, + "loss": 0.97207499, + "num_input_tokens_seen": 77417975, + "router_z_loss_clip": 1.01513672, + "router_z_loss_mlp": 0.20959473, + "step": 2751, + "time_per_iteration": 2.82804799079895 + }, + { + "auxiliary_loss_clip": 0.01172983, + "auxiliary_loss_mlp": 0.01047543, + "balance_loss_clip": 1.07219529, + "balance_loss_mlp": 1.02825522, + "epoch": 0.0798560733561604, + "flos": 42380300897280.0, + "grad_norm": 1.8962773189062971, + "language_loss": 0.76855278, + "learning_rate": 3.9739875294337795e-06, + "loss": 0.79075801, + "num_input_tokens_seen": 77439400, + "router_z_loss_clip": 1.00732422, + "router_z_loss_mlp": 0.19287109, + "step": 2752, + "time_per_iteration": 2.699171543121338 + }, + { + "auxiliary_loss_clip": 0.01052011, + "auxiliary_loss_mlp": 0.01001872, + "balance_loss_clip": 1.02612829, + "balance_loss_mlp": 1.00056672, + "epoch": 0.07988509082467646, + "flos": 62988825859200.0, + "grad_norm": 0.7732967452229991, + "language_loss": 0.51035464, + "learning_rate": 3.973957304260002e-06, + "loss": 0.53089345, + "num_input_tokens_seen": 77492195, + "router_z_loss_clip": 0.2578125, + "router_z_loss_mlp": 0.01306152, + "step": 2753, + "time_per_iteration": 2.9671294689178467 + }, + { + "auxiliary_loss_clip": 0.01156917, + "auxiliary_loss_mlp": 0.0103765, + "balance_loss_clip": 1.06898117, + "balance_loss_mlp": 1.02333891, + "epoch": 0.0799141082931925, + "flos": 16206700965120.0, + "grad_norm": 2.459671933604948, + "language_loss": 0.74400294, + "learning_rate": 3.973927061651435e-06, + "loss": 0.76594865, + "num_input_tokens_seen": 77502680, + "router_z_loss_clip": 0.87890625, + "router_z_loss_mlp": 0.14294434, + "step": 2754, + "time_per_iteration": 2.487298011779785 + }, + { + "auxiliary_loss_clip": 0.01177281, + "auxiliary_loss_mlp": 0.01047844, + "balance_loss_clip": 1.07300878, + "balance_loss_mlp": 1.02822208, + "epoch": 0.07994312576170855, + "flos": 12962746857600.0, + "grad_norm": 2.4938705589064023, + "language_loss": 0.66976815, + "learning_rate": 3.973896801608347e-06, + "loss": 0.69201934, + "num_input_tokens_seen": 77516035, + "router_z_loss_clip": 1.04394531, + "router_z_loss_mlp": 0.19641113, + "step": 2755, + "time_per_iteration": 2.5179853439331055 + }, + { + "auxiliary_loss_clip": 0.0117097, + "auxiliary_loss_mlp": 0.0104701, + "balance_loss_clip": 1.06907368, + "balance_loss_mlp": 1.02830637, + "epoch": 0.0799721432302246, + "flos": 20515514693760.0, + "grad_norm": 2.339814236021073, + "language_loss": 0.85870802, + "learning_rate": 3.9738665241310016e-06, + "loss": 0.88088775, + "num_input_tokens_seen": 77531290, + "router_z_loss_clip": 1.02001953, + "router_z_loss_mlp": 0.18688965, + "step": 2756, + "time_per_iteration": 2.523080587387085 + }, + { + "auxiliary_loss_clip": 0.01162853, + "auxiliary_loss_mlp": 0.01051471, + "balance_loss_clip": 1.06821775, + "balance_loss_mlp": 1.03381038, + "epoch": 0.08000116069874064, + "flos": 39928786686720.0, + "grad_norm": 2.2740017972285993, + "language_loss": 0.98747492, + "learning_rate": 3.9738362292196695e-06, + "loss": 1.00961828, + "num_input_tokens_seen": 77553525, + "router_z_loss_clip": 0.9453125, + "router_z_loss_mlp": 0.17669678, + "step": 2757, + "time_per_iteration": 2.692309856414795 + }, + { + "auxiliary_loss_clip": 0.01169517, + "auxiliary_loss_mlp": 0.01047185, + "balance_loss_clip": 1.06879663, + "balance_loss_mlp": 1.02744341, + "epoch": 0.08003017816725669, + "flos": 34344066977280.0, + "grad_norm": 2.7190965229212383, + "language_loss": 0.86275738, + "learning_rate": 3.973805916874616e-06, + "loss": 0.88492435, + "num_input_tokens_seen": 77570860, + "router_z_loss_clip": 1.00732422, + "router_z_loss_mlp": 0.19750977, + "step": 2758, + "time_per_iteration": 2.623020887374878 + }, + { + "auxiliary_loss_clip": 0.01049593, + "auxiliary_loss_mlp": 0.01005009, + "balance_loss_clip": 1.02369928, + "balance_loss_mlp": 1.00370967, + "epoch": 0.08005919563577274, + "flos": 74772301299840.0, + "grad_norm": 0.6969594548576505, + "language_loss": 0.54269516, + "learning_rate": 3.973775587096112e-06, + "loss": 0.56324118, + "num_input_tokens_seen": 77631570, + "router_z_loss_clip": 0.2578125, + "router_z_loss_mlp": 0.01300049, + "step": 2759, + "time_per_iteration": 3.0864269733428955 + }, + { + "auxiliary_loss_clip": 0.01163005, + "auxiliary_loss_mlp": 0.01041268, + "balance_loss_clip": 1.06558776, + "balance_loss_mlp": 1.02296948, + "epoch": 0.08008821310428878, + "flos": 16173735258240.0, + "grad_norm": 2.464613951822974, + "language_loss": 0.76476318, + "learning_rate": 3.973745239884422e-06, + "loss": 0.78680587, + "num_input_tokens_seen": 77645030, + "router_z_loss_clip": 0.97607422, + "router_z_loss_mlp": 0.1829834, + "step": 2760, + "time_per_iteration": 2.453590154647827 + }, + { + "auxiliary_loss_clip": 0.01177306, + "auxiliary_loss_mlp": 0.01049331, + "balance_loss_clip": 1.07290852, + "balance_loss_mlp": 1.02925634, + "epoch": 0.08011723057280483, + "flos": 30476200608000.0, + "grad_norm": 2.494899039520093, + "language_loss": 0.85290623, + "learning_rate": 3.973714875239815e-06, + "loss": 0.87517256, + "num_input_tokens_seen": 77661270, + "router_z_loss_clip": 1.04492188, + "router_z_loss_mlp": 0.20056152, + "step": 2761, + "time_per_iteration": 2.613293170928955 + }, + { + "auxiliary_loss_clip": 0.0116838, + "auxiliary_loss_mlp": 0.01053294, + "balance_loss_clip": 1.06999969, + "balance_loss_mlp": 1.0342803, + "epoch": 0.08014624804132088, + "flos": 29418595534080.0, + "grad_norm": 1.8188998541021475, + "language_loss": 0.81210881, + "learning_rate": 3.973684493162559e-06, + "loss": 0.83432555, + "num_input_tokens_seen": 77681110, + "router_z_loss_clip": 0.984375, + "router_z_loss_mlp": 0.19030762, + "step": 2762, + "time_per_iteration": 2.54182505607605 + }, + { + "auxiliary_loss_clip": 0.01172231, + "auxiliary_loss_mlp": 0.01051149, + "balance_loss_clip": 1.07197595, + "balance_loss_mlp": 1.03213501, + "epoch": 0.08017526550983692, + "flos": 70062542006400.0, + "grad_norm": 2.4277814945253287, + "language_loss": 1.00660217, + "learning_rate": 3.973654093652924e-06, + "loss": 1.02883601, + "num_input_tokens_seen": 77702940, + "router_z_loss_clip": 1.00244141, + "router_z_loss_mlp": 0.19018555, + "step": 2763, + "time_per_iteration": 2.9291937351226807 + }, + { + "auxiliary_loss_clip": 0.01173702, + "auxiliary_loss_mlp": 0.0104595, + "balance_loss_clip": 1.06916595, + "balance_loss_mlp": 1.02622032, + "epoch": 0.08020428297835297, + "flos": 29889958734720.0, + "grad_norm": 3.7362227862824207, + "language_loss": 0.96740639, + "learning_rate": 3.973623676711178e-06, + "loss": 0.98960292, + "num_input_tokens_seen": 77720095, + "router_z_loss_clip": 1.04394531, + "router_z_loss_mlp": 0.19702148, + "step": 2764, + "time_per_iteration": 2.548410654067993 + }, + { + "auxiliary_loss_clip": 0.01047563, + "auxiliary_loss_mlp": 0.01004261, + "balance_loss_clip": 1.02136397, + "balance_loss_mlp": 1.00295532, + "epoch": 0.08023330044686902, + "flos": 63820767738240.0, + "grad_norm": 0.6561521524031557, + "language_loss": 0.46520466, + "learning_rate": 3.973593242337587e-06, + "loss": 0.4857229, + "num_input_tokens_seen": 77780995, + "router_z_loss_clip": 0.26171875, + "router_z_loss_mlp": 0.01306152, + "step": 2765, + "time_per_iteration": 3.184051752090454 + }, + { + "auxiliary_loss_clip": 0.01158148, + "auxiliary_loss_mlp": 0.01041677, + "balance_loss_clip": 1.06483674, + "balance_loss_mlp": 1.02588773, + "epoch": 0.08026231791538506, + "flos": 15296688875520.0, + "grad_norm": 2.8593005914919027, + "language_loss": 0.86051035, + "learning_rate": 3.973562790532424e-06, + "loss": 0.88250852, + "num_input_tokens_seen": 77792880, + "router_z_loss_clip": 0.93310547, + "router_z_loss_mlp": 0.15795898, + "step": 2766, + "time_per_iteration": 2.4604883193969727 + }, + { + "auxiliary_loss_clip": 0.01174792, + "auxiliary_loss_mlp": 0.01050151, + "balance_loss_clip": 1.07192254, + "balance_loss_mlp": 1.03008854, + "epoch": 0.08029133538390111, + "flos": 29490452691840.0, + "grad_norm": 2.3047480534416214, + "language_loss": 0.82115972, + "learning_rate": 3.973532321295955e-06, + "loss": 0.84340918, + "num_input_tokens_seen": 77807655, + "router_z_loss_clip": 1.03027344, + "router_z_loss_mlp": 0.20043945, + "step": 2767, + "time_per_iteration": 2.5994961261749268 + }, + { + "auxiliary_loss_clip": 0.01171286, + "auxiliary_loss_mlp": 0.01042876, + "balance_loss_clip": 1.0686233, + "balance_loss_mlp": 1.02459562, + "epoch": 0.08032035285241715, + "flos": 45625440153600.0, + "grad_norm": 2.661578495278046, + "language_loss": 0.92655408, + "learning_rate": 3.973501834628449e-06, + "loss": 0.94869566, + "num_input_tokens_seen": 77829060, + "router_z_loss_clip": 1.02587891, + "router_z_loss_mlp": 0.18280029, + "step": 2768, + "time_per_iteration": 2.7521262168884277 + }, + { + "auxiliary_loss_clip": 0.01176151, + "auxiliary_loss_mlp": 0.01051441, + "balance_loss_clip": 1.0699861, + "balance_loss_mlp": 1.03088915, + "epoch": 0.0803493703209332, + "flos": 20479568158080.0, + "grad_norm": 2.6295638011659865, + "language_loss": 0.93620324, + "learning_rate": 3.9734713305301775e-06, + "loss": 0.95847917, + "num_input_tokens_seen": 77844735, + "router_z_loss_clip": 1.06103516, + "router_z_loss_mlp": 0.20532227, + "step": 2769, + "time_per_iteration": 2.5677013397216797 + }, + { + "auxiliary_loss_clip": 0.01177419, + "auxiliary_loss_mlp": 0.01049289, + "balance_loss_clip": 1.06907547, + "balance_loss_mlp": 1.03077579, + "epoch": 0.08037838778944925, + "flos": 27335956043520.0, + "grad_norm": 2.535628827688164, + "language_loss": 0.93277359, + "learning_rate": 3.973440809001408e-06, + "loss": 0.95504069, + "num_input_tokens_seen": 77858655, + "router_z_loss_clip": 1.08251953, + "router_z_loss_mlp": 0.1852417, + "step": 2770, + "time_per_iteration": 2.4801700115203857 + }, + { + "auxiliary_loss_clip": 0.01184542, + "auxiliary_loss_mlp": 0.0105271, + "balance_loss_clip": 1.07670355, + "balance_loss_mlp": 1.03149092, + "epoch": 0.08040740525796529, + "flos": 15771679349760.0, + "grad_norm": 2.429569136442609, + "language_loss": 0.83740222, + "learning_rate": 3.973410270042411e-06, + "loss": 0.85977477, + "num_input_tokens_seen": 77872280, + "router_z_loss_clip": 1.07910156, + "router_z_loss_mlp": 0.21179199, + "step": 2771, + "time_per_iteration": 2.4735195636749268 + }, + { + "auxiliary_loss_clip": 0.01050882, + "auxiliary_loss_mlp": 0.01002324, + "balance_loss_clip": 1.02463388, + "balance_loss_mlp": 1.00089359, + "epoch": 0.08043642272648134, + "flos": 62914418835840.0, + "grad_norm": 0.6724011815493469, + "language_loss": 0.48890632, + "learning_rate": 3.973379713653455e-06, + "loss": 0.5094384, + "num_input_tokens_seen": 77932040, + "router_z_loss_clip": 0.26171875, + "router_z_loss_mlp": 0.01428223, + "step": 2772, + "time_per_iteration": 3.045945405960083 + }, + { + "auxiliary_loss_clip": 0.01173402, + "auxiliary_loss_mlp": 0.01048214, + "balance_loss_clip": 1.06989193, + "balance_loss_mlp": 1.03079748, + "epoch": 0.08046544019499739, + "flos": 32742990149760.0, + "grad_norm": 2.413503981443279, + "language_loss": 1.00646544, + "learning_rate": 3.973349139834812e-06, + "loss": 1.02868164, + "num_input_tokens_seen": 77949865, + "router_z_loss_clip": 1.03564453, + "router_z_loss_mlp": 0.17401123, + "step": 2773, + "time_per_iteration": 2.589071035385132 + }, + { + "auxiliary_loss_clip": 0.01177509, + "auxiliary_loss_mlp": 0.01050132, + "balance_loss_clip": 1.07214904, + "balance_loss_mlp": 1.03037286, + "epoch": 0.08049445766351343, + "flos": 32081407499520.0, + "grad_norm": 4.107972192020829, + "language_loss": 0.89256668, + "learning_rate": 3.97331854858675e-06, + "loss": 0.91484308, + "num_input_tokens_seen": 77964835, + "router_z_loss_clip": 1.05371094, + "router_z_loss_mlp": 0.19769287, + "step": 2774, + "time_per_iteration": 2.585984230041504 + }, + { + "auxiliary_loss_clip": 0.01160702, + "auxiliary_loss_mlp": 0.01045894, + "balance_loss_clip": 1.06512094, + "balance_loss_mlp": 1.02944934, + "epoch": 0.08052347513202948, + "flos": 25183686038400.0, + "grad_norm": 2.7855772033099386, + "language_loss": 0.89458776, + "learning_rate": 3.9732879399095416e-06, + "loss": 0.91665375, + "num_input_tokens_seen": 77979095, + "router_z_loss_clip": 0.95605469, + "router_z_loss_mlp": 0.16455078, + "step": 2775, + "time_per_iteration": 2.545945405960083 + }, + { + "auxiliary_loss_clip": 0.01174793, + "auxiliary_loss_mlp": 0.0105861, + "balance_loss_clip": 1.07538378, + "balance_loss_mlp": 1.04038262, + "epoch": 0.08055249260054553, + "flos": 74735633514240.0, + "grad_norm": 2.731686535837139, + "language_loss": 0.79213482, + "learning_rate": 3.973257313803454e-06, + "loss": 0.8144688, + "num_input_tokens_seen": 78011060, + "router_z_loss_clip": 0.99560547, + "router_z_loss_mlp": 0.18237305, + "step": 2776, + "time_per_iteration": 5.413174390792847 + }, + { + "auxiliary_loss_clip": 0.01181632, + "auxiliary_loss_mlp": 0.01054887, + "balance_loss_clip": 1.07647514, + "balance_loss_mlp": 1.03277349, + "epoch": 0.08058151006906157, + "flos": 18182255034240.0, + "grad_norm": 2.610671999723865, + "language_loss": 0.79557371, + "learning_rate": 3.97322667026876e-06, + "loss": 0.81793892, + "num_input_tokens_seen": 78023655, + "router_z_loss_clip": 1.05078125, + "router_z_loss_mlp": 0.22131348, + "step": 2777, + "time_per_iteration": 4.767325401306152 + }, + { + "auxiliary_loss_clip": 0.0105308, + "auxiliary_loss_mlp": 0.01002011, + "balance_loss_clip": 1.02702165, + "balance_loss_mlp": 1.00066364, + "epoch": 0.08061052753757762, + "flos": 51331179761280.0, + "grad_norm": 1.437784900482963, + "language_loss": 0.56690723, + "learning_rate": 3.973196009305729e-06, + "loss": 0.58745813, + "num_input_tokens_seen": 78068045, + "router_z_loss_clip": 0.26171875, + "router_z_loss_mlp": 0.01348877, + "step": 2778, + "time_per_iteration": 2.7969374656677246 + }, + { + "auxiliary_loss_clip": 0.01180258, + "auxiliary_loss_mlp": 0.0105476, + "balance_loss_clip": 1.07299316, + "balance_loss_mlp": 1.03460145, + "epoch": 0.08063954500609367, + "flos": 15921391236480.0, + "grad_norm": 3.427233350380797, + "language_loss": 0.90605402, + "learning_rate": 3.9731653309146335e-06, + "loss": 0.92840415, + "num_input_tokens_seen": 78081020, + "router_z_loss_clip": 1.07128906, + "router_z_loss_mlp": 0.20153809, + "step": 2779, + "time_per_iteration": 4.921347141265869 + }, + { + "auxiliary_loss_clip": 0.01180305, + "auxiliary_loss_mlp": 0.01051259, + "balance_loss_clip": 1.07257414, + "balance_loss_mlp": 1.03040898, + "epoch": 0.08066856247460971, + "flos": 30927347429760.0, + "grad_norm": 2.0213060175845374, + "language_loss": 0.74499106, + "learning_rate": 3.973134635095742e-06, + "loss": 0.76730669, + "num_input_tokens_seen": 78101145, + "router_z_loss_clip": 1.078125, + "router_z_loss_mlp": 0.20837402, + "step": 2780, + "time_per_iteration": 2.6061770915985107 + }, + { + "auxiliary_loss_clip": 0.01176819, + "auxiliary_loss_mlp": 0.01052572, + "balance_loss_clip": 1.07449615, + "balance_loss_mlp": 1.0341661, + "epoch": 0.08069757994312576, + "flos": 36828798289920.0, + "grad_norm": 1.9775551998486824, + "language_loss": 0.74555016, + "learning_rate": 3.973103921849328e-06, + "loss": 0.76784408, + "num_input_tokens_seen": 78127865, + "router_z_loss_clip": 1.02294922, + "router_z_loss_mlp": 0.18432617, + "step": 2781, + "time_per_iteration": 2.687290668487549 + }, + { + "auxiliary_loss_clip": 0.0116673, + "auxiliary_loss_mlp": 0.01047168, + "balance_loss_clip": 1.07077622, + "balance_loss_mlp": 1.02910733, + "epoch": 0.08072659741164182, + "flos": 18550734704640.0, + "grad_norm": 1.9556300865605354, + "language_loss": 0.68382162, + "learning_rate": 3.973073191175661e-06, + "loss": 0.70596063, + "num_input_tokens_seen": 78143890, + "router_z_loss_clip": 0.95800781, + "router_z_loss_mlp": 0.18066406, + "step": 2782, + "time_per_iteration": 2.5159428119659424 + }, + { + "auxiliary_loss_clip": 0.01169381, + "auxiliary_loss_mlp": 0.01044113, + "balance_loss_clip": 1.06879282, + "balance_loss_mlp": 1.02676833, + "epoch": 0.08075561488015785, + "flos": 19311609525120.0, + "grad_norm": 2.4254795451262767, + "language_loss": 0.84294522, + "learning_rate": 3.973042443075013e-06, + "loss": 0.86508012, + "num_input_tokens_seen": 78161225, + "router_z_loss_clip": 1.00683594, + "router_z_loss_mlp": 0.17340088, + "step": 2783, + "time_per_iteration": 2.5013976097106934 + }, + { + "auxiliary_loss_clip": 0.0105581, + "auxiliary_loss_mlp": 0.01004867, + "balance_loss_clip": 1.02976155, + "balance_loss_mlp": 1.00338316, + "epoch": 0.0807846323486739, + "flos": 74794424077440.0, + "grad_norm": 0.635871628021716, + "language_loss": 0.46935409, + "learning_rate": 3.973011677547657e-06, + "loss": 0.48996091, + "num_input_tokens_seen": 78227595, + "router_z_loss_clip": 0.25976562, + "router_z_loss_mlp": 0.01483154, + "step": 2784, + "time_per_iteration": 3.320852279663086 + }, + { + "auxiliary_loss_clip": 0.01175684, + "auxiliary_loss_mlp": 0.01055421, + "balance_loss_clip": 1.07056451, + "balance_loss_mlp": 1.03451133, + "epoch": 0.08081364981718994, + "flos": 25841892810240.0, + "grad_norm": 2.064321559816848, + "language_loss": 0.86672819, + "learning_rate": 3.972980894593863e-06, + "loss": 0.88903922, + "num_input_tokens_seen": 78245215, + "router_z_loss_clip": 1.05126953, + "router_z_loss_mlp": 0.2088623, + "step": 2785, + "time_per_iteration": 2.60768985748291 + }, + { + "auxiliary_loss_clip": 0.01165632, + "auxiliary_loss_mlp": 0.01036251, + "balance_loss_clip": 1.06816387, + "balance_loss_mlp": 1.01822686, + "epoch": 0.080842667285706, + "flos": 16282293137280.0, + "grad_norm": 3.5238751975511606, + "language_loss": 0.86894333, + "learning_rate": 3.9729500942139024e-06, + "loss": 0.89096218, + "num_input_tokens_seen": 78257420, + "router_z_loss_clip": 0.97363281, + "router_z_loss_mlp": 0.18023682, + "step": 2786, + "time_per_iteration": 2.46427583694458 + }, + { + "auxiliary_loss_clip": 0.01178391, + "auxiliary_loss_mlp": 0.01048357, + "balance_loss_clip": 1.07447672, + "balance_loss_mlp": 1.02933097, + "epoch": 0.08087168475422205, + "flos": 29892005809920.0, + "grad_norm": 2.173010322457783, + "language_loss": 0.7745136, + "learning_rate": 3.9729192764080485e-06, + "loss": 0.79678112, + "num_input_tokens_seen": 78277050, + "router_z_loss_clip": 1.04003906, + "router_z_loss_mlp": 0.19012451, + "step": 2787, + "time_per_iteration": 2.6140615940093994 + }, + { + "auxiliary_loss_clip": 0.01052828, + "auxiliary_loss_mlp": 0.01002948, + "balance_loss_clip": 1.02667987, + "balance_loss_mlp": 1.00149965, + "epoch": 0.08090070222273808, + "flos": 58431259468800.0, + "grad_norm": 0.7098260595400099, + "language_loss": 0.53190589, + "learning_rate": 3.972888441176574e-06, + "loss": 0.55246365, + "num_input_tokens_seen": 78337715, + "router_z_loss_clip": 0.26171875, + "router_z_loss_mlp": 0.01446533, + "step": 2788, + "time_per_iteration": 2.9875967502593994 + }, + { + "auxiliary_loss_clip": 0.0117024, + "auxiliary_loss_mlp": 0.01045464, + "balance_loss_clip": 1.07152581, + "balance_loss_mlp": 1.02777362, + "epoch": 0.08092971969125413, + "flos": 22922463104640.0, + "grad_norm": 2.0927562163531923, + "language_loss": 0.92794561, + "learning_rate": 3.97285758851975e-06, + "loss": 0.95010269, + "num_input_tokens_seen": 78352180, + "router_z_loss_clip": 0.98632812, + "router_z_loss_mlp": 0.17700195, + "step": 2789, + "time_per_iteration": 2.524705171585083 + }, + { + "auxiliary_loss_clip": 0.01178176, + "auxiliary_loss_mlp": 0.01051925, + "balance_loss_clip": 1.07076359, + "balance_loss_mlp": 1.02965641, + "epoch": 0.08095873715977019, + "flos": 14238006393600.0, + "grad_norm": 3.029541063829244, + "language_loss": 0.80900443, + "learning_rate": 3.972826718437849e-06, + "loss": 0.8313055, + "num_input_tokens_seen": 78365905, + "router_z_loss_clip": 1.07470703, + "router_z_loss_mlp": 0.22253418, + "step": 2790, + "time_per_iteration": 2.5288307666778564 + }, + { + "auxiliary_loss_clip": 0.01177205, + "auxiliary_loss_mlp": 0.01055228, + "balance_loss_clip": 1.07203341, + "balance_loss_mlp": 1.03484344, + "epoch": 0.08098775462828622, + "flos": 17487311627520.0, + "grad_norm": 2.4994187598412014, + "language_loss": 0.93428308, + "learning_rate": 3.972795830931145e-06, + "loss": 0.9566074, + "num_input_tokens_seen": 78380245, + "router_z_loss_clip": 1.05175781, + "router_z_loss_mlp": 0.20397949, + "step": 2791, + "time_per_iteration": 2.4690215587615967 + }, + { + "auxiliary_loss_clip": 0.01172245, + "auxiliary_loss_mlp": 0.0104776, + "balance_loss_clip": 1.0713532, + "balance_loss_mlp": 1.03055251, + "epoch": 0.08101677209680228, + "flos": 24675155239680.0, + "grad_norm": 2.2260107648369174, + "language_loss": 0.87518626, + "learning_rate": 3.972764925999909e-06, + "loss": 0.89738631, + "num_input_tokens_seen": 78395665, + "router_z_loss_clip": 1.00976562, + "router_z_loss_mlp": 0.171875, + "step": 2792, + "time_per_iteration": 2.5530107021331787 + }, + { + "auxiliary_loss_clip": 0.01051429, + "auxiliary_loss_mlp": 0.0100015, + "balance_loss_clip": 1.0250473, + "balance_loss_mlp": 0.99872571, + "epoch": 0.08104578956531833, + "flos": 65206596314880.0, + "grad_norm": 0.6399466252190532, + "language_loss": 0.45499557, + "learning_rate": 3.972734003644415e-06, + "loss": 0.47551137, + "num_input_tokens_seen": 78455080, + "router_z_loss_clip": 0.26367188, + "router_z_loss_mlp": 0.01422119, + "step": 2793, + "time_per_iteration": 3.0692970752716064 + }, + { + "auxiliary_loss_clip": 0.01179754, + "auxiliary_loss_mlp": 0.01048924, + "balance_loss_clip": 1.07155287, + "balance_loss_mlp": 1.02923083, + "epoch": 0.08107480703383436, + "flos": 12815297527680.0, + "grad_norm": 3.4894813416746264, + "language_loss": 0.84619439, + "learning_rate": 3.9727030638649366e-06, + "loss": 0.86848116, + "num_input_tokens_seen": 78466485, + "router_z_loss_clip": 1.08203125, + "router_z_loss_mlp": 0.19689941, + "step": 2794, + "time_per_iteration": 2.4673428535461426 + }, + { + "auxiliary_loss_clip": 0.01171152, + "auxiliary_loss_mlp": 0.0104397, + "balance_loss_clip": 1.07376957, + "balance_loss_mlp": 1.02708983, + "epoch": 0.08110382450235042, + "flos": 31826369957760.0, + "grad_norm": 2.0841856495571247, + "language_loss": 0.61369944, + "learning_rate": 3.9726721066617465e-06, + "loss": 0.63585067, + "num_input_tokens_seen": 78483535, + "router_z_loss_clip": 0.97460938, + "router_z_loss_mlp": 0.16882324, + "step": 2795, + "time_per_iteration": 2.596036672592163 + }, + { + "auxiliary_loss_clip": 0.01171265, + "auxiliary_loss_mlp": 0.01050322, + "balance_loss_clip": 1.06899452, + "balance_loss_mlp": 1.02886474, + "epoch": 0.08113284197086647, + "flos": 33177113925120.0, + "grad_norm": 1.9960766490292705, + "language_loss": 0.8202523, + "learning_rate": 3.972641132035118e-06, + "loss": 0.84246814, + "num_input_tokens_seen": 78501735, + "router_z_loss_clip": 1.0234375, + "router_z_loss_mlp": 0.21496582, + "step": 2796, + "time_per_iteration": 2.609495162963867 + }, + { + "auxiliary_loss_clip": 0.01052504, + "auxiliary_loss_mlp": 0.01004214, + "balance_loss_clip": 1.0263555, + "balance_loss_mlp": 1.0028317, + "epoch": 0.0811618594393825, + "flos": 68200648525440.0, + "grad_norm": 0.653136143317561, + "language_loss": 0.48992375, + "learning_rate": 3.972610139985324e-06, + "loss": 0.51049089, + "num_input_tokens_seen": 78561620, + "router_z_loss_clip": 0.26171875, + "router_z_loss_mlp": 0.01385498, + "step": 2797, + "time_per_iteration": 3.056628704071045 + }, + { + "auxiliary_loss_clip": 0.01170172, + "auxiliary_loss_mlp": 0.01045909, + "balance_loss_clip": 1.06961226, + "balance_loss_mlp": 1.02682352, + "epoch": 0.08119087690789856, + "flos": 20075932051200.0, + "grad_norm": 2.582932990557547, + "language_loss": 0.90299737, + "learning_rate": 3.97257913051264e-06, + "loss": 0.92515814, + "num_input_tokens_seen": 78578090, + "router_z_loss_clip": 1.00683594, + "router_z_loss_mlp": 0.1907959, + "step": 2798, + "time_per_iteration": 2.573943614959717 + }, + { + "auxiliary_loss_clip": 0.01166412, + "auxiliary_loss_mlp": 0.01049381, + "balance_loss_clip": 1.06673217, + "balance_loss_mlp": 1.03053427, + "epoch": 0.0812198943764146, + "flos": 39232370822400.0, + "grad_norm": 2.1992304725312883, + "language_loss": 0.71680504, + "learning_rate": 3.972548103617338e-06, + "loss": 0.73896295, + "num_input_tokens_seen": 78597045, + "router_z_loss_clip": 0.99804688, + "router_z_loss_mlp": 0.18878174, + "step": 2799, + "time_per_iteration": 2.642673969268799 + }, + { + "auxiliary_loss_clip": 0.01174239, + "auxiliary_loss_mlp": 0.01057773, + "balance_loss_clip": 1.07199061, + "balance_loss_mlp": 1.03861618, + "epoch": 0.08124891184493065, + "flos": 33688302330240.0, + "grad_norm": 2.2580246449978874, + "language_loss": 0.95109606, + "learning_rate": 3.972517059299694e-06, + "loss": 0.97341627, + "num_input_tokens_seen": 78615905, + "router_z_loss_clip": 1.02197266, + "router_z_loss_mlp": 0.19165039, + "step": 2800, + "time_per_iteration": 2.625523805618286 + }, + { + "auxiliary_loss_clip": 0.01159993, + "auxiliary_loss_mlp": 0.01042717, + "balance_loss_clip": 1.06778693, + "balance_loss_mlp": 1.02762532, + "epoch": 0.0812779293134467, + "flos": 16248214108800.0, + "grad_norm": 2.076371714965731, + "language_loss": 0.76536709, + "learning_rate": 3.972485997559981e-06, + "loss": 0.78739417, + "num_input_tokens_seen": 78631130, + "router_z_loss_clip": 0.92285156, + "router_z_loss_mlp": 0.15106201, + "step": 2801, + "time_per_iteration": 2.4768869876861572 + }, + { + "auxiliary_loss_clip": 0.01173832, + "auxiliary_loss_mlp": 0.01051675, + "balance_loss_clip": 1.07180524, + "balance_loss_mlp": 1.0311954, + "epoch": 0.08130694678196274, + "flos": 31936687603200.0, + "grad_norm": 2.070583247641769, + "language_loss": 0.97125745, + "learning_rate": 3.972454918398473e-06, + "loss": 0.99351251, + "num_input_tokens_seen": 78648255, + "router_z_loss_clip": 1.01904297, + "router_z_loss_mlp": 0.20483398, + "step": 2802, + "time_per_iteration": 2.6225945949554443 + }, + { + "auxiliary_loss_clip": 0.01169032, + "auxiliary_loss_mlp": 0.01053905, + "balance_loss_clip": 1.06713533, + "balance_loss_mlp": 1.03423572, + "epoch": 0.08133596425047879, + "flos": 18362490503040.0, + "grad_norm": 1.9763725572806587, + "language_loss": 0.74285859, + "learning_rate": 3.972423821815445e-06, + "loss": 0.76508796, + "num_input_tokens_seen": 78661715, + "router_z_loss_clip": 1.01953125, + "router_z_loss_mlp": 0.1965332, + "step": 2803, + "time_per_iteration": 2.443005084991455 + }, + { + "auxiliary_loss_clip": 0.01164232, + "auxiliary_loss_mlp": 0.01042352, + "balance_loss_clip": 1.0669955, + "balance_loss_mlp": 1.02524567, + "epoch": 0.08136498171899484, + "flos": 31395622060800.0, + "grad_norm": 1.832494775146006, + "language_loss": 0.70484942, + "learning_rate": 3.9723927078111715e-06, + "loss": 0.72691524, + "num_input_tokens_seen": 78680170, + "router_z_loss_clip": 0.97167969, + "router_z_loss_mlp": 0.17126465, + "step": 2804, + "time_per_iteration": 2.6066360473632812 + }, + { + "auxiliary_loss_clip": 0.01189144, + "auxiliary_loss_mlp": 0.01062045, + "balance_loss_clip": 1.07263196, + "balance_loss_mlp": 1.03994393, + "epoch": 0.08139399918751088, + "flos": 12086777882880.0, + "grad_norm": 3.5624589486225164, + "language_loss": 0.90284717, + "learning_rate": 3.9723615763859275e-06, + "loss": 0.92535901, + "num_input_tokens_seen": 78692995, + "router_z_loss_clip": 1.16503906, + "router_z_loss_mlp": 0.22119141, + "step": 2805, + "time_per_iteration": 2.4603121280670166 + }, + { + "auxiliary_loss_clip": 0.0105082, + "auxiliary_loss_mlp": 0.01012491, + "balance_loss_clip": 1.02501559, + "balance_loss_mlp": 1.01120901, + "epoch": 0.08142301665602693, + "flos": 54317729007360.0, + "grad_norm": 0.6822788012744979, + "language_loss": 0.4551096, + "learning_rate": 3.972330427539988e-06, + "loss": 0.4757427, + "num_input_tokens_seen": 78753245, + "router_z_loss_clip": 0.2578125, + "router_z_loss_mlp": 0.01281738, + "step": 2806, + "time_per_iteration": 3.144620418548584 + }, + { + "auxiliary_loss_clip": 0.01175082, + "auxiliary_loss_mlp": 0.01053106, + "balance_loss_clip": 1.06687129, + "balance_loss_mlp": 1.03140378, + "epoch": 0.08145203412454298, + "flos": 28872861223680.0, + "grad_norm": 2.1120534943822524, + "language_loss": 1.02016664, + "learning_rate": 3.972299261273628e-06, + "loss": 1.04244852, + "num_input_tokens_seen": 78770310, + "router_z_loss_clip": 1.08007812, + "router_z_loss_mlp": 0.21722412, + "step": 2807, + "time_per_iteration": 2.6152267456054688 + }, + { + "auxiliary_loss_clip": 0.01161908, + "auxiliary_loss_mlp": 0.01047661, + "balance_loss_clip": 1.06641805, + "balance_loss_mlp": 1.02902877, + "epoch": 0.08148105159305902, + "flos": 20662784455680.0, + "grad_norm": 2.849745023470496, + "language_loss": 0.79449296, + "learning_rate": 3.972268077587123e-06, + "loss": 0.8165887, + "num_input_tokens_seen": 78785440, + "router_z_loss_clip": 0.95410156, + "router_z_loss_mlp": 0.18621826, + "step": 2808, + "time_per_iteration": 2.5289626121520996 + }, + { + "auxiliary_loss_clip": 0.01164862, + "auxiliary_loss_mlp": 0.01050546, + "balance_loss_clip": 1.06777024, + "balance_loss_mlp": 1.03222942, + "epoch": 0.08151006906157507, + "flos": 12342677351040.0, + "grad_norm": 3.1278971154611837, + "language_loss": 0.77532315, + "learning_rate": 3.972236876480748e-06, + "loss": 0.79747725, + "num_input_tokens_seen": 78798000, + "router_z_loss_clip": 0.97070312, + "router_z_loss_mlp": 0.18310547, + "step": 2809, + "time_per_iteration": 2.513322114944458 + }, + { + "auxiliary_loss_clip": 0.01051557, + "auxiliary_loss_mlp": 0.01004936, + "balance_loss_clip": 1.02577138, + "balance_loss_mlp": 1.00367856, + "epoch": 0.08153908653009112, + "flos": 72003265839360.0, + "grad_norm": 1.1733492466006366, + "language_loss": 0.525316, + "learning_rate": 3.972205657954779e-06, + "loss": 0.54588097, + "num_input_tokens_seen": 78861820, + "router_z_loss_clip": 0.2578125, + "router_z_loss_mlp": 0.01257324, + "step": 2810, + "time_per_iteration": 3.1513965129852295 + }, + { + "auxiliary_loss_clip": 0.01172856, + "auxiliary_loss_mlp": 0.01054953, + "balance_loss_clip": 1.06846058, + "balance_loss_mlp": 1.03280365, + "epoch": 0.08156810399860716, + "flos": 24499193489280.0, + "grad_norm": 2.0017876458139177, + "language_loss": 0.84186554, + "learning_rate": 3.972174422009492e-06, + "loss": 0.86414361, + "num_input_tokens_seen": 78876665, + "router_z_loss_clip": 1.04443359, + "router_z_loss_mlp": 0.22155762, + "step": 2811, + "time_per_iteration": 2.558354616165161 + }, + { + "auxiliary_loss_clip": 0.01171331, + "auxiliary_loss_mlp": 0.01054777, + "balance_loss_clip": 1.07081366, + "balance_loss_mlp": 1.03545308, + "epoch": 0.08159712146712321, + "flos": 47148518597760.0, + "grad_norm": 2.3476613217634172, + "language_loss": 0.80225986, + "learning_rate": 3.972143168645162e-06, + "loss": 0.82452095, + "num_input_tokens_seen": 78896085, + "router_z_loss_clip": 1.00390625, + "router_z_loss_mlp": 0.19311523, + "step": 2812, + "time_per_iteration": 2.6300065517425537 + }, + { + "auxiliary_loss_clip": 0.01173183, + "auxiliary_loss_mlp": 0.01046016, + "balance_loss_clip": 1.06714761, + "balance_loss_mlp": 1.02473748, + "epoch": 0.08162613893563926, + "flos": 21756767028480.0, + "grad_norm": 2.300028919719449, + "language_loss": 0.97118485, + "learning_rate": 3.972111897862065e-06, + "loss": 0.99337685, + "num_input_tokens_seen": 78912310, + "router_z_loss_clip": 1.06005859, + "router_z_loss_mlp": 0.21264648, + "step": 2813, + "time_per_iteration": 2.513054132461548 + }, + { + "auxiliary_loss_clip": 0.01168633, + "auxiliary_loss_mlp": 0.01050083, + "balance_loss_clip": 1.0679388, + "balance_loss_mlp": 1.03038931, + "epoch": 0.0816551564041553, + "flos": 23506550161920.0, + "grad_norm": 2.4446520136391423, + "language_loss": 0.8264479, + "learning_rate": 3.972080609660478e-06, + "loss": 0.84863508, + "num_input_tokens_seen": 78926950, + "router_z_loss_clip": 1.00634766, + "router_z_loss_mlp": 0.19702148, + "step": 2814, + "time_per_iteration": 2.524972677230835 + }, + { + "auxiliary_loss_clip": 0.01164086, + "auxiliary_loss_mlp": 0.01039657, + "balance_loss_clip": 1.06691015, + "balance_loss_mlp": 1.02126288, + "epoch": 0.08168417387267135, + "flos": 21354818860800.0, + "grad_norm": 2.6980813201534755, + "language_loss": 0.76918244, + "learning_rate": 3.972049304040678e-06, + "loss": 0.79121989, + "num_input_tokens_seen": 78941075, + "router_z_loss_clip": 0.97216797, + "router_z_loss_mlp": 0.18383789, + "step": 2815, + "time_per_iteration": 2.5193369388580322 + }, + { + "auxiliary_loss_clip": 0.01176789, + "auxiliary_loss_mlp": 0.01058329, + "balance_loss_clip": 1.06898403, + "balance_loss_mlp": 1.03746796, + "epoch": 0.08171319134118739, + "flos": 12779458732800.0, + "grad_norm": 4.850957828794976, + "language_loss": 0.87300307, + "learning_rate": 3.972017981002939e-06, + "loss": 0.89535427, + "num_input_tokens_seen": 78952065, + "router_z_loss_clip": 1.07714844, + "router_z_loss_mlp": 0.20861816, + "step": 2816, + "time_per_iteration": 2.4334089756011963 + }, + { + "auxiliary_loss_clip": 0.01162091, + "auxiliary_loss_mlp": 0.01049968, + "balance_loss_clip": 1.06622076, + "balance_loss_mlp": 1.03230739, + "epoch": 0.08174220880970344, + "flos": 25769712430080.0, + "grad_norm": 2.501088346888451, + "language_loss": 0.81980979, + "learning_rate": 3.971986640547541e-06, + "loss": 0.84193027, + "num_input_tokens_seen": 78967760, + "router_z_loss_clip": 0.95898438, + "router_z_loss_mlp": 0.17669678, + "step": 2817, + "time_per_iteration": 2.5567660331726074 + }, + { + "auxiliary_loss_clip": 0.01173083, + "auxiliary_loss_mlp": 0.01045504, + "balance_loss_clip": 1.06847167, + "balance_loss_mlp": 1.02639508, + "epoch": 0.08177122627821949, + "flos": 16976482358400.0, + "grad_norm": 2.6822772634155863, + "language_loss": 0.85210216, + "learning_rate": 3.971955282674758e-06, + "loss": 0.87428808, + "num_input_tokens_seen": 78984445, + "router_z_loss_clip": 1.046875, + "router_z_loss_mlp": 0.19116211, + "step": 2818, + "time_per_iteration": 2.490243911743164 + }, + { + "auxiliary_loss_clip": 0.01170231, + "auxiliary_loss_mlp": 0.01048996, + "balance_loss_clip": 1.06534195, + "balance_loss_mlp": 1.02844465, + "epoch": 0.08180024374673553, + "flos": 32554674120960.0, + "grad_norm": 2.7369770566822664, + "language_loss": 0.88927168, + "learning_rate": 3.971923907384868e-06, + "loss": 0.91146398, + "num_input_tokens_seen": 78999140, + "router_z_loss_clip": 1.04785156, + "router_z_loss_mlp": 0.20581055, + "step": 2819, + "time_per_iteration": 2.5995025634765625 + }, + { + "auxiliary_loss_clip": 0.01054939, + "auxiliary_loss_mlp": 0.01000218, + "balance_loss_clip": 1.02892458, + "balance_loss_mlp": 0.99890691, + "epoch": 0.08182926121525158, + "flos": 64373541114240.0, + "grad_norm": 0.7169007301872299, + "language_loss": 0.52956855, + "learning_rate": 3.971892514678147e-06, + "loss": 0.55012012, + "num_input_tokens_seen": 79055315, + "router_z_loss_clip": 0.26171875, + "router_z_loss_mlp": 0.01312256, + "step": 2820, + "time_per_iteration": 2.99794340133667 + }, + { + "auxiliary_loss_clip": 0.01053721, + "auxiliary_loss_mlp": 0.0100106, + "balance_loss_clip": 1.02785754, + "balance_loss_mlp": 0.99973053, + "epoch": 0.08185827868376763, + "flos": 61718845622400.0, + "grad_norm": 0.7232060992992316, + "language_loss": 0.52376455, + "learning_rate": 3.971861104554876e-06, + "loss": 0.54431236, + "num_input_tokens_seen": 79121095, + "router_z_loss_clip": 0.2578125, + "router_z_loss_mlp": 0.01330566, + "step": 2821, + "time_per_iteration": 3.2528865337371826 + }, + { + "auxiliary_loss_clip": 0.01170113, + "auxiliary_loss_mlp": 0.01048649, + "balance_loss_clip": 1.06863356, + "balance_loss_mlp": 1.03066003, + "epoch": 0.08188729615228367, + "flos": 65719361940480.0, + "grad_norm": 2.848966222993878, + "language_loss": 0.76445329, + "learning_rate": 3.971829677015328e-06, + "loss": 0.78664094, + "num_input_tokens_seen": 79140545, + "router_z_loss_clip": 1.01416016, + "router_z_loss_mlp": 0.17980957, + "step": 2822, + "time_per_iteration": 2.8913779258728027 + }, + { + "auxiliary_loss_clip": 0.01171972, + "auxiliary_loss_mlp": 0.01049901, + "balance_loss_clip": 1.06986594, + "balance_loss_mlp": 1.02956426, + "epoch": 0.08191631362079972, + "flos": 11977393991040.0, + "grad_norm": 3.7833864976519362, + "language_loss": 0.96311533, + "learning_rate": 3.971798232059782e-06, + "loss": 0.98533404, + "num_input_tokens_seen": 79151650, + "router_z_loss_clip": 1.02050781, + "router_z_loss_mlp": 0.20355225, + "step": 2823, + "time_per_iteration": 2.492622137069702 + }, + { + "auxiliary_loss_clip": 0.0117094, + "auxiliary_loss_mlp": 0.01048552, + "balance_loss_clip": 1.07044959, + "balance_loss_mlp": 1.02990723, + "epoch": 0.08194533108931577, + "flos": 15626241181440.0, + "grad_norm": 2.1632200201769396, + "language_loss": 0.74120533, + "learning_rate": 3.9717667696885165e-06, + "loss": 0.7634002, + "num_input_tokens_seen": 79166095, + "router_z_loss_clip": 1.00488281, + "router_z_loss_mlp": 0.18640137, + "step": 2824, + "time_per_iteration": 2.5299270153045654 + }, + { + "auxiliary_loss_clip": 0.01177665, + "auxiliary_loss_mlp": 0.01051985, + "balance_loss_clip": 1.06972647, + "balance_loss_mlp": 1.03309059, + "epoch": 0.08197434855783181, + "flos": 34126053379200.0, + "grad_norm": 2.2115716928770777, + "language_loss": 0.98327148, + "learning_rate": 3.97173528990181e-06, + "loss": 1.00556803, + "num_input_tokens_seen": 79186530, + "router_z_loss_clip": 1.08007812, + "router_z_loss_mlp": 0.18884277, + "step": 2825, + "time_per_iteration": 2.614232063293457 + }, + { + "auxiliary_loss_clip": 0.0104601, + "auxiliary_loss_mlp": 0.00999086, + "balance_loss_clip": 1.02033138, + "balance_loss_mlp": 0.99776882, + "epoch": 0.08200336602634786, + "flos": 63831182682240.0, + "grad_norm": 0.8156477328244912, + "language_loss": 0.54935813, + "learning_rate": 3.971703792699938e-06, + "loss": 0.56980908, + "num_input_tokens_seen": 79244290, + "router_z_loss_clip": 0.2578125, + "router_z_loss_mlp": 0.01318359, + "step": 2826, + "time_per_iteration": 3.026719808578491 + }, + { + "auxiliary_loss_clip": 0.01166439, + "auxiliary_loss_mlp": 0.01059322, + "balance_loss_clip": 1.06549859, + "balance_loss_mlp": 1.03878212, + "epoch": 0.08203238349486391, + "flos": 13218179448960.0, + "grad_norm": 2.405458114958897, + "language_loss": 0.9663614, + "learning_rate": 3.971672278083181e-06, + "loss": 0.98861897, + "num_input_tokens_seen": 79256600, + "router_z_loss_clip": 1.00927734, + "router_z_loss_mlp": 0.20556641, + "step": 2827, + "time_per_iteration": 2.4712302684783936 + }, + { + "auxiliary_loss_clip": 0.01168735, + "auxiliary_loss_mlp": 0.01060425, + "balance_loss_clip": 1.06975329, + "balance_loss_mlp": 1.04024315, + "epoch": 0.08206140096337995, + "flos": 74732078067840.0, + "grad_norm": 3.029315087476581, + "language_loss": 0.70374918, + "learning_rate": 3.971640746051817e-06, + "loss": 0.72604084, + "num_input_tokens_seen": 79277260, + "router_z_loss_clip": 0.98925781, + "router_z_loss_mlp": 0.2019043, + "step": 2828, + "time_per_iteration": 2.9735779762268066 + }, + { + "auxiliary_loss_clip": 0.01175319, + "auxiliary_loss_mlp": 0.01076225, + "balance_loss_clip": 1.07162857, + "balance_loss_mlp": 1.05502915, + "epoch": 0.082090418431896, + "flos": 19346658220800.0, + "grad_norm": 2.239367978018831, + "language_loss": 0.97331548, + "learning_rate": 3.971609196606123e-06, + "loss": 0.99583089, + "num_input_tokens_seen": 79289820, + "router_z_loss_clip": 1.03564453, + "router_z_loss_mlp": 0.21179199, + "step": 2829, + "time_per_iteration": 2.50496244430542 + }, + { + "auxiliary_loss_clip": 0.01046186, + "auxiliary_loss_mlp": 0.01004337, + "balance_loss_clip": 1.0205617, + "balance_loss_mlp": 1.00313902, + "epoch": 0.08211943590041205, + "flos": 69302244781440.0, + "grad_norm": 0.7812308613282982, + "language_loss": 0.50105286, + "learning_rate": 3.97157762974638e-06, + "loss": 0.52155811, + "num_input_tokens_seen": 79348430, + "router_z_loss_clip": 0.25585938, + "router_z_loss_mlp": 0.01196289, + "step": 2830, + "time_per_iteration": 3.0832738876342773 + }, + { + "auxiliary_loss_clip": 0.01173144, + "auxiliary_loss_mlp": 0.01053222, + "balance_loss_clip": 1.0659256, + "balance_loss_mlp": 1.03231287, + "epoch": 0.08214845336892809, + "flos": 24893456146560.0, + "grad_norm": 2.6198823263758477, + "language_loss": 0.78275728, + "learning_rate": 3.9715460454728655e-06, + "loss": 0.80502099, + "num_input_tokens_seen": 79361765, + "router_z_loss_clip": 1.07324219, + "router_z_loss_mlp": 0.20898438, + "step": 2831, + "time_per_iteration": 2.5269107818603516 + }, + { + "auxiliary_loss_clip": 0.01165965, + "auxiliary_loss_mlp": 0.01064737, + "balance_loss_clip": 1.06870675, + "balance_loss_mlp": 1.04481685, + "epoch": 0.08217747083744414, + "flos": 35036999222400.0, + "grad_norm": 2.1422606489439326, + "language_loss": 0.88778079, + "learning_rate": 3.971514443785858e-06, + "loss": 0.91008782, + "num_input_tokens_seen": 79380390, + "router_z_loss_clip": 0.97363281, + "router_z_loss_mlp": 0.19909668, + "step": 2832, + "time_per_iteration": 2.702880620956421 + }, + { + "auxiliary_loss_clip": 0.01165548, + "auxiliary_loss_mlp": 0.01055176, + "balance_loss_clip": 1.06660581, + "balance_loss_mlp": 1.03709829, + "epoch": 0.08220648830596018, + "flos": 10806454529280.0, + "grad_norm": 2.670829623363462, + "language_loss": 0.78591549, + "learning_rate": 3.971482824685637e-06, + "loss": 0.80812269, + "num_input_tokens_seen": 79390070, + "router_z_loss_clip": 0.98828125, + "router_z_loss_mlp": 0.18078613, + "step": 2833, + "time_per_iteration": 2.554717540740967 + }, + { + "auxiliary_loss_clip": 0.01169098, + "auxiliary_loss_mlp": 0.01053303, + "balance_loss_clip": 1.06446922, + "balance_loss_mlp": 1.03534997, + "epoch": 0.08223550577447623, + "flos": 32416562327040.0, + "grad_norm": 2.283097326199179, + "language_loss": 0.85213554, + "learning_rate": 3.971451188172482e-06, + "loss": 0.87435949, + "num_input_tokens_seen": 79408015, + "router_z_loss_clip": 1.04736328, + "router_z_loss_mlp": 0.17944336, + "step": 2834, + "time_per_iteration": 2.49122953414917 + }, + { + "auxiliary_loss_clip": 0.01177152, + "auxiliary_loss_mlp": 0.01064616, + "balance_loss_clip": 1.07001889, + "balance_loss_mlp": 1.04066658, + "epoch": 0.08226452324299229, + "flos": 20075716569600.0, + "grad_norm": 5.032624876896704, + "language_loss": 1.06788421, + "learning_rate": 3.971419534246673e-06, + "loss": 1.09030199, + "num_input_tokens_seen": 79419590, + "router_z_loss_clip": 1.07226562, + "router_z_loss_mlp": 0.23986816, + "step": 2835, + "time_per_iteration": 2.498002767562866 + }, + { + "auxiliary_loss_clip": 0.01161965, + "auxiliary_loss_mlp": 0.01041381, + "balance_loss_clip": 1.06471062, + "balance_loss_mlp": 1.02379727, + "epoch": 0.08229354071150832, + "flos": 26206457898240.0, + "grad_norm": 2.595564769718072, + "language_loss": 0.90999496, + "learning_rate": 3.971387862908488e-06, + "loss": 0.93202841, + "num_input_tokens_seen": 79433355, + "router_z_loss_clip": 0.97314453, + "router_z_loss_mlp": 0.17596436, + "step": 2836, + "time_per_iteration": 2.567929744720459 + }, + { + "auxiliary_loss_clip": 0.01159394, + "auxiliary_loss_mlp": 0.01053153, + "balance_loss_clip": 1.06697297, + "balance_loss_mlp": 1.03388321, + "epoch": 0.08232255818002437, + "flos": 31644697944960.0, + "grad_norm": 2.322530762338982, + "language_loss": 0.98786837, + "learning_rate": 3.971356174158207e-06, + "loss": 1.00999391, + "num_input_tokens_seen": 79449785, + "router_z_loss_clip": 0.92382812, + "router_z_loss_mlp": 0.19281006, + "step": 2837, + "time_per_iteration": 2.6232378482818604 + }, + { + "auxiliary_loss_clip": 0.01045865, + "auxiliary_loss_mlp": 0.01001751, + "balance_loss_clip": 1.02025294, + "balance_loss_mlp": 1.00041568, + "epoch": 0.08235157564854043, + "flos": 60792851980800.0, + "grad_norm": 0.7931094042483734, + "language_loss": 0.54640627, + "learning_rate": 3.971324467996112e-06, + "loss": 0.56688249, + "num_input_tokens_seen": 79501635, + "router_z_loss_clip": 0.2578125, + "router_z_loss_mlp": 0.0133667, + "step": 2838, + "time_per_iteration": 2.892758369445801 + }, + { + "auxiliary_loss_clip": 0.0104384, + "auxiliary_loss_mlp": 0.01002245, + "balance_loss_clip": 1.01830101, + "balance_loss_mlp": 1.00091612, + "epoch": 0.08238059311705646, + "flos": 65361372019200.0, + "grad_norm": 0.7184477534834943, + "language_loss": 0.51445359, + "learning_rate": 3.971292744422481e-06, + "loss": 0.53491443, + "num_input_tokens_seen": 79560665, + "router_z_loss_clip": 0.25585938, + "router_z_loss_mlp": 0.01330566, + "step": 2839, + "time_per_iteration": 3.0191848278045654 + }, + { + "auxiliary_loss_clip": 0.01161918, + "auxiliary_loss_mlp": 0.01054671, + "balance_loss_clip": 1.06423593, + "balance_loss_mlp": 1.03524601, + "epoch": 0.08240961058557252, + "flos": 21793395922560.0, + "grad_norm": 2.1388152761119934, + "language_loss": 0.8183105, + "learning_rate": 3.971261003437595e-06, + "loss": 0.84047639, + "num_input_tokens_seen": 79575580, + "router_z_loss_clip": 0.9765625, + "router_z_loss_mlp": 0.19421387, + "step": 2840, + "time_per_iteration": 2.4620273113250732 + }, + { + "auxiliary_loss_clip": 0.01041705, + "auxiliary_loss_mlp": 0.01001637, + "balance_loss_clip": 1.01606166, + "balance_loss_mlp": 1.00021827, + "epoch": 0.08243862805408857, + "flos": 62265441859200.0, + "grad_norm": 0.649628943901843, + "language_loss": 0.50002718, + "learning_rate": 3.9712292450417345e-06, + "loss": 0.52046061, + "num_input_tokens_seen": 79639355, + "router_z_loss_clip": 0.2578125, + "router_z_loss_mlp": 0.01416016, + "step": 2841, + "time_per_iteration": 3.127969741821289 + }, + { + "auxiliary_loss_clip": 0.01168423, + "auxiliary_loss_mlp": 0.01053609, + "balance_loss_clip": 1.0694356, + "balance_loss_mlp": 1.03421402, + "epoch": 0.0824676455226046, + "flos": 37189556536320.0, + "grad_norm": 2.2712517060311828, + "language_loss": 0.59210479, + "learning_rate": 3.971197469235179e-06, + "loss": 0.61432511, + "num_input_tokens_seen": 79656910, + "router_z_loss_clip": 0.98974609, + "router_z_loss_mlp": 0.1940918, + "step": 2842, + "time_per_iteration": 2.6163368225097656 + }, + { + "auxiliary_loss_clip": 0.01170989, + "auxiliary_loss_mlp": 0.01051911, + "balance_loss_clip": 1.06965494, + "balance_loss_mlp": 1.03307557, + "epoch": 0.08249666299112066, + "flos": 20877242607360.0, + "grad_norm": 2.3982959009555476, + "language_loss": 0.85687959, + "learning_rate": 3.97116567601821e-06, + "loss": 0.87910861, + "num_input_tokens_seen": 79670550, + "router_z_loss_clip": 1.01318359, + "router_z_loss_mlp": 0.18835449, + "step": 2843, + "time_per_iteration": 2.5408201217651367 + }, + { + "auxiliary_loss_clip": 0.01162355, + "auxiliary_loss_mlp": 0.01051165, + "balance_loss_clip": 1.06661415, + "balance_loss_mlp": 1.0330925, + "epoch": 0.08252568045963671, + "flos": 11429037987840.0, + "grad_norm": 2.271917959125553, + "language_loss": 0.75408959, + "learning_rate": 3.971133865391108e-06, + "loss": 0.77622479, + "num_input_tokens_seen": 79683340, + "router_z_loss_clip": 0.95751953, + "router_z_loss_mlp": 0.18078613, + "step": 2844, + "time_per_iteration": 2.5261640548706055 + }, + { + "auxiliary_loss_clip": 0.01165797, + "auxiliary_loss_mlp": 0.01053407, + "balance_loss_clip": 1.06131911, + "balance_loss_mlp": 1.0326165, + "epoch": 0.08255469792815275, + "flos": 23396052948480.0, + "grad_norm": 2.423390058847291, + "language_loss": 0.91569531, + "learning_rate": 3.971102037354154e-06, + "loss": 0.93788731, + "num_input_tokens_seen": 79699075, + "router_z_loss_clip": 1.04492188, + "router_z_loss_mlp": 0.20788574, + "step": 2845, + "time_per_iteration": 2.5829596519470215 + }, + { + "auxiliary_loss_clip": 0.01040992, + "auxiliary_loss_mlp": 0.00999435, + "balance_loss_clip": 1.01538658, + "balance_loss_mlp": 0.99806434, + "epoch": 0.0825837153966688, + "flos": 68680954212480.0, + "grad_norm": 0.7073481799391471, + "language_loss": 0.49923033, + "learning_rate": 3.97107019190763e-06, + "loss": 0.5196346, + "num_input_tokens_seen": 79764135, + "router_z_loss_clip": 0.2578125, + "router_z_loss_mlp": 0.01373291, + "step": 2846, + "time_per_iteration": 3.229438543319702 + }, + { + "auxiliary_loss_clip": 0.01165409, + "auxiliary_loss_mlp": 0.01047867, + "balance_loss_clip": 1.06822181, + "balance_loss_mlp": 1.02925277, + "epoch": 0.08261273286518483, + "flos": 16502030588160.0, + "grad_norm": 2.7507899661048163, + "language_loss": 0.77716976, + "learning_rate": 3.971038329051816e-06, + "loss": 0.79930246, + "num_input_tokens_seen": 79776670, + "router_z_loss_clip": 0.97265625, + "router_z_loss_mlp": 0.18615723, + "step": 2847, + "time_per_iteration": 7.275501728057861 + }, + { + "auxiliary_loss_clip": 0.01154861, + "auxiliary_loss_mlp": 0.01049001, + "balance_loss_clip": 1.061957, + "balance_loss_mlp": 1.03083372, + "epoch": 0.08264175033370089, + "flos": 45106673978880.0, + "grad_norm": 2.1844372256972036, + "language_loss": 0.89123917, + "learning_rate": 3.971006448786993e-06, + "loss": 0.9132778, + "num_input_tokens_seen": 79796380, + "router_z_loss_clip": 0.9296875, + "router_z_loss_mlp": 0.18170166, + "step": 2848, + "time_per_iteration": 4.987576007843018 + }, + { + "auxiliary_loss_clip": 0.01165208, + "auxiliary_loss_mlp": 0.01055682, + "balance_loss_clip": 1.06445169, + "balance_loss_mlp": 1.03580415, + "epoch": 0.08267076780221694, + "flos": 41317236956160.0, + "grad_norm": 1.9538738720255915, + "language_loss": 0.9335115, + "learning_rate": 3.970974551113444e-06, + "loss": 0.95572042, + "num_input_tokens_seen": 79818490, + "router_z_loss_clip": 1.00732422, + "router_z_loss_mlp": 0.19891357, + "step": 2849, + "time_per_iteration": 2.710655927658081 + }, + { + "auxiliary_loss_clip": 0.01172829, + "auxiliary_loss_mlp": 0.01052169, + "balance_loss_clip": 1.07048106, + "balance_loss_mlp": 1.03400135, + "epoch": 0.08269978527073298, + "flos": 10515937328640.0, + "grad_norm": 2.781313088625408, + "language_loss": 0.99993974, + "learning_rate": 3.970942636031451e-06, + "loss": 1.02218974, + "num_input_tokens_seen": 79829675, + "router_z_loss_clip": 1.02294922, + "router_z_loss_mlp": 0.1817627, + "step": 2850, + "time_per_iteration": 4.826336860656738 + }, + { + "auxiliary_loss_clip": 0.01168013, + "auxiliary_loss_mlp": 0.01053142, + "balance_loss_clip": 1.06706977, + "balance_loss_mlp": 1.03234029, + "epoch": 0.08272880273924903, + "flos": 29856310669440.0, + "grad_norm": 2.0846737684613084, + "language_loss": 0.81683636, + "learning_rate": 3.970910703541295e-06, + "loss": 0.83904791, + "num_input_tokens_seen": 79846265, + "router_z_loss_clip": 1.0078125, + "router_z_loss_mlp": 0.20788574, + "step": 2851, + "time_per_iteration": 2.5480661392211914 + }, + { + "auxiliary_loss_clip": 0.01158606, + "auxiliary_loss_mlp": 0.01044178, + "balance_loss_clip": 1.06481099, + "balance_loss_mlp": 1.02663016, + "epoch": 0.08275782020776508, + "flos": 41463177914880.0, + "grad_norm": 2.060732004170301, + "language_loss": 0.8929792, + "learning_rate": 3.970878753643257e-06, + "loss": 0.91500711, + "num_input_tokens_seen": 79872505, + "router_z_loss_clip": 0.93798828, + "router_z_loss_mlp": 0.17553711, + "step": 2852, + "time_per_iteration": 2.76092267036438 + }, + { + "auxiliary_loss_clip": 0.01174291, + "auxiliary_loss_mlp": 0.01057274, + "balance_loss_clip": 1.06882334, + "balance_loss_mlp": 1.03737783, + "epoch": 0.08278683767628112, + "flos": 10882477664640.0, + "grad_norm": 2.2295769310975024, + "language_loss": 0.73903203, + "learning_rate": 3.970846786337621e-06, + "loss": 0.76134765, + "num_input_tokens_seen": 79884075, + "router_z_loss_clip": 1.0546875, + "router_z_loss_mlp": 0.19891357, + "step": 2853, + "time_per_iteration": 2.4742624759674072 + }, + { + "auxiliary_loss_clip": 0.01169723, + "auxiliary_loss_mlp": 0.01046621, + "balance_loss_clip": 1.07275319, + "balance_loss_mlp": 1.02822721, + "epoch": 0.08281585514479717, + "flos": 16392826264320.0, + "grad_norm": 2.87092063135059, + "language_loss": 0.90306532, + "learning_rate": 3.970814801624668e-06, + "loss": 0.92522877, + "num_input_tokens_seen": 79896940, + "router_z_loss_clip": 0.96923828, + "router_z_loss_mlp": 0.18395996, + "step": 2854, + "time_per_iteration": 2.4846363067626953 + }, + { + "auxiliary_loss_clip": 0.01043458, + "auxiliary_loss_mlp": 0.01003109, + "balance_loss_clip": 1.01723146, + "balance_loss_mlp": 1.0016669, + "epoch": 0.08284487261331322, + "flos": 59146167859200.0, + "grad_norm": 0.7497696051043361, + "language_loss": 0.52317864, + "learning_rate": 3.970782799504682e-06, + "loss": 0.54364431, + "num_input_tokens_seen": 79952015, + "router_z_loss_clip": 0.26171875, + "router_z_loss_mlp": 0.0144043, + "step": 2855, + "time_per_iteration": 3.1348888874053955 + }, + { + "auxiliary_loss_clip": 0.0117156, + "auxiliary_loss_mlp": 0.0105321, + "balance_loss_clip": 1.06837249, + "balance_loss_mlp": 1.03400576, + "epoch": 0.08287389008182926, + "flos": 16796354630400.0, + "grad_norm": 2.887168093855374, + "language_loss": 0.90193892, + "learning_rate": 3.970750779977944e-06, + "loss": 0.92418659, + "num_input_tokens_seen": 79965315, + "router_z_loss_clip": 1.03222656, + "router_z_loss_mlp": 0.1920166, + "step": 2856, + "time_per_iteration": 2.507638931274414 + }, + { + "auxiliary_loss_clip": 0.01161157, + "auxiliary_loss_mlp": 0.01039521, + "balance_loss_clip": 1.06242549, + "balance_loss_mlp": 1.0223664, + "epoch": 0.08290290755034531, + "flos": 24201709050240.0, + "grad_norm": 2.99168621096394, + "language_loss": 0.5844444, + "learning_rate": 3.9707187430447384e-06, + "loss": 0.60645115, + "num_input_tokens_seen": 79983950, + "router_z_loss_clip": 0.98632812, + "router_z_loss_mlp": 0.17156982, + "step": 2857, + "time_per_iteration": 2.4589650630950928 + }, + { + "auxiliary_loss_clip": 0.01164303, + "auxiliary_loss_mlp": 0.01058529, + "balance_loss_clip": 1.06920671, + "balance_loss_mlp": 1.03751266, + "epoch": 0.08293192501886136, + "flos": 20113386958080.0, + "grad_norm": 2.321145105477115, + "language_loss": 0.82595748, + "learning_rate": 3.970686688705347e-06, + "loss": 0.8481859, + "num_input_tokens_seen": 79997820, + "router_z_loss_clip": 0.95068359, + "router_z_loss_mlp": 0.21020508, + "step": 2858, + "time_per_iteration": 2.493480920791626 + }, + { + "auxiliary_loss_clip": 0.01171068, + "auxiliary_loss_mlp": 0.01047989, + "balance_loss_clip": 1.06894541, + "balance_loss_mlp": 1.02827215, + "epoch": 0.0829609424873774, + "flos": 27994557864960.0, + "grad_norm": 2.2894935059397667, + "language_loss": 0.66939592, + "learning_rate": 3.970654616960054e-06, + "loss": 0.69158649, + "num_input_tokens_seen": 80011150, + "router_z_loss_clip": 1.02001953, + "router_z_loss_mlp": 0.19714355, + "step": 2859, + "time_per_iteration": 2.4911508560180664 + }, + { + "auxiliary_loss_clip": 0.01161276, + "auxiliary_loss_mlp": 0.0104734, + "balance_loss_clip": 1.0656414, + "balance_loss_mlp": 1.02811193, + "epoch": 0.08298995995589345, + "flos": 11758590293760.0, + "grad_norm": 2.62834000563037, + "language_loss": 0.79983288, + "learning_rate": 3.970622527809142e-06, + "loss": 0.82191908, + "num_input_tokens_seen": 80022510, + "router_z_loss_clip": 0.95507812, + "router_z_loss_mlp": 0.19250488, + "step": 2860, + "time_per_iteration": 2.4587645530700684 + }, + { + "auxiliary_loss_clip": 0.01047051, + "auxiliary_loss_mlp": 0.01002772, + "balance_loss_clip": 1.02043867, + "balance_loss_mlp": 1.00143647, + "epoch": 0.0830189774244095, + "flos": 66557448023040.0, + "grad_norm": 0.6556371109338044, + "language_loss": 0.47983462, + "learning_rate": 3.970590421252893e-06, + "loss": 0.50033283, + "num_input_tokens_seen": 80079365, + "router_z_loss_clip": 0.265625, + "router_z_loss_mlp": 0.0133667, + "step": 2861, + "time_per_iteration": 3.0704967975616455 + }, + { + "auxiliary_loss_clip": 0.01175414, + "auxiliary_loss_mlp": 0.01051894, + "balance_loss_clip": 1.07000756, + "balance_loss_mlp": 1.03024578, + "epoch": 0.08304799489292554, + "flos": 48907854748800.0, + "grad_norm": 2.5566686433353976, + "language_loss": 0.85321587, + "learning_rate": 3.970558297291593e-06, + "loss": 0.87548894, + "num_input_tokens_seen": 80099900, + "router_z_loss_clip": 1.05419922, + "router_z_loss_mlp": 0.21655273, + "step": 2862, + "time_per_iteration": 2.6861519813537598 + }, + { + "auxiliary_loss_clip": 0.01169475, + "auxiliary_loss_mlp": 0.0104405, + "balance_loss_clip": 1.0668925, + "balance_loss_mlp": 1.02448201, + "epoch": 0.08307701236144159, + "flos": 21061931362560.0, + "grad_norm": 2.4242492588394313, + "language_loss": 1.00318575, + "learning_rate": 3.9705261559255246e-06, + "loss": 1.02532113, + "num_input_tokens_seen": 80112670, + "router_z_loss_clip": 1.02587891, + "router_z_loss_mlp": 0.19586182, + "step": 2863, + "time_per_iteration": 2.5078017711639404 + }, + { + "auxiliary_loss_clip": 0.01171906, + "auxiliary_loss_mlp": 0.01047942, + "balance_loss_clip": 1.06556034, + "balance_loss_mlp": 1.02801645, + "epoch": 0.08310602982995763, + "flos": 74732113981440.0, + "grad_norm": 1.7563118915232165, + "language_loss": 0.84693354, + "learning_rate": 3.970493997154972e-06, + "loss": 0.86913204, + "num_input_tokens_seen": 80141800, + "router_z_loss_clip": 1.06445312, + "router_z_loss_mlp": 0.19952393, + "step": 2864, + "time_per_iteration": 2.936858892440796 + }, + { + "auxiliary_loss_clip": 0.01175657, + "auxiliary_loss_mlp": 0.01057932, + "balance_loss_clip": 1.06807184, + "balance_loss_mlp": 1.03405464, + "epoch": 0.08313504729847368, + "flos": 16246777564800.0, + "grad_norm": 2.0998012792512784, + "language_loss": 0.84117699, + "learning_rate": 3.970461820980218e-06, + "loss": 0.86351287, + "num_input_tokens_seen": 80155880, + "router_z_loss_clip": 1.07568359, + "router_z_loss_mlp": 0.23895264, + "step": 2865, + "time_per_iteration": 2.472717523574829 + }, + { + "auxiliary_loss_clip": 0.01159232, + "auxiliary_loss_mlp": 0.01042341, + "balance_loss_clip": 1.06391811, + "balance_loss_mlp": 1.02354217, + "epoch": 0.08316406476698973, + "flos": 16353934813440.0, + "grad_norm": 3.3993372961724897, + "language_loss": 0.79008281, + "learning_rate": 3.97042962740155e-06, + "loss": 0.8120985, + "num_input_tokens_seen": 80165950, + "router_z_loss_clip": 0.953125, + "router_z_loss_mlp": 0.18780518, + "step": 2866, + "time_per_iteration": 2.4701731204986572 + }, + { + "auxiliary_loss_clip": 0.0117341, + "auxiliary_loss_mlp": 0.01057524, + "balance_loss_clip": 1.07103443, + "balance_loss_mlp": 1.03597069, + "epoch": 0.08319308223550577, + "flos": 13690332748800.0, + "grad_norm": 2.642850864438424, + "language_loss": 0.84113169, + "learning_rate": 3.970397416419248e-06, + "loss": 0.86344105, + "num_input_tokens_seen": 80176790, + "router_z_loss_clip": 1.02246094, + "router_z_loss_mlp": 0.2154541, + "step": 2867, + "time_per_iteration": 2.4589579105377197 + }, + { + "auxiliary_loss_clip": 0.01045385, + "auxiliary_loss_mlp": 0.00999812, + "balance_loss_clip": 1.01923323, + "balance_loss_mlp": 0.99844664, + "epoch": 0.08322209970402182, + "flos": 65590626574080.0, + "grad_norm": 0.6613996436686809, + "language_loss": 0.50175124, + "learning_rate": 3.9703651880336e-06, + "loss": 0.52220321, + "num_input_tokens_seen": 80244220, + "router_z_loss_clip": 0.26171875, + "router_z_loss_mlp": 0.01367188, + "step": 2868, + "time_per_iteration": 3.2716500759124756 + }, + { + "auxiliary_loss_clip": 0.01164219, + "auxiliary_loss_mlp": 0.01045951, + "balance_loss_clip": 1.06948757, + "balance_loss_mlp": 1.0283854, + "epoch": 0.08325111717253787, + "flos": 31206408192000.0, + "grad_norm": 1.7391583995248674, + "language_loss": 0.84098423, + "learning_rate": 3.9703329422448884e-06, + "loss": 0.86308581, + "num_input_tokens_seen": 80264805, + "router_z_loss_clip": 0.94677734, + "router_z_loss_mlp": 0.17559814, + "step": 2869, + "time_per_iteration": 2.618091583251953 + }, + { + "auxiliary_loss_clip": 0.01164635, + "auxiliary_loss_mlp": 0.0105271, + "balance_loss_clip": 1.06549382, + "balance_loss_mlp": 1.03523421, + "epoch": 0.08328013464105391, + "flos": 15663013729920.0, + "grad_norm": 2.3703912473356605, + "language_loss": 0.82228971, + "learning_rate": 3.970300679053399e-06, + "loss": 0.84446317, + "num_input_tokens_seen": 80279240, + "router_z_loss_clip": 0.99169922, + "router_z_loss_mlp": 0.17456055, + "step": 2870, + "time_per_iteration": 2.510850429534912 + }, + { + "auxiliary_loss_clip": 0.01167007, + "auxiliary_loss_mlp": 0.01060927, + "balance_loss_clip": 1.07117152, + "balance_loss_mlp": 1.04185903, + "epoch": 0.08330915210956996, + "flos": 16062412032000.0, + "grad_norm": 2.940268775622006, + "language_loss": 0.73787165, + "learning_rate": 3.970268398459417e-06, + "loss": 0.76015103, + "num_input_tokens_seen": 80291680, + "router_z_loss_clip": 0.95703125, + "router_z_loss_mlp": 0.19067383, + "step": 2871, + "time_per_iteration": 2.500556230545044 + }, + { + "auxiliary_loss_clip": 0.01153612, + "auxiliary_loss_mlp": 0.01040392, + "balance_loss_clip": 1.06105757, + "balance_loss_mlp": 1.02488852, + "epoch": 0.08333816957808601, + "flos": 43100452673280.0, + "grad_norm": 2.603428532874577, + "language_loss": 0.76480639, + "learning_rate": 3.970236100463228e-06, + "loss": 0.78674638, + "num_input_tokens_seen": 80309670, + "router_z_loss_clip": 0.92626953, + "router_z_loss_mlp": 0.1552124, + "step": 2872, + "time_per_iteration": 2.7119436264038086 + }, + { + "auxiliary_loss_clip": 0.0116855, + "auxiliary_loss_mlp": 0.01047673, + "balance_loss_clip": 1.0658828, + "balance_loss_mlp": 1.02936888, + "epoch": 0.08336718704660205, + "flos": 16610875776000.0, + "grad_norm": 3.3298318480161404, + "language_loss": 0.72962213, + "learning_rate": 3.970203785065116e-06, + "loss": 0.75178432, + "num_input_tokens_seen": 80322605, + "router_z_loss_clip": 1.02539062, + "router_z_loss_mlp": 0.18304443, + "step": 2873, + "time_per_iteration": 2.4966275691986084 + }, + { + "auxiliary_loss_clip": 0.01166299, + "auxiliary_loss_mlp": 0.01046329, + "balance_loss_clip": 1.06443143, + "balance_loss_mlp": 1.02696908, + "epoch": 0.0833962045151181, + "flos": 18910487370240.0, + "grad_norm": 2.4165767997004948, + "language_loss": 0.72258925, + "learning_rate": 3.970171452265366e-06, + "loss": 0.74471551, + "num_input_tokens_seen": 80338735, + "router_z_loss_clip": 1.01953125, + "router_z_loss_mlp": 0.19360352, + "step": 2874, + "time_per_iteration": 2.593238592147827 + }, + { + "auxiliary_loss_clip": 0.01042589, + "auxiliary_loss_mlp": 0.01004229, + "balance_loss_clip": 1.01629758, + "balance_loss_mlp": 1.0029, + "epoch": 0.08342522198363415, + "flos": 66132338561280.0, + "grad_norm": 0.6397977345155227, + "language_loss": 0.46889532, + "learning_rate": 3.970139102064265e-06, + "loss": 0.48936349, + "num_input_tokens_seen": 80402755, + "router_z_loss_clip": 0.26171875, + "router_z_loss_mlp": 0.01330566, + "step": 2875, + "time_per_iteration": 3.2413294315338135 + }, + { + "auxiliary_loss_clip": 0.0104135, + "auxiliary_loss_mlp": 0.0100463, + "balance_loss_clip": 1.01509643, + "balance_loss_mlp": 1.00330687, + "epoch": 0.08345423945215019, + "flos": 61082974131840.0, + "grad_norm": 0.7983981619272001, + "language_loss": 0.48862278, + "learning_rate": 3.970106734462099e-06, + "loss": 0.50908256, + "num_input_tokens_seen": 80460090, + "router_z_loss_clip": 0.26171875, + "router_z_loss_mlp": 0.01324463, + "step": 2876, + "time_per_iteration": 2.911073684692383 + }, + { + "auxiliary_loss_clip": 0.01167131, + "auxiliary_loss_mlp": 0.01052983, + "balance_loss_clip": 1.06720078, + "balance_loss_mlp": 1.03476739, + "epoch": 0.08348325692066624, + "flos": 38466970888320.0, + "grad_norm": 1.9711979315895938, + "language_loss": 0.88689148, + "learning_rate": 3.970074349459152e-06, + "loss": 0.90909266, + "num_input_tokens_seen": 80481000, + "router_z_loss_clip": 0.99951172, + "router_z_loss_mlp": 0.18212891, + "step": 2877, + "time_per_iteration": 2.6559784412384033 + }, + { + "auxiliary_loss_clip": 0.0116665, + "auxiliary_loss_mlp": 0.01066448, + "balance_loss_clip": 1.06565595, + "balance_loss_mlp": 1.04714847, + "epoch": 0.08351227438918228, + "flos": 20114751674880.0, + "grad_norm": 2.485696403232717, + "language_loss": 0.73295462, + "learning_rate": 3.970041947055712e-06, + "loss": 0.75528562, + "num_input_tokens_seen": 80496045, + "router_z_loss_clip": 1.01025391, + "router_z_loss_mlp": 0.1932373, + "step": 2878, + "time_per_iteration": 2.483325719833374 + }, + { + "auxiliary_loss_clip": 0.01180122, + "auxiliary_loss_mlp": 0.0105337, + "balance_loss_clip": 1.06881249, + "balance_loss_mlp": 1.03269863, + "epoch": 0.08354129185769833, + "flos": 27633548223360.0, + "grad_norm": 2.0011867398770664, + "language_loss": 1.09241748, + "learning_rate": 3.970009527252064e-06, + "loss": 1.11475229, + "num_input_tokens_seen": 80520330, + "router_z_loss_clip": 1.11328125, + "router_z_loss_mlp": 0.20666504, + "step": 2879, + "time_per_iteration": 2.741806745529175 + }, + { + "auxiliary_loss_clip": 0.01170996, + "auxiliary_loss_mlp": 0.01052797, + "balance_loss_clip": 1.06361032, + "balance_loss_mlp": 1.03225672, + "epoch": 0.08357030932621438, + "flos": 25405255082880.0, + "grad_norm": 2.9217933794708473, + "language_loss": 1.04955959, + "learning_rate": 3.969977090048495e-06, + "loss": 1.07179749, + "num_input_tokens_seen": 80535250, + "router_z_loss_clip": 1.07226562, + "router_z_loss_mlp": 0.20544434, + "step": 2880, + "time_per_iteration": 2.557312250137329 + }, + { + "auxiliary_loss_clip": 0.01039111, + "auxiliary_loss_mlp": 0.01003415, + "balance_loss_clip": 1.01279497, + "balance_loss_mlp": 1.00207984, + "epoch": 0.08359932679473042, + "flos": 74771223891840.0, + "grad_norm": 0.7535687288758434, + "language_loss": 0.56371999, + "learning_rate": 3.9699446354452904e-06, + "loss": 0.58414519, + "num_input_tokens_seen": 80592605, + "router_z_loss_clip": 0.26367188, + "router_z_loss_mlp": 0.0133667, + "step": 2881, + "time_per_iteration": 3.0379066467285156 + }, + { + "auxiliary_loss_clip": 0.01038729, + "auxiliary_loss_mlp": 0.0100105, + "balance_loss_clip": 1.01238132, + "balance_loss_mlp": 0.99975663, + "epoch": 0.08362834426324647, + "flos": 68503160868480.0, + "grad_norm": 0.6317213653427752, + "language_loss": 0.47551972, + "learning_rate": 3.969912163442738e-06, + "loss": 0.49591753, + "num_input_tokens_seen": 80654020, + "router_z_loss_clip": 0.26367188, + "router_z_loss_mlp": 0.01293945, + "step": 2882, + "time_per_iteration": 3.079193353652954 + }, + { + "auxiliary_loss_clip": 0.01036964, + "auxiliary_loss_mlp": 0.01000522, + "balance_loss_clip": 1.01056659, + "balance_loss_mlp": 0.99919254, + "epoch": 0.08365736173176253, + "flos": 74790042618240.0, + "grad_norm": 0.6451097393135555, + "language_loss": 0.52955216, + "learning_rate": 3.969879674041125e-06, + "loss": 0.549927, + "num_input_tokens_seen": 80721415, + "router_z_loss_clip": 0.265625, + "router_z_loss_mlp": 0.01330566, + "step": 2883, + "time_per_iteration": 3.2383651733398438 + }, + { + "auxiliary_loss_clip": 0.01153885, + "auxiliary_loss_mlp": 0.01043467, + "balance_loss_clip": 1.05971074, + "balance_loss_mlp": 1.02711177, + "epoch": 0.08368637920027856, + "flos": 40509677433600.0, + "grad_norm": 2.7335894667917064, + "language_loss": 0.83731687, + "learning_rate": 3.969847167240736e-06, + "loss": 0.85929042, + "num_input_tokens_seen": 80737960, + "router_z_loss_clip": 0.94238281, + "router_z_loss_mlp": 0.16351318, + "step": 2884, + "time_per_iteration": 2.5846898555755615 + }, + { + "auxiliary_loss_clip": 0.01169824, + "auxiliary_loss_mlp": 0.01056824, + "balance_loss_clip": 1.06396914, + "balance_loss_mlp": 1.03573632, + "epoch": 0.08371539666879461, + "flos": 34308012700800.0, + "grad_norm": 2.9964795803798103, + "language_loss": 0.88932037, + "learning_rate": 3.969814643041861e-06, + "loss": 0.91158688, + "num_input_tokens_seen": 80755925, + "router_z_loss_clip": 1.05859375, + "router_z_loss_mlp": 0.21069336, + "step": 2885, + "time_per_iteration": 2.6293694972991943 + }, + { + "auxiliary_loss_clip": 0.01164238, + "auxiliary_loss_mlp": 0.01049448, + "balance_loss_clip": 1.06152678, + "balance_loss_mlp": 1.03170967, + "epoch": 0.08374441413731067, + "flos": 17780378693760.0, + "grad_norm": 2.349759055029016, + "language_loss": 0.80321491, + "learning_rate": 3.969782101444785e-06, + "loss": 0.82535172, + "num_input_tokens_seen": 80770785, + "router_z_loss_clip": 1.02587891, + "router_z_loss_mlp": 0.17749023, + "step": 2886, + "time_per_iteration": 2.539280652999878 + }, + { + "auxiliary_loss_clip": 0.01165677, + "auxiliary_loss_mlp": 0.01050114, + "balance_loss_clip": 1.0620507, + "balance_loss_mlp": 1.02978253, + "epoch": 0.0837734316058267, + "flos": 26461998230400.0, + "grad_norm": 2.4159811530165634, + "language_loss": 0.8273918, + "learning_rate": 3.969749542449797e-06, + "loss": 0.84954971, + "num_input_tokens_seen": 80787430, + "router_z_loss_clip": 1.03759766, + "router_z_loss_mlp": 0.20330811, + "step": 2887, + "time_per_iteration": 2.5232391357421875 + }, + { + "auxiliary_loss_clip": 0.01037777, + "auxiliary_loss_mlp": 0.01013535, + "balance_loss_clip": 1.01134276, + "balance_loss_mlp": 1.01215219, + "epoch": 0.08380244907434276, + "flos": 74780453687040.0, + "grad_norm": 0.7029304162849286, + "language_loss": 0.504287, + "learning_rate": 3.969716966057184e-06, + "loss": 0.52480018, + "num_input_tokens_seen": 80850515, + "router_z_loss_clip": 0.26367188, + "router_z_loss_mlp": 0.01385498, + "step": 2888, + "time_per_iteration": 3.1454241275787354 + }, + { + "auxiliary_loss_clip": 0.011625, + "auxiliary_loss_mlp": 0.01048885, + "balance_loss_clip": 1.0569756, + "balance_loss_mlp": 1.02779686, + "epoch": 0.0838314665428588, + "flos": 21099924973440.0, + "grad_norm": 2.4984284052378425, + "language_loss": 1.07892847, + "learning_rate": 3.969684372267235e-06, + "loss": 1.10104227, + "num_input_tokens_seen": 80864320, + "router_z_loss_clip": 1.05664062, + "router_z_loss_mlp": 0.21081543, + "step": 2889, + "time_per_iteration": 2.5036816596984863 + }, + { + "auxiliary_loss_clip": 0.01162037, + "auxiliary_loss_mlp": 0.01057612, + "balance_loss_clip": 1.05995822, + "balance_loss_mlp": 1.03961706, + "epoch": 0.08386048401137484, + "flos": 17084501533440.0, + "grad_norm": 3.0338023643672702, + "language_loss": 0.99940634, + "learning_rate": 3.9696517610802345e-06, + "loss": 1.02160287, + "num_input_tokens_seen": 80877340, + "router_z_loss_clip": 1.02001953, + "router_z_loss_mlp": 0.17999268, + "step": 2890, + "time_per_iteration": 2.508784770965576 + }, + { + "auxiliary_loss_clip": 0.01167488, + "auxiliary_loss_mlp": 0.01058299, + "balance_loss_clip": 1.06410146, + "balance_loss_mlp": 1.03934455, + "epoch": 0.0838895014798909, + "flos": 11765485704960.0, + "grad_norm": 2.6193794643441795, + "language_loss": 0.88869798, + "learning_rate": 3.969619132496473e-06, + "loss": 0.91095585, + "num_input_tokens_seen": 80888470, + "router_z_loss_clip": 1.03417969, + "router_z_loss_mlp": 0.18963623, + "step": 2891, + "time_per_iteration": 2.501286745071411 + }, + { + "auxiliary_loss_clip": 0.01035499, + "auxiliary_loss_mlp": 0.01007609, + "balance_loss_clip": 1.00872052, + "balance_loss_mlp": 1.00625575, + "epoch": 0.08391851894840695, + "flos": 65801313797760.0, + "grad_norm": 0.7044139351330698, + "language_loss": 0.53205538, + "learning_rate": 3.969586486516239e-06, + "loss": 0.55248648, + "num_input_tokens_seen": 80946685, + "router_z_loss_clip": 0.26953125, + "router_z_loss_mlp": 0.0135498, + "step": 2892, + "time_per_iteration": 3.0266036987304688 + }, + { + "auxiliary_loss_clip": 0.01164985, + "auxiliary_loss_mlp": 0.01047056, + "balance_loss_clip": 1.06189668, + "balance_loss_mlp": 1.02847147, + "epoch": 0.08394753641692299, + "flos": 26137689310080.0, + "grad_norm": 1.947367251497586, + "language_loss": 0.91196239, + "learning_rate": 3.96955382313982e-06, + "loss": 0.93408275, + "num_input_tokens_seen": 80966870, + "router_z_loss_clip": 1.03027344, + "router_z_loss_mlp": 0.18597412, + "step": 2893, + "time_per_iteration": 2.5906424522399902 + }, + { + "auxiliary_loss_clip": 0.01158887, + "auxiliary_loss_mlp": 0.01047863, + "balance_loss_clip": 1.05866671, + "balance_loss_mlp": 1.030375, + "epoch": 0.08397655388543904, + "flos": 17450754560640.0, + "grad_norm": 2.428454212555609, + "language_loss": 0.81455624, + "learning_rate": 3.969521142367504e-06, + "loss": 0.83662367, + "num_input_tokens_seen": 80984620, + "router_z_loss_clip": 1.00439453, + "router_z_loss_mlp": 0.17510986, + "step": 2894, + "time_per_iteration": 2.4705429077148438 + }, + { + "auxiliary_loss_clip": 0.01157137, + "auxiliary_loss_mlp": 0.01054919, + "balance_loss_clip": 1.05845523, + "balance_loss_mlp": 1.03624535, + "epoch": 0.08400557135395507, + "flos": 26133666986880.0, + "grad_norm": 4.510988157078783, + "language_loss": 0.80095398, + "learning_rate": 3.969488444199581e-06, + "loss": 0.82307452, + "num_input_tokens_seen": 81000215, + "router_z_loss_clip": 0.98779297, + "router_z_loss_mlp": 0.18652344, + "step": 2895, + "time_per_iteration": 2.5271830558776855 + }, + { + "auxiliary_loss_clip": 0.01165034, + "auxiliary_loss_mlp": 0.01040797, + "balance_loss_clip": 1.06636333, + "balance_loss_mlp": 1.02336872, + "epoch": 0.08403458882247113, + "flos": 23543825500800.0, + "grad_norm": 2.384552884990417, + "language_loss": 0.88953257, + "learning_rate": 3.969455728636339e-06, + "loss": 0.91159087, + "num_input_tokens_seen": 81015230, + "router_z_loss_clip": 0.98632812, + "router_z_loss_mlp": 0.17419434, + "step": 2896, + "time_per_iteration": 2.499079704284668 + }, + { + "auxiliary_loss_clip": 0.01160143, + "auxiliary_loss_mlp": 0.01054391, + "balance_loss_clip": 1.0593257, + "balance_loss_mlp": 1.03555036, + "epoch": 0.08406360629098718, + "flos": 27556088544000.0, + "grad_norm": 3.268823844734532, + "language_loss": 0.77009344, + "learning_rate": 3.969422995678067e-06, + "loss": 0.79223877, + "num_input_tokens_seen": 81033775, + "router_z_loss_clip": 1.0078125, + "router_z_loss_mlp": 0.18847656, + "step": 2897, + "time_per_iteration": 2.542773723602295 + }, + { + "auxiliary_loss_clip": 0.01155402, + "auxiliary_loss_mlp": 0.01042145, + "balance_loss_clip": 1.06102777, + "balance_loss_mlp": 1.02534795, + "epoch": 0.08409262375950322, + "flos": 16214637870720.0, + "grad_norm": 2.6176633764232333, + "language_loss": 0.62387699, + "learning_rate": 3.969390245325053e-06, + "loss": 0.64585245, + "num_input_tokens_seen": 81048230, + "router_z_loss_clip": 0.94335938, + "router_z_loss_mlp": 0.16796875, + "step": 2898, + "time_per_iteration": 2.475376605987549 + }, + { + "auxiliary_loss_clip": 0.01157121, + "auxiliary_loss_mlp": 0.01042019, + "balance_loss_clip": 1.06148648, + "balance_loss_mlp": 1.02600932, + "epoch": 0.08412164122801927, + "flos": 44083938032640.0, + "grad_norm": 2.110537391469996, + "language_loss": 0.74272156, + "learning_rate": 3.969357477577589e-06, + "loss": 0.76471299, + "num_input_tokens_seen": 81067855, + "router_z_loss_clip": 0.95605469, + "router_z_loss_mlp": 0.16009521, + "step": 2899, + "time_per_iteration": 2.7017626762390137 + }, + { + "auxiliary_loss_clip": 0.01167674, + "auxiliary_loss_mlp": 0.01062335, + "balance_loss_clip": 1.06581926, + "balance_loss_mlp": 1.04242682, + "epoch": 0.08415065869653532, + "flos": 74731359795840.0, + "grad_norm": 2.17787848775502, + "language_loss": 0.76336277, + "learning_rate": 3.969324692435962e-06, + "loss": 0.78566289, + "num_input_tokens_seen": 81089800, + "router_z_loss_clip": 1.01855469, + "router_z_loss_mlp": 0.19897461, + "step": 2900, + "time_per_iteration": 2.8839073181152344 + }, + { + "auxiliary_loss_clip": 0.01168302, + "auxiliary_loss_mlp": 0.01045264, + "balance_loss_clip": 1.06419039, + "balance_loss_mlp": 1.02725148, + "epoch": 0.08417967616505136, + "flos": 25512879208320.0, + "grad_norm": 3.066924106427263, + "language_loss": 0.98719859, + "learning_rate": 3.969291889900463e-06, + "loss": 1.00933433, + "num_input_tokens_seen": 81105500, + "router_z_loss_clip": 1.04052734, + "router_z_loss_mlp": 0.18005371, + "step": 2901, + "time_per_iteration": 2.537652015686035 + }, + { + "auxiliary_loss_clip": 0.01035804, + "auxiliary_loss_mlp": 0.01008679, + "balance_loss_clip": 1.00930548, + "balance_loss_mlp": 1.00745094, + "epoch": 0.08420869363356741, + "flos": 59120492613120.0, + "grad_norm": 0.6680225815899946, + "language_loss": 0.4833706, + "learning_rate": 3.969259069971381e-06, + "loss": 0.50381541, + "num_input_tokens_seen": 81168915, + "router_z_loss_clip": 0.265625, + "router_z_loss_mlp": 0.01226807, + "step": 2902, + "time_per_iteration": 3.1048166751861572 + }, + { + "auxiliary_loss_clip": 0.0116082, + "auxiliary_loss_mlp": 0.01051538, + "balance_loss_clip": 1.06025219, + "balance_loss_mlp": 1.03214288, + "epoch": 0.08423771110208346, + "flos": 30038234077440.0, + "grad_norm": 2.9285491587631043, + "language_loss": 0.87325311, + "learning_rate": 3.9692262326490054e-06, + "loss": 0.89537668, + "num_input_tokens_seen": 81184685, + "router_z_loss_clip": 1.00488281, + "router_z_loss_mlp": 0.1940918, + "step": 2903, + "time_per_iteration": 2.56111478805542 + }, + { + "auxiliary_loss_clip": 0.01159083, + "auxiliary_loss_mlp": 0.01044681, + "balance_loss_clip": 1.06188774, + "balance_loss_mlp": 1.0276103, + "epoch": 0.0842667285705995, + "flos": 11612074717440.0, + "grad_norm": 3.7396672625892537, + "language_loss": 0.89710826, + "learning_rate": 3.969193377933628e-06, + "loss": 0.91914588, + "num_input_tokens_seen": 81196130, + "router_z_loss_clip": 0.97167969, + "router_z_loss_mlp": 0.17071533, + "step": 2904, + "time_per_iteration": 2.425121307373047 + }, + { + "auxiliary_loss_clip": 0.0116244, + "auxiliary_loss_mlp": 0.01050693, + "balance_loss_clip": 1.0637598, + "balance_loss_mlp": 1.03265643, + "epoch": 0.08429574603911555, + "flos": 22885331420160.0, + "grad_norm": 2.537101059259879, + "language_loss": 0.9286176, + "learning_rate": 3.969160505825536e-06, + "loss": 0.95074892, + "num_input_tokens_seen": 81209205, + "router_z_loss_clip": 0.98632812, + "router_z_loss_mlp": 0.18048096, + "step": 2905, + "time_per_iteration": 2.511457920074463 + }, + { + "auxiliary_loss_clip": 0.01161169, + "auxiliary_loss_mlp": 0.01052849, + "balance_loss_clip": 1.06194496, + "balance_loss_mlp": 1.03449082, + "epoch": 0.0843247635076316, + "flos": 32627824168320.0, + "grad_norm": 1.8935997316961084, + "language_loss": 0.86129069, + "learning_rate": 3.9691276163250235e-06, + "loss": 0.88343084, + "num_input_tokens_seen": 81231660, + "router_z_loss_clip": 0.99169922, + "router_z_loss_mlp": 0.18359375, + "step": 2906, + "time_per_iteration": 2.6813113689422607 + }, + { + "auxiliary_loss_clip": 0.01155966, + "auxiliary_loss_mlp": 0.01038756, + "balance_loss_clip": 1.05907202, + "balance_loss_mlp": 1.0201714, + "epoch": 0.08435378097614764, + "flos": 74736818663040.0, + "grad_norm": 1.7064570781246102, + "language_loss": 0.84246397, + "learning_rate": 3.969094709432378e-06, + "loss": 0.86441123, + "num_input_tokens_seen": 81258125, + "router_z_loss_clip": 0.96777344, + "router_z_loss_mlp": 0.18597412, + "step": 2907, + "time_per_iteration": 2.9096362590789795 + }, + { + "auxiliary_loss_clip": 0.0115831, + "auxiliary_loss_mlp": 0.01048141, + "balance_loss_clip": 1.06037033, + "balance_loss_mlp": 1.02799428, + "epoch": 0.08438279844466369, + "flos": 25367512867200.0, + "grad_norm": 2.6649006999901146, + "language_loss": 0.76824594, + "learning_rate": 3.9690617851478915e-06, + "loss": 0.7903105, + "num_input_tokens_seen": 81273645, + "router_z_loss_clip": 0.97949219, + "router_z_loss_mlp": 0.20147705, + "step": 2908, + "time_per_iteration": 2.52925181388855 + }, + { + "auxiliary_loss_clip": 0.01173814, + "auxiliary_loss_mlp": 0.01048782, + "balance_loss_clip": 1.06638265, + "balance_loss_mlp": 1.02869546, + "epoch": 0.08441181591317973, + "flos": 50143648216320.0, + "grad_norm": 2.106705446811621, + "language_loss": 1.05449617, + "learning_rate": 3.969028843471854e-06, + "loss": 1.07672215, + "num_input_tokens_seen": 81298960, + "router_z_loss_clip": 1.07421875, + "router_z_loss_mlp": 0.20092773, + "step": 2909, + "time_per_iteration": 2.7910549640655518 + }, + { + "auxiliary_loss_clip": 0.01040333, + "auxiliary_loss_mlp": 0.01006582, + "balance_loss_clip": 1.01336336, + "balance_loss_mlp": 1.00533581, + "epoch": 0.08444083338169578, + "flos": 61345553529600.0, + "grad_norm": 0.6896925415053167, + "language_loss": 0.50086224, + "learning_rate": 3.968995884404558e-06, + "loss": 0.52133137, + "num_input_tokens_seen": 81355070, + "router_z_loss_clip": 0.26953125, + "router_z_loss_mlp": 0.01245117, + "step": 2910, + "time_per_iteration": 3.025477170944214 + }, + { + "auxiliary_loss_clip": 0.01164983, + "auxiliary_loss_mlp": 0.01069083, + "balance_loss_clip": 1.06265569, + "balance_loss_mlp": 1.04721975, + "epoch": 0.08446985085021183, + "flos": 10371432913920.0, + "grad_norm": 3.3449289594327616, + "language_loss": 1.0227778, + "learning_rate": 3.968962907946293e-06, + "loss": 1.04511833, + "num_input_tokens_seen": 81364915, + "router_z_loss_clip": 1.02246094, + "router_z_loss_mlp": 0.21862793, + "step": 2911, + "time_per_iteration": 2.4502651691436768 + }, + { + "auxiliary_loss_clip": 0.01038937, + "auxiliary_loss_mlp": 0.01001026, + "balance_loss_clip": 1.01206148, + "balance_loss_mlp": 0.99981004, + "epoch": 0.08449886831872787, + "flos": 74779699501440.0, + "grad_norm": 0.7033708155024534, + "language_loss": 0.45233524, + "learning_rate": 3.968929914097351e-06, + "loss": 0.47273487, + "num_input_tokens_seen": 81431215, + "router_z_loss_clip": 0.26953125, + "router_z_loss_mlp": 0.012146, + "step": 2912, + "time_per_iteration": 3.234984874725342 + }, + { + "auxiliary_loss_clip": 0.01037938, + "auxiliary_loss_mlp": 0.01000954, + "balance_loss_clip": 1.01136231, + "balance_loss_mlp": 0.99980336, + "epoch": 0.08452788578724392, + "flos": 74781567008640.0, + "grad_norm": 0.7067215013794893, + "language_loss": 0.53287625, + "learning_rate": 3.968896902858023e-06, + "loss": 0.55326509, + "num_input_tokens_seen": 81501990, + "router_z_loss_clip": 0.265625, + "router_z_loss_mlp": 0.01147461, + "step": 2913, + "time_per_iteration": 3.256112575531006 + }, + { + "auxiliary_loss_clip": 0.01036974, + "auxiliary_loss_mlp": 0.00999759, + "balance_loss_clip": 1.01044273, + "balance_loss_mlp": 0.99863845, + "epoch": 0.08455690325575997, + "flos": 74778765747840.0, + "grad_norm": 0.6941646196100828, + "language_loss": 0.53647143, + "learning_rate": 3.968863874228601e-06, + "loss": 0.55683875, + "num_input_tokens_seen": 81569450, + "router_z_loss_clip": 0.265625, + "router_z_loss_mlp": 0.01123047, + "step": 2914, + "time_per_iteration": 3.253899097442627 + }, + { + "auxiliary_loss_clip": 0.01159108, + "auxiliary_loss_mlp": 0.01049852, + "balance_loss_clip": 1.06106687, + "balance_loss_mlp": 1.03251886, + "epoch": 0.08458592072427601, + "flos": 25075738690560.0, + "grad_norm": 2.8991336458019674, + "language_loss": 0.85236436, + "learning_rate": 3.968830828209377e-06, + "loss": 0.8744539, + "num_input_tokens_seen": 81585420, + "router_z_loss_clip": 0.98095703, + "router_z_loss_mlp": 0.17333984, + "step": 2915, + "time_per_iteration": 2.5344834327697754 + }, + { + "auxiliary_loss_clip": 0.01161719, + "auxiliary_loss_mlp": 0.01055526, + "balance_loss_clip": 1.06362808, + "balance_loss_mlp": 1.03762114, + "epoch": 0.08461493819279206, + "flos": 33504834637440.0, + "grad_norm": 2.207387306095762, + "language_loss": 0.81338811, + "learning_rate": 3.968797764800642e-06, + "loss": 0.83556056, + "num_input_tokens_seen": 81603545, + "router_z_loss_clip": 0.98144531, + "router_z_loss_mlp": 0.17907715, + "step": 2916, + "time_per_iteration": 2.6295166015625 + }, + { + "auxiliary_loss_clip": 0.01158658, + "auxiliary_loss_mlp": 0.01047445, + "balance_loss_clip": 1.06028688, + "balance_loss_mlp": 1.02875257, + "epoch": 0.08464395566130811, + "flos": 32704565575680.0, + "grad_norm": 4.033989958976574, + "language_loss": 0.89300072, + "learning_rate": 3.968764684002688e-06, + "loss": 0.91506171, + "num_input_tokens_seen": 81619200, + "router_z_loss_clip": 0.98339844, + "router_z_loss_mlp": 0.18682861, + "step": 2917, + "time_per_iteration": 2.5926499366760254 + }, + { + "auxiliary_loss_clip": 0.01164508, + "auxiliary_loss_mlp": 0.01055747, + "balance_loss_clip": 1.06340921, + "balance_loss_mlp": 1.03488529, + "epoch": 0.08467297312982415, + "flos": 24201026691840.0, + "grad_norm": 2.8541615012763244, + "language_loss": 0.96698403, + "learning_rate": 3.968731585815808e-06, + "loss": 0.98918658, + "num_input_tokens_seen": 81634375, + "router_z_loss_clip": 1.01220703, + "router_z_loss_mlp": 0.20874023, + "step": 2918, + "time_per_iteration": 2.4905049800872803 + }, + { + "auxiliary_loss_clip": 0.01164628, + "auxiliary_loss_mlp": 0.0106227, + "balance_loss_clip": 1.06280982, + "balance_loss_mlp": 1.04304123, + "epoch": 0.0847019905983402, + "flos": 30478750473600.0, + "grad_norm": 2.32284690373774, + "language_loss": 0.91083956, + "learning_rate": 3.968698470240294e-06, + "loss": 0.93310851, + "num_input_tokens_seen": 81650875, + "router_z_loss_clip": 1.02099609, + "router_z_loss_mlp": 0.19213867, + "step": 2919, + "time_per_iteration": 7.319621801376343 + }, + { + "auxiliary_loss_clip": 0.01156315, + "auxiliary_loss_mlp": 0.01048099, + "balance_loss_clip": 1.05875897, + "balance_loss_mlp": 1.02926338, + "epoch": 0.08473100806685625, + "flos": 22631012150400.0, + "grad_norm": 2.7045637136335787, + "language_loss": 0.81059039, + "learning_rate": 3.968665337276439e-06, + "loss": 0.83263451, + "num_input_tokens_seen": 81662870, + "router_z_loss_clip": 0.97607422, + "router_z_loss_mlp": 0.18835449, + "step": 2920, + "time_per_iteration": 4.815542459487915 + }, + { + "auxiliary_loss_clip": 0.01164241, + "auxiliary_loss_mlp": 0.01045878, + "balance_loss_clip": 1.06101465, + "balance_loss_mlp": 1.02570772, + "epoch": 0.08476002553537229, + "flos": 31940638099200.0, + "grad_norm": 3.1441560356739418, + "language_loss": 0.8121649, + "learning_rate": 3.968632186924534e-06, + "loss": 0.83426607, + "num_input_tokens_seen": 81687295, + "router_z_loss_clip": 1.03271484, + "router_z_loss_mlp": 0.20141602, + "step": 2921, + "time_per_iteration": 4.945970773696899 + }, + { + "auxiliary_loss_clip": 0.01042658, + "auxiliary_loss_mlp": 0.01023244, + "balance_loss_clip": 1.01611984, + "balance_loss_mlp": 1.02210546, + "epoch": 0.08478904300388834, + "flos": 74778550266240.0, + "grad_norm": 0.637758627182964, + "language_loss": 0.48854405, + "learning_rate": 3.968599019184874e-06, + "loss": 0.50920308, + "num_input_tokens_seen": 81756140, + "router_z_loss_clip": 0.265625, + "router_z_loss_mlp": 0.01141357, + "step": 2922, + "time_per_iteration": 3.308703899383545 + }, + { + "auxiliary_loss_clip": 0.01042054, + "auxiliary_loss_mlp": 0.01015899, + "balance_loss_clip": 1.01560771, + "balance_loss_mlp": 1.01476693, + "epoch": 0.0848180604724044, + "flos": 59487643480320.0, + "grad_norm": 0.6534293520500567, + "language_loss": 0.4418056, + "learning_rate": 3.96856583405775e-06, + "loss": 0.46238515, + "num_input_tokens_seen": 81815875, + "router_z_loss_clip": 0.265625, + "router_z_loss_mlp": 0.01135254, + "step": 2923, + "time_per_iteration": 3.090297222137451 + }, + { + "auxiliary_loss_clip": 0.01038289, + "auxiliary_loss_mlp": 0.01006692, + "balance_loss_clip": 1.01191688, + "balance_loss_mlp": 1.00553548, + "epoch": 0.08484707794092043, + "flos": 58351860455040.0, + "grad_norm": 0.6158309176174391, + "language_loss": 0.46706033, + "learning_rate": 3.968532631543457e-06, + "loss": 0.48751014, + "num_input_tokens_seen": 81879675, + "router_z_loss_clip": 0.26367188, + "router_z_loss_mlp": 0.01153564, + "step": 2924, + "time_per_iteration": 3.129243850708008 + }, + { + "auxiliary_loss_clip": 0.01036606, + "auxiliary_loss_mlp": 0.01003761, + "balance_loss_clip": 1.01033854, + "balance_loss_mlp": 1.0025332, + "epoch": 0.08487609540943648, + "flos": 53233296474240.0, + "grad_norm": 0.7325949710028506, + "language_loss": 0.49128872, + "learning_rate": 3.9684994116422855e-06, + "loss": 0.5116924, + "num_input_tokens_seen": 81928060, + "router_z_loss_clip": 0.26171875, + "router_z_loss_mlp": 0.01226807, + "step": 2925, + "time_per_iteration": 2.8207077980041504 + }, + { + "auxiliary_loss_clip": 0.01162155, + "auxiliary_loss_mlp": 0.01050031, + "balance_loss_clip": 1.06188321, + "balance_loss_mlp": 1.02845395, + "epoch": 0.08490511287795252, + "flos": 24381872691840.0, + "grad_norm": 2.792675436791864, + "language_loss": 0.82019871, + "learning_rate": 3.968466174354532e-06, + "loss": 0.84232056, + "num_input_tokens_seen": 81940890, + "router_z_loss_clip": 1.00292969, + "router_z_loss_mlp": 0.21582031, + "step": 2926, + "time_per_iteration": 2.5313198566436768 + }, + { + "auxiliary_loss_clip": 0.01156281, + "auxiliary_loss_mlp": 0.01047585, + "balance_loss_clip": 1.06277823, + "balance_loss_mlp": 1.03263021, + "epoch": 0.08493413034646857, + "flos": 14827875540480.0, + "grad_norm": 4.233120124781573, + "language_loss": 0.86161137, + "learning_rate": 3.968432919680489e-06, + "loss": 0.88365006, + "num_input_tokens_seen": 81954135, + "router_z_loss_clip": 0.93554688, + "router_z_loss_mlp": 0.1494751, + "step": 2927, + "time_per_iteration": 2.4400486946105957 + }, + { + "auxiliary_loss_clip": 0.01038499, + "auxiliary_loss_mlp": 0.01012171, + "balance_loss_clip": 1.01213288, + "balance_loss_mlp": 1.01099038, + "epoch": 0.08496314781498462, + "flos": 74770792928640.0, + "grad_norm": 0.6966802349991797, + "language_loss": 0.50825173, + "learning_rate": 3.968399647620449e-06, + "loss": 0.52875841, + "num_input_tokens_seen": 82015780, + "router_z_loss_clip": 0.26367188, + "router_z_loss_mlp": 0.01177979, + "step": 2928, + "time_per_iteration": 3.0998501777648926 + }, + { + "auxiliary_loss_clip": 0.01038575, + "auxiliary_loss_mlp": 0.01007067, + "balance_loss_clip": 1.01207066, + "balance_loss_mlp": 1.00589263, + "epoch": 0.08499216528350066, + "flos": 63829243347840.0, + "grad_norm": 0.6858295317550097, + "language_loss": 0.49361473, + "learning_rate": 3.9683663581747075e-06, + "loss": 0.51407117, + "num_input_tokens_seen": 82075535, + "router_z_loss_clip": 0.265625, + "router_z_loss_mlp": 0.01171875, + "step": 2929, + "time_per_iteration": 3.0154776573181152 + }, + { + "auxiliary_loss_clip": 0.01166701, + "auxiliary_loss_mlp": 0.01059303, + "balance_loss_clip": 1.06271362, + "balance_loss_mlp": 1.03806543, + "epoch": 0.08502118275201671, + "flos": 22413142206720.0, + "grad_norm": 2.434885224011045, + "language_loss": 0.72066343, + "learning_rate": 3.968333051343557e-06, + "loss": 0.7429235, + "num_input_tokens_seen": 82091210, + "router_z_loss_clip": 1.0390625, + "router_z_loss_mlp": 0.21221924, + "step": 2930, + "time_per_iteration": 2.4756786823272705 + }, + { + "auxiliary_loss_clip": 0.01036121, + "auxiliary_loss_mlp": 0.01000208, + "balance_loss_clip": 1.00968611, + "balance_loss_mlp": 0.99895614, + "epoch": 0.08505020022053277, + "flos": 51962669792640.0, + "grad_norm": 0.7111923457756285, + "language_loss": 0.51726604, + "learning_rate": 3.968299727127292e-06, + "loss": 0.53762925, + "num_input_tokens_seen": 82140880, + "router_z_loss_clip": 0.265625, + "router_z_loss_mlp": 0.01251221, + "step": 2931, + "time_per_iteration": 2.8021514415740967 + }, + { + "auxiliary_loss_clip": 0.01161093, + "auxiliary_loss_mlp": 0.01048159, + "balance_loss_clip": 1.06331062, + "balance_loss_mlp": 1.02989578, + "epoch": 0.0850792176890488, + "flos": 16392359387520.0, + "grad_norm": 3.3755543496996876, + "language_loss": 0.70005631, + "learning_rate": 3.968266385526209e-06, + "loss": 0.72214884, + "num_input_tokens_seen": 82154165, + "router_z_loss_clip": 0.97802734, + "router_z_loss_mlp": 0.18286133, + "step": 2932, + "time_per_iteration": 2.5461153984069824 + }, + { + "auxiliary_loss_clip": 0.01175931, + "auxiliary_loss_mlp": 0.01047416, + "balance_loss_clip": 1.0707109, + "balance_loss_mlp": 1.02856922, + "epoch": 0.08510823515756485, + "flos": 25808998930560.0, + "grad_norm": 1.8886517656521082, + "language_loss": 0.89734739, + "learning_rate": 3.9682330265406e-06, + "loss": 0.91958094, + "num_input_tokens_seen": 82170685, + "router_z_loss_clip": 1.05224609, + "router_z_loss_mlp": 0.18859863, + "step": 2933, + "time_per_iteration": 2.5600879192352295 + }, + { + "auxiliary_loss_clip": 0.01036772, + "auxiliary_loss_mlp": 0.01008968, + "balance_loss_clip": 1.01063991, + "balance_loss_mlp": 1.00776958, + "epoch": 0.0851372526260809, + "flos": 74778406611840.0, + "grad_norm": 0.7696371287233759, + "language_loss": 0.48375434, + "learning_rate": 3.96819965017076e-06, + "loss": 0.50421172, + "num_input_tokens_seen": 82233585, + "router_z_loss_clip": 0.26171875, + "router_z_loss_mlp": 0.01196289, + "step": 2934, + "time_per_iteration": 3.1510403156280518 + }, + { + "auxiliary_loss_clip": 0.01157787, + "auxiliary_loss_mlp": 0.01048894, + "balance_loss_clip": 1.0615387, + "balance_loss_mlp": 1.03025007, + "epoch": 0.08516627009459694, + "flos": 25075379554560.0, + "grad_norm": 3.4211985218606755, + "language_loss": 0.83369493, + "learning_rate": 3.968166256416985e-06, + "loss": 0.85576177, + "num_input_tokens_seen": 82246235, + "router_z_loss_clip": 0.96240234, + "router_z_loss_mlp": 0.18658447, + "step": 2935, + "time_per_iteration": 2.5305593013763428 + }, + { + "auxiliary_loss_clip": 0.01169246, + "auxiliary_loss_mlp": 0.01054829, + "balance_loss_clip": 1.06365943, + "balance_loss_mlp": 1.03326392, + "epoch": 0.085195287563113, + "flos": 21828696013440.0, + "grad_norm": 2.7751842964033875, + "language_loss": 0.93982983, + "learning_rate": 3.968132845279569e-06, + "loss": 0.96207058, + "num_input_tokens_seen": 82259175, + "router_z_loss_clip": 1.05664062, + "router_z_loss_mlp": 0.2154541, + "step": 2936, + "time_per_iteration": 2.4428932666778564 + }, + { + "auxiliary_loss_clip": 0.01155312, + "auxiliary_loss_mlp": 0.01039261, + "balance_loss_clip": 1.06400013, + "balance_loss_mlp": 1.02442515, + "epoch": 0.08522430503162905, + "flos": 19164267936000.0, + "grad_norm": 2.921289857197678, + "language_loss": 0.8244164, + "learning_rate": 3.968099416758807e-06, + "loss": 0.84636211, + "num_input_tokens_seen": 82272095, + "router_z_loss_clip": 0.91308594, + "router_z_loss_mlp": 0.14831543, + "step": 2937, + "time_per_iteration": 2.4853224754333496 + }, + { + "auxiliary_loss_clip": 0.01164732, + "auxiliary_loss_mlp": 0.010547, + "balance_loss_clip": 1.06218243, + "balance_loss_mlp": 1.03599644, + "epoch": 0.08525332250014508, + "flos": 20551712624640.0, + "grad_norm": 2.0834362753380398, + "language_loss": 0.82064039, + "learning_rate": 3.968065970854994e-06, + "loss": 0.84283471, + "num_input_tokens_seen": 82290455, + "router_z_loss_clip": 1.02539062, + "router_z_loss_mlp": 0.18701172, + "step": 2938, + "time_per_iteration": 2.5938427448272705 + }, + { + "auxiliary_loss_clip": 0.01170549, + "auxiliary_loss_mlp": 0.01062116, + "balance_loss_clip": 1.0632602, + "balance_loss_mlp": 1.04223156, + "epoch": 0.08528233996866114, + "flos": 23287207760640.0, + "grad_norm": 2.2453738492524424, + "language_loss": 0.80608213, + "learning_rate": 3.968032507568427e-06, + "loss": 0.82840884, + "num_input_tokens_seen": 82304355, + "router_z_loss_clip": 1.07421875, + "router_z_loss_mlp": 0.19873047, + "step": 2939, + "time_per_iteration": 2.5042097568511963 + }, + { + "auxiliary_loss_clip": 0.0116508, + "auxiliary_loss_mlp": 0.01045093, + "balance_loss_clip": 1.06311572, + "balance_loss_mlp": 1.02569795, + "epoch": 0.08531135743717717, + "flos": 24308650817280.0, + "grad_norm": 4.241003021285616, + "language_loss": 0.90279269, + "learning_rate": 3.9679990268994e-06, + "loss": 0.92489439, + "num_input_tokens_seen": 82323635, + "router_z_loss_clip": 1.02050781, + "router_z_loss_mlp": 0.19372559, + "step": 2940, + "time_per_iteration": 2.6181678771972656 + }, + { + "auxiliary_loss_clip": 0.01167513, + "auxiliary_loss_mlp": 0.01051696, + "balance_loss_clip": 1.06460869, + "balance_loss_mlp": 1.0324378, + "epoch": 0.08534037490569323, + "flos": 21867300155520.0, + "grad_norm": 2.5575175526835885, + "language_loss": 0.87630826, + "learning_rate": 3.96796552884821e-06, + "loss": 0.89850038, + "num_input_tokens_seen": 82337660, + "router_z_loss_clip": 1.02978516, + "router_z_loss_mlp": 0.19256592, + "step": 2941, + "time_per_iteration": 2.513448476791382 + }, + { + "auxiliary_loss_clip": 0.01157118, + "auxiliary_loss_mlp": 0.0104208, + "balance_loss_clip": 1.06174684, + "balance_loss_mlp": 1.02551603, + "epoch": 0.08536939237420928, + "flos": 25841641415040.0, + "grad_norm": 2.056575899233352, + "language_loss": 0.69500208, + "learning_rate": 3.967932013415151e-06, + "loss": 0.71699405, + "num_input_tokens_seen": 82351460, + "router_z_loss_clip": 0.95458984, + "router_z_loss_mlp": 0.16558838, + "step": 2942, + "time_per_iteration": 2.5141351222991943 + }, + { + "auxiliary_loss_clip": 0.01036062, + "auxiliary_loss_mlp": 0.01026131, + "balance_loss_clip": 1.01043212, + "balance_loss_mlp": 1.02501035, + "epoch": 0.08539840984272531, + "flos": 65438508476160.0, + "grad_norm": 0.7626605243467133, + "language_loss": 0.52733326, + "learning_rate": 3.967898480600521e-06, + "loss": 0.54795516, + "num_input_tokens_seen": 82412940, + "router_z_loss_clip": 0.25585938, + "router_z_loss_mlp": 0.01123047, + "step": 2943, + "time_per_iteration": 3.065199851989746 + }, + { + "auxiliary_loss_clip": 0.01164077, + "auxiliary_loss_mlp": 0.01053017, + "balance_loss_clip": 1.06056237, + "balance_loss_mlp": 1.03459883, + "epoch": 0.08542742731124137, + "flos": 32336265473280.0, + "grad_norm": 2.437418747655934, + "language_loss": 0.97099328, + "learning_rate": 3.967864930404615e-06, + "loss": 0.99316424, + "num_input_tokens_seen": 82432090, + "router_z_loss_clip": 1.03515625, + "router_z_loss_mlp": 0.18414307, + "step": 2944, + "time_per_iteration": 2.586176872253418 + }, + { + "auxiliary_loss_clip": 0.01155134, + "auxiliary_loss_mlp": 0.01049144, + "balance_loss_clip": 1.05939507, + "balance_loss_mlp": 1.02915263, + "epoch": 0.08545644477975742, + "flos": 14312305676160.0, + "grad_norm": 2.1490869453834787, + "language_loss": 0.86749268, + "learning_rate": 3.9678313628277295e-06, + "loss": 0.88953543, + "num_input_tokens_seen": 82445605, + "router_z_loss_clip": 0.95800781, + "router_z_loss_mlp": 0.19995117, + "step": 2945, + "time_per_iteration": 2.4923107624053955 + }, + { + "auxiliary_loss_clip": 0.01153276, + "auxiliary_loss_mlp": 0.01045709, + "balance_loss_clip": 1.05928755, + "balance_loss_mlp": 1.0296278, + "epoch": 0.08548546224827346, + "flos": 25770215220480.0, + "grad_norm": 2.429001181207054, + "language_loss": 0.90613693, + "learning_rate": 3.967797777870161e-06, + "loss": 0.92812681, + "num_input_tokens_seen": 82462585, + "router_z_loss_clip": 0.94091797, + "router_z_loss_mlp": 0.1607666, + "step": 2946, + "time_per_iteration": 2.5102384090423584 + }, + { + "auxiliary_loss_clip": 0.01175212, + "auxiliary_loss_mlp": 0.01065716, + "balance_loss_clip": 1.06547892, + "balance_loss_mlp": 1.04235101, + "epoch": 0.08551447971678951, + "flos": 13149482688000.0, + "grad_norm": 2.801514368156684, + "language_loss": 0.95834267, + "learning_rate": 3.967764175532207e-06, + "loss": 0.98075199, + "num_input_tokens_seen": 82475850, + "router_z_loss_clip": 1.09765625, + "router_z_loss_mlp": 0.23376465, + "step": 2947, + "time_per_iteration": 2.500302314758301 + }, + { + "auxiliary_loss_clip": 0.01035501, + "auxiliary_loss_mlp": 0.01000543, + "balance_loss_clip": 1.00962341, + "balance_loss_mlp": 0.99943489, + "epoch": 0.08554349718530556, + "flos": 54451064292480.0, + "grad_norm": 0.6242346468423658, + "language_loss": 0.44335616, + "learning_rate": 3.967730555814162e-06, + "loss": 0.46371657, + "num_input_tokens_seen": 82538950, + "router_z_loss_clip": 0.2578125, + "router_z_loss_mlp": 0.0111084, + "step": 2948, + "time_per_iteration": 3.309819459915161 + }, + { + "auxiliary_loss_clip": 0.01153795, + "auxiliary_loss_mlp": 0.01049821, + "balance_loss_clip": 1.06199276, + "balance_loss_mlp": 1.03281021, + "epoch": 0.0855725146538216, + "flos": 13981280912640.0, + "grad_norm": 2.4827599437413825, + "language_loss": 1.00066853, + "learning_rate": 3.967696918716326e-06, + "loss": 1.0227046, + "num_input_tokens_seen": 82549720, + "router_z_loss_clip": 0.91845703, + "router_z_loss_mlp": 0.17004395, + "step": 2949, + "time_per_iteration": 2.498927354812622 + }, + { + "auxiliary_loss_clip": 0.01155995, + "auxiliary_loss_mlp": 0.01051959, + "balance_loss_clip": 1.06122208, + "balance_loss_mlp": 1.03492415, + "epoch": 0.08560153212233765, + "flos": 16325242824960.0, + "grad_norm": 2.3621747589450917, + "language_loss": 0.75355715, + "learning_rate": 3.967663264238994e-06, + "loss": 0.77563667, + "num_input_tokens_seen": 82564730, + "router_z_loss_clip": 0.94824219, + "router_z_loss_mlp": 0.17041016, + "step": 2950, + "time_per_iteration": 2.460404634475708 + }, + { + "auxiliary_loss_clip": 0.0116226, + "auxiliary_loss_mlp": 0.01045305, + "balance_loss_clip": 1.06273985, + "balance_loss_mlp": 1.02760196, + "epoch": 0.0856305495908537, + "flos": 29672771149440.0, + "grad_norm": 2.3730458634604488, + "language_loss": 0.92169464, + "learning_rate": 3.967629592382463e-06, + "loss": 0.94377041, + "num_input_tokens_seen": 82582765, + "router_z_loss_clip": 0.99560547, + "router_z_loss_mlp": 0.17700195, + "step": 2951, + "time_per_iteration": 2.6593685150146484 + }, + { + "auxiliary_loss_clip": 0.0116088, + "auxiliary_loss_mlp": 0.01050715, + "balance_loss_clip": 1.06412673, + "balance_loss_mlp": 1.03216648, + "epoch": 0.08565956705936974, + "flos": 74733981488640.0, + "grad_norm": 2.243500727919036, + "language_loss": 0.76120424, + "learning_rate": 3.967595903147033e-06, + "loss": 0.78332019, + "num_input_tokens_seen": 82604750, + "router_z_loss_clip": 0.96777344, + "router_z_loss_mlp": 0.18554688, + "step": 2952, + "time_per_iteration": 2.886167049407959 + }, + { + "auxiliary_loss_clip": 0.01161712, + "auxiliary_loss_mlp": 0.01045568, + "balance_loss_clip": 1.06268597, + "balance_loss_mlp": 1.02695918, + "epoch": 0.08568858452788579, + "flos": 51127708193280.0, + "grad_norm": 1.9277912558559054, + "language_loss": 0.6934132, + "learning_rate": 3.9675621965329985e-06, + "loss": 0.71548605, + "num_input_tokens_seen": 82623855, + "router_z_loss_clip": 0.99169922, + "router_z_loss_mlp": 0.18609619, + "step": 2953, + "time_per_iteration": 2.672567129135132 + }, + { + "auxiliary_loss_clip": 0.01161588, + "auxiliary_loss_mlp": 0.01041692, + "balance_loss_clip": 1.06142068, + "balance_loss_mlp": 1.02254689, + "epoch": 0.08571760199640184, + "flos": 11617354016640.0, + "grad_norm": 3.079993583791305, + "language_loss": 0.91518587, + "learning_rate": 3.967528472540658e-06, + "loss": 0.93721867, + "num_input_tokens_seen": 82636850, + "router_z_loss_clip": 1.00195312, + "router_z_loss_mlp": 0.19140625, + "step": 2954, + "time_per_iteration": 2.4973196983337402 + }, + { + "auxiliary_loss_clip": 0.01173665, + "auxiliary_loss_mlp": 0.01067953, + "balance_loss_clip": 1.06604767, + "balance_loss_mlp": 1.04487371, + "epoch": 0.08574661946491788, + "flos": 15883361712000.0, + "grad_norm": 2.1352976454850303, + "language_loss": 0.79623282, + "learning_rate": 3.967494731170311e-06, + "loss": 0.81864899, + "num_input_tokens_seen": 82651110, + "router_z_loss_clip": 1.07568359, + "router_z_loss_mlp": 0.23083496, + "step": 2955, + "time_per_iteration": 2.4451181888580322 + }, + { + "auxiliary_loss_clip": 0.01163007, + "auxiliary_loss_mlp": 0.01057304, + "balance_loss_clip": 1.05880237, + "balance_loss_mlp": 1.03722954, + "epoch": 0.08577563693343393, + "flos": 27046157114880.0, + "grad_norm": 2.2389976049143034, + "language_loss": 0.94739735, + "learning_rate": 3.967460972422254e-06, + "loss": 0.96960044, + "num_input_tokens_seen": 82666475, + "router_z_loss_clip": 1.04150391, + "router_z_loss_mlp": 0.20092773, + "step": 2956, + "time_per_iteration": 2.564434289932251 + }, + { + "auxiliary_loss_clip": 0.01036552, + "auxiliary_loss_mlp": 0.01002514, + "balance_loss_clip": 1.01093018, + "balance_loss_mlp": 1.00130355, + "epoch": 0.08580465440194997, + "flos": 62475410810880.0, + "grad_norm": 0.682977104565172, + "language_loss": 0.49619985, + "learning_rate": 3.967427196296785e-06, + "loss": 0.51659048, + "num_input_tokens_seen": 82725945, + "router_z_loss_clip": 0.2578125, + "router_z_loss_mlp": 0.01208496, + "step": 2957, + "time_per_iteration": 3.056600570678711 + }, + { + "auxiliary_loss_clip": 0.0115749, + "auxiliary_loss_mlp": 0.01048873, + "balance_loss_clip": 1.05926037, + "balance_loss_mlp": 1.03176665, + "epoch": 0.08583367187046602, + "flos": 44266651539840.0, + "grad_norm": 2.010278758760275, + "language_loss": 0.69751847, + "learning_rate": 3.967393402794204e-06, + "loss": 0.71958214, + "num_input_tokens_seen": 82744660, + "router_z_loss_clip": 0.98193359, + "router_z_loss_mlp": 0.17120361, + "step": 2958, + "time_per_iteration": 2.647578477859497 + }, + { + "auxiliary_loss_clip": 0.01037319, + "auxiliary_loss_mlp": 0.0100356, + "balance_loss_clip": 1.01181757, + "balance_loss_mlp": 1.00240993, + "epoch": 0.08586268933898207, + "flos": 63342868262400.0, + "grad_norm": 0.6755227344904121, + "language_loss": 0.47450316, + "learning_rate": 3.967359591914807e-06, + "loss": 0.49491191, + "num_input_tokens_seen": 82799420, + "router_z_loss_clip": 0.25390625, + "router_z_loss_mlp": 0.01147461, + "step": 2959, + "time_per_iteration": 2.9344100952148438 + }, + { + "auxiliary_loss_clip": 0.01159152, + "auxiliary_loss_mlp": 0.01048101, + "balance_loss_clip": 1.0639168, + "balance_loss_mlp": 1.02850866, + "epoch": 0.08589170680749811, + "flos": 21976684047360.0, + "grad_norm": 2.2082332069022783, + "language_loss": 0.77320433, + "learning_rate": 3.9673257636588956e-06, + "loss": 0.79527688, + "num_input_tokens_seen": 82815605, + "router_z_loss_clip": 0.95117188, + "router_z_loss_mlp": 0.19586182, + "step": 2960, + "time_per_iteration": 2.51659893989563 + }, + { + "auxiliary_loss_clip": 0.01168148, + "auxiliary_loss_mlp": 0.01051095, + "balance_loss_clip": 1.06344342, + "balance_loss_mlp": 1.03211701, + "epoch": 0.08592072427601416, + "flos": 20954917768320.0, + "grad_norm": 4.29770670318322, + "language_loss": 0.84656179, + "learning_rate": 3.967291918026766e-06, + "loss": 0.86875427, + "num_input_tokens_seen": 82832975, + "router_z_loss_clip": 1.04882812, + "router_z_loss_mlp": 0.18951416, + "step": 2961, + "time_per_iteration": 2.593583583831787 + }, + { + "auxiliary_loss_clip": 0.01161302, + "auxiliary_loss_mlp": 0.01042009, + "balance_loss_clip": 1.06266284, + "balance_loss_mlp": 1.02440155, + "epoch": 0.08594974174453021, + "flos": 35157480416640.0, + "grad_norm": 1.7330334276370338, + "language_loss": 0.88940853, + "learning_rate": 3.967258055018719e-06, + "loss": 0.91144168, + "num_input_tokens_seen": 82855390, + "router_z_loss_clip": 0.98632812, + "router_z_loss_mlp": 0.17596436, + "step": 2962, + "time_per_iteration": 2.631638526916504 + }, + { + "auxiliary_loss_clip": 0.01168256, + "auxiliary_loss_mlp": 0.0104801, + "balance_loss_clip": 1.06503773, + "balance_loss_mlp": 1.03004551, + "epoch": 0.08597875921304625, + "flos": 29972482231680.0, + "grad_norm": 1.8035490752683012, + "language_loss": 0.92598391, + "learning_rate": 3.967224174635052e-06, + "loss": 0.94814658, + "num_input_tokens_seen": 82874040, + "router_z_loss_clip": 1.03271484, + "router_z_loss_mlp": 0.1796875, + "step": 2963, + "time_per_iteration": 2.591792583465576 + }, + { + "auxiliary_loss_clip": 0.0115849, + "auxiliary_loss_mlp": 0.01042741, + "balance_loss_clip": 1.06509137, + "balance_loss_mlp": 1.02643907, + "epoch": 0.0860077766815623, + "flos": 26060840161920.0, + "grad_norm": 3.814336508949957, + "language_loss": 0.67952466, + "learning_rate": 3.967190276876065e-06, + "loss": 0.70153695, + "num_input_tokens_seen": 82888640, + "router_z_loss_clip": 0.93554688, + "router_z_loss_mlp": 0.16308594, + "step": 2964, + "time_per_iteration": 2.542792558670044 + }, + { + "auxiliary_loss_clip": 0.01169293, + "auxiliary_loss_mlp": 0.01055568, + "balance_loss_clip": 1.06404352, + "balance_loss_mlp": 1.03447962, + "epoch": 0.08603679415007835, + "flos": 16756673080320.0, + "grad_norm": 3.2425745988816073, + "language_loss": 0.8981148, + "learning_rate": 3.967156361742057e-06, + "loss": 0.92036343, + "num_input_tokens_seen": 82901695, + "router_z_loss_clip": 1.05371094, + "router_z_loss_mlp": 0.21081543, + "step": 2965, + "time_per_iteration": 2.447894811630249 + }, + { + "auxiliary_loss_clip": 0.01164025, + "auxiliary_loss_mlp": 0.01052493, + "balance_loss_clip": 1.06696963, + "balance_loss_mlp": 1.03304982, + "epoch": 0.08606581161859439, + "flos": 17053367420160.0, + "grad_norm": 2.2396126040336193, + "language_loss": 0.70874709, + "learning_rate": 3.967122429233328e-06, + "loss": 0.73091227, + "num_input_tokens_seen": 82913445, + "router_z_loss_clip": 0.97119141, + "router_z_loss_mlp": 0.19445801, + "step": 2966, + "time_per_iteration": 2.482645034790039 + }, + { + "auxiliary_loss_clip": 0.01038758, + "auxiliary_loss_mlp": 0.01014426, + "balance_loss_clip": 1.01327872, + "balance_loss_mlp": 1.0133059, + "epoch": 0.08609482908711044, + "flos": 62482557617280.0, + "grad_norm": 0.6862384543850587, + "language_loss": 0.49085078, + "learning_rate": 3.967088479350179e-06, + "loss": 0.51138264, + "num_input_tokens_seen": 82974995, + "router_z_loss_clip": 0.25390625, + "router_z_loss_mlp": 0.01123047, + "step": 2967, + "time_per_iteration": 3.0928094387054443 + }, + { + "auxiliary_loss_clip": 0.01038898, + "auxiliary_loss_mlp": 0.01007938, + "balance_loss_clip": 1.01339567, + "balance_loss_mlp": 1.00684094, + "epoch": 0.0861238465556265, + "flos": 69376220841600.0, + "grad_norm": 0.6421194627935409, + "language_loss": 0.50967509, + "learning_rate": 3.9670545120929075e-06, + "loss": 0.53014344, + "num_input_tokens_seen": 83040845, + "router_z_loss_clip": 0.25390625, + "router_z_loss_mlp": 0.01098633, + "step": 2968, + "time_per_iteration": 3.2134807109832764 + }, + { + "auxiliary_loss_clip": 0.011588, + "auxiliary_loss_mlp": 0.01048357, + "balance_loss_clip": 1.0625751, + "balance_loss_mlp": 1.02930725, + "epoch": 0.08615286402414253, + "flos": 23506873384320.0, + "grad_norm": 2.489030502138631, + "language_loss": 0.95627445, + "learning_rate": 3.967020527461815e-06, + "loss": 0.97834599, + "num_input_tokens_seen": 83055765, + "router_z_loss_clip": 0.96142578, + "router_z_loss_mlp": 0.19042969, + "step": 2969, + "time_per_iteration": 2.535778284072876 + }, + { + "auxiliary_loss_clip": 0.01160559, + "auxiliary_loss_mlp": 0.01047649, + "balance_loss_clip": 1.06488061, + "balance_loss_mlp": 1.03062582, + "epoch": 0.08618188149265858, + "flos": 20551712624640.0, + "grad_norm": 4.165715749275266, + "language_loss": 1.05219376, + "learning_rate": 3.966986525457201e-06, + "loss": 1.07427585, + "num_input_tokens_seen": 83068405, + "router_z_loss_clip": 0.95703125, + "router_z_loss_mlp": 0.17016602, + "step": 2970, + "time_per_iteration": 2.528918504714966 + }, + { + "auxiliary_loss_clip": 0.01164083, + "auxiliary_loss_mlp": 0.01058186, + "balance_loss_clip": 1.0648303, + "balance_loss_mlp": 1.03874862, + "epoch": 0.08621089896117463, + "flos": 44047955583360.0, + "grad_norm": 2.31294733134497, + "language_loss": 0.88277233, + "learning_rate": 3.966952506079366e-06, + "loss": 0.90499508, + "num_input_tokens_seen": 83086835, + "router_z_loss_clip": 0.99316406, + "router_z_loss_mlp": 0.19439697, + "step": 2971, + "time_per_iteration": 2.7563300132751465 + }, + { + "auxiliary_loss_clip": 0.01161289, + "auxiliary_loss_mlp": 0.01058492, + "balance_loss_clip": 1.06523657, + "balance_loss_mlp": 1.03895307, + "epoch": 0.08623991642969067, + "flos": 29243962586880.0, + "grad_norm": 2.398139786374108, + "language_loss": 0.90594518, + "learning_rate": 3.96691846932861e-06, + "loss": 0.92814296, + "num_input_tokens_seen": 83103805, + "router_z_loss_clip": 0.9609375, + "router_z_loss_mlp": 0.19537354, + "step": 2972, + "time_per_iteration": 2.656506299972534 + }, + { + "auxiliary_loss_clip": 0.01160389, + "auxiliary_loss_mlp": 0.01051407, + "balance_loss_clip": 1.06346762, + "balance_loss_mlp": 1.0332278, + "epoch": 0.08626893389820672, + "flos": 37115400908160.0, + "grad_norm": 3.0744104749248033, + "language_loss": 0.94136202, + "learning_rate": 3.966884415205234e-06, + "loss": 0.96348, + "num_input_tokens_seen": 83119235, + "router_z_loss_clip": 0.96972656, + "router_z_loss_mlp": 0.1817627, + "step": 2973, + "time_per_iteration": 2.6626524925231934 + }, + { + "auxiliary_loss_clip": 0.01040345, + "auxiliary_loss_mlp": 0.01011033, + "balance_loss_clip": 1.01460004, + "balance_loss_mlp": 1.00995386, + "epoch": 0.08629795136672276, + "flos": 68535013253760.0, + "grad_norm": 0.6680762881358087, + "language_loss": 0.46960104, + "learning_rate": 3.966850343709541e-06, + "loss": 0.49011481, + "num_input_tokens_seen": 83183645, + "router_z_loss_clip": 0.2578125, + "router_z_loss_mlp": 0.01080322, + "step": 2974, + "time_per_iteration": 3.1915087699890137 + }, + { + "auxiliary_loss_clip": 0.01175446, + "auxiliary_loss_mlp": 0.01056591, + "balance_loss_clip": 1.06851363, + "balance_loss_mlp": 1.03605688, + "epoch": 0.08632696883523881, + "flos": 43465484638080.0, + "grad_norm": 2.736145277412365, + "language_loss": 0.93474418, + "learning_rate": 3.966816254841828e-06, + "loss": 0.95706451, + "num_input_tokens_seen": 83200595, + "router_z_loss_clip": 1.06835938, + "router_z_loss_mlp": 0.20526123, + "step": 2975, + "time_per_iteration": 2.728696823120117 + }, + { + "auxiliary_loss_clip": 0.01167193, + "auxiliary_loss_mlp": 0.01051365, + "balance_loss_clip": 1.06703877, + "balance_loss_mlp": 1.03214836, + "epoch": 0.08635598630375486, + "flos": 12975927148800.0, + "grad_norm": 3.037370825069237, + "language_loss": 0.99849623, + "learning_rate": 3.966782148602399e-06, + "loss": 1.02068186, + "num_input_tokens_seen": 83213930, + "router_z_loss_clip": 1.00097656, + "router_z_loss_mlp": 0.19226074, + "step": 2976, + "time_per_iteration": 2.5295426845550537 + }, + { + "auxiliary_loss_clip": 0.011602, + "auxiliary_loss_mlp": 0.01044739, + "balance_loss_clip": 1.06597769, + "balance_loss_mlp": 1.02838957, + "epoch": 0.0863850037722709, + "flos": 36239503760640.0, + "grad_norm": 2.3437214312990675, + "language_loss": 0.88824427, + "learning_rate": 3.966748024991553e-06, + "loss": 0.91029364, + "num_input_tokens_seen": 83229760, + "router_z_loss_clip": 0.94287109, + "router_z_loss_mlp": 0.16351318, + "step": 2977, + "time_per_iteration": 2.62276029586792 + }, + { + "auxiliary_loss_clip": 0.01037781, + "auxiliary_loss_mlp": 0.01002751, + "balance_loss_clip": 1.01249218, + "balance_loss_mlp": 1.00162458, + "epoch": 0.08641402124078695, + "flos": 58794100704000.0, + "grad_norm": 0.6897446329428804, + "language_loss": 0.49817798, + "learning_rate": 3.966713884009594e-06, + "loss": 0.5185833, + "num_input_tokens_seen": 83292035, + "router_z_loss_clip": 0.25390625, + "router_z_loss_mlp": 0.0112915, + "step": 2978, + "time_per_iteration": 3.0212244987487793 + }, + { + "auxiliary_loss_clip": 0.01037034, + "auxiliary_loss_mlp": 0.01002206, + "balance_loss_clip": 1.01173687, + "balance_loss_mlp": 1.00106716, + "epoch": 0.086443038709303, + "flos": 74769033162240.0, + "grad_norm": 0.7141374374328407, + "language_loss": 0.54115534, + "learning_rate": 3.96667972565682e-06, + "loss": 0.56154776, + "num_input_tokens_seen": 83345420, + "router_z_loss_clip": 0.25390625, + "router_z_loss_mlp": 0.01141357, + "step": 2979, + "time_per_iteration": 3.0543999671936035 + }, + { + "auxiliary_loss_clip": 0.0116241, + "auxiliary_loss_mlp": 0.01049071, + "balance_loss_clip": 1.06882715, + "balance_loss_mlp": 1.03167224, + "epoch": 0.08647205617781904, + "flos": 19022457041280.0, + "grad_norm": 1.9783126539461093, + "language_loss": 0.72462946, + "learning_rate": 3.966645549933537e-06, + "loss": 0.74674428, + "num_input_tokens_seen": 83365005, + "router_z_loss_clip": 0.93554688, + "router_z_loss_mlp": 0.17382812, + "step": 2980, + "time_per_iteration": 2.641110897064209 + }, + { + "auxiliary_loss_clip": 0.01168246, + "auxiliary_loss_mlp": 0.01057487, + "balance_loss_clip": 1.07017791, + "balance_loss_mlp": 1.03862786, + "epoch": 0.0865010736463351, + "flos": 12085808215680.0, + "grad_norm": 3.1966451293856317, + "language_loss": 1.17383409, + "learning_rate": 3.966611356840044e-06, + "loss": 1.19609141, + "num_input_tokens_seen": 83376525, + "router_z_loss_clip": 0.98144531, + "router_z_loss_mlp": 0.18859863, + "step": 2981, + "time_per_iteration": 2.446920394897461 + }, + { + "auxiliary_loss_clip": 0.01166336, + "auxiliary_loss_mlp": 0.01041865, + "balance_loss_clip": 1.06966591, + "balance_loss_mlp": 1.02552176, + "epoch": 0.08653009111485115, + "flos": 11393342847360.0, + "grad_norm": 3.3117079766880524, + "language_loss": 0.89324152, + "learning_rate": 3.966577146376644e-06, + "loss": 0.91532362, + "num_input_tokens_seen": 83387550, + "router_z_loss_clip": 0.96679688, + "router_z_loss_mlp": 0.16351318, + "step": 2982, + "time_per_iteration": 2.4723799228668213 + }, + { + "auxiliary_loss_clip": 0.01168245, + "auxiliary_loss_mlp": 0.01054423, + "balance_loss_clip": 1.06888318, + "balance_loss_mlp": 1.03706598, + "epoch": 0.08655910858336718, + "flos": 15544115712000.0, + "grad_norm": 2.1355710258910148, + "language_loss": 0.80373406, + "learning_rate": 3.966542918543638e-06, + "loss": 0.8259607, + "num_input_tokens_seen": 83404265, + "router_z_loss_clip": 0.99365234, + "router_z_loss_mlp": 0.17346191, + "step": 2983, + "time_per_iteration": 2.55129075050354 + }, + { + "auxiliary_loss_clip": 0.0117521, + "auxiliary_loss_mlp": 0.01057808, + "balance_loss_clip": 1.07258081, + "balance_loss_mlp": 1.03934264, + "epoch": 0.08658812605188324, + "flos": 70136733548160.0, + "grad_norm": 2.0976756893407775, + "language_loss": 0.94225234, + "learning_rate": 3.966508673341329e-06, + "loss": 0.9645825, + "num_input_tokens_seen": 83430270, + "router_z_loss_clip": 1.02636719, + "router_z_loss_mlp": 0.18481445, + "step": 2984, + "time_per_iteration": 2.874667167663574 + }, + { + "auxiliary_loss_clip": 0.01158745, + "auxiliary_loss_mlp": 0.01052061, + "balance_loss_clip": 1.0660733, + "balance_loss_mlp": 1.03577054, + "epoch": 0.08661714352039929, + "flos": 12381856110720.0, + "grad_norm": 5.179667953355504, + "language_loss": 0.77598041, + "learning_rate": 3.966474410770021e-06, + "loss": 0.79808843, + "num_input_tokens_seen": 83444340, + "router_z_loss_clip": 0.92626953, + "router_z_loss_mlp": 0.1630249, + "step": 2985, + "time_per_iteration": 2.5166192054748535 + }, + { + "auxiliary_loss_clip": 0.01160454, + "auxiliary_loss_mlp": 0.01046899, + "balance_loss_clip": 1.06557679, + "balance_loss_mlp": 1.02984643, + "epoch": 0.08664616098891532, + "flos": 17049596492160.0, + "grad_norm": 2.5905624887216314, + "language_loss": 0.89352429, + "learning_rate": 3.966440130830015e-06, + "loss": 0.91559786, + "num_input_tokens_seen": 83457715, + "router_z_loss_clip": 0.94873047, + "router_z_loss_mlp": 0.17059326, + "step": 2986, + "time_per_iteration": 2.4674336910247803 + }, + { + "auxiliary_loss_clip": 0.01169044, + "auxiliary_loss_mlp": 0.01053427, + "balance_loss_clip": 1.06708252, + "balance_loss_mlp": 1.03362644, + "epoch": 0.08667517845743138, + "flos": 28870526839680.0, + "grad_norm": 2.0201745869209624, + "language_loss": 0.97701031, + "learning_rate": 3.966405833521613e-06, + "loss": 0.99923509, + "num_input_tokens_seen": 83475890, + "router_z_loss_clip": 1.01953125, + "router_z_loss_mlp": 0.19799805, + "step": 2987, + "time_per_iteration": 2.666940689086914 + }, + { + "auxiliary_loss_clip": 0.0116752, + "auxiliary_loss_mlp": 0.01046464, + "balance_loss_clip": 1.0668788, + "balance_loss_mlp": 1.02867198, + "epoch": 0.08670419592594741, + "flos": 30293163878400.0, + "grad_norm": 2.34889981074932, + "language_loss": 1.13116741, + "learning_rate": 3.9663715188451196e-06, + "loss": 1.1533072, + "num_input_tokens_seen": 83495060, + "router_z_loss_clip": 1.00732422, + "router_z_loss_mlp": 0.17791748, + "step": 2988, + "time_per_iteration": 2.615488290786743 + }, + { + "auxiliary_loss_clip": 0.01167872, + "auxiliary_loss_mlp": 0.01052686, + "balance_loss_clip": 1.06971121, + "balance_loss_mlp": 1.03517365, + "epoch": 0.08673321339446347, + "flos": 31901351598720.0, + "grad_norm": 2.214292711681437, + "language_loss": 1.08605981, + "learning_rate": 3.966337186800837e-06, + "loss": 1.1082654, + "num_input_tokens_seen": 83510665, + "router_z_loss_clip": 0.98242188, + "router_z_loss_mlp": 0.17504883, + "step": 2989, + "time_per_iteration": 5.188557386398315 + }, + { + "auxiliary_loss_clip": 0.0104075, + "auxiliary_loss_mlp": 0.01034197, + "balance_loss_clip": 1.01543593, + "balance_loss_mlp": 1.03296936, + "epoch": 0.08676223086297952, + "flos": 60066810374400.0, + "grad_norm": 0.7046220504300935, + "language_loss": 0.53500885, + "learning_rate": 3.966302837389069e-06, + "loss": 0.5557583, + "num_input_tokens_seen": 83570370, + "router_z_loss_clip": 0.25390625, + "router_z_loss_mlp": 0.01226807, + "step": 2990, + "time_per_iteration": 5.421040773391724 + }, + { + "auxiliary_loss_clip": 0.01160375, + "auxiliary_loss_mlp": 0.01051202, + "balance_loss_clip": 1.06661534, + "balance_loss_mlp": 1.03562129, + "epoch": 0.08679124833149555, + "flos": 20049107569920.0, + "grad_norm": 1.8892840336943229, + "language_loss": 0.68219984, + "learning_rate": 3.9662684706101185e-06, + "loss": 0.70431554, + "num_input_tokens_seen": 83585960, + "router_z_loss_clip": 0.93847656, + "router_z_loss_mlp": 0.15570068, + "step": 2991, + "time_per_iteration": 4.820230484008789 + }, + { + "auxiliary_loss_clip": 0.01160295, + "auxiliary_loss_mlp": 0.01040555, + "balance_loss_clip": 1.06283069, + "balance_loss_mlp": 1.02251816, + "epoch": 0.0868202658000116, + "flos": 26548867272960.0, + "grad_norm": 2.36553167127217, + "language_loss": 0.85193503, + "learning_rate": 3.966234086464289e-06, + "loss": 0.87394357, + "num_input_tokens_seen": 83602445, + "router_z_loss_clip": 0.97509766, + "router_z_loss_mlp": 0.18035889, + "step": 2992, + "time_per_iteration": 4.823228120803833 + }, + { + "auxiliary_loss_clip": 0.01169667, + "auxiliary_loss_mlp": 0.01049748, + "balance_loss_clip": 1.06692743, + "balance_loss_mlp": 1.02999496, + "epoch": 0.08684928326852766, + "flos": 35874687277440.0, + "grad_norm": 2.5134189921761254, + "language_loss": 0.98546875, + "learning_rate": 3.966199684951885e-06, + "loss": 1.00766289, + "num_input_tokens_seen": 83617205, + "router_z_loss_clip": 1.02783203, + "router_z_loss_mlp": 0.19769287, + "step": 2993, + "time_per_iteration": 2.6177289485931396 + }, + { + "auxiliary_loss_clip": 0.01042615, + "auxiliary_loss_mlp": 0.01002497, + "balance_loss_clip": 1.01729012, + "balance_loss_mlp": 1.00115633, + "epoch": 0.0868783007370437, + "flos": 74777401031040.0, + "grad_norm": 0.6285101167290809, + "language_loss": 0.5310356, + "learning_rate": 3.9661652660732085e-06, + "loss": 0.55148673, + "num_input_tokens_seen": 83685800, + "router_z_loss_clip": 0.25390625, + "router_z_loss_mlp": 0.01342773, + "step": 2994, + "time_per_iteration": 3.232611656188965 + }, + { + "auxiliary_loss_clip": 0.0115802, + "auxiliary_loss_mlp": 0.01043535, + "balance_loss_clip": 1.06467533, + "balance_loss_mlp": 1.02562964, + "epoch": 0.08690731820555975, + "flos": 16436709705600.0, + "grad_norm": 2.333216618528503, + "language_loss": 0.77857506, + "learning_rate": 3.966130829828566e-06, + "loss": 0.80059063, + "num_input_tokens_seen": 83701440, + "router_z_loss_clip": 0.93359375, + "router_z_loss_mlp": 0.17926025, + "step": 2995, + "time_per_iteration": 2.4804182052612305 + }, + { + "auxiliary_loss_clip": 0.01162333, + "auxiliary_loss_mlp": 0.0104999, + "balance_loss_clip": 1.06964946, + "balance_loss_mlp": 1.03375959, + "epoch": 0.0869363356740758, + "flos": 45076689100800.0, + "grad_norm": 1.863296847555468, + "language_loss": 0.7602793, + "learning_rate": 3.9660963762182605e-06, + "loss": 0.78240252, + "num_input_tokens_seen": 83722085, + "router_z_loss_clip": 0.92724609, + "router_z_loss_mlp": 0.16229248, + "step": 2996, + "time_per_iteration": 2.7126708030700684 + }, + { + "auxiliary_loss_clip": 0.01169708, + "auxiliary_loss_mlp": 0.01059701, + "balance_loss_clip": 1.0696131, + "balance_loss_mlp": 1.03911376, + "epoch": 0.08696535314259184, + "flos": 25915581561600.0, + "grad_norm": 2.2127532582397222, + "language_loss": 0.89537793, + "learning_rate": 3.9660619052425955e-06, + "loss": 0.91767204, + "num_input_tokens_seen": 83736680, + "router_z_loss_clip": 1.0, + "router_z_loss_mlp": 0.20593262, + "step": 2997, + "time_per_iteration": 2.557512044906616 + }, + { + "auxiliary_loss_clip": 0.01165679, + "auxiliary_loss_mlp": 0.01048854, + "balance_loss_clip": 1.0621891, + "balance_loss_mlp": 1.02953029, + "epoch": 0.08699437061110789, + "flos": 40951809941760.0, + "grad_norm": 2.022194193095047, + "language_loss": 0.85590708, + "learning_rate": 3.966027416901876e-06, + "loss": 0.87805247, + "num_input_tokens_seen": 83754960, + "router_z_loss_clip": 1.03564453, + "router_z_loss_mlp": 0.1930542, + "step": 2998, + "time_per_iteration": 2.6558220386505127 + }, + { + "auxiliary_loss_clip": 0.01164482, + "auxiliary_loss_mlp": 0.01044326, + "balance_loss_clip": 1.06708121, + "balance_loss_mlp": 1.02811384, + "epoch": 0.08702338807962394, + "flos": 16758791982720.0, + "grad_norm": 2.0201261769089554, + "language_loss": 0.66258478, + "learning_rate": 3.965992911196407e-06, + "loss": 0.68467295, + "num_input_tokens_seen": 83771945, + "router_z_loss_clip": 0.97314453, + "router_z_loss_mlp": 0.16223145, + "step": 2999, + "time_per_iteration": 2.5059499740600586 + }, + { + "auxiliary_loss_clip": 0.01154072, + "auxiliary_loss_mlp": 0.01051551, + "balance_loss_clip": 1.06270468, + "balance_loss_mlp": 1.0358398, + "epoch": 0.08705240554813998, + "flos": 18326005263360.0, + "grad_norm": 3.146554775091389, + "language_loss": 0.70003378, + "learning_rate": 3.965958388126493e-06, + "loss": 0.72209007, + "num_input_tokens_seen": 83784165, + "router_z_loss_clip": 0.91357422, + "router_z_loss_mlp": 0.15710449, + "step": 3000, + "time_per_iteration": 2.4340405464172363 + }, + { + "auxiliary_loss_clip": 0.01164546, + "auxiliary_loss_mlp": 0.01040717, + "balance_loss_clip": 1.06511354, + "balance_loss_mlp": 1.0198977, + "epoch": 0.08708142301665603, + "flos": 17596839173760.0, + "grad_norm": 2.903514639176145, + "language_loss": 0.97606951, + "learning_rate": 3.9659238476924395e-06, + "loss": 0.99812222, + "num_input_tokens_seen": 83797305, + "router_z_loss_clip": 0.99414062, + "router_z_loss_mlp": 0.20825195, + "step": 3001, + "time_per_iteration": 2.455488681793213 + }, + { + "auxiliary_loss_clip": 0.01168275, + "auxiliary_loss_mlp": 0.01059571, + "balance_loss_clip": 1.06758189, + "balance_loss_mlp": 1.04019988, + "epoch": 0.08711044048517208, + "flos": 14677589036160.0, + "grad_norm": 3.368037699933978, + "language_loss": 0.78728402, + "learning_rate": 3.965889289894551e-06, + "loss": 0.80956256, + "num_input_tokens_seen": 83809840, + "router_z_loss_clip": 1.00683594, + "router_z_loss_mlp": 0.19366455, + "step": 3002, + "time_per_iteration": 2.4600064754486084 + }, + { + "auxiliary_loss_clip": 0.01041158, + "auxiliary_loss_mlp": 0.01001647, + "balance_loss_clip": 1.01579845, + "balance_loss_mlp": 1.00043094, + "epoch": 0.08713945795368812, + "flos": 63794374220160.0, + "grad_norm": 0.6710132900804557, + "language_loss": 0.46471342, + "learning_rate": 3.965854714733132e-06, + "loss": 0.48514146, + "num_input_tokens_seen": 83867970, + "router_z_loss_clip": 0.25390625, + "router_z_loss_mlp": 0.012146, + "step": 3003, + "time_per_iteration": 3.019496440887451 + }, + { + "auxiliary_loss_clip": 0.01167907, + "auxiliary_loss_mlp": 0.0104189, + "balance_loss_clip": 1.06803823, + "balance_loss_mlp": 1.02579093, + "epoch": 0.08716847542220417, + "flos": 46821480243840.0, + "grad_norm": 2.1995377892831747, + "language_loss": 0.7754786, + "learning_rate": 3.96582012220849e-06, + "loss": 0.79757661, + "num_input_tokens_seen": 83886625, + "router_z_loss_clip": 0.99902344, + "router_z_loss_mlp": 0.16094971, + "step": 3004, + "time_per_iteration": 2.622011184692383 + }, + { + "auxiliary_loss_clip": 0.0103938, + "auxiliary_loss_mlp": 0.00997667, + "balance_loss_clip": 1.01401353, + "balance_loss_mlp": 0.9964096, + "epoch": 0.08719749289072021, + "flos": 64193449299840.0, + "grad_norm": 0.6303993705236782, + "language_loss": 0.48066026, + "learning_rate": 3.965785512320928e-06, + "loss": 0.50103074, + "num_input_tokens_seen": 83947440, + "router_z_loss_clip": 0.25390625, + "router_z_loss_mlp": 0.01257324, + "step": 3005, + "time_per_iteration": 3.0792346000671387 + }, + { + "auxiliary_loss_clip": 0.01165809, + "auxiliary_loss_mlp": 0.01047227, + "balance_loss_clip": 1.06827664, + "balance_loss_mlp": 1.02972686, + "epoch": 0.08722651035923626, + "flos": 11356354817280.0, + "grad_norm": 3.5560319038478907, + "language_loss": 1.04694462, + "learning_rate": 3.965750885070753e-06, + "loss": 1.06907511, + "num_input_tokens_seen": 83956845, + "router_z_loss_clip": 0.97460938, + "router_z_loss_mlp": 0.17492676, + "step": 3006, + "time_per_iteration": 2.532918930053711 + }, + { + "auxiliary_loss_clip": 0.01039053, + "auxiliary_loss_mlp": 0.01010578, + "balance_loss_clip": 1.01375771, + "balance_loss_mlp": 1.00925434, + "epoch": 0.08725552782775231, + "flos": 71482380762240.0, + "grad_norm": 0.6991577083035784, + "language_loss": 0.50519729, + "learning_rate": 3.965716240458271e-06, + "loss": 0.52569366, + "num_input_tokens_seen": 84013015, + "router_z_loss_clip": 0.25390625, + "router_z_loss_mlp": 0.01324463, + "step": 3007, + "time_per_iteration": 3.029224157333374 + }, + { + "auxiliary_loss_clip": 0.0103969, + "auxiliary_loss_mlp": 0.01014011, + "balance_loss_clip": 1.01444817, + "balance_loss_mlp": 1.01280117, + "epoch": 0.08728454529626835, + "flos": 67626365880960.0, + "grad_norm": 0.7038673611072335, + "language_loss": 0.41253209, + "learning_rate": 3.965681578483788e-06, + "loss": 0.43306911, + "num_input_tokens_seen": 84080210, + "router_z_loss_clip": 0.25195312, + "router_z_loss_mlp": 0.01208496, + "step": 3008, + "time_per_iteration": 3.233942985534668 + }, + { + "auxiliary_loss_clip": 0.01169169, + "auxiliary_loss_mlp": 0.0105545, + "balance_loss_clip": 1.06658411, + "balance_loss_mlp": 1.03740788, + "epoch": 0.0873135627647844, + "flos": 37894340269440.0, + "grad_norm": 1.6918565646701735, + "language_loss": 0.8964321, + "learning_rate": 3.965646899147609e-06, + "loss": 0.91867828, + "num_input_tokens_seen": 84102595, + "router_z_loss_clip": 1.02832031, + "router_z_loss_mlp": 0.18048096, + "step": 3009, + "time_per_iteration": 2.6350808143615723 + }, + { + "auxiliary_loss_clip": 0.01165607, + "auxiliary_loss_mlp": 0.01050928, + "balance_loss_clip": 1.06102443, + "balance_loss_mlp": 1.03187871, + "epoch": 0.08734258023330045, + "flos": 30553947596160.0, + "grad_norm": 2.3500622516528438, + "language_loss": 1.10131788, + "learning_rate": 3.965612202450042e-06, + "loss": 1.1234833, + "num_input_tokens_seen": 84123360, + "router_z_loss_clip": 1.04589844, + "router_z_loss_mlp": 0.19049072, + "step": 3010, + "time_per_iteration": 2.639737129211426 + }, + { + "auxiliary_loss_clip": 0.01165992, + "auxiliary_loss_mlp": 0.01043248, + "balance_loss_clip": 1.0638485, + "balance_loss_mlp": 1.02444839, + "epoch": 0.08737159770181649, + "flos": 20076614409600.0, + "grad_norm": 2.6144333564514914, + "language_loss": 0.81774569, + "learning_rate": 3.965577488391393e-06, + "loss": 0.83983809, + "num_input_tokens_seen": 84138665, + "router_z_loss_clip": 1.02148438, + "router_z_loss_mlp": 0.18798828, + "step": 3011, + "time_per_iteration": 2.5221264362335205 + }, + { + "auxiliary_loss_clip": 0.01164867, + "auxiliary_loss_mlp": 0.01039215, + "balance_loss_clip": 1.06517935, + "balance_loss_mlp": 1.0219655, + "epoch": 0.08740061517033254, + "flos": 30475482336000.0, + "grad_norm": 2.9610891229151055, + "language_loss": 0.75253046, + "learning_rate": 3.965542756971967e-06, + "loss": 0.77457124, + "num_input_tokens_seen": 84155010, + "router_z_loss_clip": 0.99560547, + "router_z_loss_mlp": 0.17242432, + "step": 3012, + "time_per_iteration": 2.6278023719787598 + }, + { + "auxiliary_loss_clip": 0.01036942, + "auxiliary_loss_mlp": 0.01006556, + "balance_loss_clip": 1.01192069, + "balance_loss_mlp": 1.00531054, + "epoch": 0.08742963263884859, + "flos": 64887387125760.0, + "grad_norm": 0.7717953641189816, + "language_loss": 0.51955616, + "learning_rate": 3.965508008192072e-06, + "loss": 0.53999114, + "num_input_tokens_seen": 84220900, + "router_z_loss_clip": 0.25, + "router_z_loss_mlp": 0.01245117, + "step": 3013, + "time_per_iteration": 3.138521909713745 + }, + { + "auxiliary_loss_clip": 0.01156457, + "auxiliary_loss_mlp": 0.01035622, + "balance_loss_clip": 1.06084836, + "balance_loss_mlp": 1.02006531, + "epoch": 0.08745865010736463, + "flos": 12926728494720.0, + "grad_norm": 2.225832834325008, + "language_loss": 0.70127058, + "learning_rate": 3.965473242052016e-06, + "loss": 0.72319138, + "num_input_tokens_seen": 84237145, + "router_z_loss_clip": 0.95605469, + "router_z_loss_mlp": 0.15563965, + "step": 3014, + "time_per_iteration": 2.517747640609741 + }, + { + "auxiliary_loss_clip": 0.01164481, + "auxiliary_loss_mlp": 0.01054785, + "balance_loss_clip": 1.06430161, + "balance_loss_mlp": 1.03504395, + "epoch": 0.08748766757588068, + "flos": 11502583084800.0, + "grad_norm": 2.9601184436562287, + "language_loss": 0.88360316, + "learning_rate": 3.965438458552104e-06, + "loss": 0.90579581, + "num_input_tokens_seen": 84247905, + "router_z_loss_clip": 1.00390625, + "router_z_loss_mlp": 0.19750977, + "step": 3015, + "time_per_iteration": 2.479440927505493 + }, + { + "auxiliary_loss_clip": 0.01161179, + "auxiliary_loss_mlp": 0.01045523, + "balance_loss_clip": 1.06056261, + "balance_loss_mlp": 1.02510214, + "epoch": 0.08751668504439673, + "flos": 30329828686080.0, + "grad_norm": 2.843648193092998, + "language_loss": 0.92888808, + "learning_rate": 3.965403657692645e-06, + "loss": 0.95095515, + "num_input_tokens_seen": 84262340, + "router_z_loss_clip": 1.00634766, + "router_z_loss_mlp": 0.2043457, + "step": 3016, + "time_per_iteration": 2.6016244888305664 + }, + { + "auxiliary_loss_clip": 0.01158954, + "auxiliary_loss_mlp": 0.01044891, + "balance_loss_clip": 1.06559277, + "balance_loss_mlp": 1.02861929, + "epoch": 0.08754570251291277, + "flos": 23178901276800.0, + "grad_norm": 2.212158686805882, + "language_loss": 0.80160379, + "learning_rate": 3.965368839473946e-06, + "loss": 0.82364225, + "num_input_tokens_seen": 84280545, + "router_z_loss_clip": 0.93310547, + "router_z_loss_mlp": 0.16265869, + "step": 3017, + "time_per_iteration": 2.5479607582092285 + }, + { + "auxiliary_loss_clip": 0.01164808, + "auxiliary_loss_mlp": 0.01045614, + "balance_loss_clip": 1.06350756, + "balance_loss_mlp": 1.02681518, + "epoch": 0.08757471998142882, + "flos": 24893025183360.0, + "grad_norm": 3.4536219044880276, + "language_loss": 0.95855123, + "learning_rate": 3.965334003896313e-06, + "loss": 0.98065549, + "num_input_tokens_seen": 84296170, + "router_z_loss_clip": 1.01416016, + "router_z_loss_mlp": 0.18798828, + "step": 3018, + "time_per_iteration": 2.539776563644409 + }, + { + "auxiliary_loss_clip": 0.01035959, + "auxiliary_loss_mlp": 0.01004175, + "balance_loss_clip": 1.01069069, + "balance_loss_mlp": 1.00289905, + "epoch": 0.08760373744994486, + "flos": 55619166579840.0, + "grad_norm": 0.6424823425533734, + "language_loss": 0.46700102, + "learning_rate": 3.965299150960055e-06, + "loss": 0.48740238, + "num_input_tokens_seen": 84355125, + "router_z_loss_clip": 0.25390625, + "router_z_loss_mlp": 0.01275635, + "step": 3019, + "time_per_iteration": 2.989436149597168 + }, + { + "auxiliary_loss_clip": 0.01164245, + "auxiliary_loss_mlp": 0.01054592, + "balance_loss_clip": 1.07005429, + "balance_loss_mlp": 1.03700829, + "epoch": 0.08763275491846091, + "flos": 16938883797120.0, + "grad_norm": 2.6287991276578113, + "language_loss": 0.76526916, + "learning_rate": 3.96526428066548e-06, + "loss": 0.78745759, + "num_input_tokens_seen": 84367125, + "router_z_loss_clip": 0.94042969, + "router_z_loss_mlp": 0.17578125, + "step": 3020, + "time_per_iteration": 2.489668846130371 + }, + { + "auxiliary_loss_clip": 0.01161258, + "auxiliary_loss_mlp": 0.01053193, + "balance_loss_clip": 1.06122875, + "balance_loss_mlp": 1.03489447, + "epoch": 0.08766177238697696, + "flos": 16360794311040.0, + "grad_norm": 2.813235037966405, + "language_loss": 0.81690681, + "learning_rate": 3.965229393012895e-06, + "loss": 0.83905137, + "num_input_tokens_seen": 84380835, + "router_z_loss_clip": 1.00048828, + "router_z_loss_mlp": 0.18310547, + "step": 3021, + "time_per_iteration": 2.4397873878479004 + }, + { + "auxiliary_loss_clip": 0.01166127, + "auxiliary_loss_mlp": 0.01048446, + "balance_loss_clip": 1.06637776, + "balance_loss_mlp": 1.02975428, + "epoch": 0.087690789855493, + "flos": 74737429194240.0, + "grad_norm": 1.9636968265358694, + "language_loss": 0.75570321, + "learning_rate": 3.96519448800261e-06, + "loss": 0.77784896, + "num_input_tokens_seen": 84406050, + "router_z_loss_clip": 0.99804688, + "router_z_loss_mlp": 0.18695068, + "step": 3022, + "time_per_iteration": 2.9042727947235107 + }, + { + "auxiliary_loss_clip": 0.01161485, + "auxiliary_loss_mlp": 0.01053473, + "balance_loss_clip": 1.06662965, + "balance_loss_mlp": 1.03709412, + "epoch": 0.08771980732400905, + "flos": 27156331105920.0, + "grad_norm": 2.1772357796625923, + "language_loss": 0.99747378, + "learning_rate": 3.96515956563493e-06, + "loss": 1.01962328, + "num_input_tokens_seen": 84424980, + "router_z_loss_clip": 0.94775391, + "router_z_loss_mlp": 0.16351318, + "step": 3023, + "time_per_iteration": 2.7866435050964355 + }, + { + "auxiliary_loss_clip": 0.0117469, + "auxiliary_loss_mlp": 0.01045078, + "balance_loss_clip": 1.06696296, + "balance_loss_mlp": 1.02421618, + "epoch": 0.0877488247925251, + "flos": 23872408139520.0, + "grad_norm": 2.570579101436772, + "language_loss": 0.97526222, + "learning_rate": 3.965124625910168e-06, + "loss": 0.99745989, + "num_input_tokens_seen": 84442980, + "router_z_loss_clip": 1.07568359, + "router_z_loss_mlp": 0.20849609, + "step": 3024, + "time_per_iteration": 2.5239429473876953 + }, + { + "auxiliary_loss_clip": 0.01170411, + "auxiliary_loss_mlp": 0.01062936, + "balance_loss_clip": 1.06679714, + "balance_loss_mlp": 1.040465, + "epoch": 0.08777784226104114, + "flos": 30586051376640.0, + "grad_norm": 2.2302396932339277, + "language_loss": 0.7925427, + "learning_rate": 3.965089668828628e-06, + "loss": 0.8148762, + "num_input_tokens_seen": 84458860, + "router_z_loss_clip": 1.03662109, + "router_z_loss_mlp": 0.22497559, + "step": 3025, + "time_per_iteration": 2.587433099746704 + }, + { + "auxiliary_loss_clip": 0.0103721, + "auxiliary_loss_mlp": 0.0100194, + "balance_loss_clip": 1.01185548, + "balance_loss_mlp": 1.00072408, + "epoch": 0.0878068597295572, + "flos": 55632307979520.0, + "grad_norm": 0.7407455239897404, + "language_loss": 0.45675367, + "learning_rate": 3.965054694390622e-06, + "loss": 0.4771452, + "num_input_tokens_seen": 84501200, + "router_z_loss_clip": 0.25390625, + "router_z_loss_mlp": 0.012146, + "step": 3026, + "time_per_iteration": 2.849026679992676 + }, + { + "auxiliary_loss_clip": 0.01154319, + "auxiliary_loss_mlp": 0.01044069, + "balance_loss_clip": 1.06226015, + "balance_loss_mlp": 1.02761245, + "epoch": 0.08783587719807325, + "flos": 15516354499200.0, + "grad_norm": 2.5023778454412535, + "language_loss": 0.85952055, + "learning_rate": 3.9650197025964576e-06, + "loss": 0.88150454, + "num_input_tokens_seen": 84514895, + "router_z_loss_clip": 0.92089844, + "router_z_loss_mlp": 0.16479492, + "step": 3027, + "time_per_iteration": 2.4695940017700195 + }, + { + "auxiliary_loss_clip": 0.01036176, + "auxiliary_loss_mlp": 0.01000436, + "balance_loss_clip": 1.01090026, + "balance_loss_mlp": 0.9992258, + "epoch": 0.08786489466658928, + "flos": 74771654855040.0, + "grad_norm": 0.6943917138125616, + "language_loss": 0.51074004, + "learning_rate": 3.9649846934464435e-06, + "loss": 0.53110611, + "num_input_tokens_seen": 84577080, + "router_z_loss_clip": 0.25195312, + "router_z_loss_mlp": 0.01208496, + "step": 3028, + "time_per_iteration": 3.077035427093506 + }, + { + "auxiliary_loss_clip": 0.01161587, + "auxiliary_loss_mlp": 0.01054143, + "balance_loss_clip": 1.06129849, + "balance_loss_mlp": 1.03578448, + "epoch": 0.08789391213510533, + "flos": 28212894685440.0, + "grad_norm": 3.3898025558585516, + "language_loss": 0.77100813, + "learning_rate": 3.9649496669408904e-06, + "loss": 0.79316533, + "num_input_tokens_seen": 84597240, + "router_z_loss_clip": 1.00195312, + "router_z_loss_mlp": 0.18347168, + "step": 3029, + "time_per_iteration": 2.616859197616577 + }, + { + "auxiliary_loss_clip": 0.0103577, + "auxiliary_loss_mlp": 0.01006836, + "balance_loss_clip": 1.01036584, + "balance_loss_mlp": 1.00567365, + "epoch": 0.08792292960362139, + "flos": 74466954760320.0, + "grad_norm": 0.6258369296866441, + "language_loss": 0.47870797, + "learning_rate": 3.964914623080106e-06, + "loss": 0.49913403, + "num_input_tokens_seen": 84670190, + "router_z_loss_clip": 0.25390625, + "router_z_loss_mlp": 0.01159668, + "step": 3030, + "time_per_iteration": 3.32039737701416 + }, + { + "auxiliary_loss_clip": 0.01035419, + "auxiliary_loss_mlp": 0.00999397, + "balance_loss_clip": 1.0099647, + "balance_loss_mlp": 0.99821663, + "epoch": 0.08795194707213742, + "flos": 74790042618240.0, + "grad_norm": 0.6211213029491386, + "language_loss": 0.46981245, + "learning_rate": 3.9648795618644e-06, + "loss": 0.49016058, + "num_input_tokens_seen": 84735105, + "router_z_loss_clip": 0.25390625, + "router_z_loss_mlp": 0.01177979, + "step": 3031, + "time_per_iteration": 3.2525789737701416 + }, + { + "auxiliary_loss_clip": 0.01036319, + "auxiliary_loss_mlp": 0.01001064, + "balance_loss_clip": 1.01091814, + "balance_loss_mlp": 0.99990797, + "epoch": 0.08798096454065348, + "flos": 74776215882240.0, + "grad_norm": 0.6560367832522032, + "language_loss": 0.48809254, + "learning_rate": 3.964844483294084e-06, + "loss": 0.50846636, + "num_input_tokens_seen": 84792095, + "router_z_loss_clip": 0.25390625, + "router_z_loss_mlp": 0.01153564, + "step": 3032, + "time_per_iteration": 3.1057536602020264 + }, + { + "auxiliary_loss_clip": 0.01035733, + "auxiliary_loss_mlp": 0.01001943, + "balance_loss_clip": 1.01038742, + "balance_loss_mlp": 1.00080419, + "epoch": 0.08800998200916953, + "flos": 74515432164480.0, + "grad_norm": 0.6479721037310687, + "language_loss": 0.53965986, + "learning_rate": 3.964809387369466e-06, + "loss": 0.56003666, + "num_input_tokens_seen": 84857135, + "router_z_loss_clip": 0.25390625, + "router_z_loss_mlp": 0.01141357, + "step": 3033, + "time_per_iteration": 3.2881882190704346 + }, + { + "auxiliary_loss_clip": 0.01162904, + "auxiliary_loss_mlp": 0.01046316, + "balance_loss_clip": 1.06562769, + "balance_loss_mlp": 1.02843475, + "epoch": 0.08803899947768556, + "flos": 12633302292480.0, + "grad_norm": 2.785362929695142, + "language_loss": 0.9590795, + "learning_rate": 3.9647742740908555e-06, + "loss": 0.98117173, + "num_input_tokens_seen": 84868400, + "router_z_loss_clip": 0.97265625, + "router_z_loss_mlp": 0.17883301, + "step": 3034, + "time_per_iteration": 2.5073747634887695 + }, + { + "auxiliary_loss_clip": 0.01156578, + "auxiliary_loss_mlp": 0.01044429, + "balance_loss_clip": 1.06178343, + "balance_loss_mlp": 1.02726316, + "epoch": 0.08806801694620162, + "flos": 21062685548160.0, + "grad_norm": 2.5736351104018147, + "language_loss": 0.68159419, + "learning_rate": 3.964739143458564e-06, + "loss": 0.70360422, + "num_input_tokens_seen": 84881615, + "router_z_loss_clip": 0.94726562, + "router_z_loss_mlp": 0.17175293, + "step": 3035, + "time_per_iteration": 2.472881317138672 + }, + { + "auxiliary_loss_clip": 0.0115424, + "auxiliary_loss_mlp": 0.01048405, + "balance_loss_clip": 1.05851245, + "balance_loss_mlp": 1.02850294, + "epoch": 0.08809703441471765, + "flos": 12634990231680.0, + "grad_norm": 2.4079439931712288, + "language_loss": 0.82820392, + "learning_rate": 3.964703995472902e-06, + "loss": 0.8502304, + "num_input_tokens_seen": 84894280, + "router_z_loss_clip": 0.95751953, + "router_z_loss_mlp": 0.19885254, + "step": 3036, + "time_per_iteration": 2.4816792011260986 + }, + { + "auxiliary_loss_clip": 0.01161761, + "auxiliary_loss_mlp": 0.01043271, + "balance_loss_clip": 1.06118274, + "balance_loss_mlp": 1.02435291, + "epoch": 0.0881260518832337, + "flos": 13110698977920.0, + "grad_norm": 2.0456444199164605, + "language_loss": 0.68673027, + "learning_rate": 3.964668830134179e-06, + "loss": 0.70878053, + "num_input_tokens_seen": 84908825, + "router_z_loss_clip": 1.00683594, + "router_z_loss_mlp": 0.18896484, + "step": 3037, + "time_per_iteration": 2.466773748397827 + }, + { + "auxiliary_loss_clip": 0.01036185, + "auxiliary_loss_mlp": 0.01006333, + "balance_loss_clip": 1.01101053, + "balance_loss_mlp": 1.00508165, + "epoch": 0.08815506935174976, + "flos": 64660646522880.0, + "grad_norm": 0.7500386053482591, + "language_loss": 0.50383115, + "learning_rate": 3.964633647442706e-06, + "loss": 0.52425635, + "num_input_tokens_seen": 84958690, + "router_z_loss_clip": 0.25195312, + "router_z_loss_mlp": 0.01251221, + "step": 3038, + "time_per_iteration": 2.9125072956085205 + }, + { + "auxiliary_loss_clip": 0.01154202, + "auxiliary_loss_mlp": 0.0104618, + "balance_loss_clip": 1.06112683, + "balance_loss_mlp": 1.02824497, + "epoch": 0.0881840868202658, + "flos": 25004420236800.0, + "grad_norm": 2.2435235502941193, + "language_loss": 0.77387345, + "learning_rate": 3.9645984473987925e-06, + "loss": 0.79587722, + "num_input_tokens_seen": 84975220, + "router_z_loss_clip": 0.93017578, + "router_z_loss_mlp": 0.17913818, + "step": 3039, + "time_per_iteration": 2.5216894149780273 + }, + { + "auxiliary_loss_clip": 0.01034783, + "auxiliary_loss_mlp": 0.01002013, + "balance_loss_clip": 1.00964642, + "balance_loss_mlp": 1.00083876, + "epoch": 0.08821310428878185, + "flos": 65477723149440.0, + "grad_norm": 0.6413879011569703, + "language_loss": 0.50564319, + "learning_rate": 3.964563230002751e-06, + "loss": 0.52601117, + "num_input_tokens_seen": 85038360, + "router_z_loss_clip": 0.25, + "router_z_loss_mlp": 0.01171875, + "step": 3040, + "time_per_iteration": 3.105236768722534 + }, + { + "auxiliary_loss_clip": 0.01158158, + "auxiliary_loss_mlp": 0.01048675, + "balance_loss_clip": 1.06120706, + "balance_loss_mlp": 1.03002512, + "epoch": 0.0882421217572979, + "flos": 22851503786880.0, + "grad_norm": 2.223453382869299, + "language_loss": 0.83295262, + "learning_rate": 3.964527995254893e-06, + "loss": 0.85502088, + "num_input_tokens_seen": 85051770, + "router_z_loss_clip": 0.96923828, + "router_z_loss_mlp": 0.18658447, + "step": 3041, + "time_per_iteration": 2.4811758995056152 + }, + { + "auxiliary_loss_clip": 0.0115373, + "auxiliary_loss_mlp": 0.01052723, + "balance_loss_clip": 1.05952072, + "balance_loss_mlp": 1.03618836, + "epoch": 0.08827113922581394, + "flos": 33147667751040.0, + "grad_norm": 2.582297251501584, + "language_loss": 0.91406667, + "learning_rate": 3.964492743155528e-06, + "loss": 0.93613112, + "num_input_tokens_seen": 85072565, + "router_z_loss_clip": 0.94189453, + "router_z_loss_mlp": 0.1652832, + "step": 3042, + "time_per_iteration": 2.6483728885650635 + }, + { + "auxiliary_loss_clip": 0.01161802, + "auxiliary_loss_mlp": 0.01062779, + "balance_loss_clip": 1.06517267, + "balance_loss_mlp": 1.04485595, + "epoch": 0.08830015669432999, + "flos": 32738321381760.0, + "grad_norm": 2.2602087139238036, + "language_loss": 0.90459293, + "learning_rate": 3.964457473704969e-06, + "loss": 0.9268387, + "num_input_tokens_seen": 85091290, + "router_z_loss_clip": 0.96630859, + "router_z_loss_mlp": 0.17932129, + "step": 3043, + "time_per_iteration": 2.6160480976104736 + }, + { + "auxiliary_loss_clip": 0.01157707, + "auxiliary_loss_mlp": 0.01038797, + "balance_loss_clip": 1.06190705, + "balance_loss_mlp": 1.02386045, + "epoch": 0.08832917416284604, + "flos": 25813739525760.0, + "grad_norm": 2.4011135292681343, + "language_loss": 0.86668819, + "learning_rate": 3.964422186903525e-06, + "loss": 0.88865322, + "num_input_tokens_seen": 85110335, + "router_z_loss_clip": 0.95751953, + "router_z_loss_mlp": 0.14929199, + "step": 3044, + "time_per_iteration": 2.5326955318450928 + }, + { + "auxiliary_loss_clip": 0.01155197, + "auxiliary_loss_mlp": 0.01048882, + "balance_loss_clip": 1.05756032, + "balance_loss_mlp": 1.02921224, + "epoch": 0.08835819163136208, + "flos": 14056011158400.0, + "grad_norm": 2.728985984846983, + "language_loss": 1.04401648, + "learning_rate": 3.964386882751511e-06, + "loss": 1.06605732, + "num_input_tokens_seen": 85120620, + "router_z_loss_clip": 0.97705078, + "router_z_loss_mlp": 0.19677734, + "step": 3045, + "time_per_iteration": 2.4904897212982178 + }, + { + "auxiliary_loss_clip": 0.0115951, + "auxiliary_loss_mlp": 0.01052479, + "balance_loss_clip": 1.06372201, + "balance_loss_mlp": 1.03475237, + "epoch": 0.08838720909987813, + "flos": 21534982502400.0, + "grad_norm": 2.5553717613281814, + "language_loss": 0.84511983, + "learning_rate": 3.964351561249236e-06, + "loss": 0.86723977, + "num_input_tokens_seen": 85133700, + "router_z_loss_clip": 0.95800781, + "router_z_loss_mlp": 0.17736816, + "step": 3046, + "time_per_iteration": 2.516836404800415 + }, + { + "auxiliary_loss_clip": 0.01033625, + "auxiliary_loss_mlp": 0.0100943, + "balance_loss_clip": 1.00851011, + "balance_loss_mlp": 1.00829124, + "epoch": 0.08841622656839418, + "flos": 63353139552000.0, + "grad_norm": 0.8450984118704348, + "language_loss": 0.48386294, + "learning_rate": 3.964316222397014e-06, + "loss": 0.5042935, + "num_input_tokens_seen": 85190265, + "router_z_loss_clip": 0.25, + "router_z_loss_mlp": 0.01141357, + "step": 3047, + "time_per_iteration": 3.048680067062378 + }, + { + "auxiliary_loss_clip": 0.01159606, + "auxiliary_loss_mlp": 0.01040215, + "balance_loss_clip": 1.06062937, + "balance_loss_mlp": 1.02056968, + "epoch": 0.08844524403691022, + "flos": 20186644746240.0, + "grad_norm": 2.7478125427463462, + "language_loss": 0.79734069, + "learning_rate": 3.964280866195156e-06, + "loss": 0.81933892, + "num_input_tokens_seen": 85204655, + "router_z_loss_clip": 0.98828125, + "router_z_loss_mlp": 0.1965332, + "step": 3048, + "time_per_iteration": 2.4947361946105957 + }, + { + "auxiliary_loss_clip": 0.01165911, + "auxiliary_loss_mlp": 0.0105369, + "balance_loss_clip": 1.06482458, + "balance_loss_mlp": 1.0339551, + "epoch": 0.08847426150542627, + "flos": 31972849620480.0, + "grad_norm": 2.1054513302997737, + "language_loss": 0.94728714, + "learning_rate": 3.964245492643974e-06, + "loss": 0.9694832, + "num_input_tokens_seen": 85223090, + "router_z_loss_clip": 1.01074219, + "router_z_loss_mlp": 0.19714355, + "step": 3049, + "time_per_iteration": 2.5897068977355957 + }, + { + "auxiliary_loss_clip": 0.01033519, + "auxiliary_loss_mlp": 0.01000798, + "balance_loss_clip": 1.008358, + "balance_loss_mlp": 0.9996056, + "epoch": 0.0885032789739423, + "flos": 64860418990080.0, + "grad_norm": 0.6459372046792228, + "language_loss": 0.48839858, + "learning_rate": 3.964210101743781e-06, + "loss": 0.50874174, + "num_input_tokens_seen": 85288030, + "router_z_loss_clip": 0.25, + "router_z_loss_mlp": 0.01190186, + "step": 3050, + "time_per_iteration": 3.1597068309783936 + }, + { + "auxiliary_loss_clip": 0.01034005, + "auxiliary_loss_mlp": 0.01001052, + "balance_loss_clip": 1.00889337, + "balance_loss_mlp": 0.99988979, + "epoch": 0.08853229644245836, + "flos": 68020736279040.0, + "grad_norm": 0.6677696992993377, + "language_loss": 0.48580098, + "learning_rate": 3.96417469349489e-06, + "loss": 0.50615156, + "num_input_tokens_seen": 85347875, + "router_z_loss_clip": 0.25195312, + "router_z_loss_mlp": 0.01159668, + "step": 3051, + "time_per_iteration": 3.035648822784424 + }, + { + "auxiliary_loss_clip": 0.01154589, + "auxiliary_loss_mlp": 0.0105199, + "balance_loss_clip": 1.05946898, + "balance_loss_mlp": 1.03185606, + "epoch": 0.08856131391097441, + "flos": 48534921792000.0, + "grad_norm": 2.685062357912352, + "language_loss": 0.85165399, + "learning_rate": 3.964139267897613e-06, + "loss": 0.87371975, + "num_input_tokens_seen": 85365790, + "router_z_loss_clip": 0.95214844, + "router_z_loss_mlp": 0.20117188, + "step": 3052, + "time_per_iteration": 2.7529444694519043 + }, + { + "auxiliary_loss_clip": 0.01146477, + "auxiliary_loss_mlp": 0.0103398, + "balance_loss_clip": 1.05684614, + "balance_loss_mlp": 1.01885843, + "epoch": 0.08859033137949045, + "flos": 46164171312000.0, + "grad_norm": 2.619852199315772, + "language_loss": 0.68219209, + "learning_rate": 3.9641038249522634e-06, + "loss": 0.70399666, + "num_input_tokens_seen": 85384705, + "router_z_loss_clip": 0.89746094, + "router_z_loss_mlp": 0.15136719, + "step": 3053, + "time_per_iteration": 2.615351438522339 + }, + { + "auxiliary_loss_clip": 0.01154966, + "auxiliary_loss_mlp": 0.01039756, + "balance_loss_clip": 1.06102419, + "balance_loss_mlp": 1.02406788, + "epoch": 0.0886193488480065, + "flos": 28942958615040.0, + "grad_norm": 1.9490824110187666, + "language_loss": 0.70682317, + "learning_rate": 3.964068364659154e-06, + "loss": 0.72877038, + "num_input_tokens_seen": 85401310, + "router_z_loss_clip": 0.93945312, + "router_z_loss_mlp": 0.15692139, + "step": 3054, + "time_per_iteration": 2.6460957527160645 + }, + { + "auxiliary_loss_clip": 0.01035138, + "auxiliary_loss_mlp": 0.01003536, + "balance_loss_clip": 1.00974512, + "balance_loss_mlp": 1.00221276, + "epoch": 0.08864836631652255, + "flos": 61744556782080.0, + "grad_norm": 0.6912248557924995, + "language_loss": 0.45132375, + "learning_rate": 3.964032887018598e-06, + "loss": 0.47171047, + "num_input_tokens_seen": 85464090, + "router_z_loss_clip": 0.25390625, + "router_z_loss_mlp": 0.01324463, + "step": 3055, + "time_per_iteration": 3.2304747104644775 + }, + { + "auxiliary_loss_clip": 0.01035854, + "auxiliary_loss_mlp": 0.01004284, + "balance_loss_clip": 1.01044822, + "balance_loss_mlp": 1.00310993, + "epoch": 0.08867738378503859, + "flos": 60430477622400.0, + "grad_norm": 0.7197101990261777, + "language_loss": 0.48483938, + "learning_rate": 3.963997392030909e-06, + "loss": 0.50524074, + "num_input_tokens_seen": 85521015, + "router_z_loss_clip": 0.25390625, + "router_z_loss_mlp": 0.01171875, + "step": 3056, + "time_per_iteration": 2.972811222076416 + }, + { + "auxiliary_loss_clip": 0.01036293, + "auxiliary_loss_mlp": 0.01005176, + "balance_loss_clip": 1.01096463, + "balance_loss_mlp": 1.0040015, + "epoch": 0.08870640125355464, + "flos": 74783039466240.0, + "grad_norm": 0.6435587524366492, + "language_loss": 0.49539956, + "learning_rate": 3.9639618796964e-06, + "loss": 0.51581424, + "num_input_tokens_seen": 85589205, + "router_z_loss_clip": 0.25390625, + "router_z_loss_mlp": 0.01171875, + "step": 3057, + "time_per_iteration": 3.1976749897003174 + }, + { + "auxiliary_loss_clip": 0.01158643, + "auxiliary_loss_mlp": 0.01046945, + "balance_loss_clip": 1.06346011, + "balance_loss_mlp": 1.0303154, + "epoch": 0.08873541872207069, + "flos": 23251045743360.0, + "grad_norm": 2.7499165554423795, + "language_loss": 0.964091, + "learning_rate": 3.963926350015385e-06, + "loss": 0.98614693, + "num_input_tokens_seen": 85604055, + "router_z_loss_clip": 0.95166016, + "router_z_loss_mlp": 0.16619873, + "step": 3058, + "time_per_iteration": 2.5430710315704346 + }, + { + "auxiliary_loss_clip": 0.01035184, + "auxiliary_loss_mlp": 0.0100638, + "balance_loss_clip": 1.01013827, + "balance_loss_mlp": 1.0052712, + "epoch": 0.08876443619058673, + "flos": 74393334858240.0, + "grad_norm": 0.7341713738152906, + "language_loss": 0.55813253, + "learning_rate": 3.963890802988178e-06, + "loss": 0.57854819, + "num_input_tokens_seen": 85656140, + "router_z_loss_clip": 0.25, + "router_z_loss_mlp": 0.0111084, + "step": 3059, + "time_per_iteration": 3.0151054859161377 + }, + { + "auxiliary_loss_clip": 0.01035375, + "auxiliary_loss_mlp": 0.01006366, + "balance_loss_clip": 1.01034164, + "balance_loss_mlp": 1.00527525, + "epoch": 0.08879345365910278, + "flos": 72070777451520.0, + "grad_norm": 0.6821642330773746, + "language_loss": 0.5168792, + "learning_rate": 3.963855238615092e-06, + "loss": 0.53729653, + "num_input_tokens_seen": 85716500, + "router_z_loss_clip": 0.25, + "router_z_loss_mlp": 0.01092529, + "step": 3060, + "time_per_iteration": 5.680483341217041 + }, + { + "auxiliary_loss_clip": 0.01034381, + "auxiliary_loss_mlp": 0.01004407, + "balance_loss_clip": 1.0095284, + "balance_loss_mlp": 1.00329804, + "epoch": 0.08882247112761883, + "flos": 73807308466560.0, + "grad_norm": 0.6931512765109807, + "language_loss": 0.4764055, + "learning_rate": 3.963819656896443e-06, + "loss": 0.49679333, + "num_input_tokens_seen": 85768580, + "router_z_loss_clip": 0.24902344, + "router_z_loss_mlp": 0.0111084, + "step": 3061, + "time_per_iteration": 5.251219987869263 + }, + { + "auxiliary_loss_clip": 0.01033887, + "auxiliary_loss_mlp": 0.00999669, + "balance_loss_clip": 1.00894403, + "balance_loss_mlp": 0.99859583, + "epoch": 0.08885148859613487, + "flos": 74783937306240.0, + "grad_norm": 0.6731034157385578, + "language_loss": 0.53611511, + "learning_rate": 3.963784057832543e-06, + "loss": 0.55645066, + "num_input_tokens_seen": 85832920, + "router_z_loss_clip": 0.25, + "router_z_loss_mlp": 0.01074219, + "step": 3062, + "time_per_iteration": 5.4250266551971436 + }, + { + "auxiliary_loss_clip": 0.01157419, + "auxiliary_loss_mlp": 0.01050412, + "balance_loss_clip": 1.06226182, + "balance_loss_mlp": 1.03191054, + "epoch": 0.08888050606465092, + "flos": 14422192358400.0, + "grad_norm": 3.588025156689815, + "language_loss": 0.84263122, + "learning_rate": 3.963748441423708e-06, + "loss": 0.8647095, + "num_input_tokens_seen": 85846430, + "router_z_loss_clip": 0.95214844, + "router_z_loss_mlp": 0.18493652, + "step": 3063, + "time_per_iteration": 4.9911887645721436 + }, + { + "auxiliary_loss_clip": 0.01161261, + "auxiliary_loss_mlp": 0.01047692, + "balance_loss_clip": 1.06410289, + "balance_loss_mlp": 1.02960742, + "epoch": 0.08890952353316697, + "flos": 13364946420480.0, + "grad_norm": 3.472710288757648, + "language_loss": 0.68893629, + "learning_rate": 3.963712807670252e-06, + "loss": 0.71102583, + "num_input_tokens_seen": 85859975, + "router_z_loss_clip": 0.97265625, + "router_z_loss_mlp": 0.1809082, + "step": 3064, + "time_per_iteration": 2.4343929290771484 + }, + { + "auxiliary_loss_clip": 0.01157597, + "auxiliary_loss_mlp": 0.01049475, + "balance_loss_clip": 1.06394362, + "balance_loss_mlp": 1.03250003, + "epoch": 0.08893854100168301, + "flos": 31570326835200.0, + "grad_norm": 2.2189302396022037, + "language_loss": 0.88641053, + "learning_rate": 3.96367715657249e-06, + "loss": 0.9084813, + "num_input_tokens_seen": 85876465, + "router_z_loss_clip": 0.93603516, + "router_z_loss_mlp": 0.16992188, + "step": 3065, + "time_per_iteration": 2.5861432552337646 + }, + { + "auxiliary_loss_clip": 0.01167201, + "auxiliary_loss_mlp": 0.0104527, + "balance_loss_clip": 1.06563604, + "balance_loss_mlp": 1.02725148, + "epoch": 0.08896755847019906, + "flos": 12471849636480.0, + "grad_norm": 2.0621242431688263, + "language_loss": 0.66135311, + "learning_rate": 3.963641488130736e-06, + "loss": 0.68347782, + "num_input_tokens_seen": 85893830, + "router_z_loss_clip": 1.015625, + "router_z_loss_mlp": 0.18005371, + "step": 3066, + "time_per_iteration": 2.502910614013672 + }, + { + "auxiliary_loss_clip": 0.01164241, + "auxiliary_loss_mlp": 0.01047593, + "balance_loss_clip": 1.06496537, + "balance_loss_mlp": 1.02768445, + "epoch": 0.0889965759387151, + "flos": 50507351377920.0, + "grad_norm": 2.2253086992021487, + "language_loss": 0.88110948, + "learning_rate": 3.9636058023453075e-06, + "loss": 0.90322781, + "num_input_tokens_seen": 85914835, + "router_z_loss_clip": 0.99316406, + "router_z_loss_mlp": 0.19897461, + "step": 3067, + "time_per_iteration": 2.760218620300293 + }, + { + "auxiliary_loss_clip": 0.01159141, + "auxiliary_loss_mlp": 0.01053544, + "balance_loss_clip": 1.05952406, + "balance_loss_mlp": 1.0327059, + "epoch": 0.08902559340723115, + "flos": 17193382634880.0, + "grad_norm": 2.2894417920020675, + "language_loss": 0.93044531, + "learning_rate": 3.9635700992165166e-06, + "loss": 0.95257211, + "num_input_tokens_seen": 85927515, + "router_z_loss_clip": 0.99560547, + "router_z_loss_mlp": 0.20837402, + "step": 3068, + "time_per_iteration": 2.5381956100463867 + }, + { + "auxiliary_loss_clip": 0.01147913, + "auxiliary_loss_mlp": 0.01041964, + "balance_loss_clip": 1.05643439, + "balance_loss_mlp": 1.02391613, + "epoch": 0.0890546108757472, + "flos": 28652369587200.0, + "grad_norm": 2.7093695319455273, + "language_loss": 0.94601065, + "learning_rate": 3.96353437874468e-06, + "loss": 0.96790946, + "num_input_tokens_seen": 85942040, + "router_z_loss_clip": 0.91503906, + "router_z_loss_mlp": 0.18041992, + "step": 3069, + "time_per_iteration": 2.610398769378662 + }, + { + "auxiliary_loss_clip": 0.01157768, + "auxiliary_loss_mlp": 0.01040877, + "balance_loss_clip": 1.06057465, + "balance_loss_mlp": 1.02289975, + "epoch": 0.08908362834426324, + "flos": 22375974608640.0, + "grad_norm": 2.451197965185133, + "language_loss": 0.88387704, + "learning_rate": 3.963498640930114e-06, + "loss": 0.90586346, + "num_input_tokens_seen": 85958910, + "router_z_loss_clip": 0.97216797, + "router_z_loss_mlp": 0.1796875, + "step": 3070, + "time_per_iteration": 2.570585250854492 + }, + { + "auxiliary_loss_clip": 0.01033961, + "auxiliary_loss_mlp": 0.01000468, + "balance_loss_clip": 1.00942039, + "balance_loss_mlp": 0.9992401, + "epoch": 0.08911264581277929, + "flos": 53755007564160.0, + "grad_norm": 0.8802821789687219, + "language_loss": 0.56115448, + "learning_rate": 3.963462885773133e-06, + "loss": 0.58149874, + "num_input_tokens_seen": 86014985, + "router_z_loss_clip": 0.24511719, + "router_z_loss_mlp": 0.01226807, + "step": 3071, + "time_per_iteration": 2.982530117034912 + }, + { + "auxiliary_loss_clip": 0.01143553, + "auxiliary_loss_mlp": 0.01032381, + "balance_loss_clip": 1.0580492, + "balance_loss_mlp": 1.01771271, + "epoch": 0.08914166328129534, + "flos": 22374897200640.0, + "grad_norm": 2.103962114064737, + "language_loss": 0.72229135, + "learning_rate": 3.963427113274054e-06, + "loss": 0.74405068, + "num_input_tokens_seen": 86030130, + "router_z_loss_clip": 0.85546875, + "router_z_loss_mlp": 0.14672852, + "step": 3072, + "time_per_iteration": 2.5757009983062744 + }, + { + "auxiliary_loss_clip": 0.01151204, + "auxiliary_loss_mlp": 0.01051585, + "balance_loss_clip": 1.05839157, + "balance_loss_mlp": 1.03412724, + "epoch": 0.08917068074981138, + "flos": 34269408558720.0, + "grad_norm": 2.4653723638777643, + "language_loss": 0.67703593, + "learning_rate": 3.9633913234331904e-06, + "loss": 0.69906384, + "num_input_tokens_seen": 86044820, + "router_z_loss_clip": 0.92773438, + "router_z_loss_mlp": 0.17468262, + "step": 3073, + "time_per_iteration": 2.5981132984161377 + }, + { + "auxiliary_loss_clip": 0.01159557, + "auxiliary_loss_mlp": 0.01063782, + "balance_loss_clip": 1.06032681, + "balance_loss_mlp": 1.04588878, + "epoch": 0.08919969821832743, + "flos": 18478949374080.0, + "grad_norm": 2.8169463159268555, + "language_loss": 0.96148527, + "learning_rate": 3.9633555162508615e-06, + "loss": 0.98371863, + "num_input_tokens_seen": 86059850, + "router_z_loss_clip": 0.9921875, + "router_z_loss_mlp": 0.17895508, + "step": 3074, + "time_per_iteration": 2.456494092941284 + }, + { + "auxiliary_loss_clip": 0.01171084, + "auxiliary_loss_mlp": 0.01057245, + "balance_loss_clip": 1.06438351, + "balance_loss_mlp": 1.03500056, + "epoch": 0.08922871568684348, + "flos": 15700971427200.0, + "grad_norm": 2.4064783106113685, + "language_loss": 0.89645088, + "learning_rate": 3.963319691727382e-06, + "loss": 0.91873413, + "num_input_tokens_seen": 86074575, + "router_z_loss_clip": 1.06640625, + "router_z_loss_mlp": 0.22253418, + "step": 3075, + "time_per_iteration": 2.4578616619110107 + }, + { + "auxiliary_loss_clip": 0.01154958, + "auxiliary_loss_mlp": 0.01044604, + "balance_loss_clip": 1.05684948, + "balance_loss_mlp": 1.02659166, + "epoch": 0.08925773315535952, + "flos": 13144849833600.0, + "grad_norm": 2.3098745289208895, + "language_loss": 0.78831136, + "learning_rate": 3.963283849863069e-06, + "loss": 0.81030697, + "num_input_tokens_seen": 86087570, + "router_z_loss_clip": 0.98095703, + "router_z_loss_mlp": 0.18011475, + "step": 3076, + "time_per_iteration": 2.449774980545044 + }, + { + "auxiliary_loss_clip": 0.01032344, + "auxiliary_loss_mlp": 0.01002973, + "balance_loss_clip": 1.00768661, + "balance_loss_mlp": 1.00155473, + "epoch": 0.08928675062387557, + "flos": 61029289255680.0, + "grad_norm": 0.621566394341687, + "language_loss": 0.4991259, + "learning_rate": 3.963247990658238e-06, + "loss": 0.51947904, + "num_input_tokens_seen": 86152120, + "router_z_loss_clip": 0.24609375, + "router_z_loss_mlp": 0.01416016, + "step": 3077, + "time_per_iteration": 3.1371665000915527 + }, + { + "auxiliary_loss_clip": 0.01159988, + "auxiliary_loss_mlp": 0.01050774, + "balance_loss_clip": 1.06036079, + "balance_loss_mlp": 1.03298795, + "epoch": 0.08931576809239163, + "flos": 25915042857600.0, + "grad_norm": 2.1061074917873603, + "language_loss": 0.92165327, + "learning_rate": 3.963212114113206e-06, + "loss": 0.94376087, + "num_input_tokens_seen": 86170530, + "router_z_loss_clip": 0.99560547, + "router_z_loss_mlp": 0.17785645, + "step": 3078, + "time_per_iteration": 2.51973032951355 + }, + { + "auxiliary_loss_clip": 0.01032482, + "auxiliary_loss_mlp": 0.01002336, + "balance_loss_clip": 1.00787365, + "balance_loss_mlp": 1.00110781, + "epoch": 0.08934478556090766, + "flos": 63170892921600.0, + "grad_norm": 0.6752837550794463, + "language_loss": 0.48117763, + "learning_rate": 3.96317622022829e-06, + "loss": 0.50152582, + "num_input_tokens_seen": 86229355, + "router_z_loss_clip": 0.24609375, + "router_z_loss_mlp": 0.01226807, + "step": 3079, + "time_per_iteration": 2.966958522796631 + }, + { + "auxiliary_loss_clip": 0.01032141, + "auxiliary_loss_mlp": 0.01003508, + "balance_loss_clip": 1.00751698, + "balance_loss_mlp": 1.00232184, + "epoch": 0.08937380302942372, + "flos": 65623376799360.0, + "grad_norm": 0.6572877976459255, + "language_loss": 0.46512538, + "learning_rate": 3.963140309003808e-06, + "loss": 0.48548186, + "num_input_tokens_seen": 86296450, + "router_z_loss_clip": 0.24609375, + "router_z_loss_mlp": 0.01184082, + "step": 3080, + "time_per_iteration": 3.1318233013153076 + }, + { + "auxiliary_loss_clip": 0.01160595, + "auxiliary_loss_mlp": 0.01053793, + "balance_loss_clip": 1.06090522, + "balance_loss_mlp": 1.03446889, + "epoch": 0.08940282049793975, + "flos": 20076901718400.0, + "grad_norm": 2.527965815734359, + "language_loss": 0.90281492, + "learning_rate": 3.963104380440076e-06, + "loss": 0.92495883, + "num_input_tokens_seen": 86310125, + "router_z_loss_clip": 0.99609375, + "router_z_loss_mlp": 0.19329834, + "step": 3081, + "time_per_iteration": 2.4871532917022705 + }, + { + "auxiliary_loss_clip": 0.01158094, + "auxiliary_loss_mlp": 0.01042131, + "balance_loss_clip": 1.06409216, + "balance_loss_mlp": 1.02556133, + "epoch": 0.0894318379664558, + "flos": 17705468880000.0, + "grad_norm": 3.692488005691408, + "language_loss": 0.80030817, + "learning_rate": 3.963068434537413e-06, + "loss": 0.82231045, + "num_input_tokens_seen": 86322465, + "router_z_loss_clip": 0.93945312, + "router_z_loss_mlp": 0.16571045, + "step": 3082, + "time_per_iteration": 2.5125088691711426 + }, + { + "auxiliary_loss_clip": 0.01031959, + "auxiliary_loss_mlp": 0.01005853, + "balance_loss_clip": 1.00731754, + "balance_loss_mlp": 1.00473809, + "epoch": 0.08946085543497186, + "flos": 65537369683200.0, + "grad_norm": 0.7855107581063197, + "language_loss": 0.53412116, + "learning_rate": 3.9630324712961335e-06, + "loss": 0.55449927, + "num_input_tokens_seen": 86379895, + "router_z_loss_clip": 0.24609375, + "router_z_loss_mlp": 0.01116943, + "step": 3083, + "time_per_iteration": 2.9557337760925293 + }, + { + "auxiliary_loss_clip": 0.0115467, + "auxiliary_loss_mlp": 0.01050799, + "balance_loss_clip": 1.06088305, + "balance_loss_mlp": 1.03271461, + "epoch": 0.0894898729034879, + "flos": 15458647299840.0, + "grad_norm": 2.5886406394999693, + "language_loss": 0.81170321, + "learning_rate": 3.9629964907165575e-06, + "loss": 0.83375794, + "num_input_tokens_seen": 86394860, + "router_z_loss_clip": 0.9375, + "router_z_loss_mlp": 0.18078613, + "step": 3084, + "time_per_iteration": 2.478437900543213 + }, + { + "auxiliary_loss_clip": 0.01164101, + "auxiliary_loss_mlp": 0.01047811, + "balance_loss_clip": 1.06294787, + "balance_loss_mlp": 1.02745032, + "epoch": 0.08951889037200395, + "flos": 22376549226240.0, + "grad_norm": 2.3668383503727792, + "language_loss": 0.81368178, + "learning_rate": 3.962960492799002e-06, + "loss": 0.83580095, + "num_input_tokens_seen": 86409735, + "router_z_loss_clip": 1.01074219, + "router_z_loss_mlp": 0.20361328, + "step": 3085, + "time_per_iteration": 2.516507387161255 + }, + { + "auxiliary_loss_clip": 0.01158726, + "auxiliary_loss_mlp": 0.01052226, + "balance_loss_clip": 1.06134808, + "balance_loss_mlp": 1.03481603, + "epoch": 0.08954790784052, + "flos": 63351556375680.0, + "grad_norm": 2.417294501598582, + "language_loss": 0.94424164, + "learning_rate": 3.9629244775437845e-06, + "loss": 0.96635115, + "num_input_tokens_seen": 86435935, + "router_z_loss_clip": 0.97412109, + "router_z_loss_mlp": 0.17431641, + "step": 3086, + "time_per_iteration": 2.7856593132019043 + }, + { + "auxiliary_loss_clip": 0.01159116, + "auxiliary_loss_mlp": 0.01051601, + "balance_loss_clip": 1.06057203, + "balance_loss_mlp": 1.03352928, + "epoch": 0.08957692530903603, + "flos": 16683487119360.0, + "grad_norm": 2.517455585329069, + "language_loss": 0.68476737, + "learning_rate": 3.9628884449512246e-06, + "loss": 0.70687449, + "num_input_tokens_seen": 86453265, + "router_z_loss_clip": 0.98535156, + "router_z_loss_mlp": 0.18078613, + "step": 3087, + "time_per_iteration": 2.683948278427124 + }, + { + "auxiliary_loss_clip": 0.01158007, + "auxiliary_loss_mlp": 0.01048302, + "balance_loss_clip": 1.06093037, + "balance_loss_mlp": 1.02951443, + "epoch": 0.08960594277755209, + "flos": 19016710865280.0, + "grad_norm": 2.6369639046379967, + "language_loss": 0.85263348, + "learning_rate": 3.96285239502164e-06, + "loss": 0.87469649, + "num_input_tokens_seen": 86467870, + "router_z_loss_clip": 0.97119141, + "router_z_loss_mlp": 0.18786621, + "step": 3088, + "time_per_iteration": 2.465461254119873 + }, + { + "auxiliary_loss_clip": 0.01156845, + "auxiliary_loss_mlp": 0.01055796, + "balance_loss_clip": 1.06230283, + "balance_loss_mlp": 1.03715718, + "epoch": 0.08963496024606814, + "flos": 30694142378880.0, + "grad_norm": 3.019981185399188, + "language_loss": 0.7181955, + "learning_rate": 3.9628163277553484e-06, + "loss": 0.74032187, + "num_input_tokens_seen": 86481695, + "router_z_loss_clip": 0.94580078, + "router_z_loss_mlp": 0.1864624, + "step": 3089, + "time_per_iteration": 2.5722858905792236 + }, + { + "auxiliary_loss_clip": 0.0114619, + "auxiliary_loss_mlp": 0.0103976, + "balance_loss_clip": 1.05713344, + "balance_loss_mlp": 1.02422118, + "epoch": 0.08966397771458418, + "flos": 13766140402560.0, + "grad_norm": 2.648935740386907, + "language_loss": 0.70310247, + "learning_rate": 3.962780243152668e-06, + "loss": 0.72496206, + "num_input_tokens_seen": 86494805, + "router_z_loss_clip": 0.89111328, + "router_z_loss_mlp": 0.15533447, + "step": 3090, + "time_per_iteration": 2.4285786151885986 + }, + { + "auxiliary_loss_clip": 0.01155796, + "auxiliary_loss_mlp": 0.0104029, + "balance_loss_clip": 1.0589025, + "balance_loss_mlp": 1.02385139, + "epoch": 0.08969299518310023, + "flos": 17121705045120.0, + "grad_norm": 3.0105446341284825, + "language_loss": 0.86222208, + "learning_rate": 3.962744141213919e-06, + "loss": 0.88418293, + "num_input_tokens_seen": 86507990, + "router_z_loss_clip": 0.96875, + "router_z_loss_mlp": 0.16442871, + "step": 3091, + "time_per_iteration": 2.490654230117798 + }, + { + "auxiliary_loss_clip": 0.01152755, + "auxiliary_loss_mlp": 0.01049986, + "balance_loss_clip": 1.0599668, + "balance_loss_mlp": 1.03061426, + "epoch": 0.08972201265161628, + "flos": 12894193751040.0, + "grad_norm": 2.281424203436229, + "language_loss": 0.62661988, + "learning_rate": 3.962708021939419e-06, + "loss": 0.64864731, + "num_input_tokens_seen": 86521415, + "router_z_loss_clip": 0.92724609, + "router_z_loss_mlp": 0.19366455, + "step": 3092, + "time_per_iteration": 2.4443764686584473 + }, + { + "auxiliary_loss_clip": 0.01160527, + "auxiliary_loss_mlp": 0.01047826, + "balance_loss_clip": 1.06297159, + "balance_loss_mlp": 1.02968216, + "epoch": 0.08975103012013232, + "flos": 11179459313280.0, + "grad_norm": 2.5336263233462435, + "language_loss": 0.86431408, + "learning_rate": 3.962671885329488e-06, + "loss": 0.88639766, + "num_input_tokens_seen": 86535700, + "router_z_loss_clip": 0.9765625, + "router_z_loss_mlp": 0.18151855, + "step": 3093, + "time_per_iteration": 2.487745761871338 + }, + { + "auxiliary_loss_clip": 0.01157663, + "auxiliary_loss_mlp": 0.01049668, + "balance_loss_clip": 1.06026649, + "balance_loss_mlp": 1.03136921, + "epoch": 0.08978004758864837, + "flos": 15808020935040.0, + "grad_norm": 3.4860080889197813, + "language_loss": 0.85672063, + "learning_rate": 3.962635731384444e-06, + "loss": 0.87879395, + "num_input_tokens_seen": 86548790, + "router_z_loss_clip": 0.97363281, + "router_z_loss_mlp": 0.18286133, + "step": 3094, + "time_per_iteration": 2.4440431594848633 + }, + { + "auxiliary_loss_clip": 0.01041194, + "auxiliary_loss_mlp": 0.01003869, + "balance_loss_clip": 1.01630807, + "balance_loss_mlp": 1.00252199, + "epoch": 0.08980906505716442, + "flos": 72879450295680.0, + "grad_norm": 0.7226421981585526, + "language_loss": 0.53068155, + "learning_rate": 3.962599560104608e-06, + "loss": 0.55113214, + "num_input_tokens_seen": 86616390, + "router_z_loss_clip": 0.24902344, + "router_z_loss_mlp": 0.01348877, + "step": 3095, + "time_per_iteration": 3.168571949005127 + }, + { + "auxiliary_loss_clip": 0.01159694, + "auxiliary_loss_mlp": 0.01040288, + "balance_loss_clip": 1.06069589, + "balance_loss_mlp": 1.02302599, + "epoch": 0.08983808252568046, + "flos": 19822618362240.0, + "grad_norm": 3.2564027761729237, + "language_loss": 0.78506458, + "learning_rate": 3.9625633714902984e-06, + "loss": 0.80706441, + "num_input_tokens_seen": 86631400, + "router_z_loss_clip": 0.98876953, + "router_z_loss_mlp": 0.17285156, + "step": 3096, + "time_per_iteration": 2.4759128093719482 + }, + { + "auxiliary_loss_clip": 0.01037055, + "auxiliary_loss_mlp": 0.01002604, + "balance_loss_clip": 1.01211619, + "balance_loss_mlp": 1.00129271, + "epoch": 0.08986709999419651, + "flos": 64969371918720.0, + "grad_norm": 0.6697233003811175, + "language_loss": 0.45933694, + "learning_rate": 3.962527165541834e-06, + "loss": 0.47973353, + "num_input_tokens_seen": 86693760, + "router_z_loss_clip": 0.25, + "router_z_loss_mlp": 0.01312256, + "step": 3097, + "time_per_iteration": 3.1991333961486816 + }, + { + "auxiliary_loss_clip": 0.01161147, + "auxiliary_loss_mlp": 0.01050483, + "balance_loss_clip": 1.0618701, + "balance_loss_mlp": 1.03263688, + "epoch": 0.08989611746271255, + "flos": 10773201513600.0, + "grad_norm": 3.2756723949135713, + "language_loss": 0.90663099, + "learning_rate": 3.962490942259536e-06, + "loss": 0.9287473, + "num_input_tokens_seen": 86705440, + "router_z_loss_clip": 0.99267578, + "router_z_loss_mlp": 0.17858887, + "step": 3098, + "time_per_iteration": 2.4372527599334717 + }, + { + "auxiliary_loss_clip": 0.01034731, + "auxiliary_loss_mlp": 0.01015782, + "balance_loss_clip": 1.00988889, + "balance_loss_mlp": 1.01450062, + "epoch": 0.0899251349312286, + "flos": 59380486231680.0, + "grad_norm": 0.6594453554449168, + "language_loss": 0.53639448, + "learning_rate": 3.962454701643724e-06, + "loss": 0.55689967, + "num_input_tokens_seen": 86768490, + "router_z_loss_clip": 0.24804688, + "router_z_loss_mlp": 0.01281738, + "step": 3099, + "time_per_iteration": 3.0657384395599365 + }, + { + "auxiliary_loss_clip": 0.01033653, + "auxiliary_loss_mlp": 0.01007897, + "balance_loss_clip": 1.00895679, + "balance_loss_mlp": 1.00659776, + "epoch": 0.08995415239974465, + "flos": 74775389869440.0, + "grad_norm": 0.657839021332613, + "language_loss": 0.53554261, + "learning_rate": 3.962418443694717e-06, + "loss": 0.55595803, + "num_input_tokens_seen": 86827760, + "router_z_loss_clip": 0.24707031, + "router_z_loss_mlp": 0.01300049, + "step": 3100, + "time_per_iteration": 3.1276743412017822 + }, + { + "auxiliary_loss_clip": 0.01149818, + "auxiliary_loss_mlp": 0.01046068, + "balance_loss_clip": 1.05691504, + "balance_loss_mlp": 1.02925348, + "epoch": 0.08998316986826069, + "flos": 20076722150400.0, + "grad_norm": 3.5248772560782604, + "language_loss": 0.85293198, + "learning_rate": 3.962382168412838e-06, + "loss": 0.8748908, + "num_input_tokens_seen": 86840645, + "router_z_loss_clip": 0.92871094, + "router_z_loss_mlp": 0.16815186, + "step": 3101, + "time_per_iteration": 2.49712872505188 + }, + { + "auxiliary_loss_clip": 0.01160554, + "auxiliary_loss_mlp": 0.01048007, + "balance_loss_clip": 1.0584383, + "balance_loss_mlp": 1.02799201, + "epoch": 0.09001218733677674, + "flos": 15626025699840.0, + "grad_norm": 2.6756418899108154, + "language_loss": 0.79695296, + "learning_rate": 3.962345875798405e-06, + "loss": 0.81903863, + "num_input_tokens_seen": 86854340, + "router_z_loss_clip": 1.02148438, + "router_z_loss_mlp": 0.19995117, + "step": 3102, + "time_per_iteration": 2.4637532234191895 + }, + { + "auxiliary_loss_clip": 0.0103162, + "auxiliary_loss_mlp": 0.00999032, + "balance_loss_clip": 1.00674427, + "balance_loss_mlp": 0.9979769, + "epoch": 0.09004120480529279, + "flos": 60804308419200.0, + "grad_norm": 0.7135240385443788, + "language_loss": 0.50971323, + "learning_rate": 3.962309565851738e-06, + "loss": 0.53001976, + "num_input_tokens_seen": 86918005, + "router_z_loss_clip": 0.24804688, + "router_z_loss_mlp": 0.01055908, + "step": 3103, + "time_per_iteration": 3.0322763919830322 + }, + { + "auxiliary_loss_clip": 0.01160182, + "auxiliary_loss_mlp": 0.01051541, + "balance_loss_clip": 1.05932641, + "balance_loss_mlp": 1.03236008, + "epoch": 0.09007022227380883, + "flos": 23107187773440.0, + "grad_norm": 2.768225594819089, + "language_loss": 0.69444478, + "learning_rate": 3.96227323857316e-06, + "loss": 0.71656203, + "num_input_tokens_seen": 86932280, + "router_z_loss_clip": 1.00927734, + "router_z_loss_mlp": 0.19189453, + "step": 3104, + "time_per_iteration": 2.497758388519287 + }, + { + "auxiliary_loss_clip": 0.01158764, + "auxiliary_loss_mlp": 0.01046906, + "balance_loss_clip": 1.06101584, + "balance_loss_mlp": 1.02956724, + "epoch": 0.09009923974232488, + "flos": 74741523344640.0, + "grad_norm": 1.7864131766653912, + "language_loss": 0.86796582, + "learning_rate": 3.962236893962991e-06, + "loss": 0.89002252, + "num_input_tokens_seen": 86959015, + "router_z_loss_clip": 0.97802734, + "router_z_loss_mlp": 0.17321777, + "step": 3105, + "time_per_iteration": 2.9506113529205322 + }, + { + "auxiliary_loss_clip": 0.01150164, + "auxiliary_loss_mlp": 0.0104715, + "balance_loss_clip": 1.05585957, + "balance_loss_mlp": 1.02914989, + "epoch": 0.09012825721084093, + "flos": 11585142495360.0, + "grad_norm": 9.741306336195944, + "language_loss": 0.75883174, + "learning_rate": 3.962200532021551e-06, + "loss": 0.78080487, + "num_input_tokens_seen": 86971795, + "router_z_loss_clip": 0.94287109, + "router_z_loss_mlp": 0.18029785, + "step": 3106, + "time_per_iteration": 2.490943670272827 + }, + { + "auxiliary_loss_clip": 0.01162157, + "auxiliary_loss_mlp": 0.01047562, + "balance_loss_clip": 1.06356943, + "balance_loss_mlp": 1.03035355, + "epoch": 0.09015727467935697, + "flos": 19860719713920.0, + "grad_norm": 2.451170593814193, + "language_loss": 0.88287759, + "learning_rate": 3.962164152749162e-06, + "loss": 0.90497476, + "num_input_tokens_seen": 86984595, + "router_z_loss_clip": 0.98535156, + "router_z_loss_mlp": 0.17199707, + "step": 3107, + "time_per_iteration": 2.501154661178589 + }, + { + "auxiliary_loss_clip": 0.01169797, + "auxiliary_loss_mlp": 0.01049435, + "balance_loss_clip": 1.06286991, + "balance_loss_mlp": 1.02893138, + "epoch": 0.09018629214787302, + "flos": 39386284600320.0, + "grad_norm": 2.0496401377990074, + "language_loss": 0.85705614, + "learning_rate": 3.962127756146145e-06, + "loss": 0.8792485, + "num_input_tokens_seen": 87006585, + "router_z_loss_clip": 1.06933594, + "router_z_loss_mlp": 0.20495605, + "step": 3108, + "time_per_iteration": 2.672813653945923 + }, + { + "auxiliary_loss_clip": 0.01153444, + "auxiliary_loss_mlp": 0.01048085, + "balance_loss_clip": 1.05931306, + "balance_loss_mlp": 1.03110933, + "epoch": 0.09021530961638907, + "flos": 19709750851200.0, + "grad_norm": 3.7112476837596122, + "language_loss": 0.95786053, + "learning_rate": 3.962091342212822e-06, + "loss": 0.9798758, + "num_input_tokens_seen": 87018950, + "router_z_loss_clip": 0.94091797, + "router_z_loss_mlp": 0.16967773, + "step": 3109, + "time_per_iteration": 2.4679019451141357 + }, + { + "auxiliary_loss_clip": 0.01033988, + "auxiliary_loss_mlp": 0.01000347, + "balance_loss_clip": 1.0089972, + "balance_loss_mlp": 0.99932146, + "epoch": 0.09024432708490511, + "flos": 70890356799360.0, + "grad_norm": 0.692683099321333, + "language_loss": 0.45873058, + "learning_rate": 3.9620549109495145e-06, + "loss": 0.47907394, + "num_input_tokens_seen": 87072385, + "router_z_loss_clip": 0.25, + "router_z_loss_mlp": 0.01025391, + "step": 3110, + "time_per_iteration": 3.0249390602111816 + }, + { + "auxiliary_loss_clip": 0.0103324, + "auxiliary_loss_mlp": 0.00999986, + "balance_loss_clip": 1.00809813, + "balance_loss_mlp": 0.99897903, + "epoch": 0.09027334455342116, + "flos": 64324058129280.0, + "grad_norm": 0.6787744420921606, + "language_loss": 0.50475156, + "learning_rate": 3.962018462356543e-06, + "loss": 0.52508384, + "num_input_tokens_seen": 87136750, + "router_z_loss_clip": 0.25195312, + "router_z_loss_mlp": 0.0100708, + "step": 3111, + "time_per_iteration": 3.286440134048462 + }, + { + "auxiliary_loss_clip": 0.0117062, + "auxiliary_loss_mlp": 0.01055306, + "balance_loss_clip": 1.06323957, + "balance_loss_mlp": 1.03458762, + "epoch": 0.09030236202193721, + "flos": 21501765400320.0, + "grad_norm": 2.30646337007318, + "language_loss": 0.85175693, + "learning_rate": 3.961981996434231e-06, + "loss": 0.87401628, + "num_input_tokens_seen": 87156225, + "router_z_loss_clip": 1.07324219, + "router_z_loss_mlp": 0.20727539, + "step": 3112, + "time_per_iteration": 2.5667669773101807 + }, + { + "auxiliary_loss_clip": 0.01167883, + "auxiliary_loss_mlp": 0.01048511, + "balance_loss_clip": 1.06526113, + "balance_loss_mlp": 1.02943802, + "epoch": 0.09033137949045325, + "flos": 16428162268800.0, + "grad_norm": 2.7323357958644996, + "language_loss": 0.79788136, + "learning_rate": 3.9619455131829e-06, + "loss": 0.82004535, + "num_input_tokens_seen": 87169495, + "router_z_loss_clip": 1.02636719, + "router_z_loss_mlp": 0.19091797, + "step": 3113, + "time_per_iteration": 2.5051183700561523 + }, + { + "auxiliary_loss_clip": 0.0103186, + "auxiliary_loss_mlp": 0.0101026, + "balance_loss_clip": 1.00679827, + "balance_loss_mlp": 1.00915706, + "epoch": 0.0903603969589693, + "flos": 74779484019840.0, + "grad_norm": 0.6549134012561728, + "language_loss": 0.48226953, + "learning_rate": 3.961909012602873e-06, + "loss": 0.50269073, + "num_input_tokens_seen": 87237000, + "router_z_loss_clip": 0.25, + "router_z_loss_mlp": 0.01104736, + "step": 3114, + "time_per_iteration": 3.2125651836395264 + }, + { + "auxiliary_loss_clip": 0.01031088, + "auxiliary_loss_mlp": 0.01012788, + "balance_loss_clip": 1.00609589, + "balance_loss_mlp": 1.01163793, + "epoch": 0.09038941442748534, + "flos": 74780489600640.0, + "grad_norm": 0.6371198006492821, + "language_loss": 0.50521785, + "learning_rate": 3.961872494694472e-06, + "loss": 0.52565658, + "num_input_tokens_seen": 87301920, + "router_z_loss_clip": 0.25, + "router_z_loss_mlp": 0.01147461, + "step": 3115, + "time_per_iteration": 3.1656899452209473 + }, + { + "auxiliary_loss_clip": 0.01154714, + "auxiliary_loss_mlp": 0.01051603, + "balance_loss_clip": 1.0577662, + "balance_loss_mlp": 1.03107524, + "epoch": 0.09041843189600139, + "flos": 17596875087360.0, + "grad_norm": 2.3482400974863116, + "language_loss": 0.85244691, + "learning_rate": 3.961835959458018e-06, + "loss": 0.87451005, + "num_input_tokens_seen": 87316030, + "router_z_loss_clip": 0.96728516, + "router_z_loss_mlp": 0.20507812, + "step": 3116, + "time_per_iteration": 2.453834295272827 + }, + { + "auxiliary_loss_clip": 0.01155233, + "auxiliary_loss_mlp": 0.01044492, + "balance_loss_clip": 1.05899, + "balance_loss_mlp": 1.02593136, + "epoch": 0.09044744936451744, + "flos": 36022531656960.0, + "grad_norm": 2.366086144198349, + "language_loss": 0.99479282, + "learning_rate": 3.961799406893836e-06, + "loss": 1.01679015, + "num_input_tokens_seen": 87334660, + "router_z_loss_clip": 0.96289062, + "router_z_loss_mlp": 0.18566895, + "step": 3117, + "time_per_iteration": 2.6215908527374268 + }, + { + "auxiliary_loss_clip": 0.01157011, + "auxiliary_loss_mlp": 0.01048193, + "balance_loss_clip": 1.059659, + "balance_loss_mlp": 1.03172469, + "epoch": 0.09047646683303348, + "flos": 30257504651520.0, + "grad_norm": 2.063601925953571, + "language_loss": 0.7745235, + "learning_rate": 3.961762837002247e-06, + "loss": 0.79657555, + "num_input_tokens_seen": 87350205, + "router_z_loss_clip": 0.97460938, + "router_z_loss_mlp": 0.16467285, + "step": 3118, + "time_per_iteration": 2.5506722927093506 + }, + { + "auxiliary_loss_clip": 0.011595, + "auxiliary_loss_mlp": 0.01056467, + "balance_loss_clip": 1.06065536, + "balance_loss_mlp": 1.038234, + "epoch": 0.09050548430154953, + "flos": 26389315059840.0, + "grad_norm": 2.880911036692833, + "language_loss": 1.01624858, + "learning_rate": 3.961726249783575e-06, + "loss": 1.03840828, + "num_input_tokens_seen": 87367660, + "router_z_loss_clip": 0.98730469, + "router_z_loss_mlp": 0.18243408, + "step": 3119, + "time_per_iteration": 2.555321455001831 + }, + { + "auxiliary_loss_clip": 0.0115816, + "auxiliary_loss_mlp": 0.01050263, + "balance_loss_clip": 1.06076372, + "balance_loss_mlp": 1.03240526, + "epoch": 0.09053450177006558, + "flos": 19346622307200.0, + "grad_norm": 3.444114876964878, + "language_loss": 0.93533981, + "learning_rate": 3.961689645238143e-06, + "loss": 0.95742404, + "num_input_tokens_seen": 87380060, + "router_z_loss_clip": 0.97412109, + "router_z_loss_mlp": 0.17871094, + "step": 3120, + "time_per_iteration": 2.4368839263916016 + }, + { + "auxiliary_loss_clip": 0.01034562, + "auxiliary_loss_mlp": 0.01002176, + "balance_loss_clip": 1.00959444, + "balance_loss_mlp": 1.00101936, + "epoch": 0.09056351923858162, + "flos": 67308844631040.0, + "grad_norm": 0.6738167570297717, + "language_loss": 0.46290964, + "learning_rate": 3.961653023366274e-06, + "loss": 0.48327702, + "num_input_tokens_seen": 87442370, + "router_z_loss_clip": 0.25, + "router_z_loss_mlp": 0.01153564, + "step": 3121, + "time_per_iteration": 3.1973683834075928 + }, + { + "auxiliary_loss_clip": 0.01160607, + "auxiliary_loss_mlp": 0.01043215, + "balance_loss_clip": 1.05946469, + "balance_loss_mlp": 1.02545309, + "epoch": 0.09059253670709767, + "flos": 13364371802880.0, + "grad_norm": 3.4229827825280306, + "language_loss": 0.75466228, + "learning_rate": 3.9616163841682915e-06, + "loss": 0.7767005, + "num_input_tokens_seen": 87455765, + "router_z_loss_clip": 1.01123047, + "router_z_loss_mlp": 0.1776123, + "step": 3122, + "time_per_iteration": 2.507161855697632 + }, + { + "auxiliary_loss_clip": 0.01033837, + "auxiliary_loss_mlp": 0.01003461, + "balance_loss_clip": 1.00907087, + "balance_loss_mlp": 1.00235868, + "epoch": 0.09062155417561372, + "flos": 54736984552320.0, + "grad_norm": 0.6711768530164348, + "language_loss": 0.46105391, + "learning_rate": 3.96157972764452e-06, + "loss": 0.48142689, + "num_input_tokens_seen": 87511035, + "router_z_loss_clip": 0.24804688, + "router_z_loss_mlp": 0.01104736, + "step": 3123, + "time_per_iteration": 2.9731130599975586 + }, + { + "auxiliary_loss_clip": 0.01146579, + "auxiliary_loss_mlp": 0.01043131, + "balance_loss_clip": 1.05611551, + "balance_loss_mlp": 1.02904916, + "epoch": 0.09065057164412976, + "flos": 25624202434560.0, + "grad_norm": 2.211888871730366, + "language_loss": 0.81808692, + "learning_rate": 3.961543053795283e-06, + "loss": 0.83998406, + "num_input_tokens_seen": 87525145, + "router_z_loss_clip": 0.90478516, + "router_z_loss_mlp": 0.14083862, + "step": 3124, + "time_per_iteration": 2.5666275024414062 + }, + { + "auxiliary_loss_clip": 0.01147246, + "auxiliary_loss_mlp": 0.01039271, + "balance_loss_clip": 1.05954719, + "balance_loss_mlp": 1.02505851, + "epoch": 0.09067958911264581, + "flos": 24965277390720.0, + "grad_norm": 2.989242129256466, + "language_loss": 0.85434091, + "learning_rate": 3.961506362620903e-06, + "loss": 0.87620604, + "num_input_tokens_seen": 87537940, + "router_z_loss_clip": 0.87597656, + "router_z_loss_mlp": 0.14215088, + "step": 3125, + "time_per_iteration": 2.5129480361938477 + }, + { + "auxiliary_loss_clip": 0.01031955, + "auxiliary_loss_mlp": 0.00998877, + "balance_loss_clip": 1.00739527, + "balance_loss_mlp": 0.99781018, + "epoch": 0.09070860658116187, + "flos": 62846296692480.0, + "grad_norm": 0.6624753126827343, + "language_loss": 0.52240741, + "learning_rate": 3.9614696541217054e-06, + "loss": 0.54271573, + "num_input_tokens_seen": 87600230, + "router_z_loss_clip": 0.24609375, + "router_z_loss_mlp": 0.01068115, + "step": 3126, + "time_per_iteration": 3.082636594772339 + }, + { + "auxiliary_loss_clip": 0.01156847, + "auxiliary_loss_mlp": 0.01047194, + "balance_loss_clip": 1.06088805, + "balance_loss_mlp": 1.03139853, + "epoch": 0.0907376240496779, + "flos": 22702833394560.0, + "grad_norm": 2.104955586361292, + "language_loss": 0.74901044, + "learning_rate": 3.961432928298014e-06, + "loss": 0.77105087, + "num_input_tokens_seen": 87614430, + "router_z_loss_clip": 0.95800781, + "router_z_loss_mlp": 0.15802002, + "step": 3127, + "time_per_iteration": 2.4275095462799072 + }, + { + "auxiliary_loss_clip": 0.0115566, + "auxiliary_loss_mlp": 0.01050685, + "balance_loss_clip": 1.0596056, + "balance_loss_mlp": 1.03280365, + "epoch": 0.09076664151819396, + "flos": 33540709345920.0, + "grad_norm": 2.2291360919543703, + "language_loss": 0.90860438, + "learning_rate": 3.9613961851501534e-06, + "loss": 0.93066788, + "num_input_tokens_seen": 87631080, + "router_z_loss_clip": 0.95996094, + "router_z_loss_mlp": 0.17895508, + "step": 3128, + "time_per_iteration": 2.6059012413024902 + }, + { + "auxiliary_loss_clip": 0.0115361, + "auxiliary_loss_mlp": 0.01048159, + "balance_loss_clip": 1.06088722, + "balance_loss_mlp": 1.03142786, + "epoch": 0.09079565898670999, + "flos": 20077799558400.0, + "grad_norm": 2.569565966164818, + "language_loss": 0.96145344, + "learning_rate": 3.961359424678448e-06, + "loss": 0.98347116, + "num_input_tokens_seen": 87644160, + "router_z_loss_clip": 0.92626953, + "router_z_loss_mlp": 0.16723633, + "step": 3129, + "time_per_iteration": 2.4483144283294678 + }, + { + "auxiliary_loss_clip": 0.01154349, + "auxiliary_loss_mlp": 0.01051557, + "balance_loss_clip": 1.05587256, + "balance_loss_mlp": 1.03328192, + "epoch": 0.09082467645522604, + "flos": 25405542391680.0, + "grad_norm": 5.72037950624058, + "language_loss": 0.94133711, + "learning_rate": 3.961322646883222e-06, + "loss": 0.96339619, + "num_input_tokens_seen": 87656830, + "router_z_loss_clip": 0.98339844, + "router_z_loss_mlp": 0.18286133, + "step": 3130, + "time_per_iteration": 2.5284335613250732 + }, + { + "auxiliary_loss_clip": 0.01032493, + "auxiliary_loss_mlp": 0.00999986, + "balance_loss_clip": 1.00792456, + "balance_loss_mlp": 0.99901468, + "epoch": 0.0908536939237421, + "flos": 62468156263680.0, + "grad_norm": 0.7322998514089029, + "language_loss": 0.55390066, + "learning_rate": 3.961285851764801e-06, + "loss": 0.57422549, + "num_input_tokens_seen": 87715725, + "router_z_loss_clip": 0.24609375, + "router_z_loss_mlp": 0.00970459, + "step": 3131, + "time_per_iteration": 3.166165590286255 + }, + { + "auxiliary_loss_clip": 0.01154929, + "auxiliary_loss_mlp": 0.01043722, + "balance_loss_clip": 1.05927086, + "balance_loss_mlp": 1.0265497, + "epoch": 0.09088271139225813, + "flos": 31241456887680.0, + "grad_norm": 2.204473930507941, + "language_loss": 1.02745974, + "learning_rate": 3.96124903932351e-06, + "loss": 1.04944623, + "num_input_tokens_seen": 87734830, + "router_z_loss_clip": 0.95654297, + "router_z_loss_mlp": 0.17175293, + "step": 3132, + "time_per_iteration": 8.126197099685669 + }, + { + "auxiliary_loss_clip": 0.01158139, + "auxiliary_loss_mlp": 0.01051481, + "balance_loss_clip": 1.05953729, + "balance_loss_mlp": 1.03374231, + "epoch": 0.09091172886077419, + "flos": 26099552044800.0, + "grad_norm": 2.199206146608495, + "language_loss": 0.82680273, + "learning_rate": 3.961212209559674e-06, + "loss": 0.84889889, + "num_input_tokens_seen": 87749915, + "router_z_loss_clip": 0.984375, + "router_z_loss_mlp": 0.17736816, + "step": 3133, + "time_per_iteration": 4.742070198059082 + }, + { + "auxiliary_loss_clip": 0.0103234, + "auxiliary_loss_mlp": 0.01008007, + "balance_loss_clip": 1.00782609, + "balance_loss_mlp": 1.00699353, + "epoch": 0.09094074632929024, + "flos": 72658922745600.0, + "grad_norm": 0.6850020718231379, + "language_loss": 0.53809744, + "learning_rate": 3.961175362473618e-06, + "loss": 0.55850089, + "num_input_tokens_seen": 87812910, + "router_z_loss_clip": 0.24511719, + "router_z_loss_mlp": 0.01013184, + "step": 3134, + "time_per_iteration": 5.550299644470215 + }, + { + "auxiliary_loss_clip": 0.01032369, + "auxiliary_loss_mlp": 0.01002595, + "balance_loss_clip": 1.00790119, + "balance_loss_mlp": 1.00161767, + "epoch": 0.09096976379780627, + "flos": 62365667783040.0, + "grad_norm": 0.6834310191465263, + "language_loss": 0.47385994, + "learning_rate": 3.961138498065667e-06, + "loss": 0.49420956, + "num_input_tokens_seen": 87875070, + "router_z_loss_clip": 0.24414062, + "router_z_loss_mlp": 0.00976562, + "step": 3135, + "time_per_iteration": 3.0465009212493896 + }, + { + "auxiliary_loss_clip": 0.01032206, + "auxiliary_loss_mlp": 0.00999441, + "balance_loss_clip": 1.00771451, + "balance_loss_mlp": 0.99852318, + "epoch": 0.09099878126632233, + "flos": 67611174428160.0, + "grad_norm": 0.7532267747944115, + "language_loss": 0.51836711, + "learning_rate": 3.9611016163361476e-06, + "loss": 0.53868353, + "num_input_tokens_seen": 87926810, + "router_z_loss_clip": 0.24511719, + "router_z_loss_mlp": 0.00915527, + "step": 3136, + "time_per_iteration": 2.8944811820983887 + }, + { + "auxiliary_loss_clip": 0.01031419, + "auxiliary_loss_mlp": 0.00998331, + "balance_loss_clip": 1.00687015, + "balance_loss_mlp": 0.99735367, + "epoch": 0.09102779873483838, + "flos": 65100911523840.0, + "grad_norm": 0.665062744705299, + "language_loss": 0.51738107, + "learning_rate": 3.961064717285386e-06, + "loss": 0.53767854, + "num_input_tokens_seen": 87988920, + "router_z_loss_clip": 0.24609375, + "router_z_loss_mlp": 0.00976562, + "step": 3137, + "time_per_iteration": 3.0817766189575195 + }, + { + "auxiliary_loss_clip": 0.01173078, + "auxiliary_loss_mlp": 0.01065344, + "balance_loss_clip": 1.06671429, + "balance_loss_mlp": 1.04357636, + "epoch": 0.09105681620335442, + "flos": 25661226378240.0, + "grad_norm": 1.8620226900599233, + "language_loss": 0.95018232, + "learning_rate": 3.961027800913706e-06, + "loss": 0.97256655, + "num_input_tokens_seen": 88009845, + "router_z_loss_clip": 1.06445312, + "router_z_loss_mlp": 0.21777344, + "step": 3138, + "time_per_iteration": 2.5411794185638428 + }, + { + "auxiliary_loss_clip": 0.01153111, + "auxiliary_loss_mlp": 0.01047164, + "balance_loss_clip": 1.05911756, + "balance_loss_mlp": 1.02893102, + "epoch": 0.09108583367187047, + "flos": 37924648369920.0, + "grad_norm": 2.0594287742941146, + "language_loss": 0.70239562, + "learning_rate": 3.9609908672214355e-06, + "loss": 0.72439837, + "num_input_tokens_seen": 88028080, + "router_z_loss_clip": 0.94091797, + "router_z_loss_mlp": 0.18237305, + "step": 3139, + "time_per_iteration": 2.6300032138824463 + }, + { + "auxiliary_loss_clip": 0.01152658, + "auxiliary_loss_mlp": 0.01047226, + "balance_loss_clip": 1.05948234, + "balance_loss_mlp": 1.03186023, + "epoch": 0.09111485114038652, + "flos": 20624790844800.0, + "grad_norm": 3.3620011056618067, + "language_loss": 0.76954937, + "learning_rate": 3.9609539162088995e-06, + "loss": 0.79154825, + "num_input_tokens_seen": 88041190, + "router_z_loss_clip": 0.93115234, + "router_z_loss_mlp": 0.15356445, + "step": 3140, + "time_per_iteration": 2.450282096862793 + }, + { + "auxiliary_loss_clip": 0.01155718, + "auxiliary_loss_mlp": 0.01053732, + "balance_loss_clip": 1.06472051, + "balance_loss_mlp": 1.03796077, + "epoch": 0.09114386860890256, + "flos": 18143076274560.0, + "grad_norm": 2.3116526409067077, + "language_loss": 0.78003186, + "learning_rate": 3.960916947876426e-06, + "loss": 0.80212629, + "num_input_tokens_seen": 88059210, + "router_z_loss_clip": 0.91015625, + "router_z_loss_mlp": 0.15765381, + "step": 3141, + "time_per_iteration": 2.5559494495391846 + }, + { + "auxiliary_loss_clip": 0.01155422, + "auxiliary_loss_mlp": 0.01054448, + "balance_loss_clip": 1.06170774, + "balance_loss_mlp": 1.03558934, + "epoch": 0.09117288607741861, + "flos": 11428642938240.0, + "grad_norm": 3.621838044081139, + "language_loss": 0.84763819, + "learning_rate": 3.96087996222434e-06, + "loss": 0.86973685, + "num_input_tokens_seen": 88069885, + "router_z_loss_clip": 0.9375, + "router_z_loss_mlp": 0.18884277, + "step": 3142, + "time_per_iteration": 2.4453442096710205 + }, + { + "auxiliary_loss_clip": 0.01148705, + "auxiliary_loss_mlp": 0.01035314, + "balance_loss_clip": 1.05656219, + "balance_loss_mlp": 1.01935208, + "epoch": 0.09120190354593466, + "flos": 12306263938560.0, + "grad_norm": 2.2088142139972597, + "language_loss": 0.77243173, + "learning_rate": 3.960842959252969e-06, + "loss": 0.79427195, + "num_input_tokens_seen": 88083135, + "router_z_loss_clip": 0.92089844, + "router_z_loss_mlp": 0.1595459, + "step": 3143, + "time_per_iteration": 2.494915246963501 + }, + { + "auxiliary_loss_clip": 0.01153345, + "auxiliary_loss_mlp": 0.01040178, + "balance_loss_clip": 1.06147349, + "balance_loss_mlp": 1.02437043, + "epoch": 0.0912309210144507, + "flos": 39453257508480.0, + "grad_norm": 2.0929820084663753, + "language_loss": 1.10155666, + "learning_rate": 3.960805938962639e-06, + "loss": 1.12349188, + "num_input_tokens_seen": 88103805, + "router_z_loss_clip": 0.91845703, + "router_z_loss_mlp": 0.15808105, + "step": 3144, + "time_per_iteration": 2.6947872638702393 + }, + { + "auxiliary_loss_clip": 0.01164941, + "auxiliary_loss_mlp": 0.01053644, + "balance_loss_clip": 1.0631305, + "balance_loss_mlp": 1.03496361, + "epoch": 0.09125993848296675, + "flos": 26539386082560.0, + "grad_norm": 2.8877919675825336, + "language_loss": 0.78734183, + "learning_rate": 3.960768901353678e-06, + "loss": 0.80952764, + "num_input_tokens_seen": 88119810, + "router_z_loss_clip": 1.01806641, + "router_z_loss_mlp": 0.18664551, + "step": 3145, + "time_per_iteration": 2.5842254161834717 + }, + { + "auxiliary_loss_clip": 0.01157805, + "auxiliary_loss_mlp": 0.01046303, + "balance_loss_clip": 1.06088567, + "balance_loss_mlp": 1.02931595, + "epoch": 0.09128895595148279, + "flos": 21865360821120.0, + "grad_norm": 2.3444644453895425, + "language_loss": 0.70594358, + "learning_rate": 3.960731846426411e-06, + "loss": 0.72798473, + "num_input_tokens_seen": 88135180, + "router_z_loss_clip": 0.96875, + "router_z_loss_mlp": 0.16986084, + "step": 3146, + "time_per_iteration": 2.482517719268799 + }, + { + "auxiliary_loss_clip": 0.01162818, + "auxiliary_loss_mlp": 0.01047378, + "balance_loss_clip": 1.06182599, + "balance_loss_mlp": 1.02749419, + "epoch": 0.09131797341999884, + "flos": 21287343162240.0, + "grad_norm": 2.115190583664845, + "language_loss": 1.06651866, + "learning_rate": 3.960694774181169e-06, + "loss": 1.08862066, + "num_input_tokens_seen": 88154590, + "router_z_loss_clip": 1.00927734, + "router_z_loss_mlp": 0.19885254, + "step": 3147, + "time_per_iteration": 2.4988467693328857 + }, + { + "auxiliary_loss_clip": 0.01169573, + "auxiliary_loss_mlp": 0.01050311, + "balance_loss_clip": 1.06827807, + "balance_loss_mlp": 1.0304389, + "epoch": 0.09134699088851489, + "flos": 68567329537920.0, + "grad_norm": 3.3096831999465346, + "language_loss": 0.95335615, + "learning_rate": 3.960657684618277e-06, + "loss": 0.975555, + "num_input_tokens_seen": 88173435, + "router_z_loss_clip": 1.01318359, + "router_z_loss_mlp": 0.19885254, + "step": 3148, + "time_per_iteration": 2.9242465496063232 + }, + { + "auxiliary_loss_clip": 0.01164998, + "auxiliary_loss_mlp": 0.01044431, + "balance_loss_clip": 1.06327796, + "balance_loss_mlp": 1.02522635, + "epoch": 0.09137600835703093, + "flos": 12416114707200.0, + "grad_norm": 1.961620077280531, + "language_loss": 0.74841607, + "learning_rate": 3.960620577738062e-06, + "loss": 0.77051038, + "num_input_tokens_seen": 88188855, + "router_z_loss_clip": 1.01611328, + "router_z_loss_mlp": 0.19226074, + "step": 3149, + "time_per_iteration": 2.4696006774902344 + }, + { + "auxiliary_loss_clip": 0.01048227, + "auxiliary_loss_mlp": 0.01007874, + "balance_loss_clip": 1.02332616, + "balance_loss_mlp": 1.00684857, + "epoch": 0.09140502582554698, + "flos": 55626169731840.0, + "grad_norm": 0.7367559574486641, + "language_loss": 0.48675904, + "learning_rate": 3.960583453540853e-06, + "loss": 0.50732005, + "num_input_tokens_seen": 88250740, + "router_z_loss_clip": 0.24902344, + "router_z_loss_mlp": 0.01025391, + "step": 3150, + "time_per_iteration": 3.0900537967681885 + }, + { + "auxiliary_loss_clip": 0.01155965, + "auxiliary_loss_mlp": 0.01047876, + "balance_loss_clip": 1.05953765, + "balance_loss_mlp": 1.03004265, + "epoch": 0.09143404329406303, + "flos": 30587631575040.0, + "grad_norm": 1.7931923419253584, + "language_loss": 0.81293702, + "learning_rate": 3.960546312026978e-06, + "loss": 0.83497542, + "num_input_tokens_seen": 88276040, + "router_z_loss_clip": 0.96484375, + "router_z_loss_mlp": 0.17822266, + "step": 3151, + "time_per_iteration": 2.664057493209839 + }, + { + "auxiliary_loss_clip": 0.01155726, + "auxiliary_loss_mlp": 0.01040639, + "balance_loss_clip": 1.05939293, + "balance_loss_mlp": 1.02287126, + "epoch": 0.09146306076257907, + "flos": 13070766032640.0, + "grad_norm": 2.973507366942527, + "language_loss": 1.01590049, + "learning_rate": 3.960509153196764e-06, + "loss": 1.03786421, + "num_input_tokens_seen": 88286350, + "router_z_loss_clip": 0.96337891, + "router_z_loss_mlp": 0.17773438, + "step": 3152, + "time_per_iteration": 2.4589731693267822 + }, + { + "auxiliary_loss_clip": 0.01147638, + "auxiliary_loss_mlp": 0.01043482, + "balance_loss_clip": 1.05668974, + "balance_loss_mlp": 1.0261966, + "epoch": 0.09149207823109512, + "flos": 15115411912320.0, + "grad_norm": 2.9825218695702786, + "language_loss": 0.79258513, + "learning_rate": 3.960471977050541e-06, + "loss": 0.81449628, + "num_input_tokens_seen": 88300595, + "router_z_loss_clip": 0.90966797, + "router_z_loss_mlp": 0.17285156, + "step": 3153, + "time_per_iteration": 2.452763080596924 + }, + { + "auxiliary_loss_clip": 0.01038849, + "auxiliary_loss_mlp": 0.01002769, + "balance_loss_clip": 1.01414895, + "balance_loss_mlp": 1.00169623, + "epoch": 0.09152109569961117, + "flos": 74785625245440.0, + "grad_norm": 0.6666756098713553, + "language_loss": 0.4930017, + "learning_rate": 3.960434783588635e-06, + "loss": 0.5134179, + "num_input_tokens_seen": 88371290, + "router_z_loss_clip": 0.24609375, + "router_z_loss_mlp": 0.01074219, + "step": 3154, + "time_per_iteration": 3.2108922004699707 + }, + { + "auxiliary_loss_clip": 0.01151196, + "auxiliary_loss_mlp": 0.01054194, + "balance_loss_clip": 1.05589974, + "balance_loss_mlp": 1.03827906, + "epoch": 0.09155011316812721, + "flos": 24604016353920.0, + "grad_norm": 2.101108610520204, + "language_loss": 0.76834601, + "learning_rate": 3.9603975728113766e-06, + "loss": 0.79039991, + "num_input_tokens_seen": 88388385, + "router_z_loss_clip": 0.953125, + "router_z_loss_mlp": 0.15905762, + "step": 3155, + "time_per_iteration": 2.510072708129883 + }, + { + "auxiliary_loss_clip": 0.01153025, + "auxiliary_loss_mlp": 0.01050902, + "balance_loss_clip": 1.05775309, + "balance_loss_mlp": 1.03369367, + "epoch": 0.09157913063664326, + "flos": 15115016862720.0, + "grad_norm": 2.922329786099151, + "language_loss": 0.83938432, + "learning_rate": 3.960360344719092e-06, + "loss": 0.86142361, + "num_input_tokens_seen": 88401655, + "router_z_loss_clip": 0.95263672, + "router_z_loss_mlp": 0.17205811, + "step": 3156, + "time_per_iteration": 2.545861005783081 + }, + { + "auxiliary_loss_clip": 0.01034468, + "auxiliary_loss_mlp": 0.01008956, + "balance_loss_clip": 1.00987554, + "balance_loss_mlp": 1.0080204, + "epoch": 0.09160814810515931, + "flos": 52013125422720.0, + "grad_norm": 1.5031819274320841, + "language_loss": 0.4816702, + "learning_rate": 3.960323099312113e-06, + "loss": 0.50210446, + "num_input_tokens_seen": 88459675, + "router_z_loss_clip": 0.24609375, + "router_z_loss_mlp": 0.00933838, + "step": 3157, + "time_per_iteration": 3.045524835586548 + }, + { + "auxiliary_loss_clip": 0.0116333, + "auxiliary_loss_mlp": 0.01044895, + "balance_loss_clip": 1.05998445, + "balance_loss_mlp": 1.02572584, + "epoch": 0.09163716557367535, + "flos": 18509257474560.0, + "grad_norm": 3.0840544343552945, + "language_loss": 0.82456255, + "learning_rate": 3.960285836590767e-06, + "loss": 0.84664488, + "num_input_tokens_seen": 88474220, + "router_z_loss_clip": 1.03222656, + "router_z_loss_mlp": 0.19165039, + "step": 3158, + "time_per_iteration": 2.5331780910491943 + }, + { + "auxiliary_loss_clip": 0.01033244, + "auxiliary_loss_mlp": 0.01012362, + "balance_loss_clip": 1.00897753, + "balance_loss_mlp": 1.01142049, + "epoch": 0.0916661830421914, + "flos": 63610655132160.0, + "grad_norm": 0.6145900823138145, + "language_loss": 0.47561058, + "learning_rate": 3.960248556555383e-06, + "loss": 0.49606663, + "num_input_tokens_seen": 88538380, + "router_z_loss_clip": 0.2421875, + "router_z_loss_mlp": 0.00939941, + "step": 3159, + "time_per_iteration": 3.1744883060455322 + }, + { + "auxiliary_loss_clip": 0.01156836, + "auxiliary_loss_mlp": 0.01047682, + "balance_loss_clip": 1.06248808, + "balance_loss_mlp": 1.02810752, + "epoch": 0.09169520051070744, + "flos": 30549135173760.0, + "grad_norm": 1.9637454442660205, + "language_loss": 0.72173309, + "learning_rate": 3.96021125920629e-06, + "loss": 0.74377823, + "num_input_tokens_seen": 88557240, + "router_z_loss_clip": 0.94335938, + "router_z_loss_mlp": 0.19580078, + "step": 3160, + "time_per_iteration": 2.777526378631592 + }, + { + "auxiliary_loss_clip": 0.01159256, + "auxiliary_loss_mlp": 0.0105654, + "balance_loss_clip": 1.06108201, + "balance_loss_mlp": 1.03716886, + "epoch": 0.09172421797922349, + "flos": 30950831946240.0, + "grad_norm": 3.359777785525683, + "language_loss": 0.80103159, + "learning_rate": 3.960173944543819e-06, + "loss": 0.82318956, + "num_input_tokens_seen": 88574140, + "router_z_loss_clip": 0.98046875, + "router_z_loss_mlp": 0.19372559, + "step": 3161, + "time_per_iteration": 2.619823932647705 + }, + { + "auxiliary_loss_clip": 0.01165436, + "auxiliary_loss_mlp": 0.01046538, + "balance_loss_clip": 1.06183696, + "balance_loss_mlp": 1.02676153, + "epoch": 0.09175323544773954, + "flos": 27156223365120.0, + "grad_norm": 1.9444456250601105, + "language_loss": 0.94116277, + "learning_rate": 3.960136612568298e-06, + "loss": 0.96328247, + "num_input_tokens_seen": 88593970, + "router_z_loss_clip": 1.03710938, + "router_z_loss_mlp": 0.19787598, + "step": 3162, + "time_per_iteration": 2.6861560344696045 + }, + { + "auxiliary_loss_clip": 0.01164273, + "auxiliary_loss_mlp": 0.01059805, + "balance_loss_clip": 1.06185222, + "balance_loss_mlp": 1.03970599, + "epoch": 0.09178225291625558, + "flos": 11579755455360.0, + "grad_norm": 3.019317423768027, + "language_loss": 0.89689517, + "learning_rate": 3.960099263280057e-06, + "loss": 0.91913593, + "num_input_tokens_seen": 88606315, + "router_z_loss_clip": 1.02392578, + "router_z_loss_mlp": 0.20123291, + "step": 3163, + "time_per_iteration": 2.4716291427612305 + }, + { + "auxiliary_loss_clip": 0.01163315, + "auxiliary_loss_mlp": 0.01051529, + "balance_loss_clip": 1.06047344, + "balance_loss_mlp": 1.03146625, + "epoch": 0.09181127038477163, + "flos": 28586617741440.0, + "grad_norm": 2.0722755155324353, + "language_loss": 0.89772248, + "learning_rate": 3.960061896679426e-06, + "loss": 0.91987097, + "num_input_tokens_seen": 88629375, + "router_z_loss_clip": 1.02929688, + "router_z_loss_mlp": 0.20080566, + "step": 3164, + "time_per_iteration": 2.567593812942505 + }, + { + "auxiliary_loss_clip": 0.01151249, + "auxiliary_loss_mlp": 0.01041902, + "balance_loss_clip": 1.05549479, + "balance_loss_mlp": 1.02577925, + "epoch": 0.09184028785328768, + "flos": 12310358088960.0, + "grad_norm": 2.875059569770051, + "language_loss": 0.96135926, + "learning_rate": 3.960024512766736e-06, + "loss": 0.98329079, + "num_input_tokens_seen": 88642175, + "router_z_loss_clip": 0.95654297, + "router_z_loss_mlp": 0.16125488, + "step": 3165, + "time_per_iteration": 2.437899351119995 + }, + { + "auxiliary_loss_clip": 0.01034064, + "auxiliary_loss_mlp": 0.00999472, + "balance_loss_clip": 1.00979424, + "balance_loss_mlp": 0.99850672, + "epoch": 0.09186930532180372, + "flos": 66962197451520.0, + "grad_norm": 0.7275492074506273, + "language_loss": 0.49351084, + "learning_rate": 3.959987111542316e-06, + "loss": 0.51384616, + "num_input_tokens_seen": 88697525, + "router_z_loss_clip": 0.2421875, + "router_z_loss_mlp": 0.00964355, + "step": 3166, + "time_per_iteration": 2.967799186706543 + }, + { + "auxiliary_loss_clip": 0.01033551, + "auxiliary_loss_mlp": 0.00999272, + "balance_loss_clip": 1.00926781, + "balance_loss_mlp": 0.99826473, + "epoch": 0.09189832279031977, + "flos": 73616265982080.0, + "grad_norm": 0.6988723279079063, + "language_loss": 0.46553975, + "learning_rate": 3.9599496930064965e-06, + "loss": 0.48586798, + "num_input_tokens_seen": 88757840, + "router_z_loss_clip": 0.2421875, + "router_z_loss_mlp": 0.0100708, + "step": 3167, + "time_per_iteration": 3.173511028289795 + }, + { + "auxiliary_loss_clip": 0.01168333, + "auxiliary_loss_mlp": 0.01058384, + "balance_loss_clip": 1.06471825, + "balance_loss_mlp": 1.03917313, + "epoch": 0.09192734025883582, + "flos": 16027363336320.0, + "grad_norm": 3.19877868373558, + "language_loss": 0.86642802, + "learning_rate": 3.959912257159608e-06, + "loss": 0.88869518, + "num_input_tokens_seen": 88770885, + "router_z_loss_clip": 1.03515625, + "router_z_loss_mlp": 0.19213867, + "step": 3168, + "time_per_iteration": 2.4441378116607666 + }, + { + "auxiliary_loss_clip": 0.01032458, + "auxiliary_loss_mlp": 0.00998358, + "balance_loss_clip": 1.00821221, + "balance_loss_mlp": 0.99741, + "epoch": 0.09195635772735186, + "flos": 68868767450880.0, + "grad_norm": 0.6438172800957634, + "language_loss": 0.47100732, + "learning_rate": 3.959874804001982e-06, + "loss": 0.49131545, + "num_input_tokens_seen": 88837170, + "router_z_loss_clip": 0.2421875, + "router_z_loss_mlp": 0.00946045, + "step": 3169, + "time_per_iteration": 3.176398754119873 + }, + { + "auxiliary_loss_clip": 0.01032799, + "auxiliary_loss_mlp": 0.00998808, + "balance_loss_clip": 1.00849009, + "balance_loss_mlp": 0.99786025, + "epoch": 0.09198537519586791, + "flos": 66522758463360.0, + "grad_norm": 0.6934662408727754, + "language_loss": 0.51359034, + "learning_rate": 3.959837333533948e-06, + "loss": 0.53390646, + "num_input_tokens_seen": 88896175, + "router_z_loss_clip": 0.24316406, + "router_z_loss_mlp": 0.00946045, + "step": 3170, + "time_per_iteration": 3.041733503341675 + }, + { + "auxiliary_loss_clip": 0.01032005, + "auxiliary_loss_mlp": 0.00999743, + "balance_loss_clip": 1.00763202, + "balance_loss_mlp": 0.99877149, + "epoch": 0.09201439266438396, + "flos": 60720851168640.0, + "grad_norm": 0.7888460442867199, + "language_loss": 0.46594742, + "learning_rate": 3.959799845755838e-06, + "loss": 0.48626488, + "num_input_tokens_seen": 88952660, + "router_z_loss_clip": 0.24414062, + "router_z_loss_mlp": 0.00970459, + "step": 3171, + "time_per_iteration": 2.9208991527557373 + }, + { + "auxiliary_loss_clip": 0.01168136, + "auxiliary_loss_mlp": 0.0105073, + "balance_loss_clip": 1.06321967, + "balance_loss_mlp": 1.02972555, + "epoch": 0.0920434101329, + "flos": 30985629246720.0, + "grad_norm": 2.6551431398096885, + "language_loss": 0.81617111, + "learning_rate": 3.959762340667983e-06, + "loss": 0.83835977, + "num_input_tokens_seen": 88970200, + "router_z_loss_clip": 1.04785156, + "router_z_loss_mlp": 0.21020508, + "step": 3172, + "time_per_iteration": 2.6611852645874023 + }, + { + "auxiliary_loss_clip": 0.01156894, + "auxiliary_loss_mlp": 0.01055674, + "balance_loss_clip": 1.06014895, + "balance_loss_mlp": 1.03723264, + "epoch": 0.09207242760141605, + "flos": 16908036992640.0, + "grad_norm": 2.5394930473614594, + "language_loss": 0.73511791, + "learning_rate": 3.959724818270713e-06, + "loss": 0.75724357, + "num_input_tokens_seen": 88983390, + "router_z_loss_clip": 0.96630859, + "router_z_loss_mlp": 0.18457031, + "step": 3173, + "time_per_iteration": 2.471916913986206 + }, + { + "auxiliary_loss_clip": 0.01031597, + "auxiliary_loss_mlp": 0.01006917, + "balance_loss_clip": 1.00736523, + "balance_loss_mlp": 1.00599885, + "epoch": 0.0921014450699321, + "flos": 61565219153280.0, + "grad_norm": 0.7058993110277526, + "language_loss": 0.51158983, + "learning_rate": 3.959687278564361e-06, + "loss": 0.53197491, + "num_input_tokens_seen": 89042285, + "router_z_loss_clip": 0.2421875, + "router_z_loss_mlp": 0.00915527, + "step": 3174, + "time_per_iteration": 2.9658923149108887 + }, + { + "auxiliary_loss_clip": 0.0115287, + "auxiliary_loss_mlp": 0.01042483, + "balance_loss_clip": 1.0551039, + "balance_loss_mlp": 1.02483988, + "epoch": 0.09213046253844814, + "flos": 25040187204480.0, + "grad_norm": 2.224655847586368, + "language_loss": 1.0269537, + "learning_rate": 3.959649721549258e-06, + "loss": 1.04890704, + "num_input_tokens_seen": 89057525, + "router_z_loss_clip": 0.97802734, + "router_z_loss_mlp": 0.1763916, + "step": 3175, + "time_per_iteration": 2.5227363109588623 + }, + { + "auxiliary_loss_clip": 0.01156767, + "auxiliary_loss_mlp": 0.01049412, + "balance_loss_clip": 1.05909157, + "balance_loss_mlp": 1.03044605, + "epoch": 0.0921594800069642, + "flos": 33102994210560.0, + "grad_norm": 2.378083420798533, + "language_loss": 0.81348157, + "learning_rate": 3.959612147225735e-06, + "loss": 0.83554339, + "num_input_tokens_seen": 89077120, + "router_z_loss_clip": 0.97607422, + "router_z_loss_mlp": 0.1895752, + "step": 3176, + "time_per_iteration": 2.6009440422058105 + }, + { + "auxiliary_loss_clip": 0.01031186, + "auxiliary_loss_mlp": 0.01008761, + "balance_loss_clip": 1.00705028, + "balance_loss_mlp": 1.00778377, + "epoch": 0.09218849747548023, + "flos": 67815795231360.0, + "grad_norm": 0.7019336210429684, + "language_loss": 0.4890244, + "learning_rate": 3.959574555594126e-06, + "loss": 0.50942385, + "num_input_tokens_seen": 89142025, + "router_z_loss_clip": 0.2421875, + "router_z_loss_mlp": 0.00976562, + "step": 3177, + "time_per_iteration": 3.122396469116211 + }, + { + "auxiliary_loss_clip": 0.0115421, + "auxiliary_loss_mlp": 0.01043552, + "balance_loss_clip": 1.05844963, + "balance_loss_mlp": 1.02614748, + "epoch": 0.09221751494399628, + "flos": 12060312537600.0, + "grad_norm": 2.1993742606959583, + "language_loss": 0.82274663, + "learning_rate": 3.959536946654761e-06, + "loss": 0.84472418, + "num_input_tokens_seen": 89157130, + "router_z_loss_clip": 0.95605469, + "router_z_loss_mlp": 0.17401123, + "step": 3178, + "time_per_iteration": 2.476524591445923 + }, + { + "auxiliary_loss_clip": 0.01162, + "auxiliary_loss_mlp": 0.01058838, + "balance_loss_clip": 1.06320834, + "balance_loss_mlp": 1.0394665, + "epoch": 0.09224653241251234, + "flos": 23833588515840.0, + "grad_norm": 2.416861996717424, + "language_loss": 0.75637007, + "learning_rate": 3.959499320407972e-06, + "loss": 0.7785784, + "num_input_tokens_seen": 89169975, + "router_z_loss_clip": 0.98828125, + "router_z_loss_mlp": 0.19360352, + "step": 3179, + "time_per_iteration": 2.471606731414795 + }, + { + "auxiliary_loss_clip": 0.01152973, + "auxiliary_loss_mlp": 0.01040764, + "balance_loss_clip": 1.0593338, + "balance_loss_mlp": 1.0254277, + "epoch": 0.09227554988102837, + "flos": 19821540954240.0, + "grad_norm": 2.56830878765151, + "language_loss": 0.85372496, + "learning_rate": 3.959461676854092e-06, + "loss": 0.87566233, + "num_input_tokens_seen": 89184900, + "router_z_loss_clip": 0.93603516, + "router_z_loss_mlp": 0.15338135, + "step": 3180, + "time_per_iteration": 2.517465591430664 + }, + { + "auxiliary_loss_clip": 0.01170368, + "auxiliary_loss_mlp": 0.01056439, + "balance_loss_clip": 1.06347632, + "balance_loss_mlp": 1.03581572, + "epoch": 0.09230456734954443, + "flos": 23030051316480.0, + "grad_norm": 2.862889732724409, + "language_loss": 1.12894416, + "learning_rate": 3.959424015993455e-06, + "loss": 1.15121222, + "num_input_tokens_seen": 89197425, + "router_z_loss_clip": 1.06884766, + "router_z_loss_mlp": 0.20605469, + "step": 3181, + "time_per_iteration": 2.5275561809539795 + }, + { + "auxiliary_loss_clip": 0.01152476, + "auxiliary_loss_mlp": 0.01043071, + "balance_loss_clip": 1.05837822, + "balance_loss_mlp": 1.02564287, + "epoch": 0.09233358481806048, + "flos": 16585811061120.0, + "grad_norm": 2.442349640439934, + "language_loss": 0.71270525, + "learning_rate": 3.959386337826391e-06, + "loss": 0.73466074, + "num_input_tokens_seen": 89211430, + "router_z_loss_clip": 0.94140625, + "router_z_loss_mlp": 0.17425537, + "step": 3182, + "time_per_iteration": 2.4483461380004883 + }, + { + "auxiliary_loss_clip": 0.01032391, + "auxiliary_loss_mlp": 0.01001128, + "balance_loss_clip": 1.00815272, + "balance_loss_mlp": 1.00021577, + "epoch": 0.09236260228657651, + "flos": 60733025879040.0, + "grad_norm": 0.7559144732333409, + "language_loss": 0.54956806, + "learning_rate": 3.959348642353234e-06, + "loss": 0.56990319, + "num_input_tokens_seen": 89272660, + "router_z_loss_clip": 0.2421875, + "router_z_loss_mlp": 0.00909424, + "step": 3183, + "time_per_iteration": 3.151237964630127 + }, + { + "auxiliary_loss_clip": 0.01145636, + "auxiliary_loss_mlp": 0.01041776, + "balance_loss_clip": 1.05610776, + "balance_loss_mlp": 1.02532482, + "epoch": 0.09239161975509257, + "flos": 28506284974080.0, + "grad_norm": 2.0765621423366727, + "language_loss": 0.90907466, + "learning_rate": 3.959310929574317e-06, + "loss": 0.93094873, + "num_input_tokens_seen": 89294255, + "router_z_loss_clip": 0.89501953, + "router_z_loss_mlp": 0.16467285, + "step": 3184, + "time_per_iteration": 2.6006855964660645 + }, + { + "auxiliary_loss_clip": 0.01159811, + "auxiliary_loss_mlp": 0.01056639, + "balance_loss_clip": 1.06132817, + "balance_loss_mlp": 1.03739309, + "epoch": 0.09242063722360862, + "flos": 27374129222400.0, + "grad_norm": 2.1885983311642434, + "language_loss": 0.73685199, + "learning_rate": 3.959273199489974e-06, + "loss": 0.75901651, + "num_input_tokens_seen": 89310450, + "router_z_loss_clip": 0.98632812, + "router_z_loss_mlp": 0.19250488, + "step": 3185, + "time_per_iteration": 2.505434274673462 + }, + { + "auxiliary_loss_clip": 0.01158128, + "auxiliary_loss_mlp": 0.01053937, + "balance_loss_clip": 1.05945325, + "balance_loss_mlp": 1.03706861, + "epoch": 0.09244965469212466, + "flos": 20988565833600.0, + "grad_norm": 2.539680171172613, + "language_loss": 0.84729403, + "learning_rate": 3.959235452100536e-06, + "loss": 0.86941469, + "num_input_tokens_seen": 89322520, + "router_z_loss_clip": 0.98828125, + "router_z_loss_mlp": 0.16870117, + "step": 3186, + "time_per_iteration": 2.49800968170166 + }, + { + "auxiliary_loss_clip": 0.01167652, + "auxiliary_loss_mlp": 0.01063556, + "balance_loss_clip": 1.06412303, + "balance_loss_mlp": 1.04458952, + "epoch": 0.09247867216064071, + "flos": 31825364376960.0, + "grad_norm": 2.4127683433688114, + "language_loss": 1.02936804, + "learning_rate": 3.9591976874063385e-06, + "loss": 1.05168009, + "num_input_tokens_seen": 89340060, + "router_z_loss_clip": 1.03564453, + "router_z_loss_mlp": 0.18963623, + "step": 3187, + "time_per_iteration": 2.6093270778656006 + }, + { + "auxiliary_loss_clip": 0.01152814, + "auxiliary_loss_mlp": 0.01046003, + "balance_loss_clip": 1.05837607, + "balance_loss_mlp": 1.02867031, + "epoch": 0.09250768962915676, + "flos": 29202880406400.0, + "grad_norm": 2.2935914657396554, + "language_loss": 0.63165838, + "learning_rate": 3.959159905407713e-06, + "loss": 0.65364659, + "num_input_tokens_seen": 89356130, + "router_z_loss_clip": 0.94335938, + "router_z_loss_mlp": 0.17346191, + "step": 3188, + "time_per_iteration": 2.4939420223236084 + }, + { + "auxiliary_loss_clip": 0.01163127, + "auxiliary_loss_mlp": 0.0104658, + "balance_loss_clip": 1.06393623, + "balance_loss_mlp": 1.02786446, + "epoch": 0.0925367070976728, + "flos": 28104049497600.0, + "grad_norm": 2.1231344840975863, + "language_loss": 0.84129393, + "learning_rate": 3.959122106104996e-06, + "loss": 0.86339104, + "num_input_tokens_seen": 89376080, + "router_z_loss_clip": 0.99267578, + "router_z_loss_mlp": 0.18725586, + "step": 3189, + "time_per_iteration": 2.630594253540039 + }, + { + "auxiliary_loss_clip": 0.01143581, + "auxiliary_loss_mlp": 0.01047096, + "balance_loss_clip": 1.05509162, + "balance_loss_mlp": 1.03025186, + "epoch": 0.09256572456618885, + "flos": 36725986586880.0, + "grad_norm": 2.8946531930377173, + "language_loss": 0.70931596, + "learning_rate": 3.959084289498519e-06, + "loss": 0.73122275, + "num_input_tokens_seen": 89392510, + "router_z_loss_clip": 0.88427734, + "router_z_loss_mlp": 0.1685791, + "step": 3190, + "time_per_iteration": 2.5871098041534424 + }, + { + "auxiliary_loss_clip": 0.01156595, + "auxiliary_loss_mlp": 0.01041734, + "balance_loss_clip": 1.05628622, + "balance_loss_mlp": 1.02429938, + "epoch": 0.09259474203470489, + "flos": 21172105353600.0, + "grad_norm": 2.5608622383288453, + "language_loss": 0.68795568, + "learning_rate": 3.959046455588617e-06, + "loss": 0.709939, + "num_input_tokens_seen": 89407370, + "router_z_loss_clip": 1.00292969, + "router_z_loss_mlp": 0.17425537, + "step": 3191, + "time_per_iteration": 2.461585760116577 + }, + { + "auxiliary_loss_clip": 0.0115876, + "auxiliary_loss_mlp": 0.01053214, + "balance_loss_clip": 1.05807424, + "balance_loss_mlp": 1.03354454, + "epoch": 0.09262375950322094, + "flos": 23359352227200.0, + "grad_norm": 2.997333405628137, + "language_loss": 1.07391298, + "learning_rate": 3.9590086043756235e-06, + "loss": 1.09603274, + "num_input_tokens_seen": 89420665, + "router_z_loss_clip": 1.0078125, + "router_z_loss_mlp": 0.1965332, + "step": 3192, + "time_per_iteration": 2.553375720977783 + }, + { + "auxiliary_loss_clip": 0.01151194, + "auxiliary_loss_mlp": 0.01042014, + "balance_loss_clip": 1.05879688, + "balance_loss_mlp": 1.02465713, + "epoch": 0.09265277697173699, + "flos": 14128658415360.0, + "grad_norm": 2.262785875973464, + "language_loss": 0.85924792, + "learning_rate": 3.958970735859874e-06, + "loss": 0.88118005, + "num_input_tokens_seen": 89433875, + "router_z_loss_clip": 0.92480469, + "router_z_loss_mlp": 0.17370605, + "step": 3193, + "time_per_iteration": 2.481395959854126 + }, + { + "auxiliary_loss_clip": 0.01151133, + "auxiliary_loss_mlp": 0.01042019, + "balance_loss_clip": 1.0562942, + "balance_loss_mlp": 1.02444148, + "epoch": 0.09268179444025303, + "flos": 21283141271040.0, + "grad_norm": 2.580767212913829, + "language_loss": 0.84477782, + "learning_rate": 3.958932850041702e-06, + "loss": 0.86670935, + "num_input_tokens_seen": 89446795, + "router_z_loss_clip": 0.94921875, + "router_z_loss_mlp": 0.17596436, + "step": 3194, + "time_per_iteration": 2.4994559288024902 + }, + { + "auxiliary_loss_clip": 0.01150056, + "auxiliary_loss_mlp": 0.01039782, + "balance_loss_clip": 1.05833292, + "balance_loss_mlp": 1.02368879, + "epoch": 0.09271081190876908, + "flos": 32118970147200.0, + "grad_norm": 2.0508463546617226, + "language_loss": 0.82602108, + "learning_rate": 3.958894946921443e-06, + "loss": 0.84791958, + "num_input_tokens_seen": 89465550, + "router_z_loss_clip": 0.91796875, + "router_z_loss_mlp": 0.16101074, + "step": 3195, + "time_per_iteration": 2.602468490600586 + }, + { + "auxiliary_loss_clip": 0.01038272, + "auxiliary_loss_mlp": 0.01008385, + "balance_loss_clip": 1.0135026, + "balance_loss_mlp": 1.0073899, + "epoch": 0.09273982937728513, + "flos": 60347523162240.0, + "grad_norm": 0.7234608710169629, + "language_loss": 0.51823735, + "learning_rate": 3.958857026499429e-06, + "loss": 0.53870392, + "num_input_tokens_seen": 89520390, + "router_z_loss_clip": 0.24804688, + "router_z_loss_mlp": 0.00994873, + "step": 3196, + "time_per_iteration": 2.9533441066741943 + }, + { + "auxiliary_loss_clip": 0.01157679, + "auxiliary_loss_mlp": 0.01054952, + "balance_loss_clip": 1.06175041, + "balance_loss_mlp": 1.03806639, + "epoch": 0.09276884684580117, + "flos": 10406768918400.0, + "grad_norm": 2.56954802821538, + "language_loss": 0.85559833, + "learning_rate": 3.958819088775999e-06, + "loss": 0.87772465, + "num_input_tokens_seen": 89531055, + "router_z_loss_clip": 0.95898438, + "router_z_loss_mlp": 0.16888428, + "step": 3197, + "time_per_iteration": 2.4731390476226807 + }, + { + "auxiliary_loss_clip": 0.01146633, + "auxiliary_loss_mlp": 0.01046907, + "balance_loss_clip": 1.05751944, + "balance_loss_mlp": 1.03138566, + "epoch": 0.09279786431431722, + "flos": 34817692734720.0, + "grad_norm": 2.136159748901448, + "language_loss": 0.84916335, + "learning_rate": 3.958781133751486e-06, + "loss": 0.87109876, + "num_input_tokens_seen": 89547495, + "router_z_loss_clip": 0.890625, + "router_z_loss_mlp": 0.15539551, + "step": 3198, + "time_per_iteration": 2.6438634395599365 + }, + { + "auxiliary_loss_clip": 0.01158554, + "auxiliary_loss_mlp": 0.01046413, + "balance_loss_clip": 1.05934906, + "balance_loss_mlp": 1.0270654, + "epoch": 0.09282688178283327, + "flos": 48025852289280.0, + "grad_norm": 2.4133489174607647, + "language_loss": 0.94752669, + "learning_rate": 3.9587431614262245e-06, + "loss": 0.96957636, + "num_input_tokens_seen": 89567925, + "router_z_loss_clip": 0.99169922, + "router_z_loss_mlp": 0.19366455, + "step": 3199, + "time_per_iteration": 2.7856991291046143 + }, + { + "auxiliary_loss_clip": 0.01039816, + "auxiliary_loss_mlp": 0.00997635, + "balance_loss_clip": 1.01500106, + "balance_loss_mlp": 0.99659199, + "epoch": 0.09285589925134931, + "flos": 74772157645440.0, + "grad_norm": 0.654696744403786, + "language_loss": 0.51844388, + "learning_rate": 3.958705171800551e-06, + "loss": 0.53881836, + "num_input_tokens_seen": 89630450, + "router_z_loss_clip": 0.24804688, + "router_z_loss_mlp": 0.01043701, + "step": 3200, + "time_per_iteration": 3.131584882736206 + }, + { + "auxiliary_loss_clip": 0.01156406, + "auxiliary_loss_mlp": 0.01043549, + "balance_loss_clip": 1.06317878, + "balance_loss_mlp": 1.02689564, + "epoch": 0.09288491671986536, + "flos": 28287373536000.0, + "grad_norm": 2.3652937434496053, + "language_loss": 1.07462478, + "learning_rate": 3.958667164874802e-06, + "loss": 1.09662437, + "num_input_tokens_seen": 89648655, + "router_z_loss_clip": 0.93212891, + "router_z_loss_mlp": 0.16650391, + "step": 3201, + "time_per_iteration": 2.621919870376587 + }, + { + "auxiliary_loss_clip": 0.01039899, + "auxiliary_loss_mlp": 0.00998505, + "balance_loss_clip": 1.01508427, + "balance_loss_mlp": 0.99743176, + "epoch": 0.09291393418838141, + "flos": 63966241820160.0, + "grad_norm": 0.7635773512159465, + "language_loss": 0.55059648, + "learning_rate": 3.958629140649311e-06, + "loss": 0.57098055, + "num_input_tokens_seen": 89701070, + "router_z_loss_clip": 0.24804688, + "router_z_loss_mlp": 0.01074219, + "step": 3202, + "time_per_iteration": 6.361333608627319 + }, + { + "auxiliary_loss_clip": 0.01162817, + "auxiliary_loss_mlp": 0.01054415, + "balance_loss_clip": 1.06157291, + "balance_loss_mlp": 1.03410745, + "epoch": 0.09294295165689745, + "flos": 16136998623360.0, + "grad_norm": 3.0654120397708553, + "language_loss": 0.82535309, + "learning_rate": 3.958591099124415e-06, + "loss": 0.84752542, + "num_input_tokens_seen": 89714040, + "router_z_loss_clip": 1.01269531, + "router_z_loss_mlp": 0.20300293, + "step": 3203, + "time_per_iteration": 4.833655118942261 + }, + { + "auxiliary_loss_clip": 0.01037519, + "auxiliary_loss_mlp": 0.01001061, + "balance_loss_clip": 1.01278591, + "balance_loss_mlp": 1.0000422, + "epoch": 0.0929719691254135, + "flos": 61511926348800.0, + "grad_norm": 0.8080186108675799, + "language_loss": 0.47761872, + "learning_rate": 3.95855304030045e-06, + "loss": 0.49800453, + "num_input_tokens_seen": 89763855, + "router_z_loss_clip": 0.24804688, + "router_z_loss_mlp": 0.01019287, + "step": 3204, + "time_per_iteration": 2.880330801010132 + }, + { + "auxiliary_loss_clip": 0.01162909, + "auxiliary_loss_mlp": 0.010462, + "balance_loss_clip": 1.06225789, + "balance_loss_mlp": 1.02744865, + "epoch": 0.09300098659392955, + "flos": 33325532922240.0, + "grad_norm": 2.5644493815650344, + "language_loss": 0.83309239, + "learning_rate": 3.9585149641777515e-06, + "loss": 0.85518348, + "num_input_tokens_seen": 89786740, + "router_z_loss_clip": 1.00683594, + "router_z_loss_mlp": 0.1875, + "step": 3205, + "time_per_iteration": 4.896314382553101 + }, + { + "auxiliary_loss_clip": 0.01156325, + "auxiliary_loss_mlp": 0.01049825, + "balance_loss_clip": 1.06318617, + "balance_loss_mlp": 1.03366017, + "epoch": 0.09303000406244559, + "flos": 15117638555520.0, + "grad_norm": 2.8044799854022195, + "language_loss": 0.73159111, + "learning_rate": 3.958476870756657e-06, + "loss": 0.75365263, + "num_input_tokens_seen": 89798675, + "router_z_loss_clip": 0.93066406, + "router_z_loss_mlp": 0.16168213, + "step": 3206, + "time_per_iteration": 5.112726211547852 + }, + { + "auxiliary_loss_clip": 0.01145064, + "auxiliary_loss_mlp": 0.01039031, + "balance_loss_clip": 1.05904937, + "balance_loss_mlp": 1.02260399, + "epoch": 0.09305902153096164, + "flos": 18799846502400.0, + "grad_norm": 2.243727774415683, + "language_loss": 0.80890465, + "learning_rate": 3.958438760037502e-06, + "loss": 0.83074564, + "num_input_tokens_seen": 89813525, + "router_z_loss_clip": 0.85986328, + "router_z_loss_mlp": 0.16430664, + "step": 3207, + "time_per_iteration": 2.4683988094329834 + }, + { + "auxiliary_loss_clip": 0.01037474, + "auxiliary_loss_mlp": 0.01017129, + "balance_loss_clip": 1.01274562, + "balance_loss_mlp": 1.01610994, + "epoch": 0.09308803899947768, + "flos": 60907012381440.0, + "grad_norm": 0.6778251151791804, + "language_loss": 0.47512829, + "learning_rate": 3.9584006320206225e-06, + "loss": 0.49567437, + "num_input_tokens_seen": 89873600, + "router_z_loss_clip": 0.24804688, + "router_z_loss_mlp": 0.01019287, + "step": 3208, + "time_per_iteration": 3.0096213817596436 + }, + { + "auxiliary_loss_clip": 0.010371, + "auxiliary_loss_mlp": 0.01013459, + "balance_loss_clip": 1.01241553, + "balance_loss_mlp": 1.01240385, + "epoch": 0.09311705646799373, + "flos": 69815408434560.0, + "grad_norm": 0.7323162811466649, + "language_loss": 0.55502379, + "learning_rate": 3.9583624867063575e-06, + "loss": 0.57552934, + "num_input_tokens_seen": 89941210, + "router_z_loss_clip": 0.24609375, + "router_z_loss_mlp": 0.01055908, + "step": 3209, + "time_per_iteration": 3.2877683639526367 + }, + { + "auxiliary_loss_clip": 0.01156216, + "auxiliary_loss_mlp": 0.01044628, + "balance_loss_clip": 1.05765605, + "balance_loss_mlp": 1.02577519, + "epoch": 0.09314607393650978, + "flos": 36645510165120.0, + "grad_norm": 2.2099365945511935, + "language_loss": 0.72719514, + "learning_rate": 3.958324324095042e-06, + "loss": 0.74920356, + "num_input_tokens_seen": 89959630, + "router_z_loss_clip": 0.98535156, + "router_z_loss_mlp": 0.1885376, + "step": 3210, + "time_per_iteration": 2.610954523086548 + }, + { + "auxiliary_loss_clip": 0.01149793, + "auxiliary_loss_mlp": 0.01038732, + "balance_loss_clip": 1.05717862, + "balance_loss_mlp": 1.02259088, + "epoch": 0.09317509140502582, + "flos": 74731934413440.0, + "grad_norm": 2.1357265095995, + "language_loss": 0.89290082, + "learning_rate": 3.9582861441870134e-06, + "loss": 0.91478598, + "num_input_tokens_seen": 89981295, + "router_z_loss_clip": 0.92626953, + "router_z_loss_mlp": 0.16131592, + "step": 3211, + "time_per_iteration": 2.9805445671081543 + }, + { + "auxiliary_loss_clip": 0.01035508, + "auxiliary_loss_mlp": 0.0100171, + "balance_loss_clip": 1.0109024, + "balance_loss_mlp": 1.00066078, + "epoch": 0.09320410887354187, + "flos": 70621531413120.0, + "grad_norm": 0.7612416333090622, + "language_loss": 0.57129055, + "learning_rate": 3.95824794698261e-06, + "loss": 0.5916627, + "num_input_tokens_seen": 90047275, + "router_z_loss_clip": 0.24609375, + "router_z_loss_mlp": 0.01049805, + "step": 3212, + "time_per_iteration": 3.1653571128845215 + }, + { + "auxiliary_loss_clip": 0.01155934, + "auxiliary_loss_mlp": 0.01042244, + "balance_loss_clip": 1.06020975, + "balance_loss_mlp": 1.02429736, + "epoch": 0.09323312634205792, + "flos": 27226464410880.0, + "grad_norm": 3.8942380695419345, + "language_loss": 1.02941537, + "learning_rate": 3.958209732482167e-06, + "loss": 1.05139709, + "num_input_tokens_seen": 90062895, + "router_z_loss_clip": 0.95703125, + "router_z_loss_mlp": 0.17950439, + "step": 3213, + "time_per_iteration": 2.6332929134368896 + }, + { + "auxiliary_loss_clip": 0.01153132, + "auxiliary_loss_mlp": 0.01045136, + "balance_loss_clip": 1.05594206, + "balance_loss_mlp": 1.02647328, + "epoch": 0.09326214381057396, + "flos": 32049160064640.0, + "grad_norm": 2.0896024791067678, + "language_loss": 0.92104572, + "learning_rate": 3.958171500686024e-06, + "loss": 0.94302845, + "num_input_tokens_seen": 90078900, + "router_z_loss_clip": 0.97021484, + "router_z_loss_mlp": 0.1864624, + "step": 3214, + "time_per_iteration": 2.62245512008667 + }, + { + "auxiliary_loss_clip": 0.01156543, + "auxiliary_loss_mlp": 0.01049912, + "balance_loss_clip": 1.05916071, + "balance_loss_mlp": 1.03123188, + "epoch": 0.09329116127909001, + "flos": 22959522961920.0, + "grad_norm": 2.366590567810232, + "language_loss": 0.71678585, + "learning_rate": 3.958133251594518e-06, + "loss": 0.73885036, + "num_input_tokens_seen": 90094855, + "router_z_loss_clip": 0.97412109, + "router_z_loss_mlp": 0.18688965, + "step": 3215, + "time_per_iteration": 2.573620080947876 + }, + { + "auxiliary_loss_clip": 0.01154806, + "auxiliary_loss_mlp": 0.01045948, + "balance_loss_clip": 1.06031477, + "balance_loss_mlp": 1.028615, + "epoch": 0.09332017874760606, + "flos": 16501455970560.0, + "grad_norm": 2.754766751097297, + "language_loss": 0.74058378, + "learning_rate": 3.958094985207987e-06, + "loss": 0.76259136, + "num_input_tokens_seen": 90108770, + "router_z_loss_clip": 0.94628906, + "router_z_loss_mlp": 0.17333984, + "step": 3216, + "time_per_iteration": 2.5034353733062744 + }, + { + "auxiliary_loss_clip": 0.01149739, + "auxiliary_loss_mlp": 0.01041349, + "balance_loss_clip": 1.05743885, + "balance_loss_mlp": 1.02537477, + "epoch": 0.0933491962161221, + "flos": 16429131936000.0, + "grad_norm": 2.3290771950240066, + "language_loss": 0.7495054, + "learning_rate": 3.958056701526768e-06, + "loss": 0.77141625, + "num_input_tokens_seen": 90122315, + "router_z_loss_clip": 0.92480469, + "router_z_loss_mlp": 0.15966797, + "step": 3217, + "time_per_iteration": 2.549839735031128 + }, + { + "auxiliary_loss_clip": 0.01149505, + "auxiliary_loss_mlp": 0.01040186, + "balance_loss_clip": 1.05471218, + "balance_loss_mlp": 1.0229485, + "epoch": 0.09337821368463815, + "flos": 14458713511680.0, + "grad_norm": 1.9104564139521767, + "language_loss": 0.91251373, + "learning_rate": 3.9580184005512e-06, + "loss": 0.93441057, + "num_input_tokens_seen": 90137370, + "router_z_loss_clip": 0.94775391, + "router_z_loss_mlp": 0.17236328, + "step": 3218, + "time_per_iteration": 2.541332960128784 + }, + { + "auxiliary_loss_clip": 0.01153924, + "auxiliary_loss_mlp": 0.01050151, + "balance_loss_clip": 1.05735397, + "balance_loss_mlp": 1.03265131, + "epoch": 0.0934072311531542, + "flos": 27016136323200.0, + "grad_norm": 1.7966344140642756, + "language_loss": 0.85141975, + "learning_rate": 3.957980082281621e-06, + "loss": 0.87346053, + "num_input_tokens_seen": 90159530, + "router_z_loss_clip": 0.96630859, + "router_z_loss_mlp": 0.17492676, + "step": 3219, + "time_per_iteration": 2.606346607208252 + }, + { + "auxiliary_loss_clip": 0.01145007, + "auxiliary_loss_mlp": 0.01048833, + "balance_loss_clip": 1.05551553, + "balance_loss_mlp": 1.03397906, + "epoch": 0.09343624862167024, + "flos": 29569133433600.0, + "grad_norm": 1.5165411406299165, + "language_loss": 0.81815952, + "learning_rate": 3.957941746718371e-06, + "loss": 0.84009796, + "num_input_tokens_seen": 90190565, + "router_z_loss_clip": 0.89501953, + "router_z_loss_mlp": 0.14868164, + "step": 3220, + "time_per_iteration": 2.874099016189575 + }, + { + "auxiliary_loss_clip": 0.01159898, + "auxiliary_loss_mlp": 0.01059574, + "balance_loss_clip": 1.0569005, + "balance_loss_mlp": 1.04114425, + "epoch": 0.0934652660901863, + "flos": 11214112959360.0, + "grad_norm": 2.5640108260010015, + "language_loss": 0.70243692, + "learning_rate": 3.9579033938617855e-06, + "loss": 0.72463161, + "num_input_tokens_seen": 90203065, + "router_z_loss_clip": 1.02929688, + "router_z_loss_mlp": 0.18432617, + "step": 3221, + "time_per_iteration": 2.5062756538391113 + }, + { + "auxiliary_loss_clip": 0.01156202, + "auxiliary_loss_mlp": 0.01054831, + "balance_loss_clip": 1.06101334, + "balance_loss_mlp": 1.03483987, + "epoch": 0.09349428355870233, + "flos": 11174036359680.0, + "grad_norm": 3.29219788723115, + "language_loss": 0.75887072, + "learning_rate": 3.957865023712205e-06, + "loss": 0.78098106, + "num_input_tokens_seen": 90213275, + "router_z_loss_clip": 0.95214844, + "router_z_loss_mlp": 0.19995117, + "step": 3222, + "time_per_iteration": 2.5433218479156494 + }, + { + "auxiliary_loss_clip": 0.01149086, + "auxiliary_loss_mlp": 0.01041935, + "balance_loss_clip": 1.05629849, + "balance_loss_mlp": 1.02510881, + "epoch": 0.09352330102721838, + "flos": 74731611191040.0, + "grad_norm": 1.8587468631225825, + "language_loss": 0.89678252, + "learning_rate": 3.957826636269969e-06, + "loss": 0.91869277, + "num_input_tokens_seen": 90240170, + "router_z_loss_clip": 0.92773438, + "router_z_loss_mlp": 0.16827393, + "step": 3223, + "time_per_iteration": 2.948772430419922 + }, + { + "auxiliary_loss_clip": 0.01151462, + "auxiliary_loss_mlp": 0.01045239, + "balance_loss_clip": 1.05626655, + "balance_loss_mlp": 1.02792358, + "epoch": 0.09355231849573444, + "flos": 13764811599360.0, + "grad_norm": 2.8743985365241698, + "language_loss": 0.88746977, + "learning_rate": 3.957788231535416e-06, + "loss": 0.9094367, + "num_input_tokens_seen": 90250905, + "router_z_loss_clip": 0.95214844, + "router_z_loss_mlp": 0.17315674, + "step": 3224, + "time_per_iteration": 2.5243210792541504 + }, + { + "auxiliary_loss_clip": 0.01159291, + "auxiliary_loss_mlp": 0.01050938, + "balance_loss_clip": 1.05865622, + "balance_loss_mlp": 1.03142333, + "epoch": 0.09358133596425047, + "flos": 31023407376000.0, + "grad_norm": 3.0893450499927715, + "language_loss": 1.04647732, + "learning_rate": 3.9577498095088855e-06, + "loss": 1.06857967, + "num_input_tokens_seen": 90267325, + "router_z_loss_clip": 1.00634766, + "router_z_loss_mlp": 0.19482422, + "step": 3225, + "time_per_iteration": 2.612612724304199 + }, + { + "auxiliary_loss_clip": 0.01038548, + "auxiliary_loss_mlp": 0.01001125, + "balance_loss_clip": 1.01419425, + "balance_loss_mlp": 1.00004053, + "epoch": 0.09361035343276652, + "flos": 72279345772800.0, + "grad_norm": 0.7156963919309208, + "language_loss": 0.48065591, + "learning_rate": 3.957711370190715e-06, + "loss": 0.50105262, + "num_input_tokens_seen": 90318240, + "router_z_loss_clip": 0.24316406, + "router_z_loss_mlp": 0.01086426, + "step": 3226, + "time_per_iteration": 3.0208632946014404 + }, + { + "auxiliary_loss_clip": 0.01159986, + "auxiliary_loss_mlp": 0.01052534, + "balance_loss_clip": 1.05984521, + "balance_loss_mlp": 1.03441393, + "epoch": 0.09363937090128258, + "flos": 17048483170560.0, + "grad_norm": 2.5902080846621214, + "language_loss": 1.00240183, + "learning_rate": 3.957672913581247e-06, + "loss": 1.02452707, + "num_input_tokens_seen": 90330885, + "router_z_loss_clip": 1.00146484, + "router_z_loss_mlp": 0.18139648, + "step": 3227, + "time_per_iteration": 2.4941890239715576 + }, + { + "auxiliary_loss_clip": 0.01160462, + "auxiliary_loss_mlp": 0.01049631, + "balance_loss_clip": 1.05910647, + "balance_loss_mlp": 1.03047383, + "epoch": 0.09366838836979861, + "flos": 12158527299840.0, + "grad_norm": 2.373495826785044, + "language_loss": 0.79986584, + "learning_rate": 3.957634439680819e-06, + "loss": 0.82196677, + "num_input_tokens_seen": 90343200, + "router_z_loss_clip": 1.01513672, + "router_z_loss_mlp": 0.19146729, + "step": 3228, + "time_per_iteration": 2.542834758758545 + }, + { + "auxiliary_loss_clip": 0.01149835, + "auxiliary_loss_mlp": 0.0104955, + "balance_loss_clip": 1.05670965, + "balance_loss_mlp": 1.03207362, + "epoch": 0.09369740583831467, + "flos": 32118359616000.0, + "grad_norm": 2.301422947799775, + "language_loss": 0.70664299, + "learning_rate": 3.9575959484897715e-06, + "loss": 0.72863686, + "num_input_tokens_seen": 90359770, + "router_z_loss_clip": 0.93164062, + "router_z_loss_mlp": 0.17468262, + "step": 3229, + "time_per_iteration": 2.612902879714966 + }, + { + "auxiliary_loss_clip": 0.01152243, + "auxiliary_loss_mlp": 0.01048187, + "balance_loss_clip": 1.05589819, + "balance_loss_mlp": 1.02930439, + "epoch": 0.09372642330683072, + "flos": 10662740213760.0, + "grad_norm": 2.56852462405098, + "language_loss": 0.91685903, + "learning_rate": 3.957557440008444e-06, + "loss": 0.93886334, + "num_input_tokens_seen": 90369560, + "router_z_loss_clip": 0.96337891, + "router_z_loss_mlp": 0.1887207, + "step": 3230, + "time_per_iteration": 2.627274513244629 + }, + { + "auxiliary_loss_clip": 0.01037785, + "auxiliary_loss_mlp": 0.00998438, + "balance_loss_clip": 1.01340652, + "balance_loss_mlp": 0.99730533, + "epoch": 0.09375544077534675, + "flos": 69345625432320.0, + "grad_norm": 0.7083369674004739, + "language_loss": 0.52002877, + "learning_rate": 3.957518914237177e-06, + "loss": 0.54039097, + "num_input_tokens_seen": 90436565, + "router_z_loss_clip": 0.24414062, + "router_z_loss_mlp": 0.01135254, + "step": 3231, + "time_per_iteration": 3.2254557609558105 + }, + { + "auxiliary_loss_clip": 0.0114152, + "auxiliary_loss_mlp": 0.01047116, + "balance_loss_clip": 1.05396819, + "balance_loss_mlp": 1.03191638, + "epoch": 0.0937844582438628, + "flos": 26207104343040.0, + "grad_norm": 2.300940690605558, + "language_loss": 0.64357454, + "learning_rate": 3.957480371176312e-06, + "loss": 0.66546094, + "num_input_tokens_seen": 90453005, + "router_z_loss_clip": 0.87548828, + "router_z_loss_mlp": 0.15197754, + "step": 3232, + "time_per_iteration": 2.5624892711639404 + }, + { + "auxiliary_loss_clip": 0.01158032, + "auxiliary_loss_mlp": 0.01050798, + "balance_loss_clip": 1.0568099, + "balance_loss_mlp": 1.03155732, + "epoch": 0.09381347571237886, + "flos": 18363208775040.0, + "grad_norm": 2.610362818733358, + "language_loss": 0.88430572, + "learning_rate": 3.957441810826188e-06, + "loss": 0.90639406, + "num_input_tokens_seen": 90467695, + "router_z_loss_clip": 1.01220703, + "router_z_loss_mlp": 0.19226074, + "step": 3233, + "time_per_iteration": 2.5285580158233643 + }, + { + "auxiliary_loss_clip": 0.0115888, + "auxiliary_loss_mlp": 0.01050827, + "balance_loss_clip": 1.05588269, + "balance_loss_mlp": 1.02965534, + "epoch": 0.0938424931808949, + "flos": 25220099450880.0, + "grad_norm": 2.9195799203846726, + "language_loss": 1.09722269, + "learning_rate": 3.957403233187145e-06, + "loss": 1.11931968, + "num_input_tokens_seen": 90480090, + "router_z_loss_clip": 1.02880859, + "router_z_loss_mlp": 0.21179199, + "step": 3234, + "time_per_iteration": 2.5660011768341064 + }, + { + "auxiliary_loss_clip": 0.01036161, + "auxiliary_loss_mlp": 0.00999554, + "balance_loss_clip": 1.0118885, + "balance_loss_mlp": 0.99846882, + "epoch": 0.09387151064941095, + "flos": 61856741934720.0, + "grad_norm": 0.6292407171184224, + "language_loss": 0.50123125, + "learning_rate": 3.957364638259524e-06, + "loss": 0.52158839, + "num_input_tokens_seen": 90540995, + "router_z_loss_clip": 0.24316406, + "router_z_loss_mlp": 0.01086426, + "step": 3235, + "time_per_iteration": 3.0923612117767334 + }, + { + "auxiliary_loss_clip": 0.01158228, + "auxiliary_loss_mlp": 0.01046254, + "balance_loss_clip": 1.05810511, + "balance_loss_mlp": 1.02627492, + "epoch": 0.093900528117927, + "flos": 11904926302080.0, + "grad_norm": 2.8900706967046133, + "language_loss": 1.022524, + "learning_rate": 3.957326026043668e-06, + "loss": 1.0445689, + "num_input_tokens_seen": 90553155, + "router_z_loss_clip": 1.00195312, + "router_z_loss_mlp": 0.1998291, + "step": 3236, + "time_per_iteration": 2.475609302520752 + }, + { + "auxiliary_loss_clip": 0.01034831, + "auxiliary_loss_mlp": 0.00997078, + "balance_loss_clip": 1.01034713, + "balance_loss_mlp": 0.99594003, + "epoch": 0.09392954558644304, + "flos": 59448716115840.0, + "grad_norm": 0.6885515553884116, + "language_loss": 0.50530064, + "learning_rate": 3.957287396539916e-06, + "loss": 0.52561975, + "num_input_tokens_seen": 90611195, + "router_z_loss_clip": 0.24414062, + "router_z_loss_mlp": 0.01141357, + "step": 3237, + "time_per_iteration": 3.0684421062469482 + }, + { + "auxiliary_loss_clip": 0.01033791, + "auxiliary_loss_mlp": 0.00998142, + "balance_loss_clip": 1.00932479, + "balance_loss_mlp": 0.9970631, + "epoch": 0.09395856305495909, + "flos": 58362778189440.0, + "grad_norm": 0.7599529223040437, + "language_loss": 0.53709692, + "learning_rate": 3.95724874974861e-06, + "loss": 0.5574162, + "num_input_tokens_seen": 90669840, + "router_z_loss_clip": 0.24511719, + "router_z_loss_mlp": 0.01080322, + "step": 3238, + "time_per_iteration": 3.0569114685058594 + }, + { + "auxiliary_loss_clip": 0.01150416, + "auxiliary_loss_mlp": 0.01046729, + "balance_loss_clip": 1.05585647, + "balance_loss_mlp": 1.02930081, + "epoch": 0.09398758052347513, + "flos": 13476916091520.0, + "grad_norm": 2.057738621309398, + "language_loss": 0.76119173, + "learning_rate": 3.95721008567009e-06, + "loss": 0.78316319, + "num_input_tokens_seen": 90685990, + "router_z_loss_clip": 0.94628906, + "router_z_loss_mlp": 0.17437744, + "step": 3239, + "time_per_iteration": 2.52966046333313 + }, + { + "auxiliary_loss_clip": 0.01150921, + "auxiliary_loss_mlp": 0.0104634, + "balance_loss_clip": 1.0563755, + "balance_loss_mlp": 1.02745163, + "epoch": 0.09401659799199118, + "flos": 15443240365440.0, + "grad_norm": 2.7885953748138093, + "language_loss": 0.68599081, + "learning_rate": 3.9571714043047e-06, + "loss": 0.70796341, + "num_input_tokens_seen": 90698820, + "router_z_loss_clip": 0.9453125, + "router_z_loss_mlp": 0.1887207, + "step": 3240, + "time_per_iteration": 2.4730327129364014 + }, + { + "auxiliary_loss_clip": 0.01149, + "auxiliary_loss_mlp": 0.01051958, + "balance_loss_clip": 1.05536294, + "balance_loss_mlp": 1.03497708, + "epoch": 0.09404561546050723, + "flos": 20841403812480.0, + "grad_norm": 2.995959236122819, + "language_loss": 0.82181418, + "learning_rate": 3.957132705652778e-06, + "loss": 0.84382367, + "num_input_tokens_seen": 90716860, + "router_z_loss_clip": 0.93554688, + "router_z_loss_mlp": 0.16986084, + "step": 3241, + "time_per_iteration": 2.7615511417388916 + }, + { + "auxiliary_loss_clip": 0.01155839, + "auxiliary_loss_mlp": 0.01053143, + "balance_loss_clip": 1.0576601, + "balance_loss_mlp": 1.03364015, + "epoch": 0.09407463292902327, + "flos": 14021860302720.0, + "grad_norm": 4.533397762888937, + "language_loss": 1.07823396, + "learning_rate": 3.9570939897146695e-06, + "loss": 1.10032392, + "num_input_tokens_seen": 90729760, + "router_z_loss_clip": 0.98193359, + "router_z_loss_mlp": 0.19525146, + "step": 3242, + "time_per_iteration": 2.501359224319458 + }, + { + "auxiliary_loss_clip": 0.01151519, + "auxiliary_loss_mlp": 0.01056408, + "balance_loss_clip": 1.05785942, + "balance_loss_mlp": 1.03696454, + "epoch": 0.09410365039753932, + "flos": 19863377320320.0, + "grad_norm": 2.1222675459976306, + "language_loss": 0.75442004, + "learning_rate": 3.957055256490715e-06, + "loss": 0.77649927, + "num_input_tokens_seen": 90745215, + "router_z_loss_clip": 0.9375, + "router_z_loss_mlp": 0.19470215, + "step": 3243, + "time_per_iteration": 2.620244026184082 + }, + { + "auxiliary_loss_clip": 0.01033381, + "auxiliary_loss_mlp": 0.01025659, + "balance_loss_clip": 1.0089066, + "balance_loss_mlp": 1.02458596, + "epoch": 0.09413266786605537, + "flos": 61350258211200.0, + "grad_norm": 0.6065675897637028, + "language_loss": 0.4413088, + "learning_rate": 3.957016505981256e-06, + "loss": 0.46189922, + "num_input_tokens_seen": 90812625, + "router_z_loss_clip": 0.24511719, + "router_z_loss_mlp": 0.01074219, + "step": 3244, + "time_per_iteration": 3.2824184894561768 + }, + { + "auxiliary_loss_clip": 0.01143158, + "auxiliary_loss_mlp": 0.01041318, + "balance_loss_clip": 1.05364025, + "balance_loss_mlp": 1.02568924, + "epoch": 0.09416168533457141, + "flos": 11576307749760.0, + "grad_norm": 2.651349299356413, + "language_loss": 0.89103019, + "learning_rate": 3.956977738186636e-06, + "loss": 0.91287494, + "num_input_tokens_seen": 90824320, + "router_z_loss_clip": 0.89550781, + "router_z_loss_mlp": 0.15637207, + "step": 3245, + "time_per_iteration": 2.5096726417541504 + }, + { + "auxiliary_loss_clip": 0.01153061, + "auxiliary_loss_mlp": 0.01049876, + "balance_loss_clip": 1.05778337, + "balance_loss_mlp": 1.03164935, + "epoch": 0.09419070280308746, + "flos": 26426123521920.0, + "grad_norm": 2.635134951607361, + "language_loss": 0.93092012, + "learning_rate": 3.956938953107196e-06, + "loss": 0.95294952, + "num_input_tokens_seen": 90840395, + "router_z_loss_clip": 0.953125, + "router_z_loss_mlp": 0.18225098, + "step": 3246, + "time_per_iteration": 2.5844051837921143 + }, + { + "auxiliary_loss_clip": 0.01161448, + "auxiliary_loss_mlp": 0.01056081, + "balance_loss_clip": 1.05775309, + "balance_loss_mlp": 1.03459978, + "epoch": 0.09421972027160351, + "flos": 17048554997760.0, + "grad_norm": 2.2535559735379813, + "language_loss": 0.90498084, + "learning_rate": 3.956900150743279e-06, + "loss": 0.92715621, + "num_input_tokens_seen": 90854320, + "router_z_loss_clip": 1.03857422, + "router_z_loss_mlp": 0.21484375, + "step": 3247, + "time_per_iteration": 2.5280306339263916 + }, + { + "auxiliary_loss_clip": 0.01032925, + "auxiliary_loss_mlp": 0.01001234, + "balance_loss_clip": 1.00882339, + "balance_loss_mlp": 1.00017917, + "epoch": 0.09424873774011955, + "flos": 61642930227840.0, + "grad_norm": 0.7315451450830172, + "language_loss": 0.48671246, + "learning_rate": 3.956861331095229e-06, + "loss": 0.50705403, + "num_input_tokens_seen": 90910080, + "router_z_loss_clip": 0.24023438, + "router_z_loss_mlp": 0.01055908, + "step": 3248, + "time_per_iteration": 3.0382978916168213 + }, + { + "auxiliary_loss_clip": 0.01159826, + "auxiliary_loss_mlp": 0.01050256, + "balance_loss_clip": 1.05904734, + "balance_loss_mlp": 1.0310396, + "epoch": 0.0942777552086356, + "flos": 35255802919680.0, + "grad_norm": 2.1419805834999477, + "language_loss": 0.81312084, + "learning_rate": 3.956822494163387e-06, + "loss": 0.83522165, + "num_input_tokens_seen": 90926805, + "router_z_loss_clip": 1.00732422, + "router_z_loss_mlp": 0.1920166, + "step": 3249, + "time_per_iteration": 2.686239004135132 + }, + { + "auxiliary_loss_clip": 0.01147135, + "auxiliary_loss_mlp": 0.010357, + "balance_loss_clip": 1.05391514, + "balance_loss_mlp": 1.01815844, + "epoch": 0.09430677267715165, + "flos": 15224293013760.0, + "grad_norm": 2.271914926237851, + "language_loss": 0.81153721, + "learning_rate": 3.956783639948098e-06, + "loss": 0.83336556, + "num_input_tokens_seen": 90941690, + "router_z_loss_clip": 0.93310547, + "router_z_loss_mlp": 0.17541504, + "step": 3250, + "time_per_iteration": 2.4971001148223877 + }, + { + "auxiliary_loss_clip": 0.01145069, + "auxiliary_loss_mlp": 0.01047632, + "balance_loss_clip": 1.05341744, + "balance_loss_mlp": 1.03026271, + "epoch": 0.09433579014566769, + "flos": 23767980324480.0, + "grad_norm": 1.793733537591949, + "language_loss": 0.6277796, + "learning_rate": 3.956744768449703e-06, + "loss": 0.6497066, + "num_input_tokens_seen": 90958880, + "router_z_loss_clip": 0.91552734, + "router_z_loss_mlp": 0.17370605, + "step": 3251, + "time_per_iteration": 2.5483574867248535 + }, + { + "auxiliary_loss_clip": 0.01146253, + "auxiliary_loss_mlp": 0.01051707, + "balance_loss_clip": 1.05563021, + "balance_loss_mlp": 1.03303838, + "epoch": 0.09436480761418374, + "flos": 18765336510720.0, + "grad_norm": 2.341143865706204, + "language_loss": 0.78165686, + "learning_rate": 3.956705879668547e-06, + "loss": 0.80363643, + "num_input_tokens_seen": 90974000, + "router_z_loss_clip": 0.90722656, + "router_z_loss_mlp": 0.18670654, + "step": 3252, + "time_per_iteration": 2.611506938934326 + }, + { + "auxiliary_loss_clip": 0.01148147, + "auxiliary_loss_mlp": 0.01040981, + "balance_loss_clip": 1.05644262, + "balance_loss_mlp": 1.0240593, + "epoch": 0.09439382508269979, + "flos": 43908407245440.0, + "grad_norm": 1.8653220190809414, + "language_loss": 0.69424021, + "learning_rate": 3.956666973604972e-06, + "loss": 0.71613145, + "num_input_tokens_seen": 90996900, + "router_z_loss_clip": 0.91748047, + "router_z_loss_mlp": 0.16906738, + "step": 3253, + "time_per_iteration": 2.7325408458709717 + }, + { + "auxiliary_loss_clip": 0.01033266, + "auxiliary_loss_mlp": 0.01003264, + "balance_loss_clip": 1.0090493, + "balance_loss_mlp": 1.00215518, + "epoch": 0.09442284255121583, + "flos": 63457136403840.0, + "grad_norm": 0.7086319762611476, + "language_loss": 0.49557924, + "learning_rate": 3.956628050259323e-06, + "loss": 0.51594448, + "num_input_tokens_seen": 91049720, + "router_z_loss_clip": 0.2421875, + "router_z_loss_mlp": 0.0111084, + "step": 3254, + "time_per_iteration": 3.034076452255249 + }, + { + "auxiliary_loss_clip": 0.0103332, + "auxiliary_loss_mlp": 0.00998463, + "balance_loss_clip": 1.00917351, + "balance_loss_mlp": 0.99745524, + "epoch": 0.09445186001973188, + "flos": 74782464848640.0, + "grad_norm": 0.6538120019027148, + "language_loss": 0.49955007, + "learning_rate": 3.956589109631944e-06, + "loss": 0.5198679, + "num_input_tokens_seen": 91116165, + "router_z_loss_clip": 0.24121094, + "router_z_loss_mlp": 0.0100708, + "step": 3255, + "time_per_iteration": 3.236719846725464 + }, + { + "auxiliary_loss_clip": 0.01155972, + "auxiliary_loss_mlp": 0.01054272, + "balance_loss_clip": 1.05798781, + "balance_loss_mlp": 1.0333271, + "epoch": 0.09448087748824792, + "flos": 65322154368000.0, + "grad_norm": 2.3634258194053435, + "language_loss": 0.93275625, + "learning_rate": 3.956550151723178e-06, + "loss": 0.95485872, + "num_input_tokens_seen": 91140755, + "router_z_loss_clip": 0.98144531, + "router_z_loss_mlp": 0.20947266, + "step": 3256, + "time_per_iteration": 2.9341881275177 + }, + { + "auxiliary_loss_clip": 0.01031363, + "auxiliary_loss_mlp": 0.01001752, + "balance_loss_clip": 1.00734735, + "balance_loss_mlp": 1.00066745, + "epoch": 0.09450989495676397, + "flos": 67793456972160.0, + "grad_norm": 0.6589599270751342, + "language_loss": 0.46740481, + "learning_rate": 3.956511176533368e-06, + "loss": 0.48773596, + "num_input_tokens_seen": 91192920, + "router_z_loss_clip": 0.24023438, + "router_z_loss_mlp": 0.01086426, + "step": 3257, + "time_per_iteration": 2.9445128440856934 + }, + { + "auxiliary_loss_clip": 0.01147052, + "auxiliary_loss_mlp": 0.01047339, + "balance_loss_clip": 1.05528593, + "balance_loss_mlp": 1.0288502, + "epoch": 0.09453891242528002, + "flos": 21061859535360.0, + "grad_norm": 2.6269471744652777, + "language_loss": 0.73329973, + "learning_rate": 3.956472184062861e-06, + "loss": 0.75524372, + "num_input_tokens_seen": 91206595, + "router_z_loss_clip": 0.91748047, + "router_z_loss_mlp": 0.18481445, + "step": 3258, + "time_per_iteration": 2.5700433254241943 + }, + { + "auxiliary_loss_clip": 0.01030745, + "auxiliary_loss_mlp": 0.01014251, + "balance_loss_clip": 1.00669432, + "balance_loss_mlp": 1.01322556, + "epoch": 0.09456792989379606, + "flos": 70352020690560.0, + "grad_norm": 0.6824987180188158, + "language_loss": 0.47943735, + "learning_rate": 3.956433174312e-06, + "loss": 0.49988729, + "num_input_tokens_seen": 91263780, + "router_z_loss_clip": 0.24023438, + "router_z_loss_mlp": 0.01025391, + "step": 3259, + "time_per_iteration": 3.0551717281341553 + }, + { + "auxiliary_loss_clip": 0.01030929, + "auxiliary_loss_mlp": 0.01015339, + "balance_loss_clip": 1.00674844, + "balance_loss_mlp": 1.01426625, + "epoch": 0.09459694736231211, + "flos": 69967598359680.0, + "grad_norm": 0.6679211578346173, + "language_loss": 0.49073407, + "learning_rate": 3.9563941472811285e-06, + "loss": 0.51119679, + "num_input_tokens_seen": 91331495, + "router_z_loss_clip": 0.2421875, + "router_z_loss_mlp": 0.01074219, + "step": 3260, + "time_per_iteration": 3.209170341491699 + }, + { + "auxiliary_loss_clip": 0.0115073, + "auxiliary_loss_mlp": 0.01049793, + "balance_loss_clip": 1.05621529, + "balance_loss_mlp": 1.03197157, + "epoch": 0.09462596483082816, + "flos": 13656217806720.0, + "grad_norm": 3.161586446985951, + "language_loss": 1.00877309, + "learning_rate": 3.956355102970593e-06, + "loss": 1.03077829, + "num_input_tokens_seen": 91342720, + "router_z_loss_clip": 0.94580078, + "router_z_loss_mlp": 0.17822266, + "step": 3261, + "time_per_iteration": 2.509749174118042 + }, + { + "auxiliary_loss_clip": 0.01030956, + "auxiliary_loss_mlp": 0.01004822, + "balance_loss_clip": 1.00696802, + "balance_loss_mlp": 1.00376153, + "epoch": 0.0946549822993442, + "flos": 74766591037440.0, + "grad_norm": 0.6925997443803928, + "language_loss": 0.51276767, + "learning_rate": 3.956316041380737e-06, + "loss": 0.53312546, + "num_input_tokens_seen": 91401140, + "router_z_loss_clip": 0.24023438, + "router_z_loss_mlp": 0.01062012, + "step": 3262, + "time_per_iteration": 3.130945920944214 + }, + { + "auxiliary_loss_clip": 0.01150269, + "auxiliary_loss_mlp": 0.01057712, + "balance_loss_clip": 1.05615807, + "balance_loss_mlp": 1.04008055, + "epoch": 0.09468399976786025, + "flos": 11397903874560.0, + "grad_norm": 2.556549381202221, + "language_loss": 0.95868611, + "learning_rate": 3.956276962511907e-06, + "loss": 0.980766, + "num_input_tokens_seen": 91413785, + "router_z_loss_clip": 0.94140625, + "router_z_loss_mlp": 0.17645264, + "step": 3263, + "time_per_iteration": 2.5305066108703613 + }, + { + "auxiliary_loss_clip": 0.01150556, + "auxiliary_loss_mlp": 0.01047968, + "balance_loss_clip": 1.05741191, + "balance_loss_mlp": 1.03205943, + "epoch": 0.0947130172363763, + "flos": 17852343592320.0, + "grad_norm": 2.760901001422407, + "language_loss": 0.90266383, + "learning_rate": 3.956237866364446e-06, + "loss": 0.92464912, + "num_input_tokens_seen": 91428145, + "router_z_loss_clip": 0.93066406, + "router_z_loss_mlp": 0.15911865, + "step": 3264, + "time_per_iteration": 2.511242628097534 + }, + { + "auxiliary_loss_clip": 0.01147449, + "auxiliary_loss_mlp": 0.01039938, + "balance_loss_clip": 1.05494404, + "balance_loss_mlp": 1.0228138, + "epoch": 0.09474203470489234, + "flos": 53723475423360.0, + "grad_norm": 2.075569087852603, + "language_loss": 0.87863362, + "learning_rate": 3.9561987529387014e-06, + "loss": 0.90050745, + "num_input_tokens_seen": 91450925, + "router_z_loss_clip": 0.92480469, + "router_z_loss_mlp": 0.17126465, + "step": 3265, + "time_per_iteration": 2.8320038318634033 + }, + { + "auxiliary_loss_clip": 0.01147442, + "auxiliary_loss_mlp": 0.01045204, + "balance_loss_clip": 1.0568428, + "balance_loss_mlp": 1.02812171, + "epoch": 0.0947710521734084, + "flos": 32226989322240.0, + "grad_norm": 2.109387865237631, + "language_loss": 0.8411845, + "learning_rate": 3.9561596222350175e-06, + "loss": 0.86311096, + "num_input_tokens_seen": 91466895, + "router_z_loss_clip": 0.90625, + "router_z_loss_mlp": 0.17089844, + "step": 3266, + "time_per_iteration": 2.626574993133545 + }, + { + "auxiliary_loss_clip": 0.01141929, + "auxiliary_loss_mlp": 0.01040128, + "balance_loss_clip": 1.0532527, + "balance_loss_mlp": 1.02400541, + "epoch": 0.09480006964192444, + "flos": 39633959854080.0, + "grad_norm": 3.3154668291337055, + "language_loss": 0.8333559, + "learning_rate": 3.95612047425374e-06, + "loss": 0.85517639, + "num_input_tokens_seen": 91481815, + "router_z_loss_clip": 0.88671875, + "router_z_loss_mlp": 0.16131592, + "step": 3267, + "time_per_iteration": 2.5933289527893066 + }, + { + "auxiliary_loss_clip": 0.01034569, + "auxiliary_loss_mlp": 0.01008078, + "balance_loss_clip": 1.0104115, + "balance_loss_mlp": 1.0069989, + "epoch": 0.09482908711044048, + "flos": 62729837821440.0, + "grad_norm": 0.7360998570946762, + "language_loss": 0.49825507, + "learning_rate": 3.956081308995216e-06, + "loss": 0.51868153, + "num_input_tokens_seen": 91534345, + "router_z_loss_clip": 0.24121094, + "router_z_loss_mlp": 0.01080322, + "step": 3268, + "time_per_iteration": 2.9946188926696777 + }, + { + "auxiliary_loss_clip": 0.01151738, + "auxiliary_loss_mlp": 0.01050109, + "balance_loss_clip": 1.05903769, + "balance_loss_mlp": 1.03277636, + "epoch": 0.09485810457895653, + "flos": 28285901078400.0, + "grad_norm": 2.8961738297368296, + "language_loss": 0.79873967, + "learning_rate": 3.9560421264597894e-06, + "loss": 0.8207581, + "num_input_tokens_seen": 91548675, + "router_z_loss_clip": 0.92626953, + "router_z_loss_mlp": 0.17315674, + "step": 3269, + "time_per_iteration": 2.6380672454833984 + }, + { + "auxiliary_loss_clip": 0.01034177, + "auxiliary_loss_mlp": 0.00998318, + "balance_loss_clip": 1.01010466, + "balance_loss_mlp": 0.99723369, + "epoch": 0.09488712204747257, + "flos": 74775820832640.0, + "grad_norm": 1.4613529676012675, + "language_loss": 0.53194702, + "learning_rate": 3.956002926647807e-06, + "loss": 0.55227196, + "num_input_tokens_seen": 91609770, + "router_z_loss_clip": 0.24023438, + "router_z_loss_mlp": 0.01086426, + "step": 3270, + "time_per_iteration": 3.1345601081848145 + }, + { + "auxiliary_loss_clip": 0.01034479, + "auxiliary_loss_mlp": 0.0099777, + "balance_loss_clip": 1.01056242, + "balance_loss_mlp": 0.99670357, + "epoch": 0.09491613951598862, + "flos": 63205007863680.0, + "grad_norm": 0.6560906875417073, + "language_loss": 0.50717223, + "learning_rate": 3.9559637095596155e-06, + "loss": 0.52749467, + "num_input_tokens_seen": 91669870, + "router_z_loss_clip": 0.23925781, + "router_z_loss_mlp": 0.01068115, + "step": 3271, + "time_per_iteration": 3.1390380859375 + }, + { + "auxiliary_loss_clip": 0.01162344, + "auxiliary_loss_mlp": 0.01060287, + "balance_loss_clip": 1.05975509, + "balance_loss_mlp": 1.03940153, + "epoch": 0.09494515698450468, + "flos": 31497392269440.0, + "grad_norm": 2.487344323955922, + "language_loss": 0.81294775, + "learning_rate": 3.955924475195562e-06, + "loss": 0.83517408, + "num_input_tokens_seen": 91687635, + "router_z_loss_clip": 1.02539062, + "router_z_loss_mlp": 0.2088623, + "step": 3272, + "time_per_iteration": 2.632486343383789 + }, + { + "auxiliary_loss_clip": 0.01144389, + "auxiliary_loss_mlp": 0.0104979, + "balance_loss_clip": 1.05577421, + "balance_loss_mlp": 1.0317297, + "epoch": 0.09497417445302071, + "flos": 29563925961600.0, + "grad_norm": 1.9321101858979797, + "language_loss": 0.87808478, + "learning_rate": 3.955885223555991e-06, + "loss": 0.90002656, + "num_input_tokens_seen": 91703845, + "router_z_loss_clip": 0.88525391, + "router_z_loss_mlp": 0.18066406, + "step": 3273, + "time_per_iteration": 5.305896043777466 + }, + { + "auxiliary_loss_clip": 0.01147509, + "auxiliary_loss_mlp": 0.01051045, + "balance_loss_clip": 1.05663061, + "balance_loss_mlp": 1.03100562, + "epoch": 0.09500319192153676, + "flos": 27921300076800.0, + "grad_norm": 2.023675293579392, + "language_loss": 0.85104978, + "learning_rate": 3.9558459546412505e-06, + "loss": 0.87303531, + "num_input_tokens_seen": 91721240, + "router_z_loss_clip": 0.90966797, + "router_z_loss_mlp": 0.20068359, + "step": 3274, + "time_per_iteration": 4.989877223968506 + }, + { + "auxiliary_loss_clip": 0.01153759, + "auxiliary_loss_mlp": 0.01051457, + "balance_loss_clip": 1.05780888, + "balance_loss_mlp": 1.03398681, + "epoch": 0.09503220939005282, + "flos": 38065561424640.0, + "grad_norm": 2.089330455177125, + "language_loss": 0.64222348, + "learning_rate": 3.955806668451687e-06, + "loss": 0.66427559, + "num_input_tokens_seen": 91741780, + "router_z_loss_clip": 0.95898438, + "router_z_loss_mlp": 0.17486572, + "step": 3275, + "time_per_iteration": 2.7486090660095215 + }, + { + "auxiliary_loss_clip": 0.01032495, + "auxiliary_loss_mlp": 0.01039983, + "balance_loss_clip": 1.00807428, + "balance_loss_mlp": 1.0389576, + "epoch": 0.09506122685856885, + "flos": 53860369132800.0, + "grad_norm": 0.7463287268332433, + "language_loss": 0.49806583, + "learning_rate": 3.955767364987648e-06, + "loss": 0.5187906, + "num_input_tokens_seen": 91790610, + "router_z_loss_clip": 0.24414062, + "router_z_loss_mlp": 0.01025391, + "step": 3276, + "time_per_iteration": 7.608342409133911 + }, + { + "auxiliary_loss_clip": 0.01032371, + "auxiliary_loss_mlp": 0.01020469, + "balance_loss_clip": 1.00791728, + "balance_loss_mlp": 1.01941383, + "epoch": 0.0950902443270849, + "flos": 63105859347840.0, + "grad_norm": 0.6941403726884875, + "language_loss": 0.46019894, + "learning_rate": 3.955728044249479e-06, + "loss": 0.48072731, + "num_input_tokens_seen": 91850135, + "router_z_loss_clip": 0.24414062, + "router_z_loss_mlp": 0.01055908, + "step": 3277, + "time_per_iteration": 3.079057455062866 + }, + { + "auxiliary_loss_clip": 0.01150877, + "auxiliary_loss_mlp": 0.01044073, + "balance_loss_clip": 1.05696595, + "balance_loss_mlp": 1.0263586, + "epoch": 0.09511926179560096, + "flos": 31389049872000.0, + "grad_norm": 2.1164711711420985, + "language_loss": 0.81659234, + "learning_rate": 3.95568870623753e-06, + "loss": 0.83854187, + "num_input_tokens_seen": 91867730, + "router_z_loss_clip": 0.93847656, + "router_z_loss_mlp": 0.17712402, + "step": 3278, + "time_per_iteration": 2.669142723083496 + }, + { + "auxiliary_loss_clip": 0.01154235, + "auxiliary_loss_mlp": 0.01046538, + "balance_loss_clip": 1.05733061, + "balance_loss_mlp": 1.02847791, + "epoch": 0.095148279264117, + "flos": 10846531128960.0, + "grad_norm": 2.3257289889408614, + "language_loss": 0.8899684, + "learning_rate": 3.955649350952147e-06, + "loss": 0.9119761, + "num_input_tokens_seen": 91878740, + "router_z_loss_clip": 0.96777344, + "router_z_loss_mlp": 0.18054199, + "step": 3279, + "time_per_iteration": 2.497063159942627 + }, + { + "auxiliary_loss_clip": 0.01159723, + "auxiliary_loss_mlp": 0.01045217, + "balance_loss_clip": 1.05865347, + "balance_loss_mlp": 1.02486801, + "epoch": 0.09517729673263305, + "flos": 35876123821440.0, + "grad_norm": 2.060046310179357, + "language_loss": 0.9000507, + "learning_rate": 3.955609978393676e-06, + "loss": 0.92210007, + "num_input_tokens_seen": 91895525, + "router_z_loss_clip": 1.01123047, + "router_z_loss_mlp": 0.20336914, + "step": 3280, + "time_per_iteration": 2.7495412826538086 + }, + { + "auxiliary_loss_clip": 0.01031866, + "auxiliary_loss_mlp": 0.01011538, + "balance_loss_clip": 1.00768018, + "balance_loss_mlp": 1.01056623, + "epoch": 0.0952063142011491, + "flos": 62516026114560.0, + "grad_norm": 0.6569158030648752, + "language_loss": 0.48939827, + "learning_rate": 3.9555705885624675e-06, + "loss": 0.50983226, + "num_input_tokens_seen": 91956735, + "router_z_loss_clip": 0.2421875, + "router_z_loss_mlp": 0.00970459, + "step": 3281, + "time_per_iteration": 3.135586977005005 + }, + { + "auxiliary_loss_clip": 0.01142617, + "auxiliary_loss_mlp": 0.01042699, + "balance_loss_clip": 1.05567551, + "balance_loss_mlp": 1.02772021, + "epoch": 0.09523533166966514, + "flos": 23185904428800.0, + "grad_norm": 1.8252412530268614, + "language_loss": 0.67710304, + "learning_rate": 3.955531181458868e-06, + "loss": 0.69895625, + "num_input_tokens_seen": 91973680, + "router_z_loss_clip": 0.86914062, + "router_z_loss_mlp": 0.14971924, + "step": 3282, + "time_per_iteration": 2.6497769355773926 + }, + { + "auxiliary_loss_clip": 0.0113031, + "auxiliary_loss_mlp": 0.01047055, + "balance_loss_clip": 1.04869425, + "balance_loss_mlp": 1.03195739, + "epoch": 0.09526434913818119, + "flos": 35984286650880.0, + "grad_norm": 2.2286747122645076, + "language_loss": 0.72275078, + "learning_rate": 3.955491757083225e-06, + "loss": 0.74452448, + "num_input_tokens_seen": 91989470, + "router_z_loss_clip": 0.81640625, + "router_z_loss_mlp": 0.15106201, + "step": 3283, + "time_per_iteration": 2.752748727798462 + }, + { + "auxiliary_loss_clip": 0.01156885, + "auxiliary_loss_mlp": 0.01051894, + "balance_loss_clip": 1.0586803, + "balance_loss_mlp": 1.03298688, + "epoch": 0.09529336660669724, + "flos": 32264803365120.0, + "grad_norm": 2.0287156518603484, + "language_loss": 0.75152534, + "learning_rate": 3.955452315435889e-06, + "loss": 0.7736131, + "num_input_tokens_seen": 92004225, + "router_z_loss_clip": 0.98095703, + "router_z_loss_mlp": 0.18896484, + "step": 3284, + "time_per_iteration": 2.622939109802246 + }, + { + "auxiliary_loss_clip": 0.01155925, + "auxiliary_loss_mlp": 0.01052352, + "balance_loss_clip": 1.05757332, + "balance_loss_mlp": 1.0331893, + "epoch": 0.09532238407521328, + "flos": 16502246069760.0, + "grad_norm": 2.101600421849894, + "language_loss": 0.71081257, + "learning_rate": 3.955412856517205e-06, + "loss": 0.73289537, + "num_input_tokens_seen": 92018580, + "router_z_loss_clip": 0.98339844, + "router_z_loss_mlp": 0.19158936, + "step": 3285, + "time_per_iteration": 2.5333473682403564 + }, + { + "auxiliary_loss_clip": 0.01030354, + "auxiliary_loss_mlp": 0.01007651, + "balance_loss_clip": 1.00624418, + "balance_loss_mlp": 1.00661981, + "epoch": 0.09535140154372933, + "flos": 60324541436160.0, + "grad_norm": 0.6761822224183137, + "language_loss": 0.44469577, + "learning_rate": 3.9553733803275255e-06, + "loss": 0.46507585, + "num_input_tokens_seen": 92075475, + "router_z_loss_clip": 0.24121094, + "router_z_loss_mlp": 0.01031494, + "step": 3286, + "time_per_iteration": 2.9958057403564453 + }, + { + "auxiliary_loss_clip": 0.01156911, + "auxiliary_loss_mlp": 0.01044907, + "balance_loss_clip": 1.05815125, + "balance_loss_mlp": 1.02722824, + "epoch": 0.09538041901224537, + "flos": 30041214906240.0, + "grad_norm": 1.7869714774787708, + "language_loss": 0.98002887, + "learning_rate": 3.955333886867196e-06, + "loss": 1.00204706, + "num_input_tokens_seen": 92101755, + "router_z_loss_clip": 0.98779297, + "router_z_loss_mlp": 0.17675781, + "step": 3287, + "time_per_iteration": 2.7815306186676025 + }, + { + "auxiliary_loss_clip": 0.01030232, + "auxiliary_loss_mlp": 0.01004792, + "balance_loss_clip": 1.00635767, + "balance_loss_mlp": 1.0037905, + "epoch": 0.09540943648076142, + "flos": 70689904951680.0, + "grad_norm": 2.6084314278781813, + "language_loss": 0.54530978, + "learning_rate": 3.955294376136566e-06, + "loss": 0.56566, + "num_input_tokens_seen": 92166985, + "router_z_loss_clip": 0.23828125, + "router_z_loss_mlp": 0.01000977, + "step": 3288, + "time_per_iteration": 3.205627918243408 + }, + { + "auxiliary_loss_clip": 0.01152263, + "auxiliary_loss_mlp": 0.01060065, + "balance_loss_clip": 1.05814219, + "balance_loss_mlp": 1.04095614, + "epoch": 0.09543845394927747, + "flos": 12161472215040.0, + "grad_norm": 2.3488338539771454, + "language_loss": 0.73651433, + "learning_rate": 3.955254848135985e-06, + "loss": 0.75863755, + "num_input_tokens_seen": 92178560, + "router_z_loss_clip": 0.94189453, + "router_z_loss_mlp": 0.19104004, + "step": 3289, + "time_per_iteration": 2.5224344730377197 + }, + { + "auxiliary_loss_clip": 0.01152847, + "auxiliary_loss_mlp": 0.01051217, + "balance_loss_clip": 1.0535996, + "balance_loss_mlp": 1.03113651, + "epoch": 0.0954674714177935, + "flos": 33141275130240.0, + "grad_norm": 2.9458381178711437, + "language_loss": 1.58261728, + "learning_rate": 3.955215302865802e-06, + "loss": 1.60465789, + "num_input_tokens_seen": 92194390, + "router_z_loss_clip": 0.99316406, + "router_z_loss_mlp": 0.2008667, + "step": 3290, + "time_per_iteration": 2.6405627727508545 + }, + { + "auxiliary_loss_clip": 0.01155039, + "auxiliary_loss_mlp": 0.01049305, + "balance_loss_clip": 1.05739427, + "balance_loss_mlp": 1.0326457, + "epoch": 0.09549648888630956, + "flos": 19246540037760.0, + "grad_norm": 2.066051637056132, + "language_loss": 0.80630815, + "learning_rate": 3.955175740326367e-06, + "loss": 0.82835162, + "num_input_tokens_seen": 92209870, + "router_z_loss_clip": 0.97607422, + "router_z_loss_mlp": 0.16674805, + "step": 3291, + "time_per_iteration": 2.5661659240722656 + }, + { + "auxiliary_loss_clip": 0.0115571, + "auxiliary_loss_mlp": 0.0105422, + "balance_loss_clip": 1.05880117, + "balance_loss_mlp": 1.03626728, + "epoch": 0.09552550635482561, + "flos": 16610839862400.0, + "grad_norm": 1.9617238810097397, + "language_loss": 0.51279664, + "learning_rate": 3.955136160518029e-06, + "loss": 0.53489596, + "num_input_tokens_seen": 92222050, + "router_z_loss_clip": 0.96875, + "router_z_loss_mlp": 0.17956543, + "step": 3292, + "time_per_iteration": 2.5466678142547607 + }, + { + "auxiliary_loss_clip": 0.01148132, + "auxiliary_loss_mlp": 0.01058596, + "balance_loss_clip": 1.05684996, + "balance_loss_mlp": 1.03980231, + "epoch": 0.09555452382334165, + "flos": 24384422557440.0, + "grad_norm": 2.972810346561763, + "language_loss": 0.93060553, + "learning_rate": 3.9550965634411356e-06, + "loss": 0.95267284, + "num_input_tokens_seen": 92238645, + "router_z_loss_clip": 0.91308594, + "router_z_loss_mlp": 0.18774414, + "step": 3293, + "time_per_iteration": 2.6141297817230225 + }, + { + "auxiliary_loss_clip": 0.01155318, + "auxiliary_loss_mlp": 0.01051979, + "balance_loss_clip": 1.06046307, + "balance_loss_mlp": 1.03493178, + "epoch": 0.0955835412918577, + "flos": 19020445879680.0, + "grad_norm": 2.1486406273466176, + "language_loss": 0.80251503, + "learning_rate": 3.955056949096039e-06, + "loss": 0.82458794, + "num_input_tokens_seen": 92253970, + "router_z_loss_clip": 0.94775391, + "router_z_loss_mlp": 0.17041016, + "step": 3294, + "time_per_iteration": 2.586986780166626 + }, + { + "auxiliary_loss_clip": 0.01138539, + "auxiliary_loss_mlp": 0.0105796, + "balance_loss_clip": 1.05231178, + "balance_loss_mlp": 1.04341602, + "epoch": 0.09561255876037375, + "flos": 14529780570240.0, + "grad_norm": 2.179969461334035, + "language_loss": 0.70400679, + "learning_rate": 3.955017317483089e-06, + "loss": 0.72597176, + "num_input_tokens_seen": 92266600, + "router_z_loss_clip": 0.86132812, + "router_z_loss_mlp": 0.14538574, + "step": 3295, + "time_per_iteration": 2.559701681137085 + }, + { + "auxiliary_loss_clip": 0.01034041, + "auxiliary_loss_mlp": 0.01052073, + "balance_loss_clip": 1.01008248, + "balance_loss_mlp": 1.05105424, + "epoch": 0.09564157622888979, + "flos": 58570664152320.0, + "grad_norm": 0.6975667227233923, + "language_loss": 0.54333925, + "learning_rate": 3.954977668602634e-06, + "loss": 0.5642004, + "num_input_tokens_seen": 92325645, + "router_z_loss_clip": 0.23925781, + "router_z_loss_mlp": 0.01019287, + "step": 3296, + "time_per_iteration": 3.074190378189087 + }, + { + "auxiliary_loss_clip": 0.01158296, + "auxiliary_loss_mlp": 0.0106316, + "balance_loss_clip": 1.05937755, + "balance_loss_mlp": 1.04324627, + "epoch": 0.09567059369740584, + "flos": 43501287519360.0, + "grad_norm": 2.010018937426964, + "language_loss": 0.79303342, + "learning_rate": 3.954938002455025e-06, + "loss": 0.81524801, + "num_input_tokens_seen": 92345125, + "router_z_loss_clip": 0.98828125, + "router_z_loss_mlp": 0.19903564, + "step": 3297, + "time_per_iteration": 2.7407095432281494 + }, + { + "auxiliary_loss_clip": 0.01162768, + "auxiliary_loss_mlp": 0.0105021, + "balance_loss_clip": 1.06115603, + "balance_loss_mlp": 1.02978945, + "epoch": 0.09569961116592189, + "flos": 48828850784640.0, + "grad_norm": 2.1129482672043065, + "language_loss": 0.88231516, + "learning_rate": 3.954898319040613e-06, + "loss": 0.90444493, + "num_input_tokens_seen": 92367500, + "router_z_loss_clip": 1.015625, + "router_z_loss_mlp": 0.2043457, + "step": 3298, + "time_per_iteration": 2.829500913619995 + }, + { + "auxiliary_loss_clip": 0.0116496, + "auxiliary_loss_mlp": 0.01052854, + "balance_loss_clip": 1.06025302, + "balance_loss_mlp": 1.03244567, + "epoch": 0.09572862863443793, + "flos": 31278121695360.0, + "grad_norm": 2.3192371412933666, + "language_loss": 1.12842178, + "learning_rate": 3.954858618359748e-06, + "loss": 1.15059984, + "num_input_tokens_seen": 92387710, + "router_z_loss_clip": 1.04736328, + "router_z_loss_mlp": 0.20410156, + "step": 3299, + "time_per_iteration": 2.6722397804260254 + }, + { + "auxiliary_loss_clip": 0.01151393, + "auxiliary_loss_mlp": 0.01057921, + "balance_loss_clip": 1.05479717, + "balance_loss_mlp": 1.03751278, + "epoch": 0.09575764610295398, + "flos": 36864313862400.0, + "grad_norm": 2.5362231288318093, + "language_loss": 0.8850615, + "learning_rate": 3.9548189004127805e-06, + "loss": 0.90715468, + "num_input_tokens_seen": 92404830, + "router_z_loss_clip": 0.96582031, + "router_z_loss_mlp": 0.20397949, + "step": 3300, + "time_per_iteration": 2.6935741901397705 + }, + { + "auxiliary_loss_clip": 0.01153331, + "auxiliary_loss_mlp": 0.01040945, + "balance_loss_clip": 1.06273007, + "balance_loss_mlp": 1.02589476, + "epoch": 0.09578666357147002, + "flos": 27958862724480.0, + "grad_norm": 2.849804825560418, + "language_loss": 0.92923021, + "learning_rate": 3.954779165200061e-06, + "loss": 0.95117295, + "num_input_tokens_seen": 92419385, + "router_z_loss_clip": 0.90576172, + "router_z_loss_mlp": 0.15057373, + "step": 3301, + "time_per_iteration": 2.561729907989502 + }, + { + "auxiliary_loss_clip": 0.01032343, + "auxiliary_loss_mlp": 0.01004433, + "balance_loss_clip": 1.00805557, + "balance_loss_mlp": 1.00338387, + "epoch": 0.09581568103998607, + "flos": 57510976089600.0, + "grad_norm": 0.7291709540782856, + "language_loss": 0.52995366, + "learning_rate": 3.954739412721942e-06, + "loss": 0.5503214, + "num_input_tokens_seen": 92473655, + "router_z_loss_clip": 0.2421875, + "router_z_loss_mlp": 0.01049805, + "step": 3302, + "time_per_iteration": 3.0071749687194824 + }, + { + "auxiliary_loss_clip": 0.0115901, + "auxiliary_loss_mlp": 0.01052288, + "balance_loss_clip": 1.06090164, + "balance_loss_mlp": 1.03228474, + "epoch": 0.09584469850850212, + "flos": 26683782756480.0, + "grad_norm": 2.1907090926185235, + "language_loss": 0.74223328, + "learning_rate": 3.954699642978773e-06, + "loss": 0.76434624, + "num_input_tokens_seen": 92489155, + "router_z_loss_clip": 0.98144531, + "router_z_loss_mlp": 0.19970703, + "step": 3303, + "time_per_iteration": 2.6350865364074707 + }, + { + "auxiliary_loss_clip": 0.01150839, + "auxiliary_loss_mlp": 0.01042542, + "balance_loss_clip": 1.05764842, + "balance_loss_mlp": 1.02617455, + "epoch": 0.09587371597701816, + "flos": 70249313750400.0, + "grad_norm": 1.953387671380628, + "language_loss": 0.65260375, + "learning_rate": 3.954659855970905e-06, + "loss": 0.67453754, + "num_input_tokens_seen": 92512685, + "router_z_loss_clip": 0.93261719, + "router_z_loss_mlp": 0.16387939, + "step": 3304, + "time_per_iteration": 2.8986549377441406 + }, + { + "auxiliary_loss_clip": 0.01149466, + "auxiliary_loss_mlp": 0.01051827, + "balance_loss_clip": 1.05684519, + "balance_loss_mlp": 1.0336237, + "epoch": 0.09590273344553421, + "flos": 42162395040000.0, + "grad_norm": 3.079748050875844, + "language_loss": 1.02262998, + "learning_rate": 3.954620051698691e-06, + "loss": 1.04464304, + "num_input_tokens_seen": 92535245, + "router_z_loss_clip": 0.92675781, + "router_z_loss_mlp": 0.18212891, + "step": 3305, + "time_per_iteration": 2.7780120372772217 + }, + { + "auxiliary_loss_clip": 0.01144464, + "auxiliary_loss_mlp": 0.0104304, + "balance_loss_clip": 1.05484593, + "balance_loss_mlp": 1.02632725, + "epoch": 0.09593175091405026, + "flos": 13143700598400.0, + "grad_norm": 3.3684063275053315, + "language_loss": 0.95333743, + "learning_rate": 3.954580230162482e-06, + "loss": 0.97521245, + "num_input_tokens_seen": 92545905, + "router_z_loss_clip": 0.89648438, + "router_z_loss_mlp": 0.16723633, + "step": 3306, + "time_per_iteration": 2.5717267990112305 + }, + { + "auxiliary_loss_clip": 0.01152461, + "auxiliary_loss_mlp": 0.01050156, + "balance_loss_clip": 1.05423439, + "balance_loss_mlp": 1.03120184, + "epoch": 0.0959607683825663, + "flos": 51501215767680.0, + "grad_norm": 1.9183253621008205, + "language_loss": 0.92215049, + "learning_rate": 3.954540391362629e-06, + "loss": 0.94417667, + "num_input_tokens_seen": 92571900, + "router_z_loss_clip": 0.98339844, + "router_z_loss_mlp": 0.18981934, + "step": 3307, + "time_per_iteration": 2.8260912895202637 + }, + { + "auxiliary_loss_clip": 0.0115249, + "auxiliary_loss_mlp": 0.01053808, + "balance_loss_clip": 1.05404246, + "balance_loss_mlp": 1.03621888, + "epoch": 0.09598978585108235, + "flos": 16572415288320.0, + "grad_norm": 2.4687433789987865, + "language_loss": 0.83485782, + "learning_rate": 3.954500535299484e-06, + "loss": 0.85692084, + "num_input_tokens_seen": 92585070, + "router_z_loss_clip": 0.984375, + "router_z_loss_mlp": 0.17584229, + "step": 3308, + "time_per_iteration": 2.4773874282836914 + }, + { + "auxiliary_loss_clip": 0.01148805, + "auxiliary_loss_mlp": 0.01047788, + "balance_loss_clip": 1.05374336, + "balance_loss_mlp": 1.02997792, + "epoch": 0.0960188033195984, + "flos": 26426123521920.0, + "grad_norm": 2.1562135624203402, + "language_loss": 0.91771483, + "learning_rate": 3.9544606619734e-06, + "loss": 0.93968081, + "num_input_tokens_seen": 92601405, + "router_z_loss_clip": 0.95068359, + "router_z_loss_mlp": 0.17822266, + "step": 3309, + "time_per_iteration": 2.5683274269104004 + }, + { + "auxiliary_loss_clip": 0.01150884, + "auxiliary_loss_mlp": 0.01052111, + "balance_loss_clip": 1.05742264, + "balance_loss_mlp": 1.03407478, + "epoch": 0.09604782078811444, + "flos": 27638288818560.0, + "grad_norm": 2.019618364157934, + "language_loss": 0.79527974, + "learning_rate": 3.954420771384728e-06, + "loss": 0.81730968, + "num_input_tokens_seen": 92620815, + "router_z_loss_clip": 0.93505859, + "router_z_loss_mlp": 0.18041992, + "step": 3310, + "time_per_iteration": 2.593142509460449 + }, + { + "auxiliary_loss_clip": 0.01036159, + "auxiliary_loss_mlp": 0.00999708, + "balance_loss_clip": 1.0127151, + "balance_loss_mlp": 0.99868852, + "epoch": 0.09607683825663049, + "flos": 72184937852160.0, + "grad_norm": 0.6440734993141854, + "language_loss": 0.54369199, + "learning_rate": 3.954380863533821e-06, + "loss": 0.56405067, + "num_input_tokens_seen": 92684385, + "router_z_loss_clip": 0.234375, + "router_z_loss_mlp": 0.01019287, + "step": 3311, + "time_per_iteration": 3.2112345695495605 + }, + { + "auxiliary_loss_clip": 0.01152917, + "auxiliary_loss_mlp": 0.01052384, + "balance_loss_clip": 1.05514455, + "balance_loss_mlp": 1.03157592, + "epoch": 0.09610585572514654, + "flos": 38542706714880.0, + "grad_norm": 1.9721904947149513, + "language_loss": 1.03249681, + "learning_rate": 3.954340938421032e-06, + "loss": 1.05454981, + "num_input_tokens_seen": 92707340, + "router_z_loss_clip": 0.97705078, + "router_z_loss_mlp": 0.20819092, + "step": 3312, + "time_per_iteration": 2.7347044944763184 + }, + { + "auxiliary_loss_clip": 0.01035197, + "auxiliary_loss_mlp": 0.01002098, + "balance_loss_clip": 1.01149249, + "balance_loss_mlp": 1.00119197, + "epoch": 0.09613487319366258, + "flos": 66128280324480.0, + "grad_norm": 0.6700520811762569, + "language_loss": 0.50845402, + "learning_rate": 3.954300996046712e-06, + "loss": 0.52882695, + "num_input_tokens_seen": 92766060, + "router_z_loss_clip": 0.23632812, + "router_z_loss_mlp": 0.0090332, + "step": 3313, + "time_per_iteration": 3.0785961151123047 + }, + { + "auxiliary_loss_clip": 0.01146409, + "auxiliary_loss_mlp": 0.0104879, + "balance_loss_clip": 1.05295861, + "balance_loss_mlp": 1.02980018, + "epoch": 0.09616389066217863, + "flos": 34820853131520.0, + "grad_norm": 5.512610811226406, + "language_loss": 0.81343436, + "learning_rate": 3.954261036411215e-06, + "loss": 0.83538628, + "num_input_tokens_seen": 92782525, + "router_z_loss_clip": 0.93457031, + "router_z_loss_mlp": 0.18994141, + "step": 3314, + "time_per_iteration": 2.726893663406372 + }, + { + "auxiliary_loss_clip": 0.01035991, + "auxiliary_loss_mlp": 0.01007959, + "balance_loss_clip": 1.01221752, + "balance_loss_mlp": 1.00689793, + "epoch": 0.09619290813069468, + "flos": 74781243786240.0, + "grad_norm": 0.6431504322798228, + "language_loss": 0.50163829, + "learning_rate": 3.954221059514895e-06, + "loss": 0.5220778, + "num_input_tokens_seen": 92844850, + "router_z_loss_clip": 0.23828125, + "router_z_loss_mlp": 0.01062012, + "step": 3315, + "time_per_iteration": 3.1860146522521973 + }, + { + "auxiliary_loss_clip": 0.01033825, + "auxiliary_loss_mlp": 0.0100227, + "balance_loss_clip": 1.01001561, + "balance_loss_mlp": 1.00122094, + "epoch": 0.09622192559921072, + "flos": 69623249650560.0, + "grad_norm": 0.5880427089638737, + "language_loss": 0.43945158, + "learning_rate": 3.954181065358102e-06, + "loss": 0.45981252, + "num_input_tokens_seen": 92910270, + "router_z_loss_clip": 0.23828125, + "router_z_loss_mlp": 0.01049805, + "step": 3316, + "time_per_iteration": 3.369893789291382 + }, + { + "auxiliary_loss_clip": 0.01154113, + "auxiliary_loss_mlp": 0.01049359, + "balance_loss_clip": 1.05659842, + "balance_loss_mlp": 1.03011858, + "epoch": 0.09625094306772677, + "flos": 47841558583680.0, + "grad_norm": 1.9628411215869583, + "language_loss": 0.82301879, + "learning_rate": 3.954141053941192e-06, + "loss": 0.84505355, + "num_input_tokens_seen": 92930045, + "router_z_loss_clip": 0.97607422, + "router_z_loss_mlp": 0.19256592, + "step": 3317, + "time_per_iteration": 2.730069875717163 + }, + { + "auxiliary_loss_clip": 0.01149583, + "auxiliary_loss_mlp": 0.01050542, + "balance_loss_clip": 1.05393577, + "balance_loss_mlp": 1.03181458, + "epoch": 0.09627996053624281, + "flos": 54154043752320.0, + "grad_norm": 2.600901629576856, + "language_loss": 0.77745932, + "learning_rate": 3.954101025264517e-06, + "loss": 0.79946053, + "num_input_tokens_seen": 92949285, + "router_z_loss_clip": 0.95654297, + "router_z_loss_mlp": 0.18731689, + "step": 3318, + "time_per_iteration": 2.7645721435546875 + }, + { + "auxiliary_loss_clip": 0.01155069, + "auxiliary_loss_mlp": 0.01050698, + "balance_loss_clip": 1.05481887, + "balance_loss_mlp": 1.03067112, + "epoch": 0.09630897800475886, + "flos": 19237920773760.0, + "grad_norm": 2.292937953229637, + "language_loss": 0.83676976, + "learning_rate": 3.954060979328432e-06, + "loss": 0.85882747, + "num_input_tokens_seen": 92963975, + "router_z_loss_clip": 1.00244141, + "router_z_loss_mlp": 0.20019531, + "step": 3319, + "time_per_iteration": 2.556931257247925 + }, + { + "auxiliary_loss_clip": 0.01032311, + "auxiliary_loss_mlp": 0.00999442, + "balance_loss_clip": 1.00832486, + "balance_loss_mlp": 0.99848264, + "epoch": 0.09633799547327491, + "flos": 72261571518720.0, + "grad_norm": 0.6110832006988639, + "language_loss": 0.49356776, + "learning_rate": 3.954020916133289e-06, + "loss": 0.51388526, + "num_input_tokens_seen": 93028700, + "router_z_loss_clip": 0.24023438, + "router_z_loss_mlp": 0.00958252, + "step": 3320, + "time_per_iteration": 3.183810234069824 + }, + { + "auxiliary_loss_clip": 0.01152822, + "auxiliary_loss_mlp": 0.01047376, + "balance_loss_clip": 1.0555284, + "balance_loss_mlp": 1.02765822, + "epoch": 0.09636701294179095, + "flos": 18836906359680.0, + "grad_norm": 2.8159475784026924, + "language_loss": 0.94731045, + "learning_rate": 3.953980835679442e-06, + "loss": 0.96931243, + "num_input_tokens_seen": 93043220, + "router_z_loss_clip": 0.97265625, + "router_z_loss_mlp": 0.19714355, + "step": 3321, + "time_per_iteration": 2.495295763015747 + }, + { + "auxiliary_loss_clip": 0.0103117, + "auxiliary_loss_mlp": 0.01002177, + "balance_loss_clip": 1.00745869, + "balance_loss_mlp": 1.001158, + "epoch": 0.096396030410307, + "flos": 70365631944960.0, + "grad_norm": 0.6453749477064117, + "language_loss": 0.52091151, + "learning_rate": 3.953940737967247e-06, + "loss": 0.54124498, + "num_input_tokens_seen": 93111250, + "router_z_loss_clip": 0.23730469, + "router_z_loss_mlp": 0.01019287, + "step": 3322, + "time_per_iteration": 3.1713151931762695 + }, + { + "auxiliary_loss_clip": 0.01162909, + "auxiliary_loss_mlp": 0.01056927, + "balance_loss_clip": 1.05829954, + "balance_loss_mlp": 1.03622031, + "epoch": 0.09642504787882306, + "flos": 28433601803520.0, + "grad_norm": 1.8883618928973447, + "language_loss": 1.01406884, + "learning_rate": 3.9539006229970555e-06, + "loss": 1.03626728, + "num_input_tokens_seen": 93131135, + "router_z_loss_clip": 1.04589844, + "router_z_loss_mlp": 0.20715332, + "step": 3323, + "time_per_iteration": 2.615818738937378 + }, + { + "auxiliary_loss_clip": 0.01147093, + "auxiliary_loss_mlp": 0.01051115, + "balance_loss_clip": 1.05531979, + "balance_loss_mlp": 1.03411615, + "epoch": 0.0964540653473391, + "flos": 32125578249600.0, + "grad_norm": 1.8561889193765, + "language_loss": 0.61580479, + "learning_rate": 3.953860490769224e-06, + "loss": 0.63778687, + "num_input_tokens_seen": 93149115, + "router_z_loss_clip": 0.91845703, + "router_z_loss_mlp": 0.16992188, + "step": 3324, + "time_per_iteration": 2.621736526489258 + }, + { + "auxiliary_loss_clip": 0.01149311, + "auxiliary_loss_mlp": 0.01038733, + "balance_loss_clip": 1.05416918, + "balance_loss_mlp": 1.02070856, + "epoch": 0.09648308281585515, + "flos": 30444312309120.0, + "grad_norm": 2.5954317884980265, + "language_loss": 0.7718842, + "learning_rate": 3.953820341284105e-06, + "loss": 0.79376459, + "num_input_tokens_seen": 93166555, + "router_z_loss_clip": 0.95117188, + "router_z_loss_mlp": 0.18017578, + "step": 3325, + "time_per_iteration": 2.5870065689086914 + }, + { + "auxiliary_loss_clip": 0.01158761, + "auxiliary_loss_mlp": 0.01047061, + "balance_loss_clip": 1.05742812, + "balance_loss_mlp": 1.02449441, + "epoch": 0.0965121002843712, + "flos": 37663756911360.0, + "grad_norm": 1.9194419726267167, + "language_loss": 0.96487767, + "learning_rate": 3.953780174542054e-06, + "loss": 0.98693585, + "num_input_tokens_seen": 93185165, + "router_z_loss_clip": 1.01367188, + "router_z_loss_mlp": 0.22583008, + "step": 3326, + "time_per_iteration": 2.6542654037475586 + }, + { + "auxiliary_loss_clip": 0.01032318, + "auxiliary_loss_mlp": 0.01005464, + "balance_loss_clip": 1.00862503, + "balance_loss_mlp": 1.00446308, + "epoch": 0.09654111775288723, + "flos": 58971965875200.0, + "grad_norm": 0.7331128793985544, + "language_loss": 0.51520824, + "learning_rate": 3.953739990543427e-06, + "loss": 0.53558612, + "num_input_tokens_seen": 93243850, + "router_z_loss_clip": 0.23632812, + "router_z_loss_mlp": 0.01000977, + "step": 3327, + "time_per_iteration": 3.0001943111419678 + }, + { + "auxiliary_loss_clip": 0.01032555, + "auxiliary_loss_mlp": 0.01009227, + "balance_loss_clip": 1.00882411, + "balance_loss_mlp": 1.00820184, + "epoch": 0.09657013522140329, + "flos": 62155411522560.0, + "grad_norm": 0.6258371538648004, + "language_loss": 0.42989314, + "learning_rate": 3.953699789288576e-06, + "loss": 0.45031095, + "num_input_tokens_seen": 93311805, + "router_z_loss_clip": 0.23730469, + "router_z_loss_mlp": 0.01025391, + "step": 3328, + "time_per_iteration": 3.2335803508758545 + }, + { + "auxiliary_loss_clip": 0.01142025, + "auxiliary_loss_mlp": 0.01047916, + "balance_loss_clip": 1.05421209, + "balance_loss_mlp": 1.03064227, + "epoch": 0.09659915268991934, + "flos": 23216428010880.0, + "grad_norm": 2.851574522119452, + "language_loss": 0.8746475, + "learning_rate": 3.9536595707778605e-06, + "loss": 0.8965469, + "num_input_tokens_seen": 93326580, + "router_z_loss_clip": 0.87744141, + "router_z_loss_mlp": 0.17279053, + "step": 3329, + "time_per_iteration": 2.5554306507110596 + }, + { + "auxiliary_loss_clip": 0.01032144, + "auxiliary_loss_mlp": 0.01003043, + "balance_loss_clip": 1.00852394, + "balance_loss_mlp": 1.00196445, + "epoch": 0.09662817015843538, + "flos": 58100162878080.0, + "grad_norm": 0.6221455640644488, + "language_loss": 0.42929611, + "learning_rate": 3.9536193350116315e-06, + "loss": 0.44964796, + "num_input_tokens_seen": 93386880, + "router_z_loss_clip": 0.23632812, + "router_z_loss_mlp": 0.01080322, + "step": 3330, + "time_per_iteration": 3.05845046043396 + }, + { + "auxiliary_loss_clip": 0.01153219, + "auxiliary_loss_mlp": 0.01047721, + "balance_loss_clip": 1.05599463, + "balance_loss_mlp": 1.02907681, + "epoch": 0.09665718762695143, + "flos": 31643800104960.0, + "grad_norm": 2.232194675530259, + "language_loss": 0.76076245, + "learning_rate": 3.953579081990246e-06, + "loss": 0.78277183, + "num_input_tokens_seen": 93403175, + "router_z_loss_clip": 0.97216797, + "router_z_loss_mlp": 0.18640137, + "step": 3331, + "time_per_iteration": 2.646578550338745 + }, + { + "auxiliary_loss_clip": 0.01029783, + "auxiliary_loss_mlp": 0.01000652, + "balance_loss_clip": 1.00625563, + "balance_loss_mlp": 0.99961442, + "epoch": 0.09668620509546746, + "flos": 74775892659840.0, + "grad_norm": 0.6773420069640519, + "language_loss": 0.48321378, + "learning_rate": 3.95353881171406e-06, + "loss": 0.5035181, + "num_input_tokens_seen": 93468200, + "router_z_loss_clip": 0.23535156, + "router_z_loss_mlp": 0.01037598, + "step": 3332, + "time_per_iteration": 3.1691174507141113 + }, + { + "auxiliary_loss_clip": 0.01155144, + "auxiliary_loss_mlp": 0.01048293, + "balance_loss_clip": 1.05830932, + "balance_loss_mlp": 1.03110313, + "epoch": 0.09671522256398352, + "flos": 36533181358080.0, + "grad_norm": 2.2297981877164985, + "language_loss": 0.80175501, + "learning_rate": 3.953498524183429e-06, + "loss": 0.82378936, + "num_input_tokens_seen": 93486025, + "router_z_loss_clip": 0.96972656, + "router_z_loss_mlp": 0.17199707, + "step": 3333, + "time_per_iteration": 2.714571237564087 + }, + { + "auxiliary_loss_clip": 0.01164168, + "auxiliary_loss_mlp": 0.01058271, + "balance_loss_clip": 1.0592742, + "balance_loss_mlp": 1.03956676, + "epoch": 0.09674424003249957, + "flos": 35396428665600.0, + "grad_norm": 3.693314012722446, + "language_loss": 1.11066115, + "learning_rate": 3.953458219398707e-06, + "loss": 1.13288546, + "num_input_tokens_seen": 93499540, + "router_z_loss_clip": 1.04833984, + "router_z_loss_mlp": 0.18701172, + "step": 3334, + "time_per_iteration": 2.714470863342285 + }, + { + "auxiliary_loss_clip": 0.01151642, + "auxiliary_loss_mlp": 0.01047242, + "balance_loss_clip": 1.05814707, + "balance_loss_mlp": 1.02828741, + "epoch": 0.0967732575010156, + "flos": 31022976412800.0, + "grad_norm": 2.3361595027977633, + "language_loss": 0.79946434, + "learning_rate": 3.953417897360253e-06, + "loss": 0.82145309, + "num_input_tokens_seen": 93516265, + "router_z_loss_clip": 0.93554688, + "router_z_loss_mlp": 0.18945312, + "step": 3335, + "time_per_iteration": 2.627274751663208 + }, + { + "auxiliary_loss_clip": 0.01157125, + "auxiliary_loss_mlp": 0.01052776, + "balance_loss_clip": 1.05937481, + "balance_loss_mlp": 1.03431058, + "epoch": 0.09680227496953166, + "flos": 28106743017600.0, + "grad_norm": 1.9228324255159848, + "language_loss": 0.8991943, + "learning_rate": 3.953377558068421e-06, + "loss": 0.92129332, + "num_input_tokens_seen": 93534020, + "router_z_loss_clip": 0.97753906, + "router_z_loss_mlp": 0.18463135, + "step": 3336, + "time_per_iteration": 2.592224597930908 + }, + { + "auxiliary_loss_clip": 0.0103148, + "auxiliary_loss_mlp": 0.01000812, + "balance_loss_clip": 1.00782669, + "balance_loss_mlp": 0.99981707, + "epoch": 0.09683129243804771, + "flos": 74782859898240.0, + "grad_norm": 0.6382803310802551, + "language_loss": 0.50683755, + "learning_rate": 3.9533372015235685e-06, + "loss": 0.52716047, + "num_input_tokens_seen": 93605390, + "router_z_loss_clip": 0.23632812, + "router_z_loss_mlp": 0.00994873, + "step": 3337, + "time_per_iteration": 3.2632336616516113 + }, + { + "auxiliary_loss_clip": 0.0115186, + "auxiliary_loss_mlp": 0.01042359, + "balance_loss_clip": 1.0561024, + "balance_loss_mlp": 1.0253123, + "epoch": 0.09686030990656375, + "flos": 42302589822720.0, + "grad_norm": 2.482555712622499, + "language_loss": 0.93334073, + "learning_rate": 3.953296827726051e-06, + "loss": 0.95528293, + "num_input_tokens_seen": 93630345, + "router_z_loss_clip": 0.95605469, + "router_z_loss_mlp": 0.17047119, + "step": 3338, + "time_per_iteration": 2.841306686401367 + }, + { + "auxiliary_loss_clip": 0.01144954, + "auxiliary_loss_mlp": 0.01033781, + "balance_loss_clip": 1.05591166, + "balance_loss_mlp": 1.01635909, + "epoch": 0.0968893273750798, + "flos": 40619025411840.0, + "grad_norm": 2.0652800473039377, + "language_loss": 0.91255319, + "learning_rate": 3.953256436676225e-06, + "loss": 0.9343406, + "num_input_tokens_seen": 93649140, + "router_z_loss_clip": 0.88964844, + "router_z_loss_mlp": 0.17419434, + "step": 3339, + "time_per_iteration": 2.763922929763794 + }, + { + "auxiliary_loss_clip": 0.01032171, + "auxiliary_loss_mlp": 0.01002003, + "balance_loss_clip": 1.00841844, + "balance_loss_mlp": 1.00104904, + "epoch": 0.09691834484359585, + "flos": 72549000149760.0, + "grad_norm": 0.6373409116139922, + "language_loss": 0.48030496, + "learning_rate": 3.9532160283744485e-06, + "loss": 0.50064665, + "num_input_tokens_seen": 93717825, + "router_z_loss_clip": 0.23730469, + "router_z_loss_mlp": 0.00952148, + "step": 3340, + "time_per_iteration": 3.233461380004883 + }, + { + "auxiliary_loss_clip": 0.01143369, + "auxiliary_loss_mlp": 0.01038487, + "balance_loss_clip": 1.05490184, + "balance_loss_mlp": 1.02211988, + "epoch": 0.09694736231211189, + "flos": 74732473117440.0, + "grad_norm": 1.921931263978906, + "language_loss": 0.6741336, + "learning_rate": 3.953175602821077e-06, + "loss": 0.69595218, + "num_input_tokens_seen": 93741825, + "router_z_loss_clip": 0.88427734, + "router_z_loss_mlp": 0.16369629, + "step": 3341, + "time_per_iteration": 2.9799580574035645 + }, + { + "auxiliary_loss_clip": 0.01159081, + "auxiliary_loss_mlp": 0.01052658, + "balance_loss_clip": 1.06022537, + "balance_loss_mlp": 1.03228498, + "epoch": 0.09697637978062794, + "flos": 74731683018240.0, + "grad_norm": 2.5251763595907777, + "language_loss": 0.83362591, + "learning_rate": 3.95313516001647e-06, + "loss": 0.85574329, + "num_input_tokens_seen": 93764820, + "router_z_loss_clip": 0.98828125, + "router_z_loss_mlp": 0.20373535, + "step": 3342, + "time_per_iteration": 2.9345970153808594 + }, + { + "auxiliary_loss_clip": 0.01150484, + "auxiliary_loss_mlp": 0.01041231, + "balance_loss_clip": 1.05727839, + "balance_loss_mlp": 1.02333176, + "epoch": 0.09700539724914399, + "flos": 25440375605760.0, + "grad_norm": 2.1029126867160577, + "language_loss": 0.89058018, + "learning_rate": 3.953094699960981e-06, + "loss": 0.91249728, + "num_input_tokens_seen": 93781590, + "router_z_loss_clip": 0.93310547, + "router_z_loss_mlp": 0.17907715, + "step": 3343, + "time_per_iteration": 2.5694973468780518 + }, + { + "auxiliary_loss_clip": 0.01035082, + "auxiliary_loss_mlp": 0.00998548, + "balance_loss_clip": 1.01132095, + "balance_loss_mlp": 0.99758798, + "epoch": 0.09703441471766003, + "flos": 74771762595840.0, + "grad_norm": 0.6649052557892937, + "language_loss": 0.47877383, + "learning_rate": 3.9530542226549696e-06, + "loss": 0.4991101, + "num_input_tokens_seen": 93843855, + "router_z_loss_clip": 0.23730469, + "router_z_loss_mlp": 0.00958252, + "step": 3344, + "time_per_iteration": 5.5513916015625 + }, + { + "auxiliary_loss_clip": 0.01153787, + "auxiliary_loss_mlp": 0.01051345, + "balance_loss_clip": 1.05862105, + "balance_loss_mlp": 1.03324938, + "epoch": 0.09706343218617608, + "flos": 15370700849280.0, + "grad_norm": 2.5669230293183714, + "language_loss": 0.86869419, + "learning_rate": 3.953013728098793e-06, + "loss": 0.89074552, + "num_input_tokens_seen": 93857705, + "router_z_loss_clip": 0.95117188, + "router_z_loss_mlp": 0.18103027, + "step": 3345, + "time_per_iteration": 4.812266111373901 + }, + { + "auxiliary_loss_clip": 0.01144716, + "auxiliary_loss_mlp": 0.01041183, + "balance_loss_clip": 1.05647302, + "balance_loss_mlp": 1.0237546, + "epoch": 0.09709244965469213, + "flos": 34967225053440.0, + "grad_norm": 2.1086615198038885, + "language_loss": 0.74496543, + "learning_rate": 3.9529732162928095e-06, + "loss": 0.76682442, + "num_input_tokens_seen": 93877710, + "router_z_loss_clip": 0.88134766, + "router_z_loss_mlp": 0.17425537, + "step": 3346, + "time_per_iteration": 4.934031009674072 + }, + { + "auxiliary_loss_clip": 0.01153011, + "auxiliary_loss_mlp": 0.01051818, + "balance_loss_clip": 1.0592587, + "balance_loss_mlp": 1.03308988, + "epoch": 0.09712146712320817, + "flos": 17813452141440.0, + "grad_norm": 3.1624797975754633, + "language_loss": 0.88924289, + "learning_rate": 3.9529326872373755e-06, + "loss": 0.91129112, + "num_input_tokens_seen": 93892715, + "router_z_loss_clip": 0.93701172, + "router_z_loss_mlp": 0.18719482, + "step": 3347, + "time_per_iteration": 4.7674713134765625 + }, + { + "auxiliary_loss_clip": 0.0103254, + "auxiliary_loss_mlp": 0.01005055, + "balance_loss_clip": 1.00872517, + "balance_loss_mlp": 1.00416136, + "epoch": 0.09715048459172422, + "flos": 74775641264640.0, + "grad_norm": 0.7269155194265926, + "language_loss": 0.49709374, + "learning_rate": 3.952892140932851e-06, + "loss": 0.51746964, + "num_input_tokens_seen": 93958110, + "router_z_loss_clip": 0.23828125, + "router_z_loss_mlp": 0.00891113, + "step": 3348, + "time_per_iteration": 3.1991913318634033 + }, + { + "auxiliary_loss_clip": 0.01153535, + "auxiliary_loss_mlp": 0.01051093, + "balance_loss_clip": 1.05697787, + "balance_loss_mlp": 1.03217459, + "epoch": 0.09717950206024026, + "flos": 15005381575680.0, + "grad_norm": 3.0829265034997726, + "language_loss": 0.9741739, + "learning_rate": 3.952851577379591e-06, + "loss": 0.99622023, + "num_input_tokens_seen": 93971550, + "router_z_loss_clip": 0.96630859, + "router_z_loss_mlp": 0.18920898, + "step": 3349, + "time_per_iteration": 2.5135421752929688 + }, + { + "auxiliary_loss_clip": 0.01158551, + "auxiliary_loss_mlp": 0.01050332, + "balance_loss_clip": 1.05726242, + "balance_loss_mlp": 1.02858198, + "epoch": 0.09720851952875631, + "flos": 11810662035840.0, + "grad_norm": 4.308315849893839, + "language_loss": 0.97262502, + "learning_rate": 3.952810996577957e-06, + "loss": 0.9947139, + "num_input_tokens_seen": 93985550, + "router_z_loss_clip": 1.01318359, + "router_z_loss_mlp": 0.21734619, + "step": 3350, + "time_per_iteration": 2.5517046451568604 + }, + { + "auxiliary_loss_clip": 0.01163192, + "auxiliary_loss_mlp": 0.01053127, + "balance_loss_clip": 1.06564426, + "balance_loss_mlp": 1.03528118, + "epoch": 0.09723753699727236, + "flos": 12232611100800.0, + "grad_norm": 2.5862594692821337, + "language_loss": 0.95743304, + "learning_rate": 3.9527703985283055e-06, + "loss": 0.9795962, + "num_input_tokens_seen": 93997240, + "router_z_loss_clip": 0.97509766, + "router_z_loss_mlp": 0.1784668, + "step": 3351, + "time_per_iteration": 2.5020315647125244 + }, + { + "auxiliary_loss_clip": 0.01034, + "auxiliary_loss_mlp": 0.01008294, + "balance_loss_clip": 1.01016736, + "balance_loss_mlp": 1.00735271, + "epoch": 0.0972665544657884, + "flos": 67728531139200.0, + "grad_norm": 0.6198909697661487, + "language_loss": 0.46572262, + "learning_rate": 3.952729783230996e-06, + "loss": 0.48614556, + "num_input_tokens_seen": 94057605, + "router_z_loss_clip": 0.23828125, + "router_z_loss_mlp": 0.00939941, + "step": 3352, + "time_per_iteration": 3.1417124271392822 + }, + { + "auxiliary_loss_clip": 0.01154927, + "auxiliary_loss_mlp": 0.01042544, + "balance_loss_clip": 1.06079412, + "balance_loss_mlp": 1.02542543, + "epoch": 0.09729557193430445, + "flos": 25920429897600.0, + "grad_norm": 1.6576806999125682, + "language_loss": 0.80355, + "learning_rate": 3.9526891506863865e-06, + "loss": 0.82552475, + "num_input_tokens_seen": 94082725, + "router_z_loss_clip": 0.94091797, + "router_z_loss_mlp": 0.17138672, + "step": 3353, + "time_per_iteration": 2.678039312362671 + }, + { + "auxiliary_loss_clip": 0.01164759, + "auxiliary_loss_mlp": 0.01050839, + "balance_loss_clip": 1.06287754, + "balance_loss_mlp": 1.02916646, + "epoch": 0.0973245894028205, + "flos": 33868142749440.0, + "grad_norm": 2.1376173708934227, + "language_loss": 0.60124141, + "learning_rate": 3.952648500894836e-06, + "loss": 0.62339741, + "num_input_tokens_seen": 94098765, + "router_z_loss_clip": 1.01904297, + "router_z_loss_mlp": 0.2166748, + "step": 3354, + "time_per_iteration": 2.607296943664551 + }, + { + "auxiliary_loss_clip": 0.0115196, + "auxiliary_loss_mlp": 0.01045376, + "balance_loss_clip": 1.05858588, + "balance_loss_mlp": 1.02790034, + "epoch": 0.09735360687133654, + "flos": 27776364698880.0, + "grad_norm": 2.287280863122472, + "language_loss": 0.68597311, + "learning_rate": 3.952607833856704e-06, + "loss": 0.70794654, + "num_input_tokens_seen": 94114050, + "router_z_loss_clip": 0.93310547, + "router_z_loss_mlp": 0.17456055, + "step": 3355, + "time_per_iteration": 2.5320045948028564 + }, + { + "auxiliary_loss_clip": 0.01035309, + "auxiliary_loss_mlp": 0.00998426, + "balance_loss_clip": 1.01135635, + "balance_loss_mlp": 0.99755567, + "epoch": 0.09738262433985259, + "flos": 56856252936960.0, + "grad_norm": 0.721755246987215, + "language_loss": 0.54961157, + "learning_rate": 3.95256714957235e-06, + "loss": 0.56994891, + "num_input_tokens_seen": 94165965, + "router_z_loss_clip": 0.23925781, + "router_z_loss_mlp": 0.00872803, + "step": 3356, + "time_per_iteration": 2.9271717071533203 + }, + { + "auxiliary_loss_clip": 0.01155032, + "auxiliary_loss_mlp": 0.01055509, + "balance_loss_clip": 1.06082058, + "balance_loss_mlp": 1.03777087, + "epoch": 0.09741164180836864, + "flos": 23105858970240.0, + "grad_norm": 2.76313800239993, + "language_loss": 1.05311918, + "learning_rate": 3.952526448042132e-06, + "loss": 1.07522464, + "num_input_tokens_seen": 94181455, + "router_z_loss_clip": 0.94091797, + "router_z_loss_mlp": 0.17749023, + "step": 3357, + "time_per_iteration": 2.556328535079956 + }, + { + "auxiliary_loss_clip": 0.01148826, + "auxiliary_loss_mlp": 0.01041377, + "balance_loss_clip": 1.06211185, + "balance_loss_mlp": 1.02596939, + "epoch": 0.09744065927688468, + "flos": 33578056512000.0, + "grad_norm": 2.397908277532929, + "language_loss": 0.85731137, + "learning_rate": 3.952485729266411e-06, + "loss": 0.87921333, + "num_input_tokens_seen": 94198660, + "router_z_loss_clip": 0.8671875, + "router_z_loss_mlp": 0.1539917, + "step": 3358, + "time_per_iteration": 2.646723747253418 + }, + { + "auxiliary_loss_clip": 0.0116396, + "auxiliary_loss_mlp": 0.01055683, + "balance_loss_clip": 1.06548047, + "balance_loss_mlp": 1.03576279, + "epoch": 0.09746967674540073, + "flos": 27996928162560.0, + "grad_norm": 2.306139828690509, + "language_loss": 0.96790445, + "learning_rate": 3.952444993245546e-06, + "loss": 0.99010092, + "num_input_tokens_seen": 94219485, + "router_z_loss_clip": 0.98486328, + "router_z_loss_mlp": 0.19909668, + "step": 3359, + "time_per_iteration": 2.6293177604675293 + }, + { + "auxiliary_loss_clip": 0.01154554, + "auxiliary_loss_mlp": 0.01046218, + "balance_loss_clip": 1.06059492, + "balance_loss_mlp": 1.02725148, + "epoch": 0.09749869421391678, + "flos": 16317880536960.0, + "grad_norm": 2.645354923478076, + "language_loss": 0.84773517, + "learning_rate": 3.952404239979896e-06, + "loss": 0.86974287, + "num_input_tokens_seen": 94232035, + "router_z_loss_clip": 0.94091797, + "router_z_loss_mlp": 0.18969727, + "step": 3360, + "time_per_iteration": 2.508303642272949 + }, + { + "auxiliary_loss_clip": 0.01159138, + "auxiliary_loss_mlp": 0.01052444, + "balance_loss_clip": 1.06213188, + "balance_loss_mlp": 1.03266716, + "epoch": 0.09752771168243282, + "flos": 29235271495680.0, + "grad_norm": 2.559789592389449, + "language_loss": 0.92671037, + "learning_rate": 3.952363469469823e-06, + "loss": 0.94882619, + "num_input_tokens_seen": 94248280, + "router_z_loss_clip": 0.97167969, + "router_z_loss_mlp": 0.19763184, + "step": 3361, + "time_per_iteration": 2.5632424354553223 + }, + { + "auxiliary_loss_clip": 0.01155439, + "auxiliary_loss_mlp": 0.01051731, + "balance_loss_clip": 1.0617696, + "balance_loss_mlp": 1.03356397, + "epoch": 0.09755672915094887, + "flos": 25111721139840.0, + "grad_norm": 2.4726520784310178, + "language_loss": 0.89753306, + "learning_rate": 3.952322681715685e-06, + "loss": 0.91960484, + "num_input_tokens_seen": 94263490, + "router_z_loss_clip": 0.9375, + "router_z_loss_mlp": 0.18151855, + "step": 3362, + "time_per_iteration": 2.5539464950561523 + }, + { + "auxiliary_loss_clip": 0.0115974, + "auxiliary_loss_mlp": 0.0105829, + "balance_loss_clip": 1.06437099, + "balance_loss_mlp": 1.04038525, + "epoch": 0.09758574661946492, + "flos": 26608226497920.0, + "grad_norm": 2.6513840534101556, + "language_loss": 0.71298099, + "learning_rate": 3.952281876717843e-06, + "loss": 0.7351613, + "num_input_tokens_seen": 94277790, + "router_z_loss_clip": 0.953125, + "router_z_loss_mlp": 0.17895508, + "step": 3363, + "time_per_iteration": 2.5112226009368896 + }, + { + "auxiliary_loss_clip": 0.01158142, + "auxiliary_loss_mlp": 0.0104702, + "balance_loss_clip": 1.06103754, + "balance_loss_mlp": 1.02912712, + "epoch": 0.09761476408798096, + "flos": 16575180635520.0, + "grad_norm": 2.8841271016869086, + "language_loss": 0.87149537, + "learning_rate": 3.952241054476658e-06, + "loss": 0.89354694, + "num_input_tokens_seen": 94290890, + "router_z_loss_clip": 0.97167969, + "router_z_loss_mlp": 0.17895508, + "step": 3364, + "time_per_iteration": 2.4997265338897705 + }, + { + "auxiliary_loss_clip": 0.01038229, + "auxiliary_loss_mlp": 0.01006944, + "balance_loss_clip": 1.01455307, + "balance_loss_mlp": 1.00594854, + "epoch": 0.09764378155649701, + "flos": 56852769317760.0, + "grad_norm": 0.7068647789937074, + "language_loss": 0.54452777, + "learning_rate": 3.952200214992489e-06, + "loss": 0.56497955, + "num_input_tokens_seen": 94347280, + "router_z_loss_clip": 0.23632812, + "router_z_loss_mlp": 0.00994873, + "step": 3365, + "time_per_iteration": 2.9646434783935547 + }, + { + "auxiliary_loss_clip": 0.010369, + "auxiliary_loss_mlp": 0.01001901, + "balance_loss_clip": 1.01322544, + "balance_loss_mlp": 1.00098324, + "epoch": 0.09767279902501305, + "flos": 60283782478080.0, + "grad_norm": 0.6733805133150534, + "language_loss": 0.47204039, + "learning_rate": 3.9521593582656975e-06, + "loss": 0.49242839, + "num_input_tokens_seen": 94404235, + "router_z_loss_clip": 0.23632812, + "router_z_loss_mlp": 0.00915527, + "step": 3366, + "time_per_iteration": 3.004128932952881 + }, + { + "auxiliary_loss_clip": 0.01161319, + "auxiliary_loss_mlp": 0.01057551, + "balance_loss_clip": 1.06180882, + "balance_loss_mlp": 1.03655839, + "epoch": 0.0977018164935291, + "flos": 38757811311360.0, + "grad_norm": 2.902509475968711, + "language_loss": 0.75935698, + "learning_rate": 3.952118484296646e-06, + "loss": 0.78154564, + "num_input_tokens_seen": 94420050, + "router_z_loss_clip": 0.99414062, + "router_z_loss_mlp": 0.2097168, + "step": 3367, + "time_per_iteration": 2.6943743228912354 + }, + { + "auxiliary_loss_clip": 0.01154465, + "auxiliary_loss_mlp": 0.01043804, + "balance_loss_clip": 1.06021023, + "balance_loss_mlp": 1.02526689, + "epoch": 0.09773083396204515, + "flos": 32299385184000.0, + "grad_norm": 2.425963395992932, + "language_loss": 0.9469288, + "learning_rate": 3.952077593085694e-06, + "loss": 0.96891153, + "num_input_tokens_seen": 94437860, + "router_z_loss_clip": 0.94335938, + "router_z_loss_mlp": 0.1854248, + "step": 3368, + "time_per_iteration": 2.624110698699951 + }, + { + "auxiliary_loss_clip": 0.01160813, + "auxiliary_loss_mlp": 0.01056359, + "balance_loss_clip": 1.06183732, + "balance_loss_mlp": 1.03646326, + "epoch": 0.09775985143056119, + "flos": 30622752097920.0, + "grad_norm": 2.1689812013190206, + "language_loss": 0.91486394, + "learning_rate": 3.952036684633201e-06, + "loss": 0.93703562, + "num_input_tokens_seen": 94459605, + "router_z_loss_clip": 0.99023438, + "router_z_loss_mlp": 0.19897461, + "step": 3369, + "time_per_iteration": 2.684640645980835 + }, + { + "auxiliary_loss_clip": 0.01035042, + "auxiliary_loss_mlp": 0.00997551, + "balance_loss_clip": 1.01169538, + "balance_loss_mlp": 0.99654979, + "epoch": 0.09778886889907724, + "flos": 71816781404160.0, + "grad_norm": 0.6551508802138698, + "language_loss": 0.52435642, + "learning_rate": 3.951995758939532e-06, + "loss": 0.54468232, + "num_input_tokens_seen": 94524810, + "router_z_loss_clip": 0.23339844, + "router_z_loss_mlp": 0.01000977, + "step": 3370, + "time_per_iteration": 3.1967265605926514 + }, + { + "auxiliary_loss_clip": 0.01152727, + "auxiliary_loss_mlp": 0.0104438, + "balance_loss_clip": 1.06350422, + "balance_loss_mlp": 1.02815557, + "epoch": 0.0978178863675933, + "flos": 27264386194560.0, + "grad_norm": 2.113695131476839, + "language_loss": 0.71989357, + "learning_rate": 3.951954816005046e-06, + "loss": 0.74186468, + "num_input_tokens_seen": 94540165, + "router_z_loss_clip": 0.89208984, + "router_z_loss_mlp": 0.16229248, + "step": 3371, + "time_per_iteration": 2.6085386276245117 + }, + { + "auxiliary_loss_clip": 0.01166135, + "auxiliary_loss_mlp": 0.01051351, + "balance_loss_clip": 1.06323326, + "balance_loss_mlp": 1.03062034, + "epoch": 0.09784690383610933, + "flos": 21829126976640.0, + "grad_norm": 2.633578250103269, + "language_loss": 0.92202306, + "learning_rate": 3.951913855830104e-06, + "loss": 0.94419801, + "num_input_tokens_seen": 94553455, + "router_z_loss_clip": 1.02880859, + "router_z_loss_mlp": 0.20727539, + "step": 3372, + "time_per_iteration": 2.596287965774536 + }, + { + "auxiliary_loss_clip": 0.01159849, + "auxiliary_loss_mlp": 0.01049134, + "balance_loss_clip": 1.06127191, + "balance_loss_mlp": 1.03149068, + "epoch": 0.09787592130462539, + "flos": 30220301139840.0, + "grad_norm": 2.2004798777950834, + "language_loss": 0.79972649, + "learning_rate": 3.951872878415071e-06, + "loss": 0.82181627, + "num_input_tokens_seen": 94571310, + "router_z_loss_clip": 0.98583984, + "router_z_loss_mlp": 0.1763916, + "step": 3373, + "time_per_iteration": 2.688232421875 + }, + { + "auxiliary_loss_clip": 0.01144887, + "auxiliary_loss_mlp": 0.0104382, + "balance_loss_clip": 1.06037259, + "balance_loss_mlp": 1.02717817, + "epoch": 0.09790493877314144, + "flos": 35364432625920.0, + "grad_norm": 2.0678545779805746, + "language_loss": 0.75456154, + "learning_rate": 3.951831883760306e-06, + "loss": 0.77644861, + "num_input_tokens_seen": 94587680, + "router_z_loss_clip": 0.84521484, + "router_z_loss_mlp": 0.16644287, + "step": 3374, + "time_per_iteration": 2.6795809268951416 + }, + { + "auxiliary_loss_clip": 0.01152628, + "auxiliary_loss_mlp": 0.01042504, + "balance_loss_clip": 1.06100988, + "balance_loss_mlp": 1.02523077, + "epoch": 0.09793395624165747, + "flos": 42260107011840.0, + "grad_norm": 1.8834110328761233, + "language_loss": 0.77846515, + "learning_rate": 3.951790871866172e-06, + "loss": 0.80041647, + "num_input_tokens_seen": 94603940, + "router_z_loss_clip": 0.91699219, + "router_z_loss_mlp": 0.17260742, + "step": 3375, + "time_per_iteration": 2.738982677459717 + }, + { + "auxiliary_loss_clip": 0.01033539, + "auxiliary_loss_mlp": 0.00999269, + "balance_loss_clip": 1.01014757, + "balance_loss_mlp": 0.9983328, + "epoch": 0.09796297371017353, + "flos": 66382527767040.0, + "grad_norm": 0.7074144294506693, + "language_loss": 0.48602566, + "learning_rate": 3.951749842733031e-06, + "loss": 0.50635374, + "num_input_tokens_seen": 94664875, + "router_z_loss_clip": 0.234375, + "router_z_loss_mlp": 0.00933838, + "step": 3376, + "time_per_iteration": 3.101513385772705 + }, + { + "auxiliary_loss_clip": 0.01033529, + "auxiliary_loss_mlp": 0.01001045, + "balance_loss_clip": 1.01020455, + "balance_loss_mlp": 1.00003135, + "epoch": 0.09799199117868958, + "flos": 63648576915840.0, + "grad_norm": 0.706229320526355, + "language_loss": 0.52570319, + "learning_rate": 3.951708796361245e-06, + "loss": 0.54604888, + "num_input_tokens_seen": 94728080, + "router_z_loss_clip": 0.23242188, + "router_z_loss_mlp": 0.01013184, + "step": 3377, + "time_per_iteration": 3.0852808952331543 + }, + { + "auxiliary_loss_clip": 0.01144134, + "auxiliary_loss_mlp": 0.01051579, + "balance_loss_clip": 1.05620527, + "balance_loss_mlp": 1.03313756, + "epoch": 0.09802100864720562, + "flos": 22267775865600.0, + "grad_norm": 2.161908943057585, + "language_loss": 0.86340487, + "learning_rate": 3.9516677327511785e-06, + "loss": 0.88536203, + "num_input_tokens_seen": 94745225, + "router_z_loss_clip": 0.88037109, + "router_z_loss_mlp": 0.18444824, + "step": 3378, + "time_per_iteration": 2.7709195613861084 + }, + { + "auxiliary_loss_clip": 0.01033299, + "auxiliary_loss_mlp": 0.01002998, + "balance_loss_clip": 1.00985026, + "balance_loss_mlp": 1.00197303, + "epoch": 0.09805002611572167, + "flos": 68684976535680.0, + "grad_norm": 0.680941850583946, + "language_loss": 0.48760223, + "learning_rate": 3.951626651903192e-06, + "loss": 0.50796521, + "num_input_tokens_seen": 94807780, + "router_z_loss_clip": 0.234375, + "router_z_loss_mlp": 0.01025391, + "step": 3379, + "time_per_iteration": 3.1971402168273926 + }, + { + "auxiliary_loss_clip": 0.01157867, + "auxiliary_loss_mlp": 0.0106169, + "balance_loss_clip": 1.06118488, + "balance_loss_mlp": 1.04206777, + "epoch": 0.0980790435842377, + "flos": 37563997864320.0, + "grad_norm": 3.2614922937863544, + "language_loss": 0.8029176, + "learning_rate": 3.951585553817649e-06, + "loss": 0.82511318, + "num_input_tokens_seen": 94826975, + "router_z_loss_clip": 0.96728516, + "router_z_loss_mlp": 0.19628906, + "step": 3380, + "time_per_iteration": 2.7213242053985596 + }, + { + "auxiliary_loss_clip": 0.01152282, + "auxiliary_loss_mlp": 0.01050678, + "balance_loss_clip": 1.05870759, + "balance_loss_mlp": 1.03220093, + "epoch": 0.09810806105275376, + "flos": 36130766313600.0, + "grad_norm": 3.216808086054197, + "language_loss": 0.89954329, + "learning_rate": 3.9515444384949136e-06, + "loss": 0.92157286, + "num_input_tokens_seen": 94848370, + "router_z_loss_clip": 0.93603516, + "router_z_loss_mlp": 0.18481445, + "step": 3381, + "time_per_iteration": 2.7324278354644775 + }, + { + "auxiliary_loss_clip": 0.01155464, + "auxiliary_loss_mlp": 0.01057814, + "balance_loss_clip": 1.0582068, + "balance_loss_mlp": 1.03845477, + "epoch": 0.09813707852126981, + "flos": 16317844623360.0, + "grad_norm": 2.6264751860056714, + "language_loss": 0.86523342, + "learning_rate": 3.951503305935347e-06, + "loss": 0.88736618, + "num_input_tokens_seen": 94861275, + "router_z_loss_clip": 0.97167969, + "router_z_loss_mlp": 0.19360352, + "step": 3382, + "time_per_iteration": 2.4816524982452393 + }, + { + "auxiliary_loss_clip": 0.01153509, + "auxiliary_loss_mlp": 0.01052106, + "balance_loss_clip": 1.05923617, + "balance_loss_mlp": 1.03366399, + "epoch": 0.09816609598978585, + "flos": 24564550285440.0, + "grad_norm": 2.323822642647211, + "language_loss": 0.88858181, + "learning_rate": 3.951462156139314e-06, + "loss": 0.91063797, + "num_input_tokens_seen": 94876090, + "router_z_loss_clip": 0.94140625, + "router_z_loss_mlp": 0.18457031, + "step": 3383, + "time_per_iteration": 2.599431037902832 + }, + { + "auxiliary_loss_clip": 0.01030707, + "auxiliary_loss_mlp": 0.01001532, + "balance_loss_clip": 1.00740492, + "balance_loss_mlp": 1.00045347, + "epoch": 0.0981951134583019, + "flos": 48163284702720.0, + "grad_norm": 0.7252147657623723, + "language_loss": 0.46343496, + "learning_rate": 3.951420989107178e-06, + "loss": 0.48375735, + "num_input_tokens_seen": 94920730, + "router_z_loss_clip": 0.23339844, + "router_z_loss_mlp": 0.01080322, + "step": 3384, + "time_per_iteration": 2.7930262088775635 + }, + { + "auxiliary_loss_clip": 0.01030279, + "auxiliary_loss_mlp": 0.01001257, + "balance_loss_clip": 1.00702763, + "balance_loss_mlp": 1.00016654, + "epoch": 0.09822413092681795, + "flos": 71340246645120.0, + "grad_norm": 0.6500146692555884, + "language_loss": 0.50327802, + "learning_rate": 3.951379804839301e-06, + "loss": 0.52359343, + "num_input_tokens_seen": 94983110, + "router_z_loss_clip": 0.23242188, + "router_z_loss_mlp": 0.01092529, + "step": 3385, + "time_per_iteration": 3.096512794494629 + }, + { + "auxiliary_loss_clip": 0.01029656, + "auxiliary_loss_mlp": 0.00998989, + "balance_loss_clip": 1.00631785, + "balance_loss_mlp": 0.9978919, + "epoch": 0.09825314839533399, + "flos": 74772911831040.0, + "grad_norm": 0.6449419268735469, + "language_loss": 0.50772572, + "learning_rate": 3.9513386033360494e-06, + "loss": 0.52801216, + "num_input_tokens_seen": 95052360, + "router_z_loss_clip": 0.23242188, + "router_z_loss_mlp": 0.01098633, + "step": 3386, + "time_per_iteration": 3.3058536052703857 + }, + { + "auxiliary_loss_clip": 0.01028901, + "auxiliary_loss_mlp": 0.00998857, + "balance_loss_clip": 1.00560784, + "balance_loss_mlp": 0.9978323, + "epoch": 0.09828216586385004, + "flos": 62505287948160.0, + "grad_norm": 0.6900252650658694, + "language_loss": 0.49452692, + "learning_rate": 3.951297384597785e-06, + "loss": 0.51480448, + "num_input_tokens_seen": 95103805, + "router_z_loss_clip": 0.23242188, + "router_z_loss_mlp": 0.01025391, + "step": 3387, + "time_per_iteration": 2.967712163925171 + }, + { + "auxiliary_loss_clip": 0.0115115, + "auxiliary_loss_mlp": 0.01048454, + "balance_loss_clip": 1.05511677, + "balance_loss_mlp": 1.02964246, + "epoch": 0.09831118333236609, + "flos": 34232492355840.0, + "grad_norm": 2.1610545075267935, + "language_loss": 0.85246027, + "learning_rate": 3.951256148624872e-06, + "loss": 0.87445629, + "num_input_tokens_seen": 95120895, + "router_z_loss_clip": 0.95947266, + "router_z_loss_mlp": 0.18811035, + "step": 3388, + "time_per_iteration": 2.663597345352173 + }, + { + "auxiliary_loss_clip": 0.01027982, + "auxiliary_loss_mlp": 0.00999222, + "balance_loss_clip": 1.0047071, + "balance_loss_mlp": 0.99823254, + "epoch": 0.09834020080088213, + "flos": 73095309077760.0, + "grad_norm": 0.6145823599756707, + "language_loss": 0.50428438, + "learning_rate": 3.951214895417675e-06, + "loss": 0.5245564, + "num_input_tokens_seen": 95189375, + "router_z_loss_clip": 0.23242188, + "router_z_loss_mlp": 0.0098877, + "step": 3389, + "time_per_iteration": 3.24324631690979 + }, + { + "auxiliary_loss_clip": 0.01156639, + "auxiliary_loss_mlp": 0.01050324, + "balance_loss_clip": 1.05901217, + "balance_loss_mlp": 1.03078556, + "epoch": 0.09836921826939818, + "flos": 29785746401280.0, + "grad_norm": 2.1886474992709974, + "language_loss": 0.8275435, + "learning_rate": 3.951173624976559e-06, + "loss": 0.84961319, + "num_input_tokens_seen": 95206140, + "router_z_loss_clip": 0.97705078, + "router_z_loss_mlp": 0.19555664, + "step": 3390, + "time_per_iteration": 2.660438060760498 + }, + { + "auxiliary_loss_clip": 0.01027276, + "auxiliary_loss_mlp": 0.01005748, + "balance_loss_clip": 1.00411725, + "balance_loss_mlp": 1.00471139, + "epoch": 0.09839823573791423, + "flos": 74776682759040.0, + "grad_norm": 0.6515495524209114, + "language_loss": 0.47515982, + "learning_rate": 3.951132337301888e-06, + "loss": 0.4954901, + "num_input_tokens_seen": 95271020, + "router_z_loss_clip": 0.23144531, + "router_z_loss_mlp": 0.01037598, + "step": 3391, + "time_per_iteration": 3.168074607849121 + }, + { + "auxiliary_loss_clip": 0.01153274, + "auxiliary_loss_mlp": 0.0105426, + "balance_loss_clip": 1.05715096, + "balance_loss_mlp": 1.03648019, + "epoch": 0.09842725320643027, + "flos": 27741639225600.0, + "grad_norm": 2.215425455109166, + "language_loss": 0.67156994, + "learning_rate": 3.951091032394027e-06, + "loss": 0.6936453, + "num_input_tokens_seen": 95288055, + "router_z_loss_clip": 0.96142578, + "router_z_loss_mlp": 0.17803955, + "step": 3392, + "time_per_iteration": 2.6221911907196045 + }, + { + "auxiliary_loss_clip": 0.01150503, + "auxiliary_loss_mlp": 0.01039231, + "balance_loss_clip": 1.05558729, + "balance_loss_mlp": 1.02205324, + "epoch": 0.09845627067494632, + "flos": 28286942572800.0, + "grad_norm": 2.212921435915925, + "language_loss": 0.88608354, + "learning_rate": 3.95104971025334e-06, + "loss": 0.90798092, + "num_input_tokens_seen": 95304820, + "router_z_loss_clip": 0.94970703, + "router_z_loss_mlp": 0.17175293, + "step": 3393, + "time_per_iteration": 2.599867820739746 + }, + { + "auxiliary_loss_clip": 0.01160717, + "auxiliary_loss_mlp": 0.01057092, + "balance_loss_clip": 1.05752373, + "balance_loss_mlp": 1.03449011, + "epoch": 0.09848528814346237, + "flos": 18874433093760.0, + "grad_norm": 3.0610988917171627, + "language_loss": 0.93817234, + "learning_rate": 3.951008370880192e-06, + "loss": 0.96035039, + "num_input_tokens_seen": 95319580, + "router_z_loss_clip": 1.03076172, + "router_z_loss_mlp": 0.22607422, + "step": 3394, + "time_per_iteration": 2.5446836948394775 + }, + { + "auxiliary_loss_clip": 0.01027524, + "auxiliary_loss_mlp": 0.01011194, + "balance_loss_clip": 1.00451422, + "balance_loss_mlp": 1.0101037, + "epoch": 0.09851430561197841, + "flos": 62698129090560.0, + "grad_norm": 0.6563519739262949, + "language_loss": 0.48603466, + "learning_rate": 3.950967014274949e-06, + "loss": 0.5064218, + "num_input_tokens_seen": 95382455, + "router_z_loss_clip": 0.23046875, + "router_z_loss_mlp": 0.01092529, + "step": 3395, + "time_per_iteration": 3.1068947315216064 + }, + { + "auxiliary_loss_clip": 0.01027181, + "auxiliary_loss_mlp": 0.01008157, + "balance_loss_clip": 1.00392222, + "balance_loss_mlp": 1.00709653, + "epoch": 0.09854332308049446, + "flos": 55838760376320.0, + "grad_norm": 0.6958989569092201, + "language_loss": 0.5149051, + "learning_rate": 3.950925640437976e-06, + "loss": 0.53525853, + "num_input_tokens_seen": 95440210, + "router_z_loss_clip": 0.23242188, + "router_z_loss_mlp": 0.01062012, + "step": 3396, + "time_per_iteration": 3.0347843170166016 + }, + { + "auxiliary_loss_clip": 0.01156905, + "auxiliary_loss_mlp": 0.01060723, + "balance_loss_clip": 1.05927861, + "balance_loss_mlp": 1.03931284, + "epoch": 0.0985723405490105, + "flos": 18472233530880.0, + "grad_norm": 2.1897092970962073, + "language_loss": 0.74807805, + "learning_rate": 3.950884249369638e-06, + "loss": 0.77025425, + "num_input_tokens_seen": 95455670, + "router_z_loss_clip": 0.97607422, + "router_z_loss_mlp": 0.2142334, + "step": 3397, + "time_per_iteration": 2.5625393390655518 + }, + { + "auxiliary_loss_clip": 0.01157666, + "auxiliary_loss_mlp": 0.01055085, + "balance_loss_clip": 1.05983591, + "balance_loss_mlp": 1.0362438, + "epoch": 0.09860135801752655, + "flos": 26244056459520.0, + "grad_norm": 2.6121552590179293, + "language_loss": 0.87711346, + "learning_rate": 3.950842841070301e-06, + "loss": 0.89924091, + "num_input_tokens_seen": 95469555, + "router_z_loss_clip": 0.97851562, + "router_z_loss_mlp": 0.18841553, + "step": 3398, + "time_per_iteration": 2.5044119358062744 + }, + { + "auxiliary_loss_clip": 0.01152798, + "auxiliary_loss_mlp": 0.01047621, + "balance_loss_clip": 1.06115079, + "balance_loss_mlp": 1.03016269, + "epoch": 0.0986303754860426, + "flos": 34634512350720.0, + "grad_norm": 3.164455532606167, + "language_loss": 0.76801413, + "learning_rate": 3.950801415540331e-06, + "loss": 0.79001832, + "num_input_tokens_seen": 95485415, + "router_z_loss_clip": 0.91699219, + "router_z_loss_mlp": 0.17462158, + "step": 3399, + "time_per_iteration": 2.6884922981262207 + }, + { + "auxiliary_loss_clip": 0.01154751, + "auxiliary_loss_mlp": 0.01062366, + "balance_loss_clip": 1.06195068, + "balance_loss_mlp": 1.04492569, + "epoch": 0.09865939295455864, + "flos": 15953818239360.0, + "grad_norm": 3.090854064654846, + "language_loss": 0.80609417, + "learning_rate": 3.950759972780093e-06, + "loss": 0.82826531, + "num_input_tokens_seen": 95498375, + "router_z_loss_clip": 0.92724609, + "router_z_loss_mlp": 0.1741333, + "step": 3400, + "time_per_iteration": 2.594529151916504 + }, + { + "auxiliary_loss_clip": 0.01156941, + "auxiliary_loss_mlp": 0.01045606, + "balance_loss_clip": 1.0622412, + "balance_loss_mlp": 1.02785611, + "epoch": 0.09868841042307469, + "flos": 27890812408320.0, + "grad_norm": 2.3617323706453783, + "language_loss": 0.91907561, + "learning_rate": 3.9507185127899535e-06, + "loss": 0.94110107, + "num_input_tokens_seen": 95513710, + "router_z_loss_clip": 0.94677734, + "router_z_loss_mlp": 0.17749023, + "step": 3401, + "time_per_iteration": 2.61008358001709 + }, + { + "auxiliary_loss_clip": 0.01027338, + "auxiliary_loss_mlp": 0.01006413, + "balance_loss_clip": 1.00389957, + "balance_loss_mlp": 1.00541198, + "epoch": 0.09871742789159074, + "flos": 62583070849920.0, + "grad_norm": 0.7144610901094552, + "language_loss": 0.50478363, + "learning_rate": 3.950677035570279e-06, + "loss": 0.52512109, + "num_input_tokens_seen": 95570645, + "router_z_loss_clip": 0.234375, + "router_z_loss_mlp": 0.01000977, + "step": 3402, + "time_per_iteration": 3.037895679473877 + }, + { + "auxiliary_loss_clip": 0.01027994, + "auxiliary_loss_mlp": 0.01000292, + "balance_loss_clip": 1.00463498, + "balance_loss_mlp": 0.9992134, + "epoch": 0.09874644536010678, + "flos": 71162956091520.0, + "grad_norm": 0.7106988195274255, + "language_loss": 0.54155803, + "learning_rate": 3.950635541121436e-06, + "loss": 0.56184089, + "num_input_tokens_seen": 95632775, + "router_z_loss_clip": 0.234375, + "router_z_loss_mlp": 0.01080322, + "step": 3403, + "time_per_iteration": 3.132854700088501 + }, + { + "auxiliary_loss_clip": 0.01152412, + "auxiliary_loss_mlp": 0.01045971, + "balance_loss_clip": 1.0581733, + "balance_loss_mlp": 1.02922225, + "epoch": 0.09877546282862283, + "flos": 26207355738240.0, + "grad_norm": 3.571629844351958, + "language_loss": 0.86784899, + "learning_rate": 3.95059402944379e-06, + "loss": 0.88983285, + "num_input_tokens_seen": 95647375, + "router_z_loss_clip": 0.94287109, + "router_z_loss_mlp": 0.1673584, + "step": 3404, + "time_per_iteration": 2.611254930496216 + }, + { + "auxiliary_loss_clip": 0.01171792, + "auxiliary_loss_mlp": 0.01058495, + "balance_loss_clip": 1.06226218, + "balance_loss_mlp": 1.03600049, + "epoch": 0.09880448029713888, + "flos": 16319245253760.0, + "grad_norm": 2.068303517844264, + "language_loss": 0.83257872, + "learning_rate": 3.950552500537708e-06, + "loss": 0.85488158, + "num_input_tokens_seen": 95662935, + "router_z_loss_clip": 1.09570312, + "router_z_loss_mlp": 0.22497559, + "step": 3405, + "time_per_iteration": 2.5979743003845215 + }, + { + "auxiliary_loss_clip": 0.01148751, + "auxiliary_loss_mlp": 0.01048751, + "balance_loss_clip": 1.05727375, + "balance_loss_mlp": 1.03202581, + "epoch": 0.09883349776565492, + "flos": 46529885635200.0, + "grad_norm": 2.222800071551491, + "language_loss": 0.77052152, + "learning_rate": 3.950510954403557e-06, + "loss": 0.79249656, + "num_input_tokens_seen": 95680995, + "router_z_loss_clip": 0.91650391, + "router_z_loss_mlp": 0.16723633, + "step": 3406, + "time_per_iteration": 2.8141844272613525 + }, + { + "auxiliary_loss_clip": 0.01027777, + "auxiliary_loss_mlp": 0.01019311, + "balance_loss_clip": 1.0043869, + "balance_loss_mlp": 1.0183152, + "epoch": 0.09886251523417097, + "flos": 61416871983360.0, + "grad_norm": 0.6935691092254822, + "language_loss": 0.43881947, + "learning_rate": 3.950469391041705e-06, + "loss": 0.45929039, + "num_input_tokens_seen": 95740850, + "router_z_loss_clip": 0.234375, + "router_z_loss_mlp": 0.00994873, + "step": 3407, + "time_per_iteration": 3.0780227184295654 + }, + { + "auxiliary_loss_clip": 0.01149716, + "auxiliary_loss_mlp": 0.01050743, + "balance_loss_clip": 1.05502737, + "balance_loss_mlp": 1.03129375, + "epoch": 0.09889153270268702, + "flos": 11170804135680.0, + "grad_norm": 5.219703800627392, + "language_loss": 0.80893278, + "learning_rate": 3.9504278104525165e-06, + "loss": 0.83093733, + "num_input_tokens_seen": 95748675, + "router_z_loss_clip": 0.94580078, + "router_z_loss_mlp": 0.19458008, + "step": 3408, + "time_per_iteration": 2.5183753967285156 + }, + { + "auxiliary_loss_clip": 0.01146103, + "auxiliary_loss_mlp": 0.01041287, + "balance_loss_clip": 1.05681205, + "balance_loss_mlp": 1.0251224, + "epoch": 0.09892055017120306, + "flos": 28578465354240.0, + "grad_norm": 2.511553016579598, + "language_loss": 0.96204513, + "learning_rate": 3.950386212636361e-06, + "loss": 0.98391902, + "num_input_tokens_seen": 95764180, + "router_z_loss_clip": 0.89306641, + "router_z_loss_mlp": 0.16174316, + "step": 3409, + "time_per_iteration": 2.6251425743103027 + }, + { + "auxiliary_loss_clip": 0.01028622, + "auxiliary_loss_mlp": 0.01021456, + "balance_loss_clip": 1.00528538, + "balance_loss_mlp": 1.02043092, + "epoch": 0.09894956763971911, + "flos": 74786307603840.0, + "grad_norm": 0.6730111915654403, + "language_loss": 0.49058855, + "learning_rate": 3.950344597593606e-06, + "loss": 0.51108932, + "num_input_tokens_seen": 95833580, + "router_z_loss_clip": 0.234375, + "router_z_loss_mlp": 0.01025391, + "step": 3410, + "time_per_iteration": 3.249446153640747 + }, + { + "auxiliary_loss_clip": 0.01154518, + "auxiliary_loss_mlp": 0.01050862, + "balance_loss_clip": 1.05943441, + "balance_loss_mlp": 1.03314781, + "epoch": 0.09897858510823515, + "flos": 15590222818560.0, + "grad_norm": 3.424771788778163, + "language_loss": 0.97788554, + "learning_rate": 3.9503029653246175e-06, + "loss": 0.99993938, + "num_input_tokens_seen": 95846295, + "router_z_loss_clip": 0.95117188, + "router_z_loss_mlp": 0.17724609, + "step": 3411, + "time_per_iteration": 2.5160470008850098 + }, + { + "auxiliary_loss_clip": 0.01148523, + "auxiliary_loss_mlp": 0.01052669, + "balance_loss_clip": 1.05785537, + "balance_loss_mlp": 1.03516865, + "epoch": 0.0990076025767512, + "flos": 32665817779200.0, + "grad_norm": 2.063994730283669, + "language_loss": 0.79516596, + "learning_rate": 3.950261315829764e-06, + "loss": 0.81717783, + "num_input_tokens_seen": 95864395, + "router_z_loss_clip": 0.90869141, + "router_z_loss_mlp": 0.17492676, + "step": 3412, + "time_per_iteration": 2.764808177947998 + }, + { + "auxiliary_loss_clip": 0.01158775, + "auxiliary_loss_mlp": 0.01050329, + "balance_loss_clip": 1.06065226, + "balance_loss_mlp": 1.03119564, + "epoch": 0.09903662004526725, + "flos": 21210924977280.0, + "grad_norm": 2.1924751939251785, + "language_loss": 0.83638, + "learning_rate": 3.950219649109414e-06, + "loss": 0.85847104, + "num_input_tokens_seen": 95881070, + "router_z_loss_clip": 0.98291016, + "router_z_loss_mlp": 0.19134521, + "step": 3413, + "time_per_iteration": 2.6062731742858887 + }, + { + "auxiliary_loss_clip": 0.01143317, + "auxiliary_loss_mlp": 0.01041837, + "balance_loss_clip": 1.05557692, + "balance_loss_mlp": 1.02604127, + "epoch": 0.09906563751378329, + "flos": 11430223136640.0, + "grad_norm": 3.120478386906345, + "language_loss": 0.88552308, + "learning_rate": 3.950177965163934e-06, + "loss": 0.90737462, + "num_input_tokens_seen": 95891380, + "router_z_loss_clip": 0.87646484, + "router_z_loss_mlp": 0.15808105, + "step": 3414, + "time_per_iteration": 2.4952971935272217 + }, + { + "auxiliary_loss_clip": 0.01157763, + "auxiliary_loss_mlp": 0.01041768, + "balance_loss_clip": 1.05890214, + "balance_loss_mlp": 1.02177691, + "epoch": 0.09909465498229934, + "flos": 17304346725120.0, + "grad_norm": 2.2998121990586897, + "language_loss": 0.82007921, + "learning_rate": 3.950136263993694e-06, + "loss": 0.84207451, + "num_input_tokens_seen": 95905645, + "router_z_loss_clip": 0.98828125, + "router_z_loss_mlp": 0.1998291, + "step": 3415, + "time_per_iteration": 2.519496440887451 + }, + { + "auxiliary_loss_clip": 0.01158707, + "auxiliary_loss_mlp": 0.0105822, + "balance_loss_clip": 1.06124544, + "balance_loss_mlp": 1.03810978, + "epoch": 0.0991236724508154, + "flos": 24863219873280.0, + "grad_norm": 2.1874070887362476, + "language_loss": 1.01848316, + "learning_rate": 3.95009454559906e-06, + "loss": 1.04065251, + "num_input_tokens_seen": 95925500, + "router_z_loss_clip": 0.97509766, + "router_z_loss_mlp": 0.20117188, + "step": 3416, + "time_per_iteration": 7.285515069961548 + }, + { + "auxiliary_loss_clip": 0.01153156, + "auxiliary_loss_mlp": 0.01058651, + "balance_loss_clip": 1.05978179, + "balance_loss_mlp": 1.03876734, + "epoch": 0.09915268991933143, + "flos": 22522166962560.0, + "grad_norm": 2.3623206782750894, + "language_loss": 0.79937506, + "learning_rate": 3.950052809980403e-06, + "loss": 0.82149315, + "num_input_tokens_seen": 95939250, + "router_z_loss_clip": 0.93359375, + "router_z_loss_mlp": 0.19885254, + "step": 3417, + "time_per_iteration": 4.8917553424835205 + }, + { + "auxiliary_loss_clip": 0.0102988, + "auxiliary_loss_mlp": 0.01005893, + "balance_loss_clip": 1.0065167, + "balance_loss_mlp": 1.00480866, + "epoch": 0.09918170738784748, + "flos": 65339464723200.0, + "grad_norm": 0.7891839351838782, + "language_loss": 0.52228755, + "learning_rate": 3.95001105713809e-06, + "loss": 0.54264522, + "num_input_tokens_seen": 95985410, + "router_z_loss_clip": 0.234375, + "router_z_loss_mlp": 0.01086426, + "step": 3418, + "time_per_iteration": 5.331165790557861 + }, + { + "auxiliary_loss_clip": 0.01156011, + "auxiliary_loss_mlp": 0.01048694, + "balance_loss_clip": 1.06191552, + "balance_loss_mlp": 1.03148007, + "epoch": 0.09921072485636354, + "flos": 27884204305920.0, + "grad_norm": 2.2816774016098447, + "language_loss": 0.77718818, + "learning_rate": 3.949969287072491e-06, + "loss": 0.79923522, + "num_input_tokens_seen": 95999710, + "router_z_loss_clip": 0.94091797, + "router_z_loss_mlp": 0.17211914, + "step": 3419, + "time_per_iteration": 2.594146490097046 + }, + { + "auxiliary_loss_clip": 0.01159236, + "auxiliary_loss_mlp": 0.01060454, + "balance_loss_clip": 1.06175792, + "balance_loss_mlp": 1.0384599, + "epoch": 0.09923974232487957, + "flos": 43902122365440.0, + "grad_norm": 2.2744793844134574, + "language_loss": 0.71944046, + "learning_rate": 3.949927499783973e-06, + "loss": 0.74163735, + "num_input_tokens_seen": 96017785, + "router_z_loss_clip": 0.97412109, + "router_z_loss_mlp": 0.21984863, + "step": 3420, + "time_per_iteration": 2.646855592727661 + }, + { + "auxiliary_loss_clip": 0.01148254, + "auxiliary_loss_mlp": 0.01044528, + "balance_loss_clip": 1.05731332, + "balance_loss_mlp": 1.02507281, + "epoch": 0.09926875979339563, + "flos": 16101052087680.0, + "grad_norm": 2.633037225931982, + "language_loss": 0.96408445, + "learning_rate": 3.949885695272908e-06, + "loss": 0.98601228, + "num_input_tokens_seen": 96030455, + "router_z_loss_clip": 0.90966797, + "router_z_loss_mlp": 0.19458008, + "step": 3421, + "time_per_iteration": 2.5327277183532715 + }, + { + "auxiliary_loss_clip": 0.01029027, + "auxiliary_loss_mlp": 0.00998037, + "balance_loss_clip": 1.00572085, + "balance_loss_mlp": 0.99709576, + "epoch": 0.09929777726191168, + "flos": 74770685187840.0, + "grad_norm": 0.6880264060333038, + "language_loss": 0.48056102, + "learning_rate": 3.949843873539662e-06, + "loss": 0.50083166, + "num_input_tokens_seen": 96091435, + "router_z_loss_clip": 0.23242188, + "router_z_loss_mlp": 0.00939941, + "step": 3422, + "time_per_iteration": 3.0921616554260254 + }, + { + "auxiliary_loss_clip": 0.01156258, + "auxiliary_loss_mlp": 0.01054239, + "balance_loss_clip": 1.06081235, + "balance_loss_mlp": 1.03508162, + "epoch": 0.09932679473042771, + "flos": 11831524859520.0, + "grad_norm": 2.944696884024858, + "language_loss": 0.88798821, + "learning_rate": 3.949802034584606e-06, + "loss": 0.91009319, + "num_input_tokens_seen": 96105620, + "router_z_loss_clip": 0.95605469, + "router_z_loss_mlp": 0.19165039, + "step": 3423, + "time_per_iteration": 2.498542547225952 + }, + { + "auxiliary_loss_clip": 0.01153365, + "auxiliary_loss_mlp": 0.01055389, + "balance_loss_clip": 1.06007147, + "balance_loss_mlp": 1.03556406, + "epoch": 0.09935581219894377, + "flos": 13219077288960.0, + "grad_norm": 3.0901551197152157, + "language_loss": 0.88033986, + "learning_rate": 3.94976017840811e-06, + "loss": 0.90242738, + "num_input_tokens_seen": 96117765, + "router_z_loss_clip": 0.93261719, + "router_z_loss_mlp": 0.19824219, + "step": 3424, + "time_per_iteration": 2.53189754486084 + }, + { + "auxiliary_loss_clip": 0.01028541, + "auxiliary_loss_mlp": 0.01014111, + "balance_loss_clip": 1.00505543, + "balance_loss_mlp": 1.01313937, + "epoch": 0.09938482966745982, + "flos": 62627492995200.0, + "grad_norm": 0.6373846560206197, + "language_loss": 0.48975492, + "learning_rate": 3.9497183050105425e-06, + "loss": 0.51018155, + "num_input_tokens_seen": 96181040, + "router_z_loss_clip": 0.234375, + "router_z_loss_mlp": 0.00970459, + "step": 3425, + "time_per_iteration": 3.131359815597534 + }, + { + "auxiliary_loss_clip": 0.01155438, + "auxiliary_loss_mlp": 0.01048629, + "balance_loss_clip": 1.06145525, + "balance_loss_mlp": 1.02881074, + "epoch": 0.09941384713597586, + "flos": 23544507859200.0, + "grad_norm": 1.941825137489006, + "language_loss": 0.79221541, + "learning_rate": 3.949676414392274e-06, + "loss": 0.81425607, + "num_input_tokens_seen": 96199450, + "router_z_loss_clip": 0.93896484, + "router_z_loss_mlp": 0.19836426, + "step": 3426, + "time_per_iteration": 2.621624708175659 + }, + { + "auxiliary_loss_clip": 0.01027881, + "auxiliary_loss_mlp": 0.01017055, + "balance_loss_clip": 1.00463176, + "balance_loss_mlp": 1.01607132, + "epoch": 0.0994428646044919, + "flos": 58681376847360.0, + "grad_norm": 0.677084822653691, + "language_loss": 0.50829476, + "learning_rate": 3.949634506553675e-06, + "loss": 0.5287441, + "num_input_tokens_seen": 96261305, + "router_z_loss_clip": 0.23242188, + "router_z_loss_mlp": 0.00982666, + "step": 3427, + "time_per_iteration": 3.133615255355835 + }, + { + "auxiliary_loss_clip": 0.01027779, + "auxiliary_loss_mlp": 0.01011146, + "balance_loss_clip": 1.00452805, + "balance_loss_mlp": 1.01018643, + "epoch": 0.09947188207300794, + "flos": 74773270967040.0, + "grad_norm": 0.6571626255077611, + "language_loss": 0.49899155, + "learning_rate": 3.949592581495115e-06, + "loss": 0.51938081, + "num_input_tokens_seen": 96325650, + "router_z_loss_clip": 0.23242188, + "router_z_loss_mlp": 0.00958252, + "step": 3428, + "time_per_iteration": 3.2467966079711914 + }, + { + "auxiliary_loss_clip": 0.01154695, + "auxiliary_loss_mlp": 0.01051659, + "balance_loss_clip": 1.05952585, + "balance_loss_mlp": 1.03272879, + "epoch": 0.099500899541524, + "flos": 32533775383680.0, + "grad_norm": 1.8803268779276388, + "language_loss": 0.75116467, + "learning_rate": 3.949550639216964e-06, + "loss": 0.77322823, + "num_input_tokens_seen": 96342530, + "router_z_loss_clip": 0.95166016, + "router_z_loss_mlp": 0.18933105, + "step": 3429, + "time_per_iteration": 2.6220622062683105 + }, + { + "auxiliary_loss_clip": 0.01028714, + "auxiliary_loss_mlp": 0.01001255, + "balance_loss_clip": 1.00570583, + "balance_loss_mlp": 1.000278, + "epoch": 0.09952991701004005, + "flos": 63232550616960.0, + "grad_norm": 0.6998257729731958, + "language_loss": 0.47360843, + "learning_rate": 3.949508679719593e-06, + "loss": 0.49390811, + "num_input_tokens_seen": 96391330, + "router_z_loss_clip": 0.23046875, + "router_z_loss_mlp": 0.00976562, + "step": 3430, + "time_per_iteration": 2.8753840923309326 + }, + { + "auxiliary_loss_clip": 0.01150886, + "auxiliary_loss_mlp": 0.01052727, + "balance_loss_clip": 1.06001544, + "balance_loss_mlp": 1.03697968, + "epoch": 0.09955893447855609, + "flos": 54811568165760.0, + "grad_norm": 2.16259010327936, + "language_loss": 1.20779693, + "learning_rate": 3.949466703003373e-06, + "loss": 1.22983313, + "num_input_tokens_seen": 96413550, + "router_z_loss_clip": 0.90820312, + "router_z_loss_mlp": 0.15759277, + "step": 3431, + "time_per_iteration": 2.7633237838745117 + }, + { + "auxiliary_loss_clip": 0.01147716, + "auxiliary_loss_mlp": 0.01041575, + "balance_loss_clip": 1.05531478, + "balance_loss_mlp": 1.02434921, + "epoch": 0.09958795194707214, + "flos": 74731683018240.0, + "grad_norm": 1.646207931209905, + "language_loss": 0.76958174, + "learning_rate": 3.949424709068674e-06, + "loss": 0.7914747, + "num_input_tokens_seen": 96438410, + "router_z_loss_clip": 0.92285156, + "router_z_loss_mlp": 0.17230225, + "step": 3432, + "time_per_iteration": 2.921847105026245 + }, + { + "auxiliary_loss_clip": 0.01157252, + "auxiliary_loss_mlp": 0.01053395, + "balance_loss_clip": 1.05942416, + "balance_loss_mlp": 1.0318656, + "epoch": 0.09961696941558819, + "flos": 17559922970880.0, + "grad_norm": 2.3808324220967902, + "language_loss": 0.8283242, + "learning_rate": 3.949382697915866e-06, + "loss": 0.85043073, + "num_input_tokens_seen": 96452550, + "router_z_loss_clip": 0.97753906, + "router_z_loss_mlp": 0.2154541, + "step": 3433, + "time_per_iteration": 2.491593360900879 + }, + { + "auxiliary_loss_clip": 0.01027962, + "auxiliary_loss_mlp": 0.00998051, + "balance_loss_clip": 1.00468957, + "balance_loss_mlp": 0.99700242, + "epoch": 0.09964598688410423, + "flos": 63243576092160.0, + "grad_norm": 0.6985707405581003, + "language_loss": 0.51390338, + "learning_rate": 3.949340669545323e-06, + "loss": 0.53416353, + "num_input_tokens_seen": 96515705, + "router_z_loss_clip": 0.23242188, + "router_z_loss_mlp": 0.01049805, + "step": 3434, + "time_per_iteration": 3.2650671005249023 + }, + { + "auxiliary_loss_clip": 0.01143146, + "auxiliary_loss_mlp": 0.01035214, + "balance_loss_clip": 1.05419874, + "balance_loss_mlp": 1.01884604, + "epoch": 0.09967500435262028, + "flos": 31206444105600.0, + "grad_norm": 2.6055124597837307, + "language_loss": 0.78410554, + "learning_rate": 3.9492986239574134e-06, + "loss": 0.80588907, + "num_input_tokens_seen": 96535760, + "router_z_loss_clip": 0.890625, + "router_z_loss_mlp": 0.16363525, + "step": 3435, + "time_per_iteration": 2.664370536804199 + }, + { + "auxiliary_loss_clip": 0.0102772, + "auxiliary_loss_mlp": 0.01000076, + "balance_loss_clip": 1.00467491, + "balance_loss_mlp": 0.99904436, + "epoch": 0.09970402182113633, + "flos": 71353283281920.0, + "grad_norm": 0.672749269226079, + "language_loss": 0.46381158, + "learning_rate": 3.949256561152509e-06, + "loss": 0.48408952, + "num_input_tokens_seen": 96602780, + "router_z_loss_clip": 0.23046875, + "router_z_loss_mlp": 0.01031494, + "step": 3436, + "time_per_iteration": 3.168912887573242 + }, + { + "auxiliary_loss_clip": 0.01026925, + "auxiliary_loss_mlp": 0.01003679, + "balance_loss_clip": 1.003896, + "balance_loss_mlp": 1.00258863, + "epoch": 0.09973303928965237, + "flos": 74770792928640.0, + "grad_norm": 0.66715211529481, + "language_loss": 0.52270645, + "learning_rate": 3.949214481130983e-06, + "loss": 0.54301244, + "num_input_tokens_seen": 96668775, + "router_z_loss_clip": 0.23046875, + "router_z_loss_mlp": 0.01092529, + "step": 3437, + "time_per_iteration": 3.2415478229522705 + }, + { + "auxiliary_loss_clip": 0.01026703, + "auxiliary_loss_mlp": 0.01004365, + "balance_loss_clip": 1.00374997, + "balance_loss_mlp": 1.00335205, + "epoch": 0.09976205675816842, + "flos": 58346832551040.0, + "grad_norm": 0.7632000220373706, + "language_loss": 0.52370369, + "learning_rate": 3.949172383893205e-06, + "loss": 0.54401433, + "num_input_tokens_seen": 96721320, + "router_z_loss_clip": 0.22949219, + "router_z_loss_mlp": 0.01013184, + "step": 3438, + "time_per_iteration": 2.887594699859619 + }, + { + "auxiliary_loss_clip": 0.0102573, + "auxiliary_loss_mlp": 0.0100607, + "balance_loss_clip": 1.0027566, + "balance_loss_mlp": 1.00506294, + "epoch": 0.09979107422668447, + "flos": 54668323704960.0, + "grad_norm": 0.6326990031420695, + "language_loss": 0.4955456, + "learning_rate": 3.949130269439549e-06, + "loss": 0.51586366, + "num_input_tokens_seen": 96783485, + "router_z_loss_clip": 0.23046875, + "router_z_loss_mlp": 0.0100708, + "step": 3439, + "time_per_iteration": 3.197070837020874 + }, + { + "auxiliary_loss_clip": 0.01025691, + "auxiliary_loss_mlp": 0.01004658, + "balance_loss_clip": 1.00279164, + "balance_loss_mlp": 1.00363278, + "epoch": 0.09982009169520051, + "flos": 61124846411520.0, + "grad_norm": 0.6743297921437759, + "language_loss": 0.49990344, + "learning_rate": 3.949088137770385e-06, + "loss": 0.52020693, + "num_input_tokens_seen": 96839995, + "router_z_loss_clip": 0.22851562, + "router_z_loss_mlp": 0.01025391, + "step": 3440, + "time_per_iteration": 3.013274908065796 + }, + { + "auxiliary_loss_clip": 0.01025307, + "auxiliary_loss_mlp": 0.01001506, + "balance_loss_clip": 1.00226855, + "balance_loss_mlp": 1.00043869, + "epoch": 0.09984910916371656, + "flos": 74768709939840.0, + "grad_norm": 0.6902458851340803, + "language_loss": 0.47088391, + "learning_rate": 3.949045988886086e-06, + "loss": 0.49115202, + "num_input_tokens_seen": 96894965, + "router_z_loss_clip": 0.23046875, + "router_z_loss_mlp": 0.01068115, + "step": 3441, + "time_per_iteration": 3.0709681510925293 + }, + { + "auxiliary_loss_clip": 0.01024892, + "auxiliary_loss_mlp": 0.00999204, + "balance_loss_clip": 1.00201595, + "balance_loss_mlp": 0.99805319, + "epoch": 0.0998781266322326, + "flos": 72181166924160.0, + "grad_norm": 0.6613994898174727, + "language_loss": 0.45901054, + "learning_rate": 3.9490038227870235e-06, + "loss": 0.47925147, + "num_input_tokens_seen": 96953490, + "router_z_loss_clip": 0.22851562, + "router_z_loss_mlp": 0.01147461, + "step": 3442, + "time_per_iteration": 3.1303935050964355 + }, + { + "auxiliary_loss_clip": 0.01025233, + "auxiliary_loss_mlp": 0.01002526, + "balance_loss_clip": 1.00242984, + "balance_loss_mlp": 1.00142956, + "epoch": 0.09990714410074865, + "flos": 64520236258560.0, + "grad_norm": 0.6314863658231445, + "language_loss": 0.47279206, + "learning_rate": 3.94896163947357e-06, + "loss": 0.49306968, + "num_input_tokens_seen": 97019635, + "router_z_loss_clip": 0.22851562, + "router_z_loss_mlp": 0.01098633, + "step": 3443, + "time_per_iteration": 3.089440107345581 + }, + { + "auxiliary_loss_clip": 0.01162586, + "auxiliary_loss_mlp": 0.01050404, + "balance_loss_clip": 1.06216621, + "balance_loss_mlp": 1.0279808, + "epoch": 0.0999361615692647, + "flos": 28543488485760.0, + "grad_norm": 2.524433021305801, + "language_loss": 1.13179123, + "learning_rate": 3.948919438946101e-06, + "loss": 1.15392113, + "num_input_tokens_seen": 97041470, + "router_z_loss_clip": 1.00488281, + "router_z_loss_mlp": 0.22399902, + "step": 3444, + "time_per_iteration": 2.634446859359741 + }, + { + "auxiliary_loss_clip": 0.01146599, + "auxiliary_loss_mlp": 0.0104637, + "balance_loss_clip": 1.05569649, + "balance_loss_mlp": 1.0293287, + "epoch": 0.09996517903778074, + "flos": 11648452216320.0, + "grad_norm": 2.2985647516910848, + "language_loss": 0.79447579, + "learning_rate": 3.948877221204984e-06, + "loss": 0.81640548, + "num_input_tokens_seen": 97052950, + "router_z_loss_clip": 0.91015625, + "router_z_loss_mlp": 0.17053223, + "step": 3445, + "time_per_iteration": 2.512882947921753 + }, + { + "auxiliary_loss_clip": 0.01162512, + "auxiliary_loss_mlp": 0.01066371, + "balance_loss_clip": 1.06327105, + "balance_loss_mlp": 1.04647517, + "epoch": 0.09999419650629679, + "flos": 20187650327040.0, + "grad_norm": 2.2039764020820494, + "language_loss": 0.93126577, + "learning_rate": 3.948834986250597e-06, + "loss": 0.95355463, + "num_input_tokens_seen": 97070740, + "router_z_loss_clip": 0.9921875, + "router_z_loss_mlp": 0.19909668, + "step": 3446, + "time_per_iteration": 2.6390531063079834 + }, + { + "auxiliary_loss_clip": 0.01140607, + "auxiliary_loss_mlp": 0.01041291, + "balance_loss_clip": 1.05447698, + "balance_loss_mlp": 1.0236541, + "epoch": 0.10002321397481284, + "flos": 74734340624640.0, + "grad_norm": 2.6537956700769207, + "language_loss": 0.82984436, + "learning_rate": 3.94879273408331e-06, + "loss": 0.85166335, + "num_input_tokens_seen": 97094640, + "router_z_loss_clip": 0.86132812, + "router_z_loss_mlp": 0.17645264, + "step": 3447, + "time_per_iteration": 2.9609813690185547 + }, + { + "auxiliary_loss_clip": 0.01144139, + "auxiliary_loss_mlp": 0.01039668, + "balance_loss_clip": 1.05947113, + "balance_loss_mlp": 1.02585208, + "epoch": 0.10005223144332888, + "flos": 29054317754880.0, + "grad_norm": 1.9638994829801562, + "language_loss": 0.67834079, + "learning_rate": 3.948750464703497e-06, + "loss": 0.70017886, + "num_input_tokens_seen": 97112150, + "router_z_loss_clip": 0.84716797, + "router_z_loss_mlp": 0.13812256, + "step": 3448, + "time_per_iteration": 2.600389003753662 + }, + { + "auxiliary_loss_clip": 0.01166523, + "auxiliary_loss_mlp": 0.01062337, + "balance_loss_clip": 1.06575799, + "balance_loss_mlp": 1.04211903, + "epoch": 0.10008124891184493, + "flos": 14749589848320.0, + "grad_norm": 4.157536815511498, + "language_loss": 0.8412146, + "learning_rate": 3.948708178111531e-06, + "loss": 0.86350322, + "num_input_tokens_seen": 97124200, + "router_z_loss_clip": 1.0078125, + "router_z_loss_mlp": 0.2019043, + "step": 3449, + "time_per_iteration": 2.516768455505371 + }, + { + "auxiliary_loss_clip": 0.01149502, + "auxiliary_loss_mlp": 0.01053389, + "balance_loss_clip": 1.05664134, + "balance_loss_mlp": 1.0360384, + "epoch": 0.10011026638036098, + "flos": 14931800565120.0, + "grad_norm": 3.5499723002354617, + "language_loss": 0.98918223, + "learning_rate": 3.948665874307787e-06, + "loss": 1.01121116, + "num_input_tokens_seen": 97134175, + "router_z_loss_clip": 0.92822266, + "router_z_loss_mlp": 0.17370605, + "step": 3450, + "time_per_iteration": 2.518019199371338 + }, + { + "auxiliary_loss_clip": 0.01155569, + "auxiliary_loss_mlp": 0.01049521, + "balance_loss_clip": 1.0612781, + "balance_loss_mlp": 1.02938628, + "epoch": 0.10013928384887702, + "flos": 25625495324160.0, + "grad_norm": 2.3272264491958983, + "language_loss": 0.83864927, + "learning_rate": 3.948623553292636e-06, + "loss": 0.86070025, + "num_input_tokens_seen": 97152730, + "router_z_loss_clip": 0.94287109, + "router_z_loss_mlp": 0.20153809, + "step": 3451, + "time_per_iteration": 2.591535806655884 + }, + { + "auxiliary_loss_clip": 0.01152083, + "auxiliary_loss_mlp": 0.01052927, + "balance_loss_clip": 1.05746424, + "balance_loss_mlp": 1.03397238, + "epoch": 0.10016830131739307, + "flos": 34019111612160.0, + "grad_norm": 1.8364794822913546, + "language_loss": 0.845258, + "learning_rate": 3.948581215066454e-06, + "loss": 0.86730808, + "num_input_tokens_seen": 97174390, + "router_z_loss_clip": 0.94726562, + "router_z_loss_mlp": 0.18969727, + "step": 3452, + "time_per_iteration": 2.6872150897979736 + }, + { + "auxiliary_loss_clip": 0.01158346, + "auxiliary_loss_mlp": 0.01055327, + "balance_loss_clip": 1.06006336, + "balance_loss_mlp": 1.03653932, + "epoch": 0.10019731878590912, + "flos": 12855158645760.0, + "grad_norm": 2.563438539059836, + "language_loss": 0.76075482, + "learning_rate": 3.948538859629614e-06, + "loss": 0.78289151, + "num_input_tokens_seen": 97186215, + "router_z_loss_clip": 0.98339844, + "router_z_loss_mlp": 0.18774414, + "step": 3453, + "time_per_iteration": 2.612060070037842 + }, + { + "auxiliary_loss_clip": 0.01028158, + "auxiliary_loss_mlp": 0.01006101, + "balance_loss_clip": 1.0052079, + "balance_loss_mlp": 1.00502229, + "epoch": 0.10022633625442516, + "flos": 62109911969280.0, + "grad_norm": 0.7430734658502142, + "language_loss": 0.5435617, + "learning_rate": 3.948496486982491e-06, + "loss": 0.56390429, + "num_input_tokens_seen": 97248085, + "router_z_loss_clip": 0.22949219, + "router_z_loss_mlp": 0.01080322, + "step": 3454, + "time_per_iteration": 3.05059814453125 + }, + { + "auxiliary_loss_clip": 0.01028092, + "auxiliary_loss_mlp": 0.01004194, + "balance_loss_clip": 1.00526428, + "balance_loss_mlp": 1.00319815, + "epoch": 0.10025535372294121, + "flos": 65869328200320.0, + "grad_norm": 0.6912986795353394, + "language_loss": 0.45744902, + "learning_rate": 3.948454097125458e-06, + "loss": 0.47777188, + "num_input_tokens_seen": 97302770, + "router_z_loss_clip": 0.22851562, + "router_z_loss_mlp": 0.00994873, + "step": 3455, + "time_per_iteration": 3.0362203121185303 + }, + { + "auxiliary_loss_clip": 0.01154104, + "auxiliary_loss_mlp": 0.0105572, + "balance_loss_clip": 1.05696189, + "balance_loss_mlp": 1.03627706, + "epoch": 0.10028437119145726, + "flos": 23512116769920.0, + "grad_norm": 2.2540985543240004, + "language_loss": 0.86936283, + "learning_rate": 3.948411690058889e-06, + "loss": 0.89146107, + "num_input_tokens_seen": 97317655, + "router_z_loss_clip": 0.97167969, + "router_z_loss_mlp": 0.19433594, + "step": 3456, + "time_per_iteration": 2.620198965072632 + }, + { + "auxiliary_loss_clip": 0.01150563, + "auxiliary_loss_mlp": 0.01058803, + "balance_loss_clip": 1.05987489, + "balance_loss_mlp": 1.04148245, + "epoch": 0.1003133886599733, + "flos": 24346680341760.0, + "grad_norm": 2.0886235378458644, + "language_loss": 0.63174951, + "learning_rate": 3.948369265783161e-06, + "loss": 0.65384322, + "num_input_tokens_seen": 97339495, + "router_z_loss_clip": 0.90820312, + "router_z_loss_mlp": 0.17321777, + "step": 3457, + "time_per_iteration": 2.8564634323120117 + }, + { + "auxiliary_loss_clip": 0.01150848, + "auxiliary_loss_mlp": 0.01051072, + "balance_loss_clip": 1.05771756, + "balance_loss_mlp": 1.03117633, + "epoch": 0.10034240612848935, + "flos": 21099170787840.0, + "grad_norm": 2.1386513389265844, + "language_loss": 0.92828405, + "learning_rate": 3.948326824298646e-06, + "loss": 0.95030326, + "num_input_tokens_seen": 97355320, + "router_z_loss_clip": 0.93066406, + "router_z_loss_mlp": 0.19897461, + "step": 3458, + "time_per_iteration": 2.5731961727142334 + }, + { + "auxiliary_loss_clip": 0.01151931, + "auxiliary_loss_mlp": 0.0104141, + "balance_loss_clip": 1.05826008, + "balance_loss_mlp": 1.02339733, + "epoch": 0.10037142359700539, + "flos": 14420935382400.0, + "grad_norm": 2.9278221786101093, + "language_loss": 0.8678292, + "learning_rate": 3.948284365605721e-06, + "loss": 0.88976264, + "num_input_tokens_seen": 97368000, + "router_z_loss_clip": 0.93652344, + "router_z_loss_mlp": 0.17993164, + "step": 3459, + "time_per_iteration": 2.462667942047119 + }, + { + "auxiliary_loss_clip": 0.01148167, + "auxiliary_loss_mlp": 0.01050493, + "balance_loss_clip": 1.05482256, + "balance_loss_mlp": 1.03107977, + "epoch": 0.10040044106552144, + "flos": 23688581310720.0, + "grad_norm": 2.5915047625093472, + "language_loss": 0.81782109, + "learning_rate": 3.94824188970476e-06, + "loss": 0.83980769, + "num_input_tokens_seen": 97382320, + "router_z_loss_clip": 0.93310547, + "router_z_loss_mlp": 0.19433594, + "step": 3460, + "time_per_iteration": 2.5951778888702393 + }, + { + "auxiliary_loss_clip": 0.01148689, + "auxiliary_loss_mlp": 0.01046221, + "balance_loss_clip": 1.05683398, + "balance_loss_mlp": 1.02749324, + "epoch": 0.1004294585340375, + "flos": 67034662162560.0, + "grad_norm": 2.152644978841647, + "language_loss": 0.85472441, + "learning_rate": 3.948199396596138e-06, + "loss": 0.87667352, + "num_input_tokens_seen": 97406935, + "router_z_loss_clip": 0.91845703, + "router_z_loss_mlp": 0.18743896, + "step": 3461, + "time_per_iteration": 2.831759214401245 + }, + { + "auxiliary_loss_clip": 0.01027934, + "auxiliary_loss_mlp": 0.00999537, + "balance_loss_clip": 1.00511265, + "balance_loss_mlp": 0.99844629, + "epoch": 0.10045847600255353, + "flos": 74777760167040.0, + "grad_norm": 0.6898461891351385, + "language_loss": 0.54258072, + "learning_rate": 3.94815688628023e-06, + "loss": 0.56285548, + "num_input_tokens_seen": 97468345, + "router_z_loss_clip": 0.22851562, + "router_z_loss_mlp": 0.01092529, + "step": 3462, + "time_per_iteration": 3.2063350677490234 + }, + { + "auxiliary_loss_clip": 0.01027282, + "auxiliary_loss_mlp": 0.0099992, + "balance_loss_clip": 1.00440598, + "balance_loss_mlp": 0.99887085, + "epoch": 0.10048749347106958, + "flos": 65328190830720.0, + "grad_norm": 0.697137754134777, + "language_loss": 0.49963993, + "learning_rate": 3.948114358757414e-06, + "loss": 0.51991194, + "num_input_tokens_seen": 97526525, + "router_z_loss_clip": 0.22851562, + "router_z_loss_mlp": 0.01049805, + "step": 3463, + "time_per_iteration": 3.098878860473633 + }, + { + "auxiliary_loss_clip": 0.01026574, + "auxiliary_loss_mlp": 0.01001844, + "balance_loss_clip": 1.00379539, + "balance_loss_mlp": 1.00074089, + "epoch": 0.10051651093958563, + "flos": 74775641264640.0, + "grad_norm": 0.647106776548214, + "language_loss": 0.45876333, + "learning_rate": 3.948071814028061e-06, + "loss": 0.47904751, + "num_input_tokens_seen": 97589950, + "router_z_loss_clip": 0.22851562, + "router_z_loss_mlp": 0.01104736, + "step": 3464, + "time_per_iteration": 3.1640427112579346 + }, + { + "auxiliary_loss_clip": 0.01159995, + "auxiliary_loss_mlp": 0.01048877, + "balance_loss_clip": 1.06170964, + "balance_loss_mlp": 1.02911186, + "epoch": 0.10054552840810167, + "flos": 42734558782080.0, + "grad_norm": 3.952797393740193, + "language_loss": 0.80985355, + "learning_rate": 3.948029252092551e-06, + "loss": 0.83194232, + "num_input_tokens_seen": 97607580, + "router_z_loss_clip": 0.98242188, + "router_z_loss_mlp": 0.19775391, + "step": 3465, + "time_per_iteration": 2.746271848678589 + }, + { + "auxiliary_loss_clip": 0.01141302, + "auxiliary_loss_mlp": 0.01039161, + "balance_loss_clip": 1.05599177, + "balance_loss_mlp": 1.02180386, + "epoch": 0.10057454587661772, + "flos": 15626600317440.0, + "grad_norm": 2.3124637497534795, + "language_loss": 0.67881215, + "learning_rate": 3.947986672951258e-06, + "loss": 0.70061672, + "num_input_tokens_seen": 97621440, + "router_z_loss_clip": 0.85449219, + "router_z_loss_mlp": 0.17370605, + "step": 3466, + "time_per_iteration": 2.5579028129577637 + }, + { + "auxiliary_loss_clip": 0.01028294, + "auxiliary_loss_mlp": 0.01004162, + "balance_loss_clip": 1.00568366, + "balance_loss_mlp": 1.00310087, + "epoch": 0.10060356334513378, + "flos": 63730777190400.0, + "grad_norm": 0.7052060947655714, + "language_loss": 0.55226612, + "learning_rate": 3.947944076604559e-06, + "loss": 0.57259059, + "num_input_tokens_seen": 97684785, + "router_z_loss_clip": 0.2265625, + "router_z_loss_mlp": 0.01062012, + "step": 3467, + "time_per_iteration": 3.187592029571533 + }, + { + "auxiliary_loss_clip": 0.01144222, + "auxiliary_loss_mlp": 0.01041632, + "balance_loss_clip": 1.05331981, + "balance_loss_mlp": 1.02419138, + "epoch": 0.10063258081364981, + "flos": 16790752108800.0, + "grad_norm": 3.3664210253433646, + "language_loss": 0.83197308, + "learning_rate": 3.947901463052829e-06, + "loss": 0.85383159, + "num_input_tokens_seen": 97696155, + "router_z_loss_clip": 0.90869141, + "router_z_loss_mlp": 0.17443848, + "step": 3468, + "time_per_iteration": 2.5326902866363525 + }, + { + "auxiliary_loss_clip": 0.01156663, + "auxiliary_loss_mlp": 0.01050451, + "balance_loss_clip": 1.05565464, + "balance_loss_mlp": 1.02980423, + "epoch": 0.10066159828216587, + "flos": 19790335013760.0, + "grad_norm": 2.355122149731086, + "language_loss": 0.77727342, + "learning_rate": 3.947858832296445e-06, + "loss": 0.79934454, + "num_input_tokens_seen": 97711315, + "router_z_loss_clip": 1.00878906, + "router_z_loss_mlp": 0.20629883, + "step": 3469, + "time_per_iteration": 2.570093870162964 + }, + { + "auxiliary_loss_clip": 0.01154469, + "auxiliary_loss_mlp": 0.01054514, + "balance_loss_clip": 1.05744922, + "balance_loss_mlp": 1.03625083, + "epoch": 0.10069061575068192, + "flos": 19784373356160.0, + "grad_norm": 2.0950357344744126, + "language_loss": 0.71576762, + "learning_rate": 3.947816184335784e-06, + "loss": 0.7378574, + "num_input_tokens_seen": 97723990, + "router_z_loss_clip": 0.97070312, + "router_z_loss_mlp": 0.18261719, + "step": 3470, + "time_per_iteration": 2.5194597244262695 + }, + { + "auxiliary_loss_clip": 0.01027612, + "auxiliary_loss_mlp": 0.01003881, + "balance_loss_clip": 1.00494385, + "balance_loss_mlp": 1.00281417, + "epoch": 0.10071963321919795, + "flos": 74768674026240.0, + "grad_norm": 0.7481030621035163, + "language_loss": 0.50452608, + "learning_rate": 3.947773519171222e-06, + "loss": 0.52484101, + "num_input_tokens_seen": 97781205, + "router_z_loss_clip": 0.2265625, + "router_z_loss_mlp": 0.01068115, + "step": 3471, + "time_per_iteration": 3.088653802871704 + }, + { + "auxiliary_loss_clip": 0.01162727, + "auxiliary_loss_mlp": 0.01042744, + "balance_loss_clip": 1.0623852, + "balance_loss_mlp": 1.021227, + "epoch": 0.100748650687714, + "flos": 32629188885120.0, + "grad_norm": 2.1808268022967447, + "language_loss": 0.88854396, + "learning_rate": 3.947730836803137e-06, + "loss": 0.91059864, + "num_input_tokens_seen": 97802440, + "router_z_loss_clip": 1.00244141, + "router_z_loss_mlp": 0.21496582, + "step": 3472, + "time_per_iteration": 2.6203954219818115 + }, + { + "auxiliary_loss_clip": 0.01151849, + "auxiliary_loss_mlp": 0.01057819, + "balance_loss_clip": 1.05708671, + "balance_loss_mlp": 1.03997922, + "epoch": 0.10077766815623004, + "flos": 24929761818240.0, + "grad_norm": 2.447613087952483, + "language_loss": 0.85004854, + "learning_rate": 3.947688137231904e-06, + "loss": 0.87214524, + "num_input_tokens_seen": 97817815, + "router_z_loss_clip": 0.94775391, + "router_z_loss_mlp": 0.1784668, + "step": 3473, + "time_per_iteration": 2.6216728687286377 + }, + { + "auxiliary_loss_clip": 0.01027968, + "auxiliary_loss_mlp": 0.01006481, + "balance_loss_clip": 1.00517178, + "balance_loss_mlp": 1.00542641, + "epoch": 0.1008066856247461, + "flos": 72261104641920.0, + "grad_norm": 0.6202432835173881, + "language_loss": 0.51623762, + "learning_rate": 3.947645420457901e-06, + "loss": 0.53658211, + "num_input_tokens_seen": 97889575, + "router_z_loss_clip": 0.22851562, + "router_z_loss_mlp": 0.01055908, + "step": 3474, + "time_per_iteration": 3.3102424144744873 + }, + { + "auxiliary_loss_clip": 0.01144735, + "auxiliary_loss_mlp": 0.01046329, + "balance_loss_clip": 1.05514324, + "balance_loss_mlp": 1.03040254, + "epoch": 0.10083570309326215, + "flos": 39378850485120.0, + "grad_norm": 2.210703007593349, + "language_loss": 0.86550605, + "learning_rate": 3.947602686481507e-06, + "loss": 0.88741678, + "num_input_tokens_seen": 97908785, + "router_z_loss_clip": 0.89599609, + "router_z_loss_mlp": 0.15948486, + "step": 3475, + "time_per_iteration": 2.6849141120910645 + }, + { + "auxiliary_loss_clip": 0.01150364, + "auxiliary_loss_mlp": 0.01039003, + "balance_loss_clip": 1.05459225, + "balance_loss_mlp": 1.02172923, + "epoch": 0.10086472056177818, + "flos": 37193291550720.0, + "grad_norm": 3.1223520235669033, + "language_loss": 0.71154511, + "learning_rate": 3.9475599353030965e-06, + "loss": 0.73343879, + "num_input_tokens_seen": 97935000, + "router_z_loss_clip": 0.95898438, + "router_z_loss_mlp": 0.17260742, + "step": 3476, + "time_per_iteration": 2.7527801990509033 + }, + { + "auxiliary_loss_clip": 0.0102921, + "auxiliary_loss_mlp": 0.01004583, + "balance_loss_clip": 1.00639391, + "balance_loss_mlp": 1.00351036, + "epoch": 0.10089373803029424, + "flos": 60257748096000.0, + "grad_norm": 0.7016414912952852, + "language_loss": 0.45112956, + "learning_rate": 3.947517166923049e-06, + "loss": 0.47146749, + "num_input_tokens_seen": 98004655, + "router_z_loss_clip": 0.22851562, + "router_z_loss_mlp": 0.01074219, + "step": 3477, + "time_per_iteration": 3.28826642036438 + }, + { + "auxiliary_loss_clip": 0.01029663, + "auxiliary_loss_mlp": 0.01000295, + "balance_loss_clip": 1.00666773, + "balance_loss_mlp": 0.9992342, + "epoch": 0.10092275549881029, + "flos": 70577181095040.0, + "grad_norm": 0.6306799076949939, + "language_loss": 0.50163096, + "learning_rate": 3.947474381341741e-06, + "loss": 0.52193058, + "num_input_tokens_seen": 98066750, + "router_z_loss_clip": 0.23046875, + "router_z_loss_mlp": 0.01062012, + "step": 3478, + "time_per_iteration": 3.1733970642089844 + }, + { + "auxiliary_loss_clip": 0.01148333, + "auxiliary_loss_mlp": 0.01047419, + "balance_loss_clip": 1.05778742, + "balance_loss_mlp": 1.02987134, + "epoch": 0.10095177296732633, + "flos": 26869082042880.0, + "grad_norm": 3.601622332345608, + "language_loss": 0.82513386, + "learning_rate": 3.947431578559553e-06, + "loss": 0.84709132, + "num_input_tokens_seen": 98087100, + "router_z_loss_clip": 0.90527344, + "router_z_loss_mlp": 0.17559814, + "step": 3479, + "time_per_iteration": 2.6405043601989746 + }, + { + "auxiliary_loss_clip": 0.0115976, + "auxiliary_loss_mlp": 0.01050852, + "balance_loss_clip": 1.06015706, + "balance_loss_mlp": 1.03074169, + "epoch": 0.10098079043584238, + "flos": 38107756926720.0, + "grad_norm": 2.144284035458354, + "language_loss": 1.07722628, + "learning_rate": 3.94738875857686e-06, + "loss": 1.09933233, + "num_input_tokens_seen": 98112000, + "router_z_loss_clip": 0.99609375, + "router_z_loss_mlp": 0.20129395, + "step": 3480, + "time_per_iteration": 2.7217843532562256 + }, + { + "auxiliary_loss_clip": 0.01031454, + "auxiliary_loss_mlp": 0.01004862, + "balance_loss_clip": 1.00854516, + "balance_loss_mlp": 1.00386095, + "epoch": 0.10100980790435843, + "flos": 74778909402240.0, + "grad_norm": 0.6950842226553293, + "language_loss": 0.49253193, + "learning_rate": 3.947345921394042e-06, + "loss": 0.51289505, + "num_input_tokens_seen": 98177410, + "router_z_loss_clip": 0.22851562, + "router_z_loss_mlp": 0.01000977, + "step": 3481, + "time_per_iteration": 3.194187879562378 + }, + { + "auxiliary_loss_clip": 0.01032065, + "auxiliary_loss_mlp": 0.01004582, + "balance_loss_clip": 1.00909817, + "balance_loss_mlp": 1.00350285, + "epoch": 0.10103882537287447, + "flos": 67623528706560.0, + "grad_norm": 0.6876209033012973, + "language_loss": 0.46973884, + "learning_rate": 3.947303067011477e-06, + "loss": 0.49010533, + "num_input_tokens_seen": 98235730, + "router_z_loss_clip": 0.23046875, + "router_z_loss_mlp": 0.01080322, + "step": 3482, + "time_per_iteration": 3.08191180229187 + }, + { + "auxiliary_loss_clip": 0.01156824, + "auxiliary_loss_mlp": 0.0106431, + "balance_loss_clip": 1.05651736, + "balance_loss_mlp": 1.04357934, + "epoch": 0.10106784284139052, + "flos": 24965277390720.0, + "grad_norm": 2.47204443453855, + "language_loss": 1.07371497, + "learning_rate": 3.947260195429542e-06, + "loss": 1.09592628, + "num_input_tokens_seen": 98250320, + "router_z_loss_clip": 1.00390625, + "router_z_loss_mlp": 0.20721436, + "step": 3483, + "time_per_iteration": 2.616805076599121 + }, + { + "auxiliary_loss_clip": 0.01152587, + "auxiliary_loss_mlp": 0.01054675, + "balance_loss_clip": 1.05656195, + "balance_loss_mlp": 1.034374, + "epoch": 0.10109686030990657, + "flos": 24090565392000.0, + "grad_norm": 2.7145727506552113, + "language_loss": 0.91702724, + "learning_rate": 3.947217306648619e-06, + "loss": 0.93909991, + "num_input_tokens_seen": 98263665, + "router_z_loss_clip": 0.95947266, + "router_z_loss_mlp": 0.203125, + "step": 3484, + "time_per_iteration": 2.5333638191223145 + }, + { + "auxiliary_loss_clip": 0.01030737, + "auxiliary_loss_mlp": 0.01001492, + "balance_loss_clip": 1.00790668, + "balance_loss_mlp": 1.00042522, + "epoch": 0.10112587777842261, + "flos": 74651999673600.0, + "grad_norm": 0.7420210930299215, + "language_loss": 0.5195626, + "learning_rate": 3.947174400669083e-06, + "loss": 0.53988492, + "num_input_tokens_seen": 98318110, + "router_z_loss_clip": 0.22851562, + "router_z_loss_mlp": 0.01068115, + "step": 3485, + "time_per_iteration": 3.031423807144165 + }, + { + "auxiliary_loss_clip": 0.01134161, + "auxiliary_loss_mlp": 0.01046039, + "balance_loss_clip": 1.04997253, + "balance_loss_mlp": 1.03023767, + "epoch": 0.10115489524693866, + "flos": 26898025426560.0, + "grad_norm": 2.440755353471316, + "language_loss": 0.86021602, + "learning_rate": 3.947131477491315e-06, + "loss": 0.88201797, + "num_input_tokens_seen": 98332110, + "router_z_loss_clip": 0.84228516, + "router_z_loss_mlp": 0.15808105, + "step": 3486, + "time_per_iteration": 4.835872173309326 + }, + { + "auxiliary_loss_clip": 0.01143993, + "auxiliary_loss_mlp": 0.01048032, + "balance_loss_clip": 1.05563378, + "balance_loss_mlp": 1.03313684, + "epoch": 0.10118391271545471, + "flos": 30845901340800.0, + "grad_norm": 1.7764566594113218, + "language_loss": 0.68499684, + "learning_rate": 3.947088537115695e-06, + "loss": 0.70691705, + "num_input_tokens_seen": 98356955, + "router_z_loss_clip": 0.88378906, + "router_z_loss_mlp": 0.14898682, + "step": 3487, + "time_per_iteration": 5.104656457901001 + }, + { + "auxiliary_loss_clip": 0.01148276, + "auxiliary_loss_mlp": 0.01054124, + "balance_loss_clip": 1.0526402, + "balance_loss_mlp": 1.03586125, + "epoch": 0.10121293018397075, + "flos": 35218455753600.0, + "grad_norm": 2.2351342221286905, + "language_loss": 1.0006839, + "learning_rate": 3.947045579542601e-06, + "loss": 1.02270782, + "num_input_tokens_seen": 98374795, + "router_z_loss_clip": 0.95703125, + "router_z_loss_mlp": 0.18292236, + "step": 3488, + "time_per_iteration": 5.033925771713257 + }, + { + "auxiliary_loss_clip": 0.01149186, + "auxiliary_loss_mlp": 0.01049051, + "balance_loss_clip": 1.05711019, + "balance_loss_mlp": 1.03146768, + "epoch": 0.1012419476524868, + "flos": 16536971543040.0, + "grad_norm": 2.936983561756567, + "language_loss": 0.8628664, + "learning_rate": 3.947002604772411e-06, + "loss": 0.88484877, + "num_input_tokens_seen": 98387880, + "router_z_loss_clip": 0.92089844, + "router_z_loss_mlp": 0.17596436, + "step": 3489, + "time_per_iteration": 5.018002986907959 + }, + { + "auxiliary_loss_clip": 0.01138489, + "auxiliary_loss_mlp": 0.01046455, + "balance_loss_clip": 1.04980183, + "balance_loss_mlp": 1.03006375, + "epoch": 0.10127096512100284, + "flos": 11209875154560.0, + "grad_norm": 3.393732238027748, + "language_loss": 1.00821912, + "learning_rate": 3.946959612805507e-06, + "loss": 1.03006864, + "num_input_tokens_seen": 98400095, + "router_z_loss_clip": 0.88623047, + "router_z_loss_mlp": 0.16363525, + "step": 3490, + "time_per_iteration": 2.4912986755371094 + }, + { + "auxiliary_loss_clip": 0.0115491, + "auxiliary_loss_mlp": 0.01044862, + "balance_loss_clip": 1.05706453, + "balance_loss_mlp": 1.02659321, + "epoch": 0.10129998258951889, + "flos": 29824206888960.0, + "grad_norm": 3.0225305423258972, + "language_loss": 0.83770436, + "learning_rate": 3.946916603642268e-06, + "loss": 0.85970211, + "num_input_tokens_seen": 98417980, + "router_z_loss_clip": 0.97851562, + "router_z_loss_mlp": 0.18273926, + "step": 3491, + "time_per_iteration": 2.6012322902679443 + }, + { + "auxiliary_loss_clip": 0.01147748, + "auxiliary_loss_mlp": 0.01045479, + "balance_loss_clip": 1.05375791, + "balance_loss_mlp": 1.02845037, + "epoch": 0.10132900005803494, + "flos": 17304203070720.0, + "grad_norm": 2.1971079573651924, + "language_loss": 0.81963134, + "learning_rate": 3.946873577283074e-06, + "loss": 0.84156358, + "num_input_tokens_seen": 98431265, + "router_z_loss_clip": 0.94042969, + "router_z_loss_mlp": 0.17028809, + "step": 3492, + "time_per_iteration": 2.5427756309509277 + }, + { + "auxiliary_loss_clip": 0.0115213, + "auxiliary_loss_mlp": 0.01053759, + "balance_loss_clip": 1.05563617, + "balance_loss_mlp": 1.03431559, + "epoch": 0.10135801752655098, + "flos": 36860435193600.0, + "grad_norm": 2.2489251435527753, + "language_loss": 1.01185131, + "learning_rate": 3.946830533728304e-06, + "loss": 1.03391027, + "num_input_tokens_seen": 98447820, + "router_z_loss_clip": 0.96289062, + "router_z_loss_mlp": 0.19439697, + "step": 3493, + "time_per_iteration": 2.73252272605896 + }, + { + "auxiliary_loss_clip": 0.0114963, + "auxiliary_loss_mlp": 0.01046473, + "balance_loss_clip": 1.05857921, + "balance_loss_mlp": 1.02882969, + "epoch": 0.10138703499506703, + "flos": 35781033542400.0, + "grad_norm": 1.8497062436036509, + "language_loss": 0.8863771, + "learning_rate": 3.9467874729783395e-06, + "loss": 0.90833807, + "num_input_tokens_seen": 98470995, + "router_z_loss_clip": 0.91015625, + "router_z_loss_mlp": 0.1763916, + "step": 3494, + "time_per_iteration": 2.6723926067352295 + }, + { + "auxiliary_loss_clip": 0.01150897, + "auxiliary_loss_mlp": 0.01045796, + "balance_loss_clip": 1.05898285, + "balance_loss_mlp": 1.02901149, + "epoch": 0.10141605246358308, + "flos": 16466048138880.0, + "grad_norm": 2.191676851667646, + "language_loss": 0.98215413, + "learning_rate": 3.94674439503356e-06, + "loss": 1.00412107, + "num_input_tokens_seen": 98486550, + "router_z_loss_clip": 0.91943359, + "router_z_loss_mlp": 0.16796875, + "step": 3495, + "time_per_iteration": 2.484576940536499 + }, + { + "auxiliary_loss_clip": 0.01144215, + "auxiliary_loss_mlp": 0.01044988, + "balance_loss_clip": 1.05762434, + "balance_loss_mlp": 1.02869856, + "epoch": 0.10144506993209912, + "flos": 26572818666240.0, + "grad_norm": 1.7197205742007298, + "language_loss": 0.77255821, + "learning_rate": 3.946701299894347e-06, + "loss": 0.79445028, + "num_input_tokens_seen": 98508275, + "router_z_loss_clip": 0.86669922, + "router_z_loss_mlp": 0.16290283, + "step": 3496, + "time_per_iteration": 2.7156944274902344 + }, + { + "auxiliary_loss_clip": 0.01143249, + "auxiliary_loss_mlp": 0.01045788, + "balance_loss_clip": 1.05484653, + "balance_loss_mlp": 1.02938461, + "epoch": 0.10147408740061517, + "flos": 40626459527040.0, + "grad_norm": 1.880041444468092, + "language_loss": 0.64283764, + "learning_rate": 3.94665818756108e-06, + "loss": 0.66472793, + "num_input_tokens_seen": 98529480, + "router_z_loss_clip": 0.88378906, + "router_z_loss_mlp": 0.1640625, + "step": 3497, + "time_per_iteration": 2.7089314460754395 + }, + { + "auxiliary_loss_clip": 0.01148451, + "auxiliary_loss_mlp": 0.01042809, + "balance_loss_clip": 1.05957675, + "balance_loss_mlp": 1.02775848, + "epoch": 0.10150310486913122, + "flos": 22739785511040.0, + "grad_norm": 2.4538591363406472, + "language_loss": 0.90257746, + "learning_rate": 3.9466150580341395e-06, + "loss": 0.92449009, + "num_input_tokens_seen": 98543400, + "router_z_loss_clip": 0.88964844, + "router_z_loss_mlp": 0.1505127, + "step": 3498, + "time_per_iteration": 2.5423431396484375 + }, + { + "auxiliary_loss_clip": 0.01150637, + "auxiliary_loss_mlp": 0.01051531, + "balance_loss_clip": 1.05686545, + "balance_loss_mlp": 1.03287506, + "epoch": 0.10153212233764726, + "flos": 16321831032960.0, + "grad_norm": 3.3255929280710683, + "language_loss": 0.85687059, + "learning_rate": 3.946571911313907e-06, + "loss": 0.87889224, + "num_input_tokens_seen": 98558500, + "router_z_loss_clip": 0.93652344, + "router_z_loss_mlp": 0.18664551, + "step": 3499, + "time_per_iteration": 2.5019164085388184 + }, + { + "auxiliary_loss_clip": 0.01150637, + "auxiliary_loss_mlp": 0.01049824, + "balance_loss_clip": 1.05814362, + "balance_loss_mlp": 1.03224683, + "epoch": 0.10156113980616331, + "flos": 22195200435840.0, + "grad_norm": 2.574979141672812, + "language_loss": 0.55122852, + "learning_rate": 3.946528747400765e-06, + "loss": 0.57323313, + "num_input_tokens_seen": 98572245, + "router_z_loss_clip": 0.92480469, + "router_z_loss_mlp": 0.17572021, + "step": 3500, + "time_per_iteration": 2.5472829341888428 + }, + { + "auxiliary_loss_clip": 0.01045729, + "auxiliary_loss_mlp": 0.010587, + "balance_loss_clip": 1.02267885, + "balance_loss_mlp": 1.0575856, + "epoch": 0.10159015727467936, + "flos": 74766591037440.0, + "grad_norm": 0.7359222985670183, + "language_loss": 0.47620457, + "learning_rate": 3.9464855662950925e-06, + "loss": 0.49724886, + "num_input_tokens_seen": 98631945, + "router_z_loss_clip": 0.23046875, + "router_z_loss_mlp": 0.01116943, + "step": 3501, + "time_per_iteration": 3.14335036277771 + }, + { + "auxiliary_loss_clip": 0.01151354, + "auxiliary_loss_mlp": 0.01054361, + "balance_loss_clip": 1.05820835, + "balance_loss_mlp": 1.03627682, + "epoch": 0.1016191747431954, + "flos": 14866228287360.0, + "grad_norm": 1.9255921769725186, + "language_loss": 0.67074847, + "learning_rate": 3.946442367997272e-06, + "loss": 0.69280559, + "num_input_tokens_seen": 98649125, + "router_z_loss_clip": 0.93115234, + "router_z_loss_mlp": 0.18078613, + "step": 3502, + "time_per_iteration": 2.7074174880981445 + }, + { + "auxiliary_loss_clip": 0.01040844, + "auxiliary_loss_mlp": 0.01024465, + "balance_loss_clip": 1.01792407, + "balance_loss_mlp": 1.02341044, + "epoch": 0.10164819221171145, + "flos": 63841884935040.0, + "grad_norm": 0.7223680190657218, + "language_loss": 0.5716753, + "learning_rate": 3.946399152507685e-06, + "loss": 0.59232843, + "num_input_tokens_seen": 98711760, + "router_z_loss_clip": 0.22949219, + "router_z_loss_mlp": 0.01055908, + "step": 3503, + "time_per_iteration": 3.1826398372650146 + }, + { + "auxiliary_loss_clip": 0.01149866, + "auxiliary_loss_mlp": 0.0105724, + "balance_loss_clip": 1.05910969, + "balance_loss_mlp": 1.03969228, + "epoch": 0.1016772096802275, + "flos": 26899677452160.0, + "grad_norm": 2.342319223264365, + "language_loss": 0.90614146, + "learning_rate": 3.9463559198267125e-06, + "loss": 0.92821252, + "num_input_tokens_seen": 98727140, + "router_z_loss_clip": 0.90771484, + "router_z_loss_mlp": 0.17547607, + "step": 3504, + "time_per_iteration": 2.579554557800293 + }, + { + "auxiliary_loss_clip": 0.01150408, + "auxiliary_loss_mlp": 0.01053139, + "balance_loss_clip": 1.0598166, + "balance_loss_mlp": 1.03627729, + "epoch": 0.10170622714874354, + "flos": 19928662289280.0, + "grad_norm": 2.0054923362505446, + "language_loss": 0.8303082, + "learning_rate": 3.946312669954737e-06, + "loss": 0.85234368, + "num_input_tokens_seen": 98740555, + "router_z_loss_clip": 0.90527344, + "router_z_loss_mlp": 0.16864014, + "step": 3505, + "time_per_iteration": 2.5672733783721924 + }, + { + "auxiliary_loss_clip": 0.01144692, + "auxiliary_loss_mlp": 0.01051969, + "balance_loss_clip": 1.05541658, + "balance_loss_mlp": 1.03651989, + "epoch": 0.1017352446172596, + "flos": 35662060719360.0, + "grad_norm": 2.149178392049341, + "language_loss": 0.84803385, + "learning_rate": 3.946269402892141e-06, + "loss": 0.87000042, + "num_input_tokens_seen": 98760755, + "router_z_loss_clip": 0.89208984, + "router_z_loss_mlp": 0.15472412, + "step": 3506, + "time_per_iteration": 2.6762731075286865 + }, + { + "auxiliary_loss_clip": 0.01149624, + "auxiliary_loss_mlp": 0.01065554, + "balance_loss_clip": 1.05710483, + "balance_loss_mlp": 1.04825699, + "epoch": 0.10176426208577563, + "flos": 18441674035200.0, + "grad_norm": 2.8090391406448716, + "language_loss": 0.70576453, + "learning_rate": 3.946226118639305e-06, + "loss": 0.72791636, + "num_input_tokens_seen": 98778370, + "router_z_loss_clip": 0.92480469, + "router_z_loss_mlp": 0.17303467, + "step": 3507, + "time_per_iteration": 2.657222270965576 + }, + { + "auxiliary_loss_clip": 0.01036031, + "auxiliary_loss_mlp": 0.01029065, + "balance_loss_clip": 1.01311374, + "balance_loss_mlp": 1.02793872, + "epoch": 0.10179327955429168, + "flos": 67217414561280.0, + "grad_norm": 0.729720174530527, + "language_loss": 0.5255518, + "learning_rate": 3.9461828171966135e-06, + "loss": 0.54620278, + "num_input_tokens_seen": 98839770, + "router_z_loss_clip": 0.22851562, + "router_z_loss_mlp": 0.0112915, + "step": 3508, + "time_per_iteration": 3.1252479553222656 + }, + { + "auxiliary_loss_clip": 0.01146972, + "auxiliary_loss_mlp": 0.01057996, + "balance_loss_clip": 1.05835056, + "balance_loss_mlp": 1.04094291, + "epoch": 0.10182229702280773, + "flos": 28620660856320.0, + "grad_norm": 2.1574647256517583, + "language_loss": 0.78432941, + "learning_rate": 3.946139498564448e-06, + "loss": 0.80637908, + "num_input_tokens_seen": 98858745, + "router_z_loss_clip": 0.88623047, + "router_z_loss_mlp": 0.17047119, + "step": 3509, + "time_per_iteration": 2.6698601245880127 + }, + { + "auxiliary_loss_clip": 0.01034761, + "auxiliary_loss_mlp": 0.01014369, + "balance_loss_clip": 1.01193333, + "balance_loss_mlp": 1.01334417, + "epoch": 0.10185131449132377, + "flos": 71087651228160.0, + "grad_norm": 0.6804930337757916, + "language_loss": 0.50781155, + "learning_rate": 3.94609616274319e-06, + "loss": 0.52830285, + "num_input_tokens_seen": 98923750, + "router_z_loss_clip": 0.22851562, + "router_z_loss_mlp": 0.01025391, + "step": 3510, + "time_per_iteration": 3.1829521656036377 + }, + { + "auxiliary_loss_clip": 0.01145766, + "auxiliary_loss_mlp": 0.01054627, + "balance_loss_clip": 1.0569458, + "balance_loss_mlp": 1.03885531, + "epoch": 0.10188033195983982, + "flos": 15845368101120.0, + "grad_norm": 2.7926560325874963, + "language_loss": 0.7220909, + "learning_rate": 3.9460528097332235e-06, + "loss": 0.74409485, + "num_input_tokens_seen": 98935470, + "router_z_loss_clip": 0.88720703, + "router_z_loss_mlp": 0.15783691, + "step": 3511, + "time_per_iteration": 2.5277395248413086 + }, + { + "auxiliary_loss_clip": 0.01034258, + "auxiliary_loss_mlp": 0.00996375, + "balance_loss_clip": 1.01139569, + "balance_loss_mlp": 0.9953739, + "epoch": 0.10190934942835587, + "flos": 53969214320640.0, + "grad_norm": 0.7186236365031531, + "language_loss": 0.54228967, + "learning_rate": 3.946009439534931e-06, + "loss": 0.56259602, + "num_input_tokens_seen": 98989080, + "router_z_loss_clip": 0.22851562, + "router_z_loss_mlp": 0.01000977, + "step": 3512, + "time_per_iteration": 2.983518600463867 + }, + { + "auxiliary_loss_clip": 0.01146038, + "auxiliary_loss_mlp": 0.01051697, + "balance_loss_clip": 1.05471003, + "balance_loss_mlp": 1.03386378, + "epoch": 0.10193836689687191, + "flos": 31387182364800.0, + "grad_norm": 2.869470766715056, + "language_loss": 1.14082575, + "learning_rate": 3.945966052148696e-06, + "loss": 1.16280317, + "num_input_tokens_seen": 99002480, + "router_z_loss_clip": 0.91455078, + "router_z_loss_mlp": 0.17834473, + "step": 3513, + "time_per_iteration": 2.6468148231506348 + }, + { + "auxiliary_loss_clip": 0.01032938, + "auxiliary_loss_mlp": 0.01019824, + "balance_loss_clip": 1.01039314, + "balance_loss_mlp": 1.01881683, + "epoch": 0.10196738436538796, + "flos": 52811922026880.0, + "grad_norm": 0.6678412797036292, + "language_loss": 0.51114899, + "learning_rate": 3.945922647574901e-06, + "loss": 0.53167659, + "num_input_tokens_seen": 99062160, + "router_z_loss_clip": 0.22558594, + "router_z_loss_mlp": 0.0100708, + "step": 3514, + "time_per_iteration": 3.0816080570220947 + }, + { + "auxiliary_loss_clip": 0.01146378, + "auxiliary_loss_mlp": 0.0104662, + "balance_loss_clip": 1.0579356, + "balance_loss_mlp": 1.03098607, + "epoch": 0.10199640183390402, + "flos": 41787307267200.0, + "grad_norm": 1.9640078449710245, + "language_loss": 0.72782791, + "learning_rate": 3.94587922581393e-06, + "loss": 0.74975789, + "num_input_tokens_seen": 99083210, + "router_z_loss_clip": 0.88525391, + "router_z_loss_mlp": 0.15625, + "step": 3515, + "time_per_iteration": 2.7230608463287354 + }, + { + "auxiliary_loss_clip": 0.01150063, + "auxiliary_loss_mlp": 0.01044514, + "balance_loss_clip": 1.05528021, + "balance_loss_mlp": 1.02710986, + "epoch": 0.10202541930242005, + "flos": 17923590218880.0, + "grad_norm": 2.468601121835928, + "language_loss": 0.96212494, + "learning_rate": 3.945835786866166e-06, + "loss": 0.98407072, + "num_input_tokens_seen": 99094380, + "router_z_loss_clip": 0.94726562, + "router_z_loss_mlp": 0.17382812, + "step": 3516, + "time_per_iteration": 2.55026912689209 + }, + { + "auxiliary_loss_clip": 0.01156787, + "auxiliary_loss_mlp": 0.01050849, + "balance_loss_clip": 1.06001043, + "balance_loss_mlp": 1.03151286, + "epoch": 0.1020544367709361, + "flos": 34087916113920.0, + "grad_norm": 1.8042609789390678, + "language_loss": 0.93134272, + "learning_rate": 3.9457923307319935e-06, + "loss": 0.95341909, + "num_input_tokens_seen": 99114625, + "router_z_loss_clip": 0.96728516, + "router_z_loss_mlp": 0.19335938, + "step": 3517, + "time_per_iteration": 2.651772975921631 + }, + { + "auxiliary_loss_clip": 0.01155773, + "auxiliary_loss_mlp": 0.01058127, + "balance_loss_clip": 1.06088316, + "balance_loss_mlp": 1.04161644, + "epoch": 0.10208345423945216, + "flos": 28253222680320.0, + "grad_norm": 1.9859143380136415, + "language_loss": 0.80552197, + "learning_rate": 3.945748857411796e-06, + "loss": 0.82766092, + "num_input_tokens_seen": 99130675, + "router_z_loss_clip": 0.94921875, + "router_z_loss_mlp": 0.16516113, + "step": 3518, + "time_per_iteration": 2.5921339988708496 + }, + { + "auxiliary_loss_clip": 0.0115699, + "auxiliary_loss_mlp": 0.01069973, + "balance_loss_clip": 1.0559926, + "balance_loss_mlp": 1.04797888, + "epoch": 0.1021124717079682, + "flos": 13626197015040.0, + "grad_norm": 2.426897436812433, + "language_loss": 0.81052357, + "learning_rate": 3.9457053669059555e-06, + "loss": 0.83279318, + "num_input_tokens_seen": 99144450, + "router_z_loss_clip": 1.01025391, + "router_z_loss_mlp": 0.2199707, + "step": 3519, + "time_per_iteration": 2.562548875808716 + }, + { + "auxiliary_loss_clip": 0.01170576, + "auxiliary_loss_mlp": 0.01066708, + "balance_loss_clip": 1.06207681, + "balance_loss_mlp": 1.04410553, + "epoch": 0.10214148917648425, + "flos": 33576943190400.0, + "grad_norm": 2.8769202903054927, + "language_loss": 0.9935714, + "learning_rate": 3.945661859214859e-06, + "loss": 1.01594424, + "num_input_tokens_seen": 99159685, + "router_z_loss_clip": 1.08496094, + "router_z_loss_mlp": 0.22619629, + "step": 3520, + "time_per_iteration": 2.692338705062866 + }, + { + "auxiliary_loss_clip": 0.01154425, + "auxiliary_loss_mlp": 0.01055668, + "balance_loss_clip": 1.05903018, + "balance_loss_mlp": 1.03827524, + "epoch": 0.10217050664500028, + "flos": 17267610090240.0, + "grad_norm": 3.1515470459730937, + "language_loss": 0.90108335, + "learning_rate": 3.94561833433889e-06, + "loss": 0.92318428, + "num_input_tokens_seen": 99172800, + "router_z_loss_clip": 0.95361328, + "router_z_loss_mlp": 0.17407227, + "step": 3521, + "time_per_iteration": 2.5242791175842285 + }, + { + "auxiliary_loss_clip": 0.0115391, + "auxiliary_loss_mlp": 0.01053925, + "balance_loss_clip": 1.05909634, + "balance_loss_mlp": 1.03465462, + "epoch": 0.10219952411351634, + "flos": 28652082278400.0, + "grad_norm": 2.217520696794171, + "language_loss": 0.84677076, + "learning_rate": 3.9455747922784324e-06, + "loss": 0.86884916, + "num_input_tokens_seen": 99188085, + "router_z_loss_clip": 0.94873047, + "router_z_loss_mlp": 0.19281006, + "step": 3522, + "time_per_iteration": 2.622211456298828 + }, + { + "auxiliary_loss_clip": 0.01147897, + "auxiliary_loss_mlp": 0.01049928, + "balance_loss_clip": 1.05946612, + "balance_loss_mlp": 1.03366184, + "epoch": 0.10222854158203239, + "flos": 35510301757440.0, + "grad_norm": 2.266359909135843, + "language_loss": 0.73222315, + "learning_rate": 3.94553123303387e-06, + "loss": 0.75420147, + "num_input_tokens_seen": 99204030, + "router_z_loss_clip": 0.88427734, + "router_z_loss_mlp": 0.16259766, + "step": 3523, + "time_per_iteration": 2.6385657787323 + }, + { + "auxiliary_loss_clip": 0.01161453, + "auxiliary_loss_mlp": 0.01048981, + "balance_loss_clip": 1.06175995, + "balance_loss_mlp": 1.02863204, + "epoch": 0.10225755905054842, + "flos": 30037695373440.0, + "grad_norm": 1.8408644532599254, + "language_loss": 0.86430967, + "learning_rate": 3.9454876566055895e-06, + "loss": 0.88641405, + "num_input_tokens_seen": 99222915, + "router_z_loss_clip": 0.99658203, + "router_z_loss_mlp": 0.20349121, + "step": 3524, + "time_per_iteration": 2.648939847946167 + }, + { + "auxiliary_loss_clip": 0.01149748, + "auxiliary_loss_mlp": 0.01051229, + "balance_loss_clip": 1.05977869, + "balance_loss_mlp": 1.03207803, + "epoch": 0.10228657651906448, + "flos": 29201120640000.0, + "grad_norm": 2.1922072150419174, + "language_loss": 0.81854022, + "learning_rate": 3.945444062993975e-06, + "loss": 0.84055001, + "num_input_tokens_seen": 99239685, + "router_z_loss_clip": 0.89892578, + "router_z_loss_mlp": 0.19177246, + "step": 3525, + "time_per_iteration": 2.6249020099639893 + }, + { + "auxiliary_loss_clip": 0.01143358, + "auxiliary_loss_mlp": 0.01052863, + "balance_loss_clip": 1.05711436, + "balance_loss_mlp": 1.03556609, + "epoch": 0.10231559398758053, + "flos": 34234503517440.0, + "grad_norm": 2.220486729471365, + "language_loss": 0.87949109, + "learning_rate": 3.94540045219941e-06, + "loss": 0.90145338, + "num_input_tokens_seen": 99258785, + "router_z_loss_clip": 0.86230469, + "router_z_loss_mlp": 0.1729126, + "step": 3526, + "time_per_iteration": 2.6558961868286133 + }, + { + "auxiliary_loss_clip": 0.01151117, + "auxiliary_loss_mlp": 0.01046029, + "balance_loss_clip": 1.0606302, + "balance_loss_mlp": 1.03046656, + "epoch": 0.10234461145609657, + "flos": 31243539876480.0, + "grad_norm": 2.009486444046063, + "language_loss": 0.93983805, + "learning_rate": 3.945356824222282e-06, + "loss": 0.96180958, + "num_input_tokens_seen": 99277965, + "router_z_loss_clip": 0.90429688, + "router_z_loss_mlp": 0.15551758, + "step": 3527, + "time_per_iteration": 2.713228940963745 + }, + { + "auxiliary_loss_clip": 0.01151097, + "auxiliary_loss_mlp": 0.01054202, + "balance_loss_clip": 1.05916369, + "balance_loss_mlp": 1.03581429, + "epoch": 0.10237362892461262, + "flos": 12196161774720.0, + "grad_norm": 2.1767171059407064, + "language_loss": 0.76496494, + "learning_rate": 3.945313179062975e-06, + "loss": 0.78701788, + "num_input_tokens_seen": 99289835, + "router_z_loss_clip": 0.91845703, + "router_z_loss_mlp": 0.18389893, + "step": 3528, + "time_per_iteration": 2.4855563640594482 + }, + { + "auxiliary_loss_clip": 0.01144388, + "auxiliary_loss_mlp": 0.01037671, + "balance_loss_clip": 1.05824006, + "balance_loss_mlp": 1.02110076, + "epoch": 0.10240264639312867, + "flos": 10659615730560.0, + "grad_norm": 3.153072893604474, + "language_loss": 0.98395675, + "learning_rate": 3.945269516721875e-06, + "loss": 1.00577736, + "num_input_tokens_seen": 99298085, + "router_z_loss_clip": 0.86132812, + "router_z_loss_mlp": 0.16577148, + "step": 3529, + "time_per_iteration": 2.4950313568115234 + }, + { + "auxiliary_loss_clip": 0.01035757, + "auxiliary_loss_mlp": 0.01050176, + "balance_loss_clip": 1.01306176, + "balance_loss_mlp": 1.04915094, + "epoch": 0.1024316638616447, + "flos": 58904921139840.0, + "grad_norm": 1.2440989876726605, + "language_loss": 0.49939001, + "learning_rate": 3.945225837199367e-06, + "loss": 0.52024937, + "num_input_tokens_seen": 99358565, + "router_z_loss_clip": 0.2265625, + "router_z_loss_mlp": 0.01025391, + "step": 3530, + "time_per_iteration": 3.0463664531707764 + }, + { + "auxiliary_loss_clip": 0.0115761, + "auxiliary_loss_mlp": 0.01072277, + "balance_loss_clip": 1.05860639, + "balance_loss_mlp": 1.05177915, + "epoch": 0.10246068133016076, + "flos": 29348929105920.0, + "grad_norm": 2.036637791653352, + "language_loss": 0.93318582, + "learning_rate": 3.945182140495838e-06, + "loss": 0.95548475, + "num_input_tokens_seen": 99378885, + "router_z_loss_clip": 0.99023438, + "router_z_loss_mlp": 0.20483398, + "step": 3531, + "time_per_iteration": 2.6201202869415283 + }, + { + "auxiliary_loss_clip": 0.01143222, + "auxiliary_loss_mlp": 0.01039737, + "balance_loss_clip": 1.05393314, + "balance_loss_mlp": 1.02362561, + "epoch": 0.10248969879867681, + "flos": 23763850260480.0, + "grad_norm": 1.9696445289755409, + "language_loss": 0.92190146, + "learning_rate": 3.945138426611672e-06, + "loss": 0.94373101, + "num_input_tokens_seen": 99394075, + "router_z_loss_clip": 0.89257812, + "router_z_loss_mlp": 0.16107178, + "step": 3532, + "time_per_iteration": 2.519582748413086 + }, + { + "auxiliary_loss_clip": 0.01149994, + "auxiliary_loss_mlp": 0.01043492, + "balance_loss_clip": 1.05610728, + "balance_loss_mlp": 1.02297664, + "epoch": 0.10251871626719285, + "flos": 32192012453760.0, + "grad_norm": 1.918847246113172, + "language_loss": 1.04812562, + "learning_rate": 3.945094695547258e-06, + "loss": 1.07006049, + "num_input_tokens_seen": 99413140, + "router_z_loss_clip": 0.93847656, + "router_z_loss_mlp": 0.2052002, + "step": 3533, + "time_per_iteration": 2.616262674331665 + }, + { + "auxiliary_loss_clip": 0.01150488, + "auxiliary_loss_mlp": 0.01053583, + "balance_loss_clip": 1.05800998, + "balance_loss_mlp": 1.03524911, + "epoch": 0.1025477337357089, + "flos": 25148385947520.0, + "grad_norm": 3.093676155822679, + "language_loss": 0.89457041, + "learning_rate": 3.945050947302979e-06, + "loss": 0.91661119, + "num_input_tokens_seen": 99427310, + "router_z_loss_clip": 0.92431641, + "router_z_loss_mlp": 0.18347168, + "step": 3534, + "time_per_iteration": 2.583709955215454 + }, + { + "auxiliary_loss_clip": 0.01147776, + "auxiliary_loss_mlp": 0.01044498, + "balance_loss_clip": 1.05748904, + "balance_loss_mlp": 1.02863169, + "epoch": 0.10257675120422495, + "flos": 24125039470080.0, + "grad_norm": 2.046411983221929, + "language_loss": 0.75735426, + "learning_rate": 3.945007181879224e-06, + "loss": 0.77927697, + "num_input_tokens_seen": 99441375, + "router_z_loss_clip": 0.90283203, + "router_z_loss_mlp": 0.15869141, + "step": 3535, + "time_per_iteration": 2.598170280456543 + }, + { + "auxiliary_loss_clip": 0.01157908, + "auxiliary_loss_mlp": 0.0105257, + "balance_loss_clip": 1.06176019, + "balance_loss_mlp": 1.03399694, + "epoch": 0.10260576867274099, + "flos": 15842961889920.0, + "grad_norm": 3.0390025685549764, + "language_loss": 1.0282315, + "learning_rate": 3.944963399276378e-06, + "loss": 1.05033624, + "num_input_tokens_seen": 99453280, + "router_z_loss_clip": 0.9609375, + "router_z_loss_mlp": 0.18579102, + "step": 3536, + "time_per_iteration": 2.5249664783477783 + }, + { + "auxiliary_loss_clip": 0.0104091, + "auxiliary_loss_mlp": 0.01000646, + "balance_loss_clip": 1.01760185, + "balance_loss_mlp": 0.99965614, + "epoch": 0.10263478614125704, + "flos": 70694394151680.0, + "grad_norm": 0.6364413747393213, + "language_loss": 0.47186506, + "learning_rate": 3.944919599494828e-06, + "loss": 0.4922806, + "num_input_tokens_seen": 99520100, + "router_z_loss_clip": 0.23339844, + "router_z_loss_mlp": 0.0098877, + "step": 3537, + "time_per_iteration": 3.1834990978240967 + }, + { + "auxiliary_loss_clip": 0.01149421, + "auxiliary_loss_mlp": 0.01047003, + "balance_loss_clip": 1.0563705, + "balance_loss_mlp": 1.02773821, + "epoch": 0.10266380360977308, + "flos": 14824643316480.0, + "grad_norm": 2.566697254688303, + "language_loss": 0.76507926, + "learning_rate": 3.944875782534962e-06, + "loss": 0.78704351, + "num_input_tokens_seen": 99532345, + "router_z_loss_clip": 0.93017578, + "router_z_loss_mlp": 0.19262695, + "step": 3538, + "time_per_iteration": 2.4816105365753174 + }, + { + "auxiliary_loss_clip": 0.01040943, + "auxiliary_loss_mlp": 0.00998455, + "balance_loss_clip": 1.01776147, + "balance_loss_mlp": 0.99747199, + "epoch": 0.10269282107828913, + "flos": 63577940820480.0, + "grad_norm": 0.6605492543093605, + "language_loss": 0.49230874, + "learning_rate": 3.9448319483971655e-06, + "loss": 0.5127027, + "num_input_tokens_seen": 99593660, + "router_z_loss_clip": 0.23242188, + "router_z_loss_mlp": 0.00982666, + "step": 3539, + "time_per_iteration": 3.1796865463256836 + }, + { + "auxiliary_loss_clip": 0.01150041, + "auxiliary_loss_mlp": 0.01040998, + "balance_loss_clip": 1.06012177, + "balance_loss_mlp": 1.02428496, + "epoch": 0.10272183854680518, + "flos": 29345696881920.0, + "grad_norm": 2.0527227316131076, + "language_loss": 1.0231719, + "learning_rate": 3.944788097081826e-06, + "loss": 1.04508233, + "num_input_tokens_seen": 99614345, + "router_z_loss_clip": 0.89794922, + "router_z_loss_mlp": 0.16723633, + "step": 3540, + "time_per_iteration": 2.7080764770507812 + }, + { + "auxiliary_loss_clip": 0.0115016, + "auxiliary_loss_mlp": 0.01041493, + "balance_loss_clip": 1.05918133, + "balance_loss_mlp": 1.02344525, + "epoch": 0.10275085601532122, + "flos": 24929941386240.0, + "grad_norm": 2.6357278447951527, + "language_loss": 0.76574576, + "learning_rate": 3.944744228589331e-06, + "loss": 0.78766227, + "num_input_tokens_seen": 99629355, + "router_z_loss_clip": 0.90917969, + "router_z_loss_mlp": 0.18041992, + "step": 3541, + "time_per_iteration": 2.5747900009155273 + }, + { + "auxiliary_loss_clip": 0.01155503, + "auxiliary_loss_mlp": 0.01058438, + "balance_loss_clip": 1.06259012, + "balance_loss_mlp": 1.04015088, + "epoch": 0.10277987348383727, + "flos": 48060757330560.0, + "grad_norm": 1.9334988308312797, + "language_loss": 0.85355747, + "learning_rate": 3.944700342920069e-06, + "loss": 0.87569684, + "num_input_tokens_seen": 99648845, + "router_z_loss_clip": 0.92871094, + "router_z_loss_mlp": 0.1829834, + "step": 3542, + "time_per_iteration": 2.789062976837158 + }, + { + "auxiliary_loss_clip": 0.01035909, + "auxiliary_loss_mlp": 0.01027892, + "balance_loss_clip": 1.01327264, + "balance_loss_mlp": 1.02683699, + "epoch": 0.10280889095235332, + "flos": 68536844847360.0, + "grad_norm": 0.6713421473684464, + "language_loss": 0.48950595, + "learning_rate": 3.9446564400744255e-06, + "loss": 0.510144, + "num_input_tokens_seen": 99710180, + "router_z_loss_clip": 0.2265625, + "router_z_loss_mlp": 0.01055908, + "step": 3543, + "time_per_iteration": 3.0719032287597656 + }, + { + "auxiliary_loss_clip": 0.0103637, + "auxiliary_loss_mlp": 0.01018197, + "balance_loss_clip": 1.01371384, + "balance_loss_mlp": 1.01710606, + "epoch": 0.10283790842086936, + "flos": 60471667543680.0, + "grad_norm": 0.6572420085321035, + "language_loss": 0.48186478, + "learning_rate": 3.94461252005279e-06, + "loss": 0.50241041, + "num_input_tokens_seen": 99767705, + "router_z_loss_clip": 0.2265625, + "router_z_loss_mlp": 0.01092529, + "step": 3544, + "time_per_iteration": 3.0022547245025635 + }, + { + "auxiliary_loss_clip": 0.01154907, + "auxiliary_loss_mlp": 0.01055816, + "balance_loss_clip": 1.05881453, + "balance_loss_mlp": 1.03631353, + "epoch": 0.10286692588938541, + "flos": 33942621600000.0, + "grad_norm": 3.352656238893852, + "language_loss": 0.73575264, + "learning_rate": 3.944568582855549e-06, + "loss": 0.75785989, + "num_input_tokens_seen": 99783540, + "router_z_loss_clip": 0.96142578, + "router_z_loss_mlp": 0.19494629, + "step": 3545, + "time_per_iteration": 2.5557281970977783 + }, + { + "auxiliary_loss_clip": 0.01159784, + "auxiliary_loss_mlp": 0.01053693, + "balance_loss_clip": 1.06050944, + "balance_loss_mlp": 1.03248572, + "epoch": 0.10289594335790146, + "flos": 27777801242880.0, + "grad_norm": 3.7747974888032343, + "language_loss": 0.87630308, + "learning_rate": 3.944524628483093e-06, + "loss": 0.8984378, + "num_input_tokens_seen": 99801740, + "router_z_loss_clip": 0.99121094, + "router_z_loss_mlp": 0.2121582, + "step": 3546, + "time_per_iteration": 2.5845651626586914 + }, + { + "auxiliary_loss_clip": 0.01148805, + "auxiliary_loss_mlp": 0.01045499, + "balance_loss_clip": 1.05622852, + "balance_loss_mlp": 1.02675354, + "epoch": 0.1029249608264175, + "flos": 10406086560000.0, + "grad_norm": 3.0554250723385517, + "language_loss": 0.86566794, + "learning_rate": 3.944480656935807e-06, + "loss": 0.88761097, + "num_input_tokens_seen": 99813705, + "router_z_loss_clip": 0.92578125, + "router_z_loss_mlp": 0.18737793, + "step": 3547, + "time_per_iteration": 2.4969544410705566 + }, + { + "auxiliary_loss_clip": 0.01035096, + "auxiliary_loss_mlp": 0.01010272, + "balance_loss_clip": 1.01199925, + "balance_loss_mlp": 1.00903249, + "epoch": 0.10295397829493355, + "flos": 74782788071040.0, + "grad_norm": 0.6000786381206719, + "language_loss": 0.50841677, + "learning_rate": 3.944436668214082e-06, + "loss": 0.52887046, + "num_input_tokens_seen": 99882985, + "router_z_loss_clip": 0.23046875, + "router_z_loss_mlp": 0.01239014, + "step": 3548, + "time_per_iteration": 3.2360994815826416 + }, + { + "auxiliary_loss_clip": 0.01158291, + "auxiliary_loss_mlp": 0.01054862, + "balance_loss_clip": 1.06196356, + "balance_loss_mlp": 1.03560996, + "epoch": 0.1029829957634496, + "flos": 28250349592320.0, + "grad_norm": 2.7641688236731325, + "language_loss": 0.93354243, + "learning_rate": 3.9443926623183045e-06, + "loss": 0.95567393, + "num_input_tokens_seen": 99898050, + "router_z_loss_clip": 0.96289062, + "router_z_loss_mlp": 0.19274902, + "step": 3549, + "time_per_iteration": 2.629789352416992 + }, + { + "auxiliary_loss_clip": 0.01142932, + "auxiliary_loss_mlp": 0.01041727, + "balance_loss_clip": 1.05529833, + "balance_loss_mlp": 1.02463865, + "epoch": 0.10301201323196564, + "flos": 11356067508480.0, + "grad_norm": 2.4318937450556186, + "language_loss": 0.69666243, + "learning_rate": 3.944348639248865e-06, + "loss": 0.71850902, + "num_input_tokens_seen": 99909700, + "router_z_loss_clip": 0.87646484, + "router_z_loss_mlp": 0.17071533, + "step": 3550, + "time_per_iteration": 2.5668540000915527 + }, + { + "auxiliary_loss_clip": 0.01034631, + "auxiliary_loss_mlp": 0.00999418, + "balance_loss_clip": 1.01167595, + "balance_loss_mlp": 0.99833888, + "epoch": 0.10304103070048169, + "flos": 74771547114240.0, + "grad_norm": 0.6591344453266108, + "language_loss": 0.53106403, + "learning_rate": 3.944304599006151e-06, + "loss": 0.55140448, + "num_input_tokens_seen": 99973710, + "router_z_loss_clip": 0.22949219, + "router_z_loss_mlp": 0.01080322, + "step": 3551, + "time_per_iteration": 3.2517313957214355 + }, + { + "auxiliary_loss_clip": 0.01034429, + "auxiliary_loss_mlp": 0.01001033, + "balance_loss_clip": 1.01158524, + "balance_loss_mlp": 0.99989492, + "epoch": 0.10307004816899773, + "flos": 74781459267840.0, + "grad_norm": 0.6724181092318309, + "language_loss": 0.5011909, + "learning_rate": 3.944260541590553e-06, + "loss": 0.52154553, + "num_input_tokens_seen": 100036985, + "router_z_loss_clip": 0.22851562, + "router_z_loss_mlp": 0.01141357, + "step": 3552, + "time_per_iteration": 3.254763603210449 + }, + { + "auxiliary_loss_clip": 0.01147258, + "auxiliary_loss_mlp": 0.01058377, + "balance_loss_clip": 1.05805409, + "balance_loss_mlp": 1.03981602, + "epoch": 0.10309906563751378, + "flos": 24055193473920.0, + "grad_norm": 2.5802207885505184, + "language_loss": 0.88728392, + "learning_rate": 3.944216467002458e-06, + "loss": 0.90934032, + "num_input_tokens_seen": 100051455, + "router_z_loss_clip": 0.89257812, + "router_z_loss_mlp": 0.18566895, + "step": 3553, + "time_per_iteration": 2.6668336391448975 + }, + { + "auxiliary_loss_clip": 0.01167964, + "auxiliary_loss_mlp": 0.0106, + "balance_loss_clip": 1.06702912, + "balance_loss_mlp": 1.03854859, + "epoch": 0.10312808310602983, + "flos": 36676392883200.0, + "grad_norm": 2.85122092049987, + "language_loss": 1.03184915, + "learning_rate": 3.944172375242258e-06, + "loss": 1.05412877, + "num_input_tokens_seen": 100065375, + "router_z_loss_clip": 1.01074219, + "router_z_loss_mlp": 0.21478271, + "step": 3554, + "time_per_iteration": 2.6881964206695557 + }, + { + "auxiliary_loss_clip": 0.01154267, + "auxiliary_loss_mlp": 0.01047079, + "balance_loss_clip": 1.05884361, + "balance_loss_mlp": 1.02715862, + "epoch": 0.10315710057454587, + "flos": 33322588007040.0, + "grad_norm": 1.9227142998697382, + "language_loss": 0.71016598, + "learning_rate": 3.944128266310339e-06, + "loss": 0.7321794, + "num_input_tokens_seen": 100084000, + "router_z_loss_clip": 0.95361328, + "router_z_loss_mlp": 0.19909668, + "step": 3555, + "time_per_iteration": 2.6935274600982666 + }, + { + "auxiliary_loss_clip": 0.01151316, + "auxiliary_loss_mlp": 0.01057282, + "balance_loss_clip": 1.0573957, + "balance_loss_mlp": 1.03881598, + "epoch": 0.10318611804306192, + "flos": 32227204803840.0, + "grad_norm": 1.9350667658724054, + "language_loss": 0.95188433, + "learning_rate": 3.944084140207093e-06, + "loss": 0.97397023, + "num_input_tokens_seen": 100105350, + "router_z_loss_clip": 0.94042969, + "router_z_loss_mlp": 0.18457031, + "step": 3556, + "time_per_iteration": 2.63975191116333 + }, + { + "auxiliary_loss_clip": 0.01150554, + "auxiliary_loss_mlp": 0.010478, + "balance_loss_clip": 1.06010675, + "balance_loss_mlp": 1.03057456, + "epoch": 0.10321513551157797, + "flos": 36829013771520.0, + "grad_norm": 2.1035780495237106, + "language_loss": 1.1804173, + "learning_rate": 3.944039996932909e-06, + "loss": 1.2024008, + "num_input_tokens_seen": 100134310, + "router_z_loss_clip": 0.90429688, + "router_z_loss_mlp": 0.17236328, + "step": 3557, + "time_per_iteration": 6.302641868591309 + }, + { + "auxiliary_loss_clip": 0.01153752, + "auxiliary_loss_mlp": 0.01050192, + "balance_loss_clip": 1.05931568, + "balance_loss_mlp": 1.03111851, + "epoch": 0.10324415298009401, + "flos": 24125434519680.0, + "grad_norm": 2.538101202966828, + "language_loss": 0.77419794, + "learning_rate": 3.9439958364881785e-06, + "loss": 0.79623735, + "num_input_tokens_seen": 100149735, + "router_z_loss_clip": 0.94482422, + "router_z_loss_mlp": 0.19073486, + "step": 3558, + "time_per_iteration": 5.016756296157837 + }, + { + "auxiliary_loss_clip": 0.01157056, + "auxiliary_loss_mlp": 0.0105562, + "balance_loss_clip": 1.05969691, + "balance_loss_mlp": 1.03733337, + "epoch": 0.10327317044861006, + "flos": 15735194110080.0, + "grad_norm": 2.160968987571617, + "language_loss": 0.88920456, + "learning_rate": 3.943951658873289e-06, + "loss": 0.9113313, + "num_input_tokens_seen": 100163390, + "router_z_loss_clip": 0.97412109, + "router_z_loss_mlp": 0.18273926, + "step": 3559, + "time_per_iteration": 4.923001766204834 + }, + { + "auxiliary_loss_clip": 0.01034407, + "auxiliary_loss_mlp": 0.01003748, + "balance_loss_clip": 1.0117178, + "balance_loss_mlp": 1.00268126, + "epoch": 0.10330218791712611, + "flos": 74766806519040.0, + "grad_norm": 0.7003010533367195, + "language_loss": 0.46598113, + "learning_rate": 3.9439074640886314e-06, + "loss": 0.4863627, + "num_input_tokens_seen": 100221550, + "router_z_loss_clip": 0.2265625, + "router_z_loss_mlp": 0.01068115, + "step": 3560, + "time_per_iteration": 5.542824745178223 + }, + { + "auxiliary_loss_clip": 0.01033599, + "auxiliary_loss_mlp": 0.01003584, + "balance_loss_clip": 1.01091254, + "balance_loss_mlp": 1.00255859, + "epoch": 0.10333120538564215, + "flos": 65039505223680.0, + "grad_norm": 0.6252894501216943, + "language_loss": 0.49439466, + "learning_rate": 3.9438632521345975e-06, + "loss": 0.51476645, + "num_input_tokens_seen": 100283810, + "router_z_loss_clip": 0.2265625, + "router_z_loss_mlp": 0.01025391, + "step": 3561, + "time_per_iteration": 3.15417218208313 + }, + { + "auxiliary_loss_clip": 0.01155564, + "auxiliary_loss_mlp": 0.01045236, + "balance_loss_clip": 1.06137514, + "balance_loss_mlp": 1.02778387, + "epoch": 0.1033602228541582, + "flos": 39342293418240.0, + "grad_norm": 2.493145775504553, + "language_loss": 0.889431, + "learning_rate": 3.943819023011576e-06, + "loss": 0.91143906, + "num_input_tokens_seen": 100303220, + "router_z_loss_clip": 0.94287109, + "router_z_loss_mlp": 0.17456055, + "step": 3562, + "time_per_iteration": 2.7456281185150146 + }, + { + "auxiliary_loss_clip": 0.0115439, + "auxiliary_loss_mlp": 0.01056684, + "balance_loss_clip": 1.05887854, + "balance_loss_mlp": 1.03749728, + "epoch": 0.10338924032267426, + "flos": 25732401177600.0, + "grad_norm": 2.4073182429784348, + "language_loss": 1.01071429, + "learning_rate": 3.943774776719959e-06, + "loss": 1.03282499, + "num_input_tokens_seen": 100316540, + "router_z_loss_clip": 0.95410156, + "router_z_loss_mlp": 0.19207764, + "step": 3563, + "time_per_iteration": 2.589451551437378 + }, + { + "auxiliary_loss_clip": 0.01157669, + "auxiliary_loss_mlp": 0.01053199, + "balance_loss_clip": 1.06197786, + "balance_loss_mlp": 1.03521633, + "epoch": 0.1034182577911903, + "flos": 30474440841600.0, + "grad_norm": 1.884646286488584, + "language_loss": 0.81122088, + "learning_rate": 3.943730513260136e-06, + "loss": 0.83332956, + "num_input_tokens_seen": 100333835, + "router_z_loss_clip": 0.95751953, + "router_z_loss_mlp": 0.17993164, + "step": 3564, + "time_per_iteration": 2.7127366065979004 + }, + { + "auxiliary_loss_clip": 0.01139706, + "auxiliary_loss_mlp": 0.01037122, + "balance_loss_clip": 1.05420661, + "balance_loss_mlp": 1.0208019, + "epoch": 0.10344727525970634, + "flos": 31497643664640.0, + "grad_norm": 2.198886614124679, + "language_loss": 0.82952332, + "learning_rate": 3.943686232632498e-06, + "loss": 0.85129166, + "num_input_tokens_seen": 100349975, + "router_z_loss_clip": 0.85449219, + "router_z_loss_mlp": 0.16308594, + "step": 3565, + "time_per_iteration": 2.6681714057922363 + }, + { + "auxiliary_loss_clip": 0.01036598, + "auxiliary_loss_mlp": 0.01005146, + "balance_loss_clip": 1.01388788, + "balance_loss_mlp": 1.00399518, + "epoch": 0.1034762927282224, + "flos": 64741697562240.0, + "grad_norm": 0.6509098316973692, + "language_loss": 0.4959482, + "learning_rate": 3.943641934837438e-06, + "loss": 0.51636559, + "num_input_tokens_seen": 100410670, + "router_z_loss_clip": 0.22753906, + "router_z_loss_mlp": 0.01147461, + "step": 3566, + "time_per_iteration": 3.1608753204345703 + }, + { + "auxiliary_loss_clip": 0.01146422, + "auxiliary_loss_mlp": 0.01046564, + "balance_loss_clip": 1.05859697, + "balance_loss_mlp": 1.02936208, + "epoch": 0.10350531019673843, + "flos": 14457959326080.0, + "grad_norm": 2.603185815793337, + "language_loss": 0.80741984, + "learning_rate": 3.943597619875345e-06, + "loss": 0.8293497, + "num_input_tokens_seen": 100423595, + "router_z_loss_clip": 0.87841797, + "router_z_loss_mlp": 0.17199707, + "step": 3567, + "time_per_iteration": 2.5624186992645264 + }, + { + "auxiliary_loss_clip": 0.01160492, + "auxiliary_loss_mlp": 0.01059282, + "balance_loss_clip": 1.05862093, + "balance_loss_mlp": 1.0394454, + "epoch": 0.10353432766525449, + "flos": 29799285828480.0, + "grad_norm": 2.6234693999773744, + "language_loss": 0.93351758, + "learning_rate": 3.9435532877466116e-06, + "loss": 0.95571536, + "num_input_tokens_seen": 100446720, + "router_z_loss_clip": 1.01904297, + "router_z_loss_mlp": 0.19842529, + "step": 3568, + "time_per_iteration": 2.6207387447357178 + }, + { + "auxiliary_loss_clip": 0.01148569, + "auxiliary_loss_mlp": 0.01046492, + "balance_loss_clip": 1.05892205, + "balance_loss_mlp": 1.03050554, + "epoch": 0.10356334513377052, + "flos": 21395362337280.0, + "grad_norm": 1.8618916855833687, + "language_loss": 0.76210839, + "learning_rate": 3.94350893845163e-06, + "loss": 0.78405905, + "num_input_tokens_seen": 100463885, + "router_z_loss_clip": 0.89794922, + "router_z_loss_mlp": 0.15991211, + "step": 3569, + "time_per_iteration": 2.5282866954803467 + }, + { + "auxiliary_loss_clip": 0.01152049, + "auxiliary_loss_mlp": 0.01048118, + "balance_loss_clip": 1.05953336, + "balance_loss_mlp": 1.02968884, + "epoch": 0.10359236260228658, + "flos": 33759333475200.0, + "grad_norm": 2.139415294293344, + "language_loss": 0.80269682, + "learning_rate": 3.94346457199079e-06, + "loss": 0.82469851, + "num_input_tokens_seen": 100479820, + "router_z_loss_clip": 0.92431641, + "router_z_loss_mlp": 0.18432617, + "step": 3570, + "time_per_iteration": 2.620570659637451 + }, + { + "auxiliary_loss_clip": 0.01154357, + "auxiliary_loss_mlp": 0.01054774, + "balance_loss_clip": 1.05907559, + "balance_loss_mlp": 1.03604579, + "epoch": 0.10362138007080263, + "flos": 34234754912640.0, + "grad_norm": 2.248310430748336, + "language_loss": 0.9151547, + "learning_rate": 3.943420188364484e-06, + "loss": 0.93724597, + "num_input_tokens_seen": 100498050, + "router_z_loss_clip": 0.95361328, + "router_z_loss_mlp": 0.18713379, + "step": 3571, + "time_per_iteration": 2.6487724781036377 + }, + { + "auxiliary_loss_clip": 0.01155851, + "auxiliary_loss_mlp": 0.01051622, + "balance_loss_clip": 1.05926442, + "balance_loss_mlp": 1.03211331, + "epoch": 0.10365039753931866, + "flos": 18471156122880.0, + "grad_norm": 2.2951324034305274, + "language_loss": 0.65019977, + "learning_rate": 3.943375787573106e-06, + "loss": 0.67227447, + "num_input_tokens_seen": 100511910, + "router_z_loss_clip": 0.96679688, + "router_z_loss_mlp": 0.1953125, + "step": 3572, + "time_per_iteration": 2.5073821544647217 + }, + { + "auxiliary_loss_clip": 0.01034329, + "auxiliary_loss_mlp": 0.01005061, + "balance_loss_clip": 1.01152515, + "balance_loss_mlp": 1.00380921, + "epoch": 0.10367941500783472, + "flos": 74775497610240.0, + "grad_norm": 0.7104871074406237, + "language_loss": 0.47146475, + "learning_rate": 3.943331369617045e-06, + "loss": 0.49185863, + "num_input_tokens_seen": 100569920, + "router_z_loss_clip": 0.22851562, + "router_z_loss_mlp": 0.01251221, + "step": 3573, + "time_per_iteration": 3.186094284057617 + }, + { + "auxiliary_loss_clip": 0.01034598, + "auxiliary_loss_mlp": 0.01001425, + "balance_loss_clip": 1.01182055, + "balance_loss_mlp": 1.00020909, + "epoch": 0.10370843247635077, + "flos": 61745634190080.0, + "grad_norm": 0.6894640080971203, + "language_loss": 0.46716541, + "learning_rate": 3.943286934496695e-06, + "loss": 0.48752564, + "num_input_tokens_seen": 100631400, + "router_z_loss_clip": 0.22851562, + "router_z_loss_mlp": 0.012146, + "step": 3574, + "time_per_iteration": 3.119309425354004 + }, + { + "auxiliary_loss_clip": 0.01033797, + "auxiliary_loss_mlp": 0.01006633, + "balance_loss_clip": 1.01099086, + "balance_loss_mlp": 1.00551879, + "epoch": 0.1037374499448668, + "flos": 53350976407680.0, + "grad_norm": 0.7519625179371753, + "language_loss": 0.51524913, + "learning_rate": 3.94324248221245e-06, + "loss": 0.53565347, + "num_input_tokens_seen": 100684225, + "router_z_loss_clip": 0.22851562, + "router_z_loss_mlp": 0.01116943, + "step": 3575, + "time_per_iteration": 2.875812292098999 + }, + { + "auxiliary_loss_clip": 0.01033526, + "auxiliary_loss_mlp": 0.01004316, + "balance_loss_clip": 1.01069403, + "balance_loss_mlp": 1.00309396, + "epoch": 0.10376646741338286, + "flos": 70070661457920.0, + "grad_norm": 0.6373946059311693, + "language_loss": 0.49531919, + "learning_rate": 3.9431980127647e-06, + "loss": 0.5156976, + "num_input_tokens_seen": 100750595, + "router_z_loss_clip": 0.22851562, + "router_z_loss_mlp": 0.01220703, + "step": 3576, + "time_per_iteration": 3.2054498195648193 + }, + { + "auxiliary_loss_clip": 0.01142258, + "auxiliary_loss_mlp": 0.01047437, + "balance_loss_clip": 1.05169368, + "balance_loss_mlp": 1.02918577, + "epoch": 0.10379548488189891, + "flos": 33793089281280.0, + "grad_norm": 2.7274807666872483, + "language_loss": 0.92445546, + "learning_rate": 3.943153526153839e-06, + "loss": 0.94635236, + "num_input_tokens_seen": 100765890, + "router_z_loss_clip": 0.90527344, + "router_z_loss_mlp": 0.18237305, + "step": 3577, + "time_per_iteration": 2.6655690670013428 + }, + { + "auxiliary_loss_clip": 0.01148318, + "auxiliary_loss_mlp": 0.0104617, + "balance_loss_clip": 1.05752254, + "balance_loss_mlp": 1.02760887, + "epoch": 0.10382450235041495, + "flos": 21355285737600.0, + "grad_norm": 2.184688122610234, + "language_loss": 0.85771787, + "learning_rate": 3.94310902238026e-06, + "loss": 0.87966269, + "num_input_tokens_seen": 100782720, + "router_z_loss_clip": 0.90820312, + "router_z_loss_mlp": 0.18560791, + "step": 3578, + "time_per_iteration": 2.5970847606658936 + }, + { + "auxiliary_loss_clip": 0.01150271, + "auxiliary_loss_mlp": 0.0104988, + "balance_loss_clip": 1.057513, + "balance_loss_mlp": 1.0313791, + "epoch": 0.103853519818931, + "flos": 66123644492160.0, + "grad_norm": 1.8107256124365796, + "language_loss": 0.85995352, + "learning_rate": 3.943064501444355e-06, + "loss": 0.88195503, + "num_input_tokens_seen": 100806760, + "router_z_loss_clip": 0.92626953, + "router_z_loss_mlp": 0.18487549, + "step": 3579, + "time_per_iteration": 2.895812511444092 + }, + { + "auxiliary_loss_clip": 0.0116499, + "auxiliary_loss_mlp": 0.01053141, + "balance_loss_clip": 1.06203663, + "balance_loss_mlp": 1.03225565, + "epoch": 0.10388253728744705, + "flos": 37633843860480.0, + "grad_norm": 1.8405517542773613, + "language_loss": 0.7561233, + "learning_rate": 3.9430199633465185e-06, + "loss": 0.77830458, + "num_input_tokens_seen": 100827400, + "router_z_loss_clip": 1.02929688, + "router_z_loss_mlp": 0.2088623, + "step": 3580, + "time_per_iteration": 2.7641329765319824 + }, + { + "auxiliary_loss_clip": 0.01163791, + "auxiliary_loss_mlp": 0.0105885, + "balance_loss_clip": 1.06432211, + "balance_loss_mlp": 1.03929949, + "epoch": 0.10391155475596309, + "flos": 25620503333760.0, + "grad_norm": 3.0913657948699242, + "language_loss": 1.09629846, + "learning_rate": 3.942975408087144e-06, + "loss": 1.11852479, + "num_input_tokens_seen": 100840280, + "router_z_loss_clip": 0.99414062, + "router_z_loss_mlp": 0.19555664, + "step": 3581, + "time_per_iteration": 2.626309871673584 + }, + { + "auxiliary_loss_clip": 0.01145501, + "auxiliary_loss_mlp": 0.01035481, + "balance_loss_clip": 1.05644608, + "balance_loss_mlp": 1.01866627, + "epoch": 0.10394057222447914, + "flos": 39960890467200.0, + "grad_norm": 1.8488187849727045, + "language_loss": 0.80385059, + "learning_rate": 3.9429308356666235e-06, + "loss": 0.82566035, + "num_input_tokens_seen": 100862655, + "router_z_loss_clip": 0.89111328, + "router_z_loss_mlp": 0.16809082, + "step": 3582, + "time_per_iteration": 2.688026189804077 + }, + { + "auxiliary_loss_clip": 0.01034195, + "auxiliary_loss_mlp": 0.01001931, + "balance_loss_clip": 1.01111746, + "balance_loss_mlp": 1.00057828, + "epoch": 0.10396958969299518, + "flos": 74779340365440.0, + "grad_norm": 0.7174921806416604, + "language_loss": 0.472653, + "learning_rate": 3.942886246085352e-06, + "loss": 0.49301422, + "num_input_tokens_seen": 100925115, + "router_z_loss_clip": 0.23046875, + "router_z_loss_mlp": 0.0135498, + "step": 3583, + "time_per_iteration": 3.210991144180298 + }, + { + "auxiliary_loss_clip": 0.01148526, + "auxiliary_loss_mlp": 0.0105102, + "balance_loss_clip": 1.05583882, + "balance_loss_mlp": 1.03247714, + "epoch": 0.10399860716151123, + "flos": 17889547104000.0, + "grad_norm": 2.8180341818299035, + "language_loss": 0.79219508, + "learning_rate": 3.942841639343723e-06, + "loss": 0.81419051, + "num_input_tokens_seen": 100938170, + "router_z_loss_clip": 0.92675781, + "router_z_loss_mlp": 0.18554688, + "step": 3584, + "time_per_iteration": 2.476087808609009 + }, + { + "auxiliary_loss_clip": 0.01030801, + "auxiliary_loss_mlp": 0.0100478, + "balance_loss_clip": 1.00782526, + "balance_loss_mlp": 1.00348091, + "epoch": 0.10402762463002728, + "flos": 57691103817600.0, + "grad_norm": 0.7296967757700689, + "language_loss": 0.49081287, + "learning_rate": 3.942797015442131e-06, + "loss": 0.51116872, + "num_input_tokens_seen": 100991815, + "router_z_loss_clip": 0.22949219, + "router_z_loss_mlp": 0.01300049, + "step": 3585, + "time_per_iteration": 2.9642179012298584 + }, + { + "auxiliary_loss_clip": 0.01154605, + "auxiliary_loss_mlp": 0.01055473, + "balance_loss_clip": 1.05849493, + "balance_loss_mlp": 1.03603017, + "epoch": 0.10405664209854332, + "flos": 15443419933440.0, + "grad_norm": 2.502068521300941, + "language_loss": 0.76517725, + "learning_rate": 3.942752374380969e-06, + "loss": 0.78727806, + "num_input_tokens_seen": 101008430, + "router_z_loss_clip": 0.96044922, + "router_z_loss_mlp": 0.19458008, + "step": 3586, + "time_per_iteration": 2.585240602493286 + }, + { + "auxiliary_loss_clip": 0.01030505, + "auxiliary_loss_mlp": 0.01002815, + "balance_loss_clip": 1.00759077, + "balance_loss_mlp": 1.00160551, + "epoch": 0.10408565956705937, + "flos": 64748916195840.0, + "grad_norm": 0.6133972042055181, + "language_loss": 0.48950177, + "learning_rate": 3.942707716160632e-06, + "loss": 0.509835, + "num_input_tokens_seen": 101076560, + "router_z_loss_clip": 0.22949219, + "router_z_loss_mlp": 0.01208496, + "step": 3587, + "time_per_iteration": 3.2135746479034424 + }, + { + "auxiliary_loss_clip": 0.01148818, + "auxiliary_loss_mlp": 0.01048887, + "balance_loss_clip": 1.05543244, + "balance_loss_mlp": 1.0309937, + "epoch": 0.10411467703557542, + "flos": 27374847494400.0, + "grad_norm": 1.861254834006367, + "language_loss": 0.85824639, + "learning_rate": 3.942663040781514e-06, + "loss": 0.88022339, + "num_input_tokens_seen": 101095995, + "router_z_loss_clip": 0.93457031, + "router_z_loss_mlp": 0.17895508, + "step": 3588, + "time_per_iteration": 2.64884352684021 + }, + { + "auxiliary_loss_clip": 0.01139242, + "auxiliary_loss_mlp": 0.01039628, + "balance_loss_clip": 1.05360913, + "balance_loss_mlp": 1.024405, + "epoch": 0.10414369450409146, + "flos": 37445743313280.0, + "grad_norm": 2.0769963043233264, + "language_loss": 0.80276269, + "learning_rate": 3.942618348244011e-06, + "loss": 0.8245514, + "num_input_tokens_seen": 101113395, + "router_z_loss_clip": 0.85644531, + "router_z_loss_mlp": 0.15228271, + "step": 3589, + "time_per_iteration": 2.687995195388794 + }, + { + "auxiliary_loss_clip": 0.01135555, + "auxiliary_loss_mlp": 0.01040613, + "balance_loss_clip": 1.0518899, + "balance_loss_mlp": 1.02458501, + "epoch": 0.10417271197260751, + "flos": 28397547527040.0, + "grad_norm": 3.1101830309355973, + "language_loss": 0.7664786, + "learning_rate": 3.942573638548515e-06, + "loss": 0.78824031, + "num_input_tokens_seen": 101128915, + "router_z_loss_clip": 0.83691406, + "router_z_loss_mlp": 0.16021729, + "step": 3590, + "time_per_iteration": 2.5807971954345703 + }, + { + "auxiliary_loss_clip": 0.01146386, + "auxiliary_loss_mlp": 0.01038861, + "balance_loss_clip": 1.0580225, + "balance_loss_mlp": 1.02117622, + "epoch": 0.10420172944112356, + "flos": 27336889797120.0, + "grad_norm": 1.7702390107182087, + "language_loss": 0.68244112, + "learning_rate": 3.9425289116954245e-06, + "loss": 0.70429355, + "num_input_tokens_seen": 101147695, + "router_z_loss_clip": 0.88427734, + "router_z_loss_mlp": 0.17663574, + "step": 3591, + "time_per_iteration": 2.577848196029663 + }, + { + "auxiliary_loss_clip": 0.01148595, + "auxiliary_loss_mlp": 0.01052212, + "balance_loss_clip": 1.0556283, + "balance_loss_mlp": 1.03573704, + "epoch": 0.1042307469096396, + "flos": 12926728494720.0, + "grad_norm": 3.0905060576944763, + "language_loss": 0.75221306, + "learning_rate": 3.942484167685131e-06, + "loss": 0.77422112, + "num_input_tokens_seen": 101160305, + "router_z_loss_clip": 0.9296875, + "router_z_loss_mlp": 0.16479492, + "step": 3592, + "time_per_iteration": 2.499276638031006 + }, + { + "auxiliary_loss_clip": 0.0114792, + "auxiliary_loss_mlp": 0.01048831, + "balance_loss_clip": 1.056916, + "balance_loss_mlp": 1.03178406, + "epoch": 0.10425976437815565, + "flos": 15152579510400.0, + "grad_norm": 2.4773666649644706, + "language_loss": 0.86238629, + "learning_rate": 3.9424394065180315e-06, + "loss": 0.88435376, + "num_input_tokens_seen": 101175425, + "router_z_loss_clip": 0.90966797, + "router_z_loss_mlp": 0.17041016, + "step": 3593, + "time_per_iteration": 2.5153393745422363 + }, + { + "auxiliary_loss_clip": 0.01147212, + "auxiliary_loss_mlp": 0.01047472, + "balance_loss_clip": 1.05972028, + "balance_loss_mlp": 1.02957821, + "epoch": 0.1042887818466717, + "flos": 19087670183040.0, + "grad_norm": 3.7440936339169655, + "language_loss": 1.03816247, + "learning_rate": 3.942394628194522e-06, + "loss": 1.06010926, + "num_input_tokens_seen": 101184420, + "router_z_loss_clip": 0.87402344, + "router_z_loss_mlp": 0.17895508, + "step": 3594, + "time_per_iteration": 2.5664432048797607 + }, + { + "auxiliary_loss_clip": 0.01030695, + "auxiliary_loss_mlp": 0.01016714, + "balance_loss_clip": 1.0075326, + "balance_loss_mlp": 1.01563501, + "epoch": 0.10431779931518774, + "flos": 67696427358720.0, + "grad_norm": 0.6987292219035073, + "language_loss": 0.53548056, + "learning_rate": 3.9423498327149965e-06, + "loss": 0.55595464, + "num_input_tokens_seen": 101248950, + "router_z_loss_clip": 0.23242188, + "router_z_loss_mlp": 0.01080322, + "step": 3595, + "time_per_iteration": 3.235276460647583 + }, + { + "auxiliary_loss_clip": 0.01155621, + "auxiliary_loss_mlp": 0.01049433, + "balance_loss_clip": 1.05977333, + "balance_loss_mlp": 1.03000224, + "epoch": 0.10434681678370379, + "flos": 29418092743680.0, + "grad_norm": 2.3356992932008294, + "language_loss": 0.89988172, + "learning_rate": 3.942305020079852e-06, + "loss": 0.92193228, + "num_input_tokens_seen": 101266210, + "router_z_loss_clip": 0.95751953, + "router_z_loss_mlp": 0.19445801, + "step": 3596, + "time_per_iteration": 2.5855400562286377 + }, + { + "auxiliary_loss_clip": 0.01151373, + "auxiliary_loss_mlp": 0.01047718, + "balance_loss_clip": 1.05889297, + "balance_loss_mlp": 1.03072429, + "epoch": 0.10437583425221984, + "flos": 33686219341440.0, + "grad_norm": 1.8064821642047624, + "language_loss": 0.80044019, + "learning_rate": 3.942260190289483e-06, + "loss": 0.82243109, + "num_input_tokens_seen": 101284895, + "router_z_loss_clip": 0.92480469, + "router_z_loss_mlp": 0.1697998, + "step": 3597, + "time_per_iteration": 2.7248024940490723 + }, + { + "auxiliary_loss_clip": 0.01149801, + "auxiliary_loss_mlp": 0.01044525, + "balance_loss_clip": 1.05710578, + "balance_loss_mlp": 1.02677476, + "epoch": 0.10440485172073588, + "flos": 74734161056640.0, + "grad_norm": 2.1210901368915884, + "language_loss": 0.7006197, + "learning_rate": 3.9422153433442854e-06, + "loss": 0.72256297, + "num_input_tokens_seen": 101307575, + "router_z_loss_clip": 0.92724609, + "router_z_loss_mlp": 0.17749023, + "step": 3598, + "time_per_iteration": 2.952883720397949 + }, + { + "auxiliary_loss_clip": 0.01031744, + "auxiliary_loss_mlp": 0.01007626, + "balance_loss_clip": 1.0089345, + "balance_loss_mlp": 1.0064702, + "epoch": 0.10443386918925193, + "flos": 69540079708800.0, + "grad_norm": 0.8615490651346951, + "language_loss": 0.50477898, + "learning_rate": 3.9421704792446565e-06, + "loss": 0.52517271, + "num_input_tokens_seen": 101364095, + "router_z_loss_clip": 0.22851562, + "router_z_loss_mlp": 0.01153564, + "step": 3599, + "time_per_iteration": 3.055358648300171 + }, + { + "auxiliary_loss_clip": 0.01155399, + "auxiliary_loss_mlp": 0.01057706, + "balance_loss_clip": 1.06000209, + "balance_loss_mlp": 1.03746414, + "epoch": 0.10446288665776797, + "flos": 34745332786560.0, + "grad_norm": 2.250081065455935, + "language_loss": 0.88759124, + "learning_rate": 3.9421255979909925e-06, + "loss": 0.90972233, + "num_input_tokens_seen": 101381965, + "router_z_loss_clip": 0.95410156, + "router_z_loss_mlp": 0.20239258, + "step": 3600, + "time_per_iteration": 2.698608636856079 + }, + { + "auxiliary_loss_clip": 0.01154392, + "auxiliary_loss_mlp": 0.01051865, + "balance_loss_clip": 1.05808735, + "balance_loss_mlp": 1.03189707, + "epoch": 0.10449190412628402, + "flos": 25513597480320.0, + "grad_norm": 2.503738576127889, + "language_loss": 0.90742809, + "learning_rate": 3.942080699583689e-06, + "loss": 0.92949063, + "num_input_tokens_seen": 101401785, + "router_z_loss_clip": 0.96337891, + "router_z_loss_mlp": 0.1998291, + "step": 3601, + "time_per_iteration": 2.732361316680908 + }, + { + "auxiliary_loss_clip": 0.01030473, + "auxiliary_loss_mlp": 0.01000144, + "balance_loss_clip": 1.00795174, + "balance_loss_mlp": 0.99893421, + "epoch": 0.10452092159480007, + "flos": 58713480627840.0, + "grad_norm": 0.7292333827783724, + "language_loss": 0.4869549, + "learning_rate": 3.9420357840231425e-06, + "loss": 0.5072611, + "num_input_tokens_seen": 101460735, + "router_z_loss_clip": 0.22558594, + "router_z_loss_mlp": 0.01208496, + "step": 3602, + "time_per_iteration": 3.2036139965057373 + }, + { + "auxiliary_loss_clip": 0.01031369, + "auxiliary_loss_mlp": 0.01005898, + "balance_loss_clip": 1.00867152, + "balance_loss_mlp": 1.00467622, + "epoch": 0.10454993906331611, + "flos": 74782105712640.0, + "grad_norm": 0.6913189356771448, + "language_loss": 0.47286579, + "learning_rate": 3.94199085130975e-06, + "loss": 0.49323845, + "num_input_tokens_seen": 101530365, + "router_z_loss_clip": 0.2265625, + "router_z_loss_mlp": 0.01220703, + "step": 3603, + "time_per_iteration": 3.2688424587249756 + }, + { + "auxiliary_loss_clip": 0.01031135, + "auxiliary_loss_mlp": 0.01007989, + "balance_loss_clip": 1.00856519, + "balance_loss_mlp": 1.00680304, + "epoch": 0.10457895653183216, + "flos": 66526601218560.0, + "grad_norm": 0.8360949337079198, + "language_loss": 0.44716659, + "learning_rate": 3.9419459014439095e-06, + "loss": 0.46755788, + "num_input_tokens_seen": 101593505, + "router_z_loss_clip": 0.22558594, + "router_z_loss_mlp": 0.01184082, + "step": 3604, + "time_per_iteration": 3.136765956878662 + }, + { + "auxiliary_loss_clip": 0.01151183, + "auxiliary_loss_mlp": 0.01043518, + "balance_loss_clip": 1.0597508, + "balance_loss_mlp": 1.02662659, + "epoch": 0.10460797400034821, + "flos": 58351570168320.0, + "grad_norm": 1.9403053028248665, + "language_loss": 0.62235296, + "learning_rate": 3.941900934426017e-06, + "loss": 0.64429998, + "num_input_tokens_seen": 101615285, + "router_z_loss_clip": 0.9140625, + "router_z_loss_mlp": 0.16906738, + "step": 3605, + "time_per_iteration": 2.7702839374542236 + }, + { + "auxiliary_loss_clip": 0.01151433, + "auxiliary_loss_mlp": 0.01044612, + "balance_loss_clip": 1.05877829, + "balance_loss_mlp": 1.0259912, + "epoch": 0.10463699146886425, + "flos": 30293451187200.0, + "grad_norm": 1.7382862003635224, + "language_loss": 0.71846128, + "learning_rate": 3.941855950256468e-06, + "loss": 0.74042171, + "num_input_tokens_seen": 101631805, + "router_z_loss_clip": 0.92626953, + "router_z_loss_mlp": 0.1862793, + "step": 3606, + "time_per_iteration": 2.605236768722534 + }, + { + "auxiliary_loss_clip": 0.01145448, + "auxiliary_loss_mlp": 0.01051368, + "balance_loss_clip": 1.05789423, + "balance_loss_mlp": 1.03609705, + "epoch": 0.1046660089373803, + "flos": 13072489885440.0, + "grad_norm": 2.132242530813434, + "language_loss": 0.62334704, + "learning_rate": 3.941810948935663e-06, + "loss": 0.64531517, + "num_input_tokens_seen": 101644270, + "router_z_loss_clip": 0.87548828, + "router_z_loss_mlp": 0.15283203, + "step": 3607, + "time_per_iteration": 2.499377489089966 + }, + { + "auxiliary_loss_clip": 0.01148157, + "auxiliary_loss_mlp": 0.01051068, + "balance_loss_clip": 1.05769229, + "balance_loss_mlp": 1.0312736, + "epoch": 0.10469502640589635, + "flos": 54628495522560.0, + "grad_norm": 1.9956887792366347, + "language_loss": 0.71221137, + "learning_rate": 3.941765930463997e-06, + "loss": 0.73420364, + "num_input_tokens_seen": 101664870, + "router_z_loss_clip": 0.90429688, + "router_z_loss_mlp": 0.19781494, + "step": 3608, + "time_per_iteration": 2.8362667560577393 + }, + { + "auxiliary_loss_clip": 0.01156183, + "auxiliary_loss_mlp": 0.01058417, + "balance_loss_clip": 1.0614295, + "balance_loss_mlp": 1.0387181, + "epoch": 0.10472404387441239, + "flos": 70246045612800.0, + "grad_norm": 2.074271881246809, + "language_loss": 0.87170243, + "learning_rate": 3.941720894841869e-06, + "loss": 0.89384842, + "num_input_tokens_seen": 101688945, + "router_z_loss_clip": 0.94677734, + "router_z_loss_mlp": 0.19702148, + "step": 3609, + "time_per_iteration": 2.929738998413086 + }, + { + "auxiliary_loss_clip": 0.01151225, + "auxiliary_loss_mlp": 0.01041661, + "balance_loss_clip": 1.05723226, + "balance_loss_mlp": 1.02501369, + "epoch": 0.10475306134292844, + "flos": 43357106327040.0, + "grad_norm": 1.8710998439442479, + "language_loss": 0.74851906, + "learning_rate": 3.941675842069676e-06, + "loss": 0.77044791, + "num_input_tokens_seen": 101709115, + "router_z_loss_clip": 0.94091797, + "router_z_loss_mlp": 0.16650391, + "step": 3610, + "time_per_iteration": 2.711150646209717 + }, + { + "auxiliary_loss_clip": 0.01156915, + "auxiliary_loss_mlp": 0.01058788, + "balance_loss_clip": 1.061221, + "balance_loss_mlp": 1.04002476, + "epoch": 0.1047820788114445, + "flos": 29163414337920.0, + "grad_norm": 2.294646373284882, + "language_loss": 0.95048869, + "learning_rate": 3.9416307721478165e-06, + "loss": 0.9726457, + "num_input_tokens_seen": 101728005, + "router_z_loss_clip": 0.95605469, + "router_z_loss_mlp": 0.18762207, + "step": 3611, + "time_per_iteration": 2.5985772609710693 + }, + { + "auxiliary_loss_clip": 0.01158424, + "auxiliary_loss_mlp": 0.01059509, + "balance_loss_clip": 1.0602926, + "balance_loss_mlp": 1.03923726, + "epoch": 0.10481109627996053, + "flos": 27082031823360.0, + "grad_norm": 2.91352989499771, + "language_loss": 1.00348878, + "learning_rate": 3.941585685076689e-06, + "loss": 1.02566814, + "num_input_tokens_seen": 101743045, + "router_z_loss_clip": 0.98193359, + "router_z_loss_mlp": 0.20263672, + "step": 3612, + "time_per_iteration": 2.625509023666382 + }, + { + "auxiliary_loss_clip": 0.0103553, + "auxiliary_loss_mlp": 0.01023162, + "balance_loss_clip": 1.01299262, + "balance_loss_mlp": 1.02198148, + "epoch": 0.10484011374847658, + "flos": 64012387818240.0, + "grad_norm": 0.7018558022173457, + "language_loss": 0.49006015, + "learning_rate": 3.9415405808566905e-06, + "loss": 0.51064706, + "num_input_tokens_seen": 101807645, + "router_z_loss_clip": 0.22558594, + "router_z_loss_mlp": 0.01177979, + "step": 3613, + "time_per_iteration": 3.152730941772461 + }, + { + "auxiliary_loss_clip": 0.01147178, + "auxiliary_loss_mlp": 0.01045859, + "balance_loss_clip": 1.05855656, + "balance_loss_mlp": 1.02854967, + "epoch": 0.10486913121699262, + "flos": 15699929932800.0, + "grad_norm": 2.090382902945913, + "language_loss": 0.60467213, + "learning_rate": 3.94149545948822e-06, + "loss": 0.62660253, + "num_input_tokens_seen": 101821330, + "router_z_loss_clip": 0.88476562, + "router_z_loss_mlp": 0.17297363, + "step": 3614, + "time_per_iteration": 2.5133814811706543 + }, + { + "auxiliary_loss_clip": 0.01144877, + "auxiliary_loss_mlp": 0.01045782, + "balance_loss_clip": 1.05546904, + "balance_loss_mlp": 1.02940285, + "epoch": 0.10489814868550867, + "flos": 33870584874240.0, + "grad_norm": 2.315135772965066, + "language_loss": 0.75840247, + "learning_rate": 3.941450320971675e-06, + "loss": 0.78030902, + "num_input_tokens_seen": 101838860, + "router_z_loss_clip": 0.89355469, + "router_z_loss_mlp": 0.16375732, + "step": 3615, + "time_per_iteration": 2.687701940536499 + }, + { + "auxiliary_loss_clip": 0.01155698, + "auxiliary_loss_mlp": 0.01052768, + "balance_loss_clip": 1.05837274, + "balance_loss_mlp": 1.03162062, + "epoch": 0.10492716615402473, + "flos": 32809172958720.0, + "grad_norm": 2.330736212961822, + "language_loss": 0.89508629, + "learning_rate": 3.941405165307456e-06, + "loss": 0.91717094, + "num_input_tokens_seen": 101854245, + "router_z_loss_clip": 0.97363281, + "router_z_loss_mlp": 0.21130371, + "step": 3616, + "time_per_iteration": 2.666072130203247 + }, + { + "auxiliary_loss_clip": 0.01036011, + "auxiliary_loss_mlp": 0.01000477, + "balance_loss_clip": 1.0136143, + "balance_loss_mlp": 0.99923116, + "epoch": 0.10495618362254076, + "flos": 67370825548800.0, + "grad_norm": 0.6767366871412155, + "language_loss": 0.51395595, + "learning_rate": 3.941359992495961e-06, + "loss": 0.53432083, + "num_input_tokens_seen": 101915255, + "router_z_loss_clip": 0.22460938, + "router_z_loss_mlp": 0.01245117, + "step": 3617, + "time_per_iteration": 3.122215986251831 + }, + { + "auxiliary_loss_clip": 0.01155854, + "auxiliary_loss_mlp": 0.01040405, + "balance_loss_clip": 1.06131637, + "balance_loss_mlp": 1.02247024, + "epoch": 0.10498520109105682, + "flos": 25371427449600.0, + "grad_norm": 1.7824318403741584, + "language_loss": 0.73546225, + "learning_rate": 3.941314802537589e-06, + "loss": 0.75742489, + "num_input_tokens_seen": 101934760, + "router_z_loss_clip": 0.94482422, + "router_z_loss_mlp": 0.17926025, + "step": 3618, + "time_per_iteration": 2.605785846710205 + }, + { + "auxiliary_loss_clip": 0.01034749, + "auxiliary_loss_mlp": 0.0100023, + "balance_loss_clip": 1.0123651, + "balance_loss_mlp": 0.99904364, + "epoch": 0.10501421855957287, + "flos": 68572288592640.0, + "grad_norm": 0.6276730598024797, + "language_loss": 0.48840359, + "learning_rate": 3.941269595432739e-06, + "loss": 0.50875342, + "num_input_tokens_seen": 101995855, + "router_z_loss_clip": 0.22363281, + "router_z_loss_mlp": 0.01184082, + "step": 3619, + "time_per_iteration": 3.1322855949401855 + }, + { + "auxiliary_loss_clip": 0.01155282, + "auxiliary_loss_mlp": 0.01041574, + "balance_loss_clip": 1.05929446, + "balance_loss_mlp": 1.02265525, + "epoch": 0.1050432360280889, + "flos": 14896823696640.0, + "grad_norm": 2.4059738764099294, + "language_loss": 0.71666771, + "learning_rate": 3.941224371181811e-06, + "loss": 0.73863631, + "num_input_tokens_seen": 102009510, + "router_z_loss_clip": 0.9609375, + "router_z_loss_mlp": 0.18920898, + "step": 3620, + "time_per_iteration": 2.516861915588379 + }, + { + "auxiliary_loss_clip": 0.01145883, + "auxiliary_loss_mlp": 0.01052704, + "balance_loss_clip": 1.05661058, + "balance_loss_mlp": 1.03391695, + "epoch": 0.10507225349660496, + "flos": 13364264062080.0, + "grad_norm": 2.3094951924765117, + "language_loss": 0.7041136, + "learning_rate": 3.9411791297852026e-06, + "loss": 0.72609949, + "num_input_tokens_seen": 102022060, + "router_z_loss_clip": 0.89453125, + "router_z_loss_mlp": 0.18811035, + "step": 3621, + "time_per_iteration": 2.563176393508911 + }, + { + "auxiliary_loss_clip": 0.01142587, + "auxiliary_loss_mlp": 0.01040957, + "balance_loss_clip": 1.05614424, + "balance_loss_mlp": 1.0246495, + "epoch": 0.10510127096512101, + "flos": 45873187234560.0, + "grad_norm": 2.804649055179859, + "language_loss": 0.72613722, + "learning_rate": 3.941133871243315e-06, + "loss": 0.74797267, + "num_input_tokens_seen": 102040600, + "router_z_loss_clip": 0.86376953, + "router_z_loss_mlp": 0.1630249, + "step": 3622, + "time_per_iteration": 2.672102451324463 + }, + { + "auxiliary_loss_clip": 0.01149951, + "auxiliary_loss_mlp": 0.01052037, + "balance_loss_clip": 1.05636835, + "balance_loss_mlp": 1.03298116, + "epoch": 0.10513028843363705, + "flos": 22413106293120.0, + "grad_norm": 2.702059526126481, + "language_loss": 0.81148225, + "learning_rate": 3.941088595556548e-06, + "loss": 0.83350217, + "num_input_tokens_seen": 102057220, + "router_z_loss_clip": 0.93554688, + "router_z_loss_mlp": 0.19042969, + "step": 3623, + "time_per_iteration": 2.684943675994873 + }, + { + "auxiliary_loss_clip": 0.01032881, + "auxiliary_loss_mlp": 0.00998649, + "balance_loss_clip": 1.01053345, + "balance_loss_mlp": 0.99757653, + "epoch": 0.1051593059021531, + "flos": 50579426995200.0, + "grad_norm": 0.7730953227486269, + "language_loss": 0.50046092, + "learning_rate": 3.9410433027253005e-06, + "loss": 0.52077621, + "num_input_tokens_seen": 102105840, + "router_z_loss_clip": 0.22363281, + "router_z_loss_mlp": 0.01074219, + "step": 3624, + "time_per_iteration": 2.9334213733673096 + }, + { + "auxiliary_loss_clip": 0.01031274, + "auxiliary_loss_mlp": 0.00999412, + "balance_loss_clip": 1.00919533, + "balance_loss_mlp": 0.99836904, + "epoch": 0.10518832337066915, + "flos": 67400630858880.0, + "grad_norm": 0.6906038734750083, + "language_loss": 0.49055406, + "learning_rate": 3.9409979927499735e-06, + "loss": 0.51086092, + "num_input_tokens_seen": 102166410, + "router_z_loss_clip": 0.22070312, + "router_z_loss_mlp": 0.01043701, + "step": 3625, + "time_per_iteration": 3.2041773796081543 + }, + { + "auxiliary_loss_clip": 0.01144045, + "auxiliary_loss_mlp": 0.01045785, + "balance_loss_clip": 1.05683565, + "balance_loss_mlp": 1.0292747, + "epoch": 0.10521734083918519, + "flos": 21610933810560.0, + "grad_norm": 2.907785689778368, + "language_loss": 0.84040284, + "learning_rate": 3.9409526656309665e-06, + "loss": 0.86230117, + "num_input_tokens_seen": 102181115, + "router_z_loss_clip": 0.87207031, + "router_z_loss_mlp": 0.16503906, + "step": 3626, + "time_per_iteration": 2.5768802165985107 + }, + { + "auxiliary_loss_clip": 0.01141663, + "auxiliary_loss_mlp": 0.01041548, + "balance_loss_clip": 1.0549953, + "balance_loss_mlp": 1.02484131, + "epoch": 0.10524635830770124, + "flos": 15845044878720.0, + "grad_norm": 2.956393644319753, + "language_loss": 0.9087137, + "learning_rate": 3.94090732136868e-06, + "loss": 0.93054593, + "num_input_tokens_seen": 102192920, + "router_z_loss_clip": 0.86767578, + "router_z_loss_mlp": 0.16705322, + "step": 3627, + "time_per_iteration": 2.538990020751953 + }, + { + "auxiliary_loss_clip": 0.01156398, + "auxiliary_loss_mlp": 0.01044563, + "balance_loss_clip": 1.06013465, + "balance_loss_mlp": 1.02568078, + "epoch": 0.10527537577621729, + "flos": 10700554256640.0, + "grad_norm": 3.0973890060697586, + "language_loss": 0.86951691, + "learning_rate": 3.940861959963516e-06, + "loss": 0.89152658, + "num_input_tokens_seen": 102206195, + "router_z_loss_clip": 0.96240234, + "router_z_loss_mlp": 0.18884277, + "step": 3628, + "time_per_iteration": 2.5120270252227783 + }, + { + "auxiliary_loss_clip": 0.01030081, + "auxiliary_loss_mlp": 0.01008682, + "balance_loss_clip": 1.0078125, + "balance_loss_mlp": 1.00758541, + "epoch": 0.10530439324473333, + "flos": 58469073511680.0, + "grad_norm": 0.6391044608380781, + "language_loss": 0.51439369, + "learning_rate": 3.940816581415872e-06, + "loss": 0.53478134, + "num_input_tokens_seen": 102266180, + "router_z_loss_clip": 0.22265625, + "router_z_loss_mlp": 0.01098633, + "step": 3629, + "time_per_iteration": 8.659159421920776 + }, + { + "auxiliary_loss_clip": 0.01151691, + "auxiliary_loss_mlp": 0.01055564, + "balance_loss_clip": 1.05687141, + "balance_loss_mlp": 1.03581142, + "epoch": 0.10533341071324938, + "flos": 33941400537600.0, + "grad_norm": 2.6420920762990154, + "language_loss": 0.86340022, + "learning_rate": 3.940771185726152e-06, + "loss": 0.88547277, + "num_input_tokens_seen": 102283595, + "router_z_loss_clip": 0.94921875, + "router_z_loss_mlp": 0.1973877, + "step": 3630, + "time_per_iteration": 5.056222915649414 + }, + { + "auxiliary_loss_clip": 0.01149092, + "auxiliary_loss_mlp": 0.01045761, + "balance_loss_clip": 1.05487013, + "balance_loss_mlp": 1.02687812, + "epoch": 0.10536242818176542, + "flos": 12779925609600.0, + "grad_norm": 4.150024356105692, + "language_loss": 1.02265739, + "learning_rate": 3.940725772894754e-06, + "loss": 1.04460597, + "num_input_tokens_seen": 102294810, + "router_z_loss_clip": 0.94140625, + "router_z_loss_mlp": 0.18896484, + "step": 3631, + "time_per_iteration": 5.019115447998047 + }, + { + "auxiliary_loss_clip": 0.01153821, + "auxiliary_loss_mlp": 0.01052955, + "balance_loss_clip": 1.06079459, + "balance_loss_mlp": 1.03568113, + "epoch": 0.10539144565028147, + "flos": 22523208456960.0, + "grad_norm": 4.727246466587651, + "language_loss": 0.88024384, + "learning_rate": 3.940680342922081e-06, + "loss": 0.90231162, + "num_input_tokens_seen": 102309360, + "router_z_loss_clip": 0.93066406, + "router_z_loss_mlp": 0.17272949, + "step": 3632, + "time_per_iteration": 2.615391969680786 + }, + { + "auxiliary_loss_clip": 0.01031615, + "auxiliary_loss_mlp": 0.00999635, + "balance_loss_clip": 1.00941563, + "balance_loss_mlp": 0.99858588, + "epoch": 0.10542046311879752, + "flos": 70440757240320.0, + "grad_norm": 0.6670013885904579, + "language_loss": 0.47966525, + "learning_rate": 3.9406348958085345e-06, + "loss": 0.49997774, + "num_input_tokens_seen": 102372010, + "router_z_loss_clip": 0.22265625, + "router_z_loss_mlp": 0.01049805, + "step": 3633, + "time_per_iteration": 3.1630218029022217 + }, + { + "auxiliary_loss_clip": 0.01141766, + "auxiliary_loss_mlp": 0.01038013, + "balance_loss_clip": 1.0545845, + "balance_loss_mlp": 1.0226289, + "epoch": 0.10544948058731356, + "flos": 42702850051200.0, + "grad_norm": 1.996293793332892, + "language_loss": 0.71510404, + "learning_rate": 3.9405894315545155e-06, + "loss": 0.73690188, + "num_input_tokens_seen": 102391550, + "router_z_loss_clip": 0.87109375, + "router_z_loss_mlp": 0.15380859, + "step": 3634, + "time_per_iteration": 2.8233609199523926 + }, + { + "auxiliary_loss_clip": 0.01152876, + "auxiliary_loss_mlp": 0.01042549, + "balance_loss_clip": 1.05815411, + "balance_loss_mlp": 1.02441692, + "epoch": 0.10547849805582961, + "flos": 33216975043200.0, + "grad_norm": 2.287425428783393, + "language_loss": 0.9218787, + "learning_rate": 3.940543950160426e-06, + "loss": 0.94383299, + "num_input_tokens_seen": 102408190, + "router_z_loss_clip": 0.94677734, + "router_z_loss_mlp": 0.18127441, + "step": 3635, + "time_per_iteration": 2.71286678314209 + }, + { + "auxiliary_loss_clip": 0.01144732, + "auxiliary_loss_mlp": 0.01047331, + "balance_loss_clip": 1.05761218, + "balance_loss_mlp": 1.02978289, + "epoch": 0.10550751552434566, + "flos": 46164135398400.0, + "grad_norm": 1.6767223037646843, + "language_loss": 0.87374878, + "learning_rate": 3.940498451626666e-06, + "loss": 0.89566934, + "num_input_tokens_seen": 102432195, + "router_z_loss_clip": 0.87207031, + "router_z_loss_mlp": 0.17572021, + "step": 3636, + "time_per_iteration": 2.7779645919799805 + }, + { + "auxiliary_loss_clip": 0.01035431, + "auxiliary_loss_mlp": 0.01003845, + "balance_loss_clip": 1.01307535, + "balance_loss_mlp": 1.00286758, + "epoch": 0.1055365329928617, + "flos": 58821858938880.0, + "grad_norm": 0.662659645085608, + "language_loss": 0.46675885, + "learning_rate": 3.940452935953639e-06, + "loss": 0.48715162, + "num_input_tokens_seen": 102488430, + "router_z_loss_clip": 0.22363281, + "router_z_loss_mlp": 0.00976562, + "step": 3637, + "time_per_iteration": 3.0068204402923584 + }, + { + "auxiliary_loss_clip": 0.01152124, + "auxiliary_loss_mlp": 0.01046089, + "balance_loss_clip": 1.0604732, + "balance_loss_mlp": 1.0279094, + "epoch": 0.10556555046137775, + "flos": 74733155475840.0, + "grad_norm": 2.124018208829272, + "language_loss": 0.84355032, + "learning_rate": 3.940407403141745e-06, + "loss": 0.86553246, + "num_input_tokens_seen": 102512410, + "router_z_loss_clip": 0.91699219, + "router_z_loss_mlp": 0.1819458, + "step": 3638, + "time_per_iteration": 2.9828615188598633 + }, + { + "auxiliary_loss_clip": 0.01145602, + "auxiliary_loss_mlp": 0.01044089, + "balance_loss_clip": 1.05666256, + "balance_loss_mlp": 1.0279901, + "epoch": 0.1055945679298938, + "flos": 36724083166080.0, + "grad_norm": 2.3040263221725734, + "language_loss": 0.86901283, + "learning_rate": 3.940361853191389e-06, + "loss": 0.89090967, + "num_input_tokens_seen": 102534150, + "router_z_loss_clip": 0.88916016, + "router_z_loss_mlp": 0.16088867, + "step": 3639, + "time_per_iteration": 2.6846303939819336 + }, + { + "auxiliary_loss_clip": 0.0115114, + "auxiliary_loss_mlp": 0.01050631, + "balance_loss_clip": 1.0558629, + "balance_loss_mlp": 1.03029966, + "epoch": 0.10562358539840984, + "flos": 34965321632640.0, + "grad_norm": 1.8907487402026302, + "language_loss": 0.93334359, + "learning_rate": 3.9403162861029715e-06, + "loss": 0.95536131, + "num_input_tokens_seen": 102556800, + "router_z_loss_clip": 0.95263672, + "router_z_loss_mlp": 0.20330811, + "step": 3640, + "time_per_iteration": 2.8529365062713623 + }, + { + "auxiliary_loss_clip": 0.0115098, + "auxiliary_loss_mlp": 0.01039885, + "balance_loss_clip": 1.05921197, + "balance_loss_mlp": 1.02223063, + "epoch": 0.10565260286692589, + "flos": 15408155756160.0, + "grad_norm": 2.612255174474213, + "language_loss": 0.84790802, + "learning_rate": 3.940270701876896e-06, + "loss": 0.86981672, + "num_input_tokens_seen": 102570985, + "router_z_loss_clip": 0.91748047, + "router_z_loss_mlp": 0.17657471, + "step": 3641, + "time_per_iteration": 2.626237154006958 + }, + { + "auxiliary_loss_clip": 0.0115549, + "auxiliary_loss_mlp": 0.01052006, + "balance_loss_clip": 1.06100512, + "balance_loss_mlp": 1.03259277, + "epoch": 0.10568162033544194, + "flos": 40217939170560.0, + "grad_norm": 2.044734344708031, + "language_loss": 0.7492106, + "learning_rate": 3.940225100513564e-06, + "loss": 0.77128553, + "num_input_tokens_seen": 102593440, + "router_z_loss_clip": 0.94433594, + "router_z_loss_mlp": 0.1940918, + "step": 3642, + "time_per_iteration": 2.7922842502593994 + }, + { + "auxiliary_loss_clip": 0.0103819, + "auxiliary_loss_mlp": 0.01017293, + "balance_loss_clip": 1.01543927, + "balance_loss_mlp": 1.01621437, + "epoch": 0.10571063780395798, + "flos": 74775353955840.0, + "grad_norm": 0.7159444525318434, + "language_loss": 0.47743961, + "learning_rate": 3.940179482013378e-06, + "loss": 0.49799442, + "num_input_tokens_seen": 102652835, + "router_z_loss_clip": 0.22753906, + "router_z_loss_mlp": 0.01080322, + "step": 3643, + "time_per_iteration": 3.163616895675659 + }, + { + "auxiliary_loss_clip": 0.01038237, + "auxiliary_loss_mlp": 0.01011174, + "balance_loss_clip": 1.0155561, + "balance_loss_mlp": 1.00998807, + "epoch": 0.10573965527247403, + "flos": 61451310147840.0, + "grad_norm": 0.6955500311017858, + "language_loss": 0.50360072, + "learning_rate": 3.940133846376742e-06, + "loss": 0.52409488, + "num_input_tokens_seen": 102710055, + "router_z_loss_clip": 0.2265625, + "router_z_loss_mlp": 0.01184082, + "step": 3644, + "time_per_iteration": 3.054140090942383 + }, + { + "auxiliary_loss_clip": 0.01037226, + "auxiliary_loss_mlp": 0.01004888, + "balance_loss_clip": 1.01457691, + "balance_loss_mlp": 1.0036658, + "epoch": 0.10576867274099008, + "flos": 74780058637440.0, + "grad_norm": 0.6753701393452886, + "language_loss": 0.52065533, + "learning_rate": 3.94008819360406e-06, + "loss": 0.54107648, + "num_input_tokens_seen": 102775945, + "router_z_loss_clip": 0.2265625, + "router_z_loss_mlp": 0.01220703, + "step": 3645, + "time_per_iteration": 3.210662364959717 + }, + { + "auxiliary_loss_clip": 0.01152593, + "auxiliary_loss_mlp": 0.01055717, + "balance_loss_clip": 1.05843472, + "balance_loss_mlp": 1.03754973, + "epoch": 0.10579769020950612, + "flos": 21317399867520.0, + "grad_norm": 3.0935291042120956, + "language_loss": 0.86946249, + "learning_rate": 3.940042523695733e-06, + "loss": 0.89154553, + "num_input_tokens_seen": 102790230, + "router_z_loss_clip": 0.94189453, + "router_z_loss_mlp": 0.1817627, + "step": 3646, + "time_per_iteration": 2.605595350265503 + }, + { + "auxiliary_loss_clip": 0.01148861, + "auxiliary_loss_mlp": 0.01047902, + "balance_loss_clip": 1.05747008, + "balance_loss_mlp": 1.03142738, + "epoch": 0.10582670767802217, + "flos": 24272596540800.0, + "grad_norm": 2.002282528764851, + "language_loss": 0.73978579, + "learning_rate": 3.939996836652166e-06, + "loss": 0.76175344, + "num_input_tokens_seen": 102804170, + "router_z_loss_clip": 0.91455078, + "router_z_loss_mlp": 0.16479492, + "step": 3647, + "time_per_iteration": 2.6326963901519775 + }, + { + "auxiliary_loss_clip": 0.01147893, + "auxiliary_loss_mlp": 0.01042324, + "balance_loss_clip": 1.05729508, + "balance_loss_mlp": 1.02439475, + "epoch": 0.10585572514653821, + "flos": 17670061048320.0, + "grad_norm": 1.9179590952275687, + "language_loss": 0.76954365, + "learning_rate": 3.939951132473761e-06, + "loss": 0.79144585, + "num_input_tokens_seen": 102819415, + "router_z_loss_clip": 0.90673828, + "router_z_loss_mlp": 0.17932129, + "step": 3648, + "time_per_iteration": 2.547661066055298 + }, + { + "auxiliary_loss_clip": 0.0114733, + "auxiliary_loss_mlp": 0.01045989, + "balance_loss_clip": 1.05560672, + "balance_loss_mlp": 1.02646244, + "epoch": 0.10588474261505426, + "flos": 29818963503360.0, + "grad_norm": 2.4717820553746663, + "language_loss": 1.05385983, + "learning_rate": 3.939905411160923e-06, + "loss": 1.07579303, + "num_input_tokens_seen": 102837610, + "router_z_loss_clip": 0.91748047, + "router_z_loss_mlp": 0.1953125, + "step": 3649, + "time_per_iteration": 2.6538519859313965 + }, + { + "auxiliary_loss_clip": 0.01146111, + "auxiliary_loss_mlp": 0.01048031, + "balance_loss_clip": 1.05581748, + "balance_loss_mlp": 1.03098977, + "epoch": 0.10591376008357031, + "flos": 29126103085440.0, + "grad_norm": 2.075500800348407, + "language_loss": 0.91848773, + "learning_rate": 3.939859672714056e-06, + "loss": 0.94042915, + "num_input_tokens_seen": 102854605, + "router_z_loss_clip": 0.90429688, + "router_z_loss_mlp": 0.17034912, + "step": 3650, + "time_per_iteration": 2.6071693897247314 + }, + { + "auxiliary_loss_clip": 0.01145031, + "auxiliary_loss_mlp": 0.01042372, + "balance_loss_clip": 1.05607986, + "balance_loss_mlp": 1.02469289, + "epoch": 0.10594277755208635, + "flos": 27665723831040.0, + "grad_norm": 2.4131561784731455, + "language_loss": 0.9180398, + "learning_rate": 3.939813917133563e-06, + "loss": 0.93991387, + "num_input_tokens_seen": 102869930, + "router_z_loss_clip": 0.89013672, + "router_z_loss_mlp": 0.17657471, + "step": 3651, + "time_per_iteration": 2.594381093978882 + }, + { + "auxiliary_loss_clip": 0.0114901, + "auxiliary_loss_mlp": 0.0104379, + "balance_loss_clip": 1.05940962, + "balance_loss_mlp": 1.02623093, + "epoch": 0.1059717950206024, + "flos": 38392204728960.0, + "grad_norm": 4.077076292190356, + "language_loss": 0.9916684, + "learning_rate": 3.939768144419848e-06, + "loss": 1.0135963, + "num_input_tokens_seen": 102884440, + "router_z_loss_clip": 0.89550781, + "router_z_loss_mlp": 0.175354, + "step": 3652, + "time_per_iteration": 2.7115211486816406 + }, + { + "auxiliary_loss_clip": 0.01139127, + "auxiliary_loss_mlp": 0.01041971, + "balance_loss_clip": 1.05156934, + "balance_loss_mlp": 1.02492452, + "epoch": 0.10600081248911845, + "flos": 21207800494080.0, + "grad_norm": 2.9485069746067945, + "language_loss": 0.74816501, + "learning_rate": 3.939722354573317e-06, + "loss": 0.76997596, + "num_input_tokens_seen": 102897590, + "router_z_loss_clip": 0.87451172, + "router_z_loss_mlp": 0.17053223, + "step": 3653, + "time_per_iteration": 2.5873451232910156 + }, + { + "auxiliary_loss_clip": 0.01034852, + "auxiliary_loss_mlp": 0.01008808, + "balance_loss_clip": 1.01201725, + "balance_loss_mlp": 1.00768185, + "epoch": 0.10602982995763449, + "flos": 59776688223360.0, + "grad_norm": 0.6785213277769797, + "language_loss": 0.49674171, + "learning_rate": 3.939676547594373e-06, + "loss": 0.5171783, + "num_input_tokens_seen": 102960385, + "router_z_loss_clip": 0.22851562, + "router_z_loss_mlp": 0.0112915, + "step": 3654, + "time_per_iteration": 3.2106337547302246 + }, + { + "auxiliary_loss_clip": 0.01141963, + "auxiliary_loss_mlp": 0.01051294, + "balance_loss_clip": 1.0535059, + "balance_loss_mlp": 1.03496802, + "epoch": 0.10605884742615054, + "flos": 15303009669120.0, + "grad_norm": 2.2517573890520874, + "language_loss": 0.55696201, + "learning_rate": 3.93963072348342e-06, + "loss": 0.57889462, + "num_input_tokens_seen": 102976680, + "router_z_loss_clip": 0.88427734, + "router_z_loss_mlp": 0.16320801, + "step": 3655, + "time_per_iteration": 2.5707199573516846 + }, + { + "auxiliary_loss_clip": 0.01034095, + "auxiliary_loss_mlp": 0.01000473, + "balance_loss_clip": 1.01124597, + "balance_loss_mlp": 0.99946004, + "epoch": 0.1060878648946666, + "flos": 57547425415680.0, + "grad_norm": 0.7656103056267473, + "language_loss": 0.51347256, + "learning_rate": 3.9395848822408635e-06, + "loss": 0.53381824, + "num_input_tokens_seen": 103032610, + "router_z_loss_clip": 0.22851562, + "router_z_loss_mlp": 0.01013184, + "step": 3656, + "time_per_iteration": 3.0055675506591797 + }, + { + "auxiliary_loss_clip": 0.0114853, + "auxiliary_loss_mlp": 0.01043793, + "balance_loss_clip": 1.05871725, + "balance_loss_mlp": 1.02606678, + "epoch": 0.10611688236318263, + "flos": 34160994334080.0, + "grad_norm": 1.8519574135616828, + "language_loss": 0.73333883, + "learning_rate": 3.939539023867109e-06, + "loss": 0.75526208, + "num_input_tokens_seen": 103051910, + "router_z_loss_clip": 0.8984375, + "router_z_loss_mlp": 0.17712402, + "step": 3657, + "time_per_iteration": 2.685887575149536 + }, + { + "auxiliary_loss_clip": 0.01145351, + "auxiliary_loss_mlp": 0.01038884, + "balance_loss_clip": 1.06080186, + "balance_loss_mlp": 1.02196836, + "epoch": 0.10614589983169868, + "flos": 26935767642240.0, + "grad_norm": 3.2485876842106824, + "language_loss": 0.85940111, + "learning_rate": 3.939493148362561e-06, + "loss": 0.88124347, + "num_input_tokens_seen": 103065945, + "router_z_loss_clip": 0.84472656, + "router_z_loss_mlp": 0.16912842, + "step": 3658, + "time_per_iteration": 2.638036012649536 + }, + { + "auxiliary_loss_clip": 0.01146794, + "auxiliary_loss_mlp": 0.01038864, + "balance_loss_clip": 1.05642319, + "balance_loss_mlp": 1.02256227, + "epoch": 0.10617491730021474, + "flos": 10734525544320.0, + "grad_norm": 2.7764489325339756, + "language_loss": 0.70775884, + "learning_rate": 3.939447255727624e-06, + "loss": 0.72961545, + "num_input_tokens_seen": 103076500, + "router_z_loss_clip": 0.90380859, + "router_z_loss_mlp": 0.1628418, + "step": 3659, + "time_per_iteration": 2.5357022285461426 + }, + { + "auxiliary_loss_clip": 0.01149192, + "auxiliary_loss_mlp": 0.01051336, + "balance_loss_clip": 1.05821741, + "balance_loss_mlp": 1.03283525, + "epoch": 0.10620393476873077, + "flos": 22266052012800.0, + "grad_norm": 2.3833956243640815, + "language_loss": 0.58210212, + "learning_rate": 3.939401345962705e-06, + "loss": 0.60410738, + "num_input_tokens_seen": 103093190, + "router_z_loss_clip": 0.90966797, + "router_z_loss_mlp": 0.18493652, + "step": 3660, + "time_per_iteration": 2.6720893383026123 + }, + { + "auxiliary_loss_clip": 0.01039378, + "auxiliary_loss_mlp": 0.01009109, + "balance_loss_clip": 1.01654887, + "balance_loss_mlp": 1.00800085, + "epoch": 0.10623295223724682, + "flos": 70620741313920.0, + "grad_norm": 0.6494480651017944, + "language_loss": 0.52601194, + "learning_rate": 3.939355419068208e-06, + "loss": 0.54649681, + "num_input_tokens_seen": 103161880, + "router_z_loss_clip": 0.22851562, + "router_z_loss_mlp": 0.0111084, + "step": 3661, + "time_per_iteration": 3.230393409729004 + }, + { + "auxiliary_loss_clip": 0.0114638, + "auxiliary_loss_mlp": 0.01044439, + "balance_loss_clip": 1.05816674, + "balance_loss_mlp": 1.02797663, + "epoch": 0.10626196970576286, + "flos": 27263775663360.0, + "grad_norm": 2.1459635260035195, + "language_loss": 0.92580962, + "learning_rate": 3.939309475044539e-06, + "loss": 0.94771785, + "num_input_tokens_seen": 103178095, + "router_z_loss_clip": 0.88232422, + "router_z_loss_mlp": 0.16455078, + "step": 3662, + "time_per_iteration": 2.634246826171875 + }, + { + "auxiliary_loss_clip": 0.01155305, + "auxiliary_loss_mlp": 0.01057887, + "balance_loss_clip": 1.06203568, + "balance_loss_mlp": 1.03722787, + "epoch": 0.10629098717427891, + "flos": 17706223065600.0, + "grad_norm": 2.3674103737658334, + "language_loss": 0.70088398, + "learning_rate": 3.939263513892105e-06, + "loss": 0.7230159, + "num_input_tokens_seen": 103192020, + "router_z_loss_clip": 0.93310547, + "router_z_loss_mlp": 0.20666504, + "step": 3663, + "time_per_iteration": 2.606966018676758 + }, + { + "auxiliary_loss_clip": 0.01138628, + "auxiliary_loss_mlp": 0.0103568, + "balance_loss_clip": 1.05547261, + "balance_loss_mlp": 1.02002811, + "epoch": 0.10632000464279497, + "flos": 24711065861760.0, + "grad_norm": 2.0527519014615425, + "language_loss": 0.82891417, + "learning_rate": 3.93921753561131e-06, + "loss": 0.85065728, + "num_input_tokens_seen": 103207820, + "router_z_loss_clip": 0.83203125, + "router_z_loss_mlp": 0.15661621, + "step": 3664, + "time_per_iteration": 2.568661689758301 + }, + { + "auxiliary_loss_clip": 0.0114654, + "auxiliary_loss_mlp": 0.01047264, + "balance_loss_clip": 1.0586915, + "balance_loss_mlp": 1.03017497, + "epoch": 0.106349022111311, + "flos": 28911357624960.0, + "grad_norm": 1.8664721189334494, + "language_loss": 0.85223037, + "learning_rate": 3.93917154020256e-06, + "loss": 0.8741684, + "num_input_tokens_seen": 103230010, + "router_z_loss_clip": 0.87841797, + "router_z_loss_mlp": 0.17102051, + "step": 3665, + "time_per_iteration": 2.681414842605591 + }, + { + "auxiliary_loss_clip": 0.01040238, + "auxiliary_loss_mlp": 0.01008988, + "balance_loss_clip": 1.01714468, + "balance_loss_mlp": 1.00791502, + "epoch": 0.10637803957982706, + "flos": 59261477495040.0, + "grad_norm": 0.908894937419008, + "language_loss": 0.52981615, + "learning_rate": 3.939125527666264e-06, + "loss": 0.55030847, + "num_input_tokens_seen": 103286485, + "router_z_loss_clip": 0.23046875, + "router_z_loss_mlp": 0.01074219, + "step": 3666, + "time_per_iteration": 3.01106858253479 + }, + { + "auxiliary_loss_clip": 0.01037914, + "auxiliary_loss_mlp": 0.01010079, + "balance_loss_clip": 1.01488745, + "balance_loss_mlp": 1.0090245, + "epoch": 0.1064070570483431, + "flos": 59966584450560.0, + "grad_norm": 0.6538789578802413, + "language_loss": 0.50512272, + "learning_rate": 3.939079498002826e-06, + "loss": 0.52560258, + "num_input_tokens_seen": 103350660, + "router_z_loss_clip": 0.23046875, + "router_z_loss_mlp": 0.01055908, + "step": 3667, + "time_per_iteration": 3.131592035293579 + }, + { + "auxiliary_loss_clip": 0.01152, + "auxiliary_loss_mlp": 0.01051232, + "balance_loss_clip": 1.05683589, + "balance_loss_mlp": 1.03147936, + "epoch": 0.10643607451685914, + "flos": 27301374224640.0, + "grad_norm": 2.2187438510644615, + "language_loss": 0.85944152, + "learning_rate": 3.939033451212654e-06, + "loss": 0.8814739, + "num_input_tokens_seen": 103365310, + "router_z_loss_clip": 0.95263672, + "router_z_loss_mlp": 0.19763184, + "step": 3668, + "time_per_iteration": 2.5406830310821533 + }, + { + "auxiliary_loss_clip": 0.01035258, + "auxiliary_loss_mlp": 0.01001648, + "balance_loss_clip": 1.01203215, + "balance_loss_mlp": 1.0005213, + "epoch": 0.1064650919853752, + "flos": 67046229319680.0, + "grad_norm": 0.6463302204818228, + "language_loss": 0.51002371, + "learning_rate": 3.938987387296152e-06, + "loss": 0.53039277, + "num_input_tokens_seen": 103430990, + "router_z_loss_clip": 0.23242188, + "router_z_loss_mlp": 0.0112915, + "step": 3669, + "time_per_iteration": 3.141803741455078 + }, + { + "auxiliary_loss_clip": 0.01154156, + "auxiliary_loss_mlp": 0.01048403, + "balance_loss_clip": 1.05955088, + "balance_loss_mlp": 1.0282805, + "epoch": 0.10649410945389125, + "flos": 15639960176640.0, + "grad_norm": 2.8057491366755247, + "language_loss": 0.88485682, + "learning_rate": 3.938941306253731e-06, + "loss": 0.90688235, + "num_input_tokens_seen": 103445470, + "router_z_loss_clip": 0.94628906, + "router_z_loss_mlp": 0.20129395, + "step": 3670, + "time_per_iteration": 2.5701358318328857 + }, + { + "auxiliary_loss_clip": 0.01034687, + "auxiliary_loss_mlp": 0.0100152, + "balance_loss_clip": 1.01154804, + "balance_loss_mlp": 1.0003165, + "epoch": 0.10652312692240729, + "flos": 65627075900160.0, + "grad_norm": 0.6656406251093492, + "language_loss": 0.48148191, + "learning_rate": 3.938895208085794e-06, + "loss": 0.50184405, + "num_input_tokens_seen": 103508490, + "router_z_loss_clip": 0.23144531, + "router_z_loss_mlp": 0.01202393, + "step": 3671, + "time_per_iteration": 3.122408390045166 + }, + { + "auxiliary_loss_clip": 0.01150533, + "auxiliary_loss_mlp": 0.01047718, + "balance_loss_clip": 1.06113935, + "balance_loss_mlp": 1.03024733, + "epoch": 0.10655214439092334, + "flos": 17816253402240.0, + "grad_norm": 2.8870068552844037, + "language_loss": 0.70908111, + "learning_rate": 3.938849092792751e-06, + "loss": 0.7310636, + "num_input_tokens_seen": 103522475, + "router_z_loss_clip": 0.89453125, + "router_z_loss_mlp": 0.17456055, + "step": 3672, + "time_per_iteration": 2.5173215866088867 + }, + { + "auxiliary_loss_clip": 0.01156951, + "auxiliary_loss_mlp": 0.01059761, + "balance_loss_clip": 1.06181204, + "balance_loss_mlp": 1.03886366, + "epoch": 0.10658116185943939, + "flos": 16587929963520.0, + "grad_norm": 2.5334159052582566, + "language_loss": 0.80558074, + "learning_rate": 3.938802960375008e-06, + "loss": 0.82774782, + "num_input_tokens_seen": 103538580, + "router_z_loss_clip": 0.95214844, + "router_z_loss_mlp": 0.20910645, + "step": 3673, + "time_per_iteration": 2.5420494079589844 + }, + { + "auxiliary_loss_clip": 0.01034255, + "auxiliary_loss_mlp": 0.00998409, + "balance_loss_clip": 1.01103377, + "balance_loss_mlp": 0.99724638, + "epoch": 0.10661017932795543, + "flos": 71445033596160.0, + "grad_norm": 0.7052667407419642, + "language_loss": 0.44859499, + "learning_rate": 3.938756810832972e-06, + "loss": 0.4689216, + "num_input_tokens_seen": 103589745, + "router_z_loss_clip": 0.23242188, + "router_z_loss_mlp": 0.01159668, + "step": 3674, + "time_per_iteration": 2.9993436336517334 + }, + { + "auxiliary_loss_clip": 0.0103382, + "auxiliary_loss_mlp": 0.00999727, + "balance_loss_clip": 1.01070368, + "balance_loss_mlp": 0.99863642, + "epoch": 0.10663919679647148, + "flos": 69457164140160.0, + "grad_norm": 0.6355989040551091, + "language_loss": 0.53117502, + "learning_rate": 3.938710644167052e-06, + "loss": 0.55151045, + "num_input_tokens_seen": 103656555, + "router_z_loss_clip": 0.23144531, + "router_z_loss_mlp": 0.01092529, + "step": 3675, + "time_per_iteration": 3.156897783279419 + }, + { + "auxiliary_loss_clip": 0.01154348, + "auxiliary_loss_mlp": 0.01061716, + "balance_loss_clip": 1.05967712, + "balance_loss_mlp": 1.04227257, + "epoch": 0.10666821426498753, + "flos": 29346594721920.0, + "grad_norm": 2.0938411249086837, + "language_loss": 1.06952977, + "learning_rate": 3.938664460377655e-06, + "loss": 1.09169042, + "num_input_tokens_seen": 103674730, + "router_z_loss_clip": 0.94775391, + "router_z_loss_mlp": 0.19433594, + "step": 3676, + "time_per_iteration": 2.6135499477386475 + }, + { + "auxiliary_loss_clip": 0.01143849, + "auxiliary_loss_mlp": 0.01036777, + "balance_loss_clip": 1.05598855, + "balance_loss_mlp": 1.02024877, + "epoch": 0.10669723173350357, + "flos": 33174492232320.0, + "grad_norm": 2.1740121421009624, + "language_loss": 0.88095921, + "learning_rate": 3.93861825946519e-06, + "loss": 0.90276551, + "num_input_tokens_seen": 103690645, + "router_z_loss_clip": 0.87939453, + "router_z_loss_mlp": 0.1652832, + "step": 3677, + "time_per_iteration": 2.638047218322754 + }, + { + "auxiliary_loss_clip": 0.01148315, + "auxiliary_loss_mlp": 0.0105352, + "balance_loss_clip": 1.05943871, + "balance_loss_mlp": 1.03615117, + "epoch": 0.10672624920201962, + "flos": 39050914291200.0, + "grad_norm": 8.25899169830328, + "language_loss": 0.84701324, + "learning_rate": 3.938572041430063e-06, + "loss": 0.86903167, + "num_input_tokens_seen": 103706385, + "router_z_loss_clip": 0.88964844, + "router_z_loss_mlp": 0.17370605, + "step": 3678, + "time_per_iteration": 2.720946788787842 + }, + { + "auxiliary_loss_clip": 0.01154656, + "auxiliary_loss_mlp": 0.01052176, + "balance_loss_clip": 1.059003, + "balance_loss_mlp": 1.03112328, + "epoch": 0.10675526667053566, + "flos": 29893047304320.0, + "grad_norm": 1.977437954221035, + "language_loss": 0.92460775, + "learning_rate": 3.938525806272682e-06, + "loss": 0.94667608, + "num_input_tokens_seen": 103722620, + "router_z_loss_clip": 0.95556641, + "router_z_loss_mlp": 0.21063232, + "step": 3679, + "time_per_iteration": 2.646696090698242 + }, + { + "auxiliary_loss_clip": 0.01146054, + "auxiliary_loss_mlp": 0.01047762, + "balance_loss_clip": 1.05433631, + "balance_loss_mlp": 1.02985644, + "epoch": 0.10678428413905171, + "flos": 19784660664960.0, + "grad_norm": 2.5334585645474825, + "language_loss": 0.83757371, + "learning_rate": 3.938479553993458e-06, + "loss": 0.85951185, + "num_input_tokens_seen": 103735605, + "router_z_loss_clip": 0.91796875, + "router_z_loss_mlp": 0.17932129, + "step": 3680, + "time_per_iteration": 2.601691722869873 + }, + { + "auxiliary_loss_clip": 0.01158157, + "auxiliary_loss_mlp": 0.01051063, + "balance_loss_clip": 1.0615238, + "balance_loss_mlp": 1.03208458, + "epoch": 0.10681330160756776, + "flos": 26794208142720.0, + "grad_norm": 2.4337351804333798, + "language_loss": 0.89304107, + "learning_rate": 3.938433284592799e-06, + "loss": 0.91513324, + "num_input_tokens_seen": 103750330, + "router_z_loss_clip": 0.96630859, + "router_z_loss_mlp": 0.18963623, + "step": 3681, + "time_per_iteration": 2.581796169281006 + }, + { + "auxiliary_loss_clip": 0.01035611, + "auxiliary_loss_mlp": 0.01015897, + "balance_loss_clip": 1.01255631, + "balance_loss_mlp": 1.01478791, + "epoch": 0.1068423190760838, + "flos": 65288078317440.0, + "grad_norm": 0.6809203066002129, + "language_loss": 0.54386842, + "learning_rate": 3.938386998071112e-06, + "loss": 0.56438351, + "num_input_tokens_seen": 103813120, + "router_z_loss_clip": 0.23046875, + "router_z_loss_mlp": 0.0111084, + "step": 3682, + "time_per_iteration": 3.133394956588745 + }, + { + "auxiliary_loss_clip": 0.01141906, + "auxiliary_loss_mlp": 0.01043955, + "balance_loss_clip": 1.05778575, + "balance_loss_mlp": 1.02881515, + "epoch": 0.10687133654459985, + "flos": 26682238471680.0, + "grad_norm": 2.214704207933495, + "language_loss": 0.58248985, + "learning_rate": 3.938340694428806e-06, + "loss": 0.60434842, + "num_input_tokens_seen": 103828230, + "router_z_loss_clip": 0.84082031, + "router_z_loss_mlp": 0.15124512, + "step": 3683, + "time_per_iteration": 2.6526107788085938 + }, + { + "auxiliary_loss_clip": 0.01145861, + "auxiliary_loss_mlp": 0.0104435, + "balance_loss_clip": 1.05745864, + "balance_loss_mlp": 1.02607489, + "epoch": 0.1069003540131159, + "flos": 25222002871680.0, + "grad_norm": 3.1034508128369227, + "language_loss": 0.84339857, + "learning_rate": 3.938294373666291e-06, + "loss": 0.86530066, + "num_input_tokens_seen": 103842790, + "router_z_loss_clip": 0.88378906, + "router_z_loss_mlp": 0.18273926, + "step": 3684, + "time_per_iteration": 2.6494393348693848 + }, + { + "auxiliary_loss_clip": 0.0114912, + "auxiliary_loss_mlp": 0.01052389, + "balance_loss_clip": 1.05502772, + "balance_loss_mlp": 1.03237343, + "epoch": 0.10692937148163194, + "flos": 16320286748160.0, + "grad_norm": 2.267272304483733, + "language_loss": 0.88704294, + "learning_rate": 3.938248035783976e-06, + "loss": 0.90905803, + "num_input_tokens_seen": 103855400, + "router_z_loss_clip": 0.94140625, + "router_z_loss_mlp": 0.20007324, + "step": 3685, + "time_per_iteration": 2.5614352226257324 + }, + { + "auxiliary_loss_clip": 0.01147521, + "auxiliary_loss_mlp": 0.01045422, + "balance_loss_clip": 1.05559659, + "balance_loss_mlp": 1.02723014, + "epoch": 0.10695838895014799, + "flos": 16793948419200.0, + "grad_norm": 2.673899882296856, + "language_loss": 0.77168047, + "learning_rate": 3.9382016807822705e-06, + "loss": 0.79360986, + "num_input_tokens_seen": 103869190, + "router_z_loss_clip": 0.91894531, + "router_z_loss_mlp": 0.1817627, + "step": 3686, + "time_per_iteration": 2.5335044860839844 + }, + { + "auxiliary_loss_clip": 0.01141904, + "auxiliary_loss_mlp": 0.01036137, + "balance_loss_clip": 1.05497074, + "balance_loss_mlp": 1.01905382, + "epoch": 0.10698740641866404, + "flos": 16503467132160.0, + "grad_norm": 2.6424153904175927, + "language_loss": 0.80623078, + "learning_rate": 3.938155308661583e-06, + "loss": 0.82801121, + "num_input_tokens_seen": 103882870, + "router_z_loss_clip": 0.86914062, + "router_z_loss_mlp": 0.17089844, + "step": 3687, + "time_per_iteration": 2.593644618988037 + }, + { + "auxiliary_loss_clip": 0.01152523, + "auxiliary_loss_mlp": 0.01048909, + "balance_loss_clip": 1.0558567, + "balance_loss_mlp": 1.02923989, + "epoch": 0.10701642388718008, + "flos": 24092181504000.0, + "grad_norm": 3.545453604470376, + "language_loss": 0.89879751, + "learning_rate": 3.938108919422323e-06, + "loss": 0.92081183, + "num_input_tokens_seen": 103899560, + "router_z_loss_clip": 0.96630859, + "router_z_loss_mlp": 0.19665527, + "step": 3688, + "time_per_iteration": 2.603346824645996 + }, + { + "auxiliary_loss_clip": 0.01142491, + "auxiliary_loss_mlp": 0.01036044, + "balance_loss_clip": 1.05461538, + "balance_loss_mlp": 1.01706588, + "epoch": 0.10704544135569613, + "flos": 27264565762560.0, + "grad_norm": 2.4027955057206567, + "language_loss": 0.91484118, + "learning_rate": 3.938062513064902e-06, + "loss": 0.93662649, + "num_input_tokens_seen": 103912745, + "router_z_loss_clip": 0.87841797, + "router_z_loss_mlp": 0.18994141, + "step": 3689, + "time_per_iteration": 2.638329029083252 + }, + { + "auxiliary_loss_clip": 0.01139062, + "auxiliary_loss_mlp": 0.01043646, + "balance_loss_clip": 1.05075264, + "balance_loss_mlp": 1.02557349, + "epoch": 0.10707445882421218, + "flos": 25258272629760.0, + "grad_norm": 3.916921846712344, + "language_loss": 1.00027072, + "learning_rate": 3.938016089589727e-06, + "loss": 1.02209783, + "num_input_tokens_seen": 103930425, + "router_z_loss_clip": 0.8828125, + "router_z_loss_mlp": 0.18078613, + "step": 3690, + "time_per_iteration": 2.6482319831848145 + }, + { + "auxiliary_loss_clip": 0.01146506, + "auxiliary_loss_mlp": 0.01052881, + "balance_loss_clip": 1.05418026, + "balance_loss_mlp": 1.03484464, + "epoch": 0.10710347629272822, + "flos": 37154184618240.0, + "grad_norm": 4.502039881210491, + "language_loss": 0.84585583, + "learning_rate": 3.9379696489972105e-06, + "loss": 0.86784971, + "num_input_tokens_seen": 103948995, + "router_z_loss_clip": 0.92431641, + "router_z_loss_mlp": 0.18029785, + "step": 3691, + "time_per_iteration": 2.650935411453247 + }, + { + "auxiliary_loss_clip": 0.01150864, + "auxiliary_loss_mlp": 0.010493, + "balance_loss_clip": 1.05689418, + "balance_loss_mlp": 1.03044152, + "epoch": 0.10713249376124427, + "flos": 14568025576320.0, + "grad_norm": 2.3803323657070865, + "language_loss": 0.76574636, + "learning_rate": 3.937923191287762e-06, + "loss": 0.78774804, + "num_input_tokens_seen": 103964050, + "router_z_loss_clip": 0.93994141, + "router_z_loss_mlp": 0.1887207, + "step": 3692, + "time_per_iteration": 2.496356964111328 + }, + { + "auxiliary_loss_clip": 0.01042943, + "auxiliary_loss_mlp": 0.0100558, + "balance_loss_clip": 1.01972103, + "balance_loss_mlp": 1.00436389, + "epoch": 0.10716151122976031, + "flos": 60760460891520.0, + "grad_norm": 0.7161867768398438, + "language_loss": 0.46987042, + "learning_rate": 3.937876716461792e-06, + "loss": 0.49035564, + "num_input_tokens_seen": 104016635, + "router_z_loss_clip": 0.23242188, + "router_z_loss_mlp": 0.012146, + "step": 3693, + "time_per_iteration": 3.005584239959717 + }, + { + "auxiliary_loss_clip": 0.01150926, + "auxiliary_loss_mlp": 0.01058204, + "balance_loss_clip": 1.05588222, + "balance_loss_mlp": 1.03898764, + "epoch": 0.10719052869827636, + "flos": 29016539625600.0, + "grad_norm": 3.2495480621832566, + "language_loss": 0.85285503, + "learning_rate": 3.93783022451971e-06, + "loss": 0.87494636, + "num_input_tokens_seen": 104030960, + "router_z_loss_clip": 0.95166016, + "router_z_loss_mlp": 0.19238281, + "step": 3694, + "time_per_iteration": 2.5756192207336426 + }, + { + "auxiliary_loss_clip": 0.01141859, + "auxiliary_loss_mlp": 0.01046947, + "balance_loss_clip": 1.05527556, + "balance_loss_mlp": 1.03010869, + "epoch": 0.10721954616679241, + "flos": 45085164710400.0, + "grad_norm": 2.253855550404315, + "language_loss": 0.62826872, + "learning_rate": 3.937783715461927e-06, + "loss": 0.65015674, + "num_input_tokens_seen": 104054930, + "router_z_loss_clip": 0.86621094, + "router_z_loss_mlp": 0.168396, + "step": 3695, + "time_per_iteration": 2.6784205436706543 + }, + { + "auxiliary_loss_clip": 0.01142991, + "auxiliary_loss_mlp": 0.01037475, + "balance_loss_clip": 1.05386937, + "balance_loss_mlp": 1.02076221, + "epoch": 0.10724856363530845, + "flos": 21794904293760.0, + "grad_norm": 2.1713613285144344, + "language_loss": 0.81122696, + "learning_rate": 3.937737189288855e-06, + "loss": 0.83303165, + "num_input_tokens_seen": 104068060, + "router_z_loss_clip": 0.89111328, + "router_z_loss_mlp": 0.16699219, + "step": 3696, + "time_per_iteration": 2.5335288047790527 + }, + { + "auxiliary_loss_clip": 0.01150029, + "auxiliary_loss_mlp": 0.01046105, + "balance_loss_clip": 1.05889034, + "balance_loss_mlp": 1.02920663, + "epoch": 0.1072775811038245, + "flos": 33540601605120.0, + "grad_norm": 2.4182909887004014, + "language_loss": 0.73546714, + "learning_rate": 3.9376906460009035e-06, + "loss": 0.75742853, + "num_input_tokens_seen": 104083525, + "router_z_loss_clip": 0.91259766, + "router_z_loss_mlp": 0.16900635, + "step": 3697, + "time_per_iteration": 2.6547458171844482 + }, + { + "auxiliary_loss_clip": 0.01139349, + "auxiliary_loss_mlp": 0.01043417, + "balance_loss_clip": 1.05582857, + "balance_loss_mlp": 1.02864683, + "epoch": 0.10730659857234055, + "flos": 21828660099840.0, + "grad_norm": 2.4796718349847153, + "language_loss": 0.81754386, + "learning_rate": 3.937644085598485e-06, + "loss": 0.83937156, + "num_input_tokens_seen": 104098820, + "router_z_loss_clip": 0.8359375, + "router_z_loss_mlp": 0.14764404, + "step": 3698, + "time_per_iteration": 2.547184944152832 + }, + { + "auxiliary_loss_clip": 0.01151234, + "auxiliary_loss_mlp": 0.01059719, + "balance_loss_clip": 1.0583334, + "balance_loss_mlp": 1.04106867, + "epoch": 0.10733561604085659, + "flos": 18667014007680.0, + "grad_norm": 2.3384027770415834, + "language_loss": 0.6627565, + "learning_rate": 3.937597508082008e-06, + "loss": 0.68486595, + "num_input_tokens_seen": 104114770, + "router_z_loss_clip": 0.9296875, + "router_z_loss_mlp": 0.18658447, + "step": 3699, + "time_per_iteration": 5.276695728302002 + }, + { + "auxiliary_loss_clip": 0.01149563, + "auxiliary_loss_mlp": 0.01060617, + "balance_loss_clip": 1.05626869, + "balance_loss_mlp": 1.04089963, + "epoch": 0.10736463350937264, + "flos": 29562633072000.0, + "grad_norm": 2.763111794422646, + "language_loss": 0.90122634, + "learning_rate": 3.937550913451887e-06, + "loss": 0.92332816, + "num_input_tokens_seen": 104131865, + "router_z_loss_clip": 0.93310547, + "router_z_loss_mlp": 0.19714355, + "step": 3700, + "time_per_iteration": 5.095210552215576 + }, + { + "auxiliary_loss_clip": 0.01140374, + "auxiliary_loss_mlp": 0.01048831, + "balance_loss_clip": 1.05635178, + "balance_loss_mlp": 1.03342903, + "epoch": 0.1073936509778887, + "flos": 11902376436480.0, + "grad_norm": 2.1956927384139946, + "language_loss": 0.65162885, + "learning_rate": 3.937504301708532e-06, + "loss": 0.67352092, + "num_input_tokens_seen": 104143640, + "router_z_loss_clip": 0.83935547, + "router_z_loss_mlp": 0.1539917, + "step": 3701, + "time_per_iteration": 4.864248037338257 + }, + { + "auxiliary_loss_clip": 0.01044035, + "auxiliary_loss_mlp": 0.01025537, + "balance_loss_clip": 1.01992297, + "balance_loss_mlp": 1.02421999, + "epoch": 0.10742266844640473, + "flos": 69306410759040.0, + "grad_norm": 0.6663988121656007, + "language_loss": 0.49668065, + "learning_rate": 3.9374576728523555e-06, + "loss": 0.51737636, + "num_input_tokens_seen": 104207295, + "router_z_loss_clip": 0.24023438, + "router_z_loss_mlp": 0.01318359, + "step": 3702, + "time_per_iteration": 3.164001703262329 + }, + { + "auxiliary_loss_clip": 0.01153975, + "auxiliary_loss_mlp": 0.01052459, + "balance_loss_clip": 1.0583899, + "balance_loss_mlp": 1.03279567, + "epoch": 0.10745168591492078, + "flos": 35300656028160.0, + "grad_norm": 2.3263760990405, + "language_loss": 0.69649625, + "learning_rate": 3.937411026883768e-06, + "loss": 0.71856058, + "num_input_tokens_seen": 104226510, + "router_z_loss_clip": 0.95507812, + "router_z_loss_mlp": 0.19647217, + "step": 3703, + "time_per_iteration": 5.170681953430176 + }, + { + "auxiliary_loss_clip": 0.01137506, + "auxiliary_loss_mlp": 0.01044029, + "balance_loss_clip": 1.05342519, + "balance_loss_mlp": 1.02873421, + "epoch": 0.10748070338343683, + "flos": 32991994206720.0, + "grad_norm": 3.068315497737337, + "language_loss": 0.94447541, + "learning_rate": 3.937364363803182e-06, + "loss": 0.96629071, + "num_input_tokens_seen": 104243580, + "router_z_loss_clip": 0.84082031, + "router_z_loss_mlp": 0.15289307, + "step": 3704, + "time_per_iteration": 2.6588144302368164 + }, + { + "auxiliary_loss_clip": 0.01038955, + "auxiliary_loss_mlp": 0.01001882, + "balance_loss_clip": 1.01491714, + "balance_loss_mlp": 1.00067759, + "epoch": 0.10750972085195287, + "flos": 62403014949120.0, + "grad_norm": 0.6710030413383498, + "language_loss": 0.49419618, + "learning_rate": 3.937317683611012e-06, + "loss": 0.51460457, + "num_input_tokens_seen": 104303375, + "router_z_loss_clip": 0.24023438, + "router_z_loss_mlp": 0.01202393, + "step": 3705, + "time_per_iteration": 2.9960756301879883 + }, + { + "auxiliary_loss_clip": 0.0113807, + "auxiliary_loss_mlp": 0.01043182, + "balance_loss_clip": 1.05234385, + "balance_loss_mlp": 1.02592659, + "epoch": 0.10753873832046892, + "flos": 23988076911360.0, + "grad_norm": 2.202214189941213, + "language_loss": 0.92240608, + "learning_rate": 3.937270986307666e-06, + "loss": 0.94421858, + "num_input_tokens_seen": 104319360, + "router_z_loss_clip": 0.85791016, + "router_z_loss_mlp": 0.17272949, + "step": 3706, + "time_per_iteration": 2.570274829864502 + }, + { + "auxiliary_loss_clip": 0.01151545, + "auxiliary_loss_mlp": 0.01042661, + "balance_loss_clip": 1.05796909, + "balance_loss_mlp": 1.02519727, + "epoch": 0.10756775578898498, + "flos": 43389928379520.0, + "grad_norm": 2.380773150242557, + "language_loss": 0.89953804, + "learning_rate": 3.93722427189356e-06, + "loss": 0.92148012, + "num_input_tokens_seen": 104335200, + "router_z_loss_clip": 0.9375, + "router_z_loss_mlp": 0.17456055, + "step": 3707, + "time_per_iteration": 2.737003803253174 + }, + { + "auxiliary_loss_clip": 0.01039507, + "auxiliary_loss_mlp": 0.01008203, + "balance_loss_clip": 1.01527524, + "balance_loss_mlp": 1.00693381, + "epoch": 0.10759677325750101, + "flos": 74780310032640.0, + "grad_norm": 1.1036085042329433, + "language_loss": 0.47874644, + "learning_rate": 3.937177540369105e-06, + "loss": 0.49922353, + "num_input_tokens_seen": 104402575, + "router_z_loss_clip": 0.2421875, + "router_z_loss_mlp": 0.01269531, + "step": 3708, + "time_per_iteration": 3.206796169281006 + }, + { + "auxiliary_loss_clip": 0.01142748, + "auxiliary_loss_mlp": 0.01052568, + "balance_loss_clip": 1.05521333, + "balance_loss_mlp": 1.03502059, + "epoch": 0.10762579072601706, + "flos": 36896633124480.0, + "grad_norm": 2.0815305540308424, + "language_loss": 0.82070374, + "learning_rate": 3.937130791734714e-06, + "loss": 0.84265697, + "num_input_tokens_seen": 104422470, + "router_z_loss_clip": 0.875, + "router_z_loss_mlp": 0.17541504, + "step": 3709, + "time_per_iteration": 2.7180731296539307 + }, + { + "auxiliary_loss_clip": 0.01145487, + "auxiliary_loss_mlp": 0.01051483, + "balance_loss_clip": 1.057109, + "balance_loss_mlp": 1.03455508, + "epoch": 0.1076548081945331, + "flos": 32082089857920.0, + "grad_norm": 3.1621995867075507, + "language_loss": 0.91417152, + "learning_rate": 3.937084025990801e-06, + "loss": 0.93614125, + "num_input_tokens_seen": 104440370, + "router_z_loss_clip": 0.88476562, + "router_z_loss_mlp": 0.16912842, + "step": 3710, + "time_per_iteration": 2.6137099266052246 + }, + { + "auxiliary_loss_clip": 0.01040503, + "auxiliary_loss_mlp": 0.0100187, + "balance_loss_clip": 1.01602387, + "balance_loss_mlp": 1.00052857, + "epoch": 0.10768382566304915, + "flos": 74081775265920.0, + "grad_norm": 0.7322719674345016, + "language_loss": 0.51017511, + "learning_rate": 3.937037243137776e-06, + "loss": 0.53059888, + "num_input_tokens_seen": 104508565, + "router_z_loss_clip": 0.24511719, + "router_z_loss_mlp": 0.01342773, + "step": 3711, + "time_per_iteration": 3.3582446575164795 + }, + { + "auxiliary_loss_clip": 0.01153905, + "auxiliary_loss_mlp": 0.01052732, + "balance_loss_clip": 1.05743194, + "balance_loss_mlp": 1.033921, + "epoch": 0.1077128431315652, + "flos": 11794716397440.0, + "grad_norm": 2.8319287622579603, + "language_loss": 0.84215569, + "learning_rate": 3.936990443176056e-06, + "loss": 0.86422205, + "num_input_tokens_seen": 104520605, + "router_z_loss_clip": 0.96435547, + "router_z_loss_mlp": 0.18817139, + "step": 3712, + "time_per_iteration": 2.46553635597229 + }, + { + "auxiliary_loss_clip": 0.01150865, + "auxiliary_loss_mlp": 0.01054415, + "balance_loss_clip": 1.05765772, + "balance_loss_mlp": 1.03662872, + "epoch": 0.10774186060008124, + "flos": 11976352496640.0, + "grad_norm": 2.8209513131130772, + "language_loss": 1.01818681, + "learning_rate": 3.936943626106052e-06, + "loss": 1.04023957, + "num_input_tokens_seen": 104532375, + "router_z_loss_clip": 0.93261719, + "router_z_loss_mlp": 0.17785645, + "step": 3713, + "time_per_iteration": 2.510324716567993 + }, + { + "auxiliary_loss_clip": 0.01138108, + "auxiliary_loss_mlp": 0.01046852, + "balance_loss_clip": 1.05191469, + "balance_loss_mlp": 1.03000164, + "epoch": 0.1077708780685973, + "flos": 36058011315840.0, + "grad_norm": 2.187775397383666, + "language_loss": 0.90834975, + "learning_rate": 3.936896791928178e-06, + "loss": 0.93019927, + "num_input_tokens_seen": 104551995, + "router_z_loss_clip": 0.86230469, + "router_z_loss_mlp": 0.16845703, + "step": 3714, + "time_per_iteration": 2.660391092300415 + }, + { + "auxiliary_loss_clip": 0.01135948, + "auxiliary_loss_mlp": 0.01045536, + "balance_loss_clip": 1.05325973, + "balance_loss_mlp": 1.02880526, + "epoch": 0.10779989553711335, + "flos": 46856995816320.0, + "grad_norm": 2.071462859016697, + "language_loss": 0.76821214, + "learning_rate": 3.936849940642848e-06, + "loss": 0.79002702, + "num_input_tokens_seen": 104569205, + "router_z_loss_clip": 0.82714844, + "router_z_loss_mlp": 0.1675415, + "step": 3715, + "time_per_iteration": 2.772343397140503 + }, + { + "auxiliary_loss_clip": 0.01136986, + "auxiliary_loss_mlp": 0.01043025, + "balance_loss_clip": 1.05319667, + "balance_loss_mlp": 1.02740848, + "epoch": 0.10782891300562938, + "flos": 74733155475840.0, + "grad_norm": 2.213478376277766, + "language_loss": 0.74376339, + "learning_rate": 3.936803072250475e-06, + "loss": 0.76556349, + "num_input_tokens_seen": 104590840, + "router_z_loss_clip": 0.83789062, + "router_z_loss_mlp": 0.15600586, + "step": 3716, + "time_per_iteration": 2.9715960025787354 + }, + { + "auxiliary_loss_clip": 0.01045142, + "auxiliary_loss_mlp": 0.01018223, + "balance_loss_clip": 1.02074265, + "balance_loss_mlp": 1.01689935, + "epoch": 0.10785793047414544, + "flos": 74774132893440.0, + "grad_norm": 0.6871218830579556, + "language_loss": 0.51721156, + "learning_rate": 3.9367561867514735e-06, + "loss": 0.53784525, + "num_input_tokens_seen": 104647695, + "router_z_loss_clip": 0.24414062, + "router_z_loss_mlp": 0.01324463, + "step": 3717, + "time_per_iteration": 3.1960883140563965 + }, + { + "auxiliary_loss_clip": 0.01145732, + "auxiliary_loss_mlp": 0.0105375, + "balance_loss_clip": 1.05664468, + "balance_loss_mlp": 1.03543913, + "epoch": 0.10788694794266149, + "flos": 16172873331840.0, + "grad_norm": 2.4952329380395404, + "language_loss": 0.79350615, + "learning_rate": 3.936709284146258e-06, + "loss": 0.81550092, + "num_input_tokens_seen": 104659490, + "router_z_loss_clip": 0.890625, + "router_z_loss_mlp": 0.18304443, + "step": 3718, + "time_per_iteration": 2.541921854019165 + }, + { + "auxiliary_loss_clip": 0.01137572, + "auxiliary_loss_mlp": 0.01039116, + "balance_loss_clip": 1.05158806, + "balance_loss_mlp": 1.02226019, + "epoch": 0.10791596541117753, + "flos": 17814960512640.0, + "grad_norm": 2.4742842704131345, + "language_loss": 0.77619153, + "learning_rate": 3.936662364435243e-06, + "loss": 0.79795837, + "num_input_tokens_seen": 104675430, + "router_z_loss_clip": 0.86083984, + "router_z_loss_mlp": 0.16876221, + "step": 3719, + "time_per_iteration": 2.5284972190856934 + }, + { + "auxiliary_loss_clip": 0.01151355, + "auxiliary_loss_mlp": 0.01042073, + "balance_loss_clip": 1.05840909, + "balance_loss_mlp": 1.02589607, + "epoch": 0.10794498287969358, + "flos": 26938497075840.0, + "grad_norm": 2.47640159204012, + "language_loss": 0.81901002, + "learning_rate": 3.936615427618841e-06, + "loss": 0.84094429, + "num_input_tokens_seen": 104691890, + "router_z_loss_clip": 0.92822266, + "router_z_loss_mlp": 0.16174316, + "step": 3720, + "time_per_iteration": 2.6336898803710938 + }, + { + "auxiliary_loss_clip": 0.01149914, + "auxiliary_loss_mlp": 0.01061078, + "balance_loss_clip": 1.05612946, + "balance_loss_mlp": 1.04252958, + "epoch": 0.10797400034820963, + "flos": 28175332037760.0, + "grad_norm": 2.8487037951532757, + "language_loss": 0.98505849, + "learning_rate": 3.936568473697469e-06, + "loss": 1.00716841, + "num_input_tokens_seen": 104707060, + "router_z_loss_clip": 0.93798828, + "router_z_loss_mlp": 0.1854248, + "step": 3721, + "time_per_iteration": 2.625176191329956 + }, + { + "auxiliary_loss_clip": 0.01138524, + "auxiliary_loss_mlp": 0.01040534, + "balance_loss_clip": 1.04950488, + "balance_loss_mlp": 1.02234256, + "epoch": 0.10800301781672567, + "flos": 13837243374720.0, + "grad_norm": 3.053255142103667, + "language_loss": 0.94109607, + "learning_rate": 3.936521502671539e-06, + "loss": 0.96288663, + "num_input_tokens_seen": 104719720, + "router_z_loss_clip": 0.890625, + "router_z_loss_mlp": 0.18188477, + "step": 3722, + "time_per_iteration": 2.562077522277832 + }, + { + "auxiliary_loss_clip": 0.01138991, + "auxiliary_loss_mlp": 0.01051352, + "balance_loss_clip": 1.0543015, + "balance_loss_mlp": 1.03423989, + "epoch": 0.10803203528524172, + "flos": 19133421131520.0, + "grad_norm": 1.5441879212223915, + "language_loss": 0.51943469, + "learning_rate": 3.936474514541469e-06, + "loss": 0.54133815, + "num_input_tokens_seen": 104738840, + "router_z_loss_clip": 0.84667969, + "router_z_loss_mlp": 0.17126465, + "step": 3723, + "time_per_iteration": 2.604236602783203 + }, + { + "auxiliary_loss_clip": 0.01149315, + "auxiliary_loss_mlp": 0.01060254, + "balance_loss_clip": 1.05564308, + "balance_loss_mlp": 1.04159784, + "epoch": 0.10806105275375776, + "flos": 30370515816960.0, + "grad_norm": 2.256850086575289, + "language_loss": 0.73486578, + "learning_rate": 3.936427509307673e-06, + "loss": 0.75696152, + "num_input_tokens_seen": 104754365, + "router_z_loss_clip": 0.93554688, + "router_z_loss_mlp": 0.18652344, + "step": 3724, + "time_per_iteration": 2.612499713897705 + }, + { + "auxiliary_loss_clip": 0.01046382, + "auxiliary_loss_mlp": 0.01002742, + "balance_loss_clip": 1.02157569, + "balance_loss_mlp": 1.00131118, + "epoch": 0.1080900702222738, + "flos": 74781028304640.0, + "grad_norm": 0.6755611910423397, + "language_loss": 0.54476333, + "learning_rate": 3.936380486970564e-06, + "loss": 0.56525457, + "num_input_tokens_seen": 104820535, + "router_z_loss_clip": 0.24804688, + "router_z_loss_mlp": 0.01428223, + "step": 3725, + "time_per_iteration": 3.2044758796691895 + }, + { + "auxiliary_loss_clip": 0.01045534, + "auxiliary_loss_mlp": 0.0100096, + "balance_loss_clip": 1.02073586, + "balance_loss_mlp": 0.99960148, + "epoch": 0.10811908769078986, + "flos": 68429687598720.0, + "grad_norm": 0.6238520550685472, + "language_loss": 0.48737058, + "learning_rate": 3.93633344753056e-06, + "loss": 0.50783551, + "num_input_tokens_seen": 104882720, + "router_z_loss_clip": 0.24804688, + "router_z_loss_mlp": 0.01361084, + "step": 3726, + "time_per_iteration": 3.111827850341797 + }, + { + "auxiliary_loss_clip": 0.01142619, + "auxiliary_loss_mlp": 0.01054359, + "balance_loss_clip": 1.05454159, + "balance_loss_mlp": 1.03684092, + "epoch": 0.1081481051593059, + "flos": 11321306121600.0, + "grad_norm": 3.671144745241637, + "language_loss": 0.811234, + "learning_rate": 3.936286390988076e-06, + "loss": 0.83320379, + "num_input_tokens_seen": 104893095, + "router_z_loss_clip": 0.88037109, + "router_z_loss_mlp": 0.17504883, + "step": 3727, + "time_per_iteration": 2.596580743789673 + }, + { + "auxiliary_loss_clip": 0.01151991, + "auxiliary_loss_mlp": 0.01046224, + "balance_loss_clip": 1.05716968, + "balance_loss_mlp": 1.02576792, + "epoch": 0.10817712262782195, + "flos": 15553845319680.0, + "grad_norm": 2.5897428173212695, + "language_loss": 0.88167143, + "learning_rate": 3.936239317343525e-06, + "loss": 0.90365356, + "num_input_tokens_seen": 104911540, + "router_z_loss_clip": 0.94726562, + "router_z_loss_mlp": 0.20458984, + "step": 3728, + "time_per_iteration": 2.6754636764526367 + }, + { + "auxiliary_loss_clip": 0.01038637, + "auxiliary_loss_mlp": 0.01003772, + "balance_loss_clip": 1.01439118, + "balance_loss_mlp": 1.00251436, + "epoch": 0.108206140096338, + "flos": 63463852247040.0, + "grad_norm": 0.6775623079096803, + "language_loss": 0.49619487, + "learning_rate": 3.936192226597327e-06, + "loss": 0.51661897, + "num_input_tokens_seen": 104969050, + "router_z_loss_clip": 0.2421875, + "router_z_loss_mlp": 0.01257324, + "step": 3729, + "time_per_iteration": 3.0113279819488525 + }, + { + "auxiliary_loss_clip": 0.01141424, + "auxiliary_loss_mlp": 0.01042848, + "balance_loss_clip": 1.05168664, + "balance_loss_mlp": 1.02458489, + "epoch": 0.10823515756485404, + "flos": 12962675030400.0, + "grad_norm": 2.682732864806161, + "language_loss": 0.87795955, + "learning_rate": 3.936145118749894e-06, + "loss": 0.89980227, + "num_input_tokens_seen": 104982655, + "router_z_loss_clip": 0.89746094, + "router_z_loss_mlp": 0.18273926, + "step": 3730, + "time_per_iteration": 2.508857250213623 + }, + { + "auxiliary_loss_clip": 0.01147282, + "auxiliary_loss_mlp": 0.01040931, + "balance_loss_clip": 1.05524349, + "balance_loss_mlp": 1.02166653, + "epoch": 0.10826417503337009, + "flos": 72837359556480.0, + "grad_norm": 1.7085762216738722, + "language_loss": 0.72568458, + "learning_rate": 3.936097993801645e-06, + "loss": 0.7475667, + "num_input_tokens_seen": 105008100, + "router_z_loss_clip": 0.91992188, + "router_z_loss_mlp": 0.19262695, + "step": 3731, + "time_per_iteration": 2.942032814025879 + }, + { + "auxiliary_loss_clip": 0.01143464, + "auxiliary_loss_mlp": 0.01054724, + "balance_loss_clip": 1.05194247, + "balance_loss_mlp": 1.03567457, + "epoch": 0.10829319250188614, + "flos": 30730160741760.0, + "grad_norm": 2.6243238655162133, + "language_loss": 0.84980369, + "learning_rate": 3.9360508517529945e-06, + "loss": 0.87178564, + "num_input_tokens_seen": 105022325, + "router_z_loss_clip": 0.91503906, + "router_z_loss_mlp": 0.19055176, + "step": 3732, + "time_per_iteration": 2.6929314136505127 + }, + { + "auxiliary_loss_clip": 0.01136701, + "auxiliary_loss_mlp": 0.01050224, + "balance_loss_clip": 1.05184197, + "balance_loss_mlp": 1.03017259, + "epoch": 0.10832220997040218, + "flos": 16720331495040.0, + "grad_norm": 2.6736789457608197, + "language_loss": 0.94575948, + "learning_rate": 3.93600369260436e-06, + "loss": 0.96762872, + "num_input_tokens_seen": 105034070, + "router_z_loss_clip": 0.84912109, + "router_z_loss_mlp": 0.20068359, + "step": 3733, + "time_per_iteration": 2.512307643890381 + }, + { + "auxiliary_loss_clip": 0.01033662, + "auxiliary_loss_mlp": 0.0100877, + "balance_loss_clip": 1.00976932, + "balance_loss_mlp": 1.00754189, + "epoch": 0.10835122743891823, + "flos": 70139609614080.0, + "grad_norm": 0.7054637327981801, + "language_loss": 0.4854995, + "learning_rate": 3.9359565163561565e-06, + "loss": 0.50592375, + "num_input_tokens_seen": 105096565, + "router_z_loss_clip": 0.23828125, + "router_z_loss_mlp": 0.01226807, + "step": 3734, + "time_per_iteration": 3.0732181072235107 + }, + { + "auxiliary_loss_clip": 0.01145358, + "auxiliary_loss_mlp": 0.01050599, + "balance_loss_clip": 1.05185008, + "balance_loss_mlp": 1.0314424, + "epoch": 0.10838024490743428, + "flos": 13255131565440.0, + "grad_norm": 2.5917542112691345, + "language_loss": 0.75446737, + "learning_rate": 3.935909323008803e-06, + "loss": 0.77642691, + "num_input_tokens_seen": 105109375, + "router_z_loss_clip": 0.93505859, + "router_z_loss_mlp": 0.19165039, + "step": 3735, + "time_per_iteration": 2.5362932682037354 + }, + { + "auxiliary_loss_clip": 0.01153298, + "auxiliary_loss_mlp": 0.01050116, + "balance_loss_clip": 1.05541635, + "balance_loss_mlp": 1.03069639, + "epoch": 0.10840926237595032, + "flos": 13254700602240.0, + "grad_norm": 2.7193799786975794, + "language_loss": 0.82584119, + "learning_rate": 3.935862112562714e-06, + "loss": 0.8478753, + "num_input_tokens_seen": 105121895, + "router_z_loss_clip": 0.97900391, + "router_z_loss_mlp": 0.1942749, + "step": 3736, + "time_per_iteration": 2.4876999855041504 + }, + { + "auxiliary_loss_clip": 0.01141813, + "auxiliary_loss_mlp": 0.01048036, + "balance_loss_clip": 1.05164731, + "balance_loss_mlp": 1.03057218, + "epoch": 0.10843827984446637, + "flos": 25891019637120.0, + "grad_norm": 1.8155137681363749, + "language_loss": 0.81096959, + "learning_rate": 3.935814885018308e-06, + "loss": 0.8328681, + "num_input_tokens_seen": 105143185, + "router_z_loss_clip": 0.90039062, + "router_z_loss_mlp": 0.17468262, + "step": 3737, + "time_per_iteration": 2.6029109954833984 + }, + { + "auxiliary_loss_clip": 0.0113935, + "auxiliary_loss_mlp": 0.01046736, + "balance_loss_clip": 1.05089331, + "balance_loss_mlp": 1.02892637, + "epoch": 0.10846729731298242, + "flos": 13438491517440.0, + "grad_norm": 2.3373075336455575, + "language_loss": 0.79607266, + "learning_rate": 3.935767640376001e-06, + "loss": 0.81793356, + "num_input_tokens_seen": 105157720, + "router_z_loss_clip": 0.88476562, + "router_z_loss_mlp": 0.17797852, + "step": 3738, + "time_per_iteration": 2.611661434173584 + }, + { + "auxiliary_loss_clip": 0.01037482, + "auxiliary_loss_mlp": 0.0100968, + "balance_loss_clip": 1.01365042, + "balance_loss_mlp": 1.00839221, + "epoch": 0.10849631478149846, + "flos": 62629575984000.0, + "grad_norm": 0.6532503831372846, + "language_loss": 0.53011918, + "learning_rate": 3.935720378636211e-06, + "loss": 0.55059081, + "num_input_tokens_seen": 105224310, + "router_z_loss_clip": 0.23828125, + "router_z_loss_mlp": 0.01287842, + "step": 3739, + "time_per_iteration": 3.180804491043091 + }, + { + "auxiliary_loss_clip": 0.01153642, + "auxiliary_loss_mlp": 0.01047438, + "balance_loss_clip": 1.0560813, + "balance_loss_mlp": 1.02915168, + "epoch": 0.10852533225001451, + "flos": 16027147854720.0, + "grad_norm": 3.0384315520654104, + "language_loss": 0.80998182, + "learning_rate": 3.935673099799355e-06, + "loss": 0.83199257, + "num_input_tokens_seen": 105237730, + "router_z_loss_clip": 0.97558594, + "router_z_loss_mlp": 0.18286133, + "step": 3740, + "time_per_iteration": 2.4945521354675293 + }, + { + "auxiliary_loss_clip": 0.01035998, + "auxiliary_loss_mlp": 0.01005502, + "balance_loss_clip": 1.01213467, + "balance_loss_mlp": 1.00418437, + "epoch": 0.10855434971853055, + "flos": 62811750787200.0, + "grad_norm": 0.6448654169358217, + "language_loss": 0.48287129, + "learning_rate": 3.935625803865852e-06, + "loss": 0.50328624, + "num_input_tokens_seen": 105303805, + "router_z_loss_clip": 0.23828125, + "router_z_loss_mlp": 0.01318359, + "step": 3741, + "time_per_iteration": 3.112370491027832 + }, + { + "auxiliary_loss_clip": 0.01137837, + "auxiliary_loss_mlp": 0.01048082, + "balance_loss_clip": 1.05031264, + "balance_loss_mlp": 1.03099334, + "epoch": 0.1085833671870466, + "flos": 35079015156480.0, + "grad_norm": 1.9234332010573174, + "language_loss": 0.74039483, + "learning_rate": 3.935578490836118e-06, + "loss": 0.762254, + "num_input_tokens_seen": 105325460, + "router_z_loss_clip": 0.875, + "router_z_loss_mlp": 0.17095947, + "step": 3742, + "time_per_iteration": 2.656585931777954 + }, + { + "auxiliary_loss_clip": 0.01138785, + "auxiliary_loss_mlp": 0.01042493, + "balance_loss_clip": 1.05161452, + "balance_loss_mlp": 1.02608323, + "epoch": 0.10861238465556265, + "flos": 34453199473920.0, + "grad_norm": 1.7873412626415177, + "language_loss": 0.84823048, + "learning_rate": 3.935531160710572e-06, + "loss": 0.87004334, + "num_input_tokens_seen": 105344780, + "router_z_loss_clip": 0.87207031, + "router_z_loss_mlp": 0.1640625, + "step": 3743, + "time_per_iteration": 2.6348001956939697 + }, + { + "auxiliary_loss_clip": 0.01145458, + "auxiliary_loss_mlp": 0.01046487, + "balance_loss_clip": 1.05069149, + "balance_loss_mlp": 1.02557755, + "epoch": 0.10864140212407869, + "flos": 25591128986880.0, + "grad_norm": 2.4626053310962113, + "language_loss": 0.98001575, + "learning_rate": 3.93548381348963e-06, + "loss": 1.00193524, + "num_input_tokens_seen": 105361105, + "router_z_loss_clip": 0.94726562, + "router_z_loss_mlp": 0.20898438, + "step": 3744, + "time_per_iteration": 2.5898797512054443 + }, + { + "auxiliary_loss_clip": 0.01032623, + "auxiliary_loss_mlp": 0.01000471, + "balance_loss_clip": 1.00887752, + "balance_loss_mlp": 0.99922526, + "epoch": 0.10867041959259474, + "flos": 49673724537600.0, + "grad_norm": 0.6778950435236187, + "language_loss": 0.46647108, + "learning_rate": 3.935436449173713e-06, + "loss": 0.48680198, + "num_input_tokens_seen": 105419590, + "router_z_loss_clip": 0.23730469, + "router_z_loss_mlp": 0.01245117, + "step": 3745, + "time_per_iteration": 2.9757237434387207 + }, + { + "auxiliary_loss_clip": 0.01147042, + "auxiliary_loss_mlp": 0.01038383, + "balance_loss_clip": 1.05610895, + "balance_loss_mlp": 1.02029884, + "epoch": 0.10869943706111079, + "flos": 28398158058240.0, + "grad_norm": 2.3384434830814023, + "language_loss": 0.76097524, + "learning_rate": 3.935389067763238e-06, + "loss": 0.78282952, + "num_input_tokens_seen": 105442125, + "router_z_loss_clip": 0.90966797, + "router_z_loss_mlp": 0.18066406, + "step": 3746, + "time_per_iteration": 2.726149797439575 + }, + { + "auxiliary_loss_clip": 0.0114987, + "auxiliary_loss_mlp": 0.01046543, + "balance_loss_clip": 1.05281854, + "balance_loss_mlp": 1.02749348, + "epoch": 0.10872845452962683, + "flos": 20007989475840.0, + "grad_norm": 2.342038684128026, + "language_loss": 0.82800019, + "learning_rate": 3.935341669258624e-06, + "loss": 0.84996426, + "num_input_tokens_seen": 105457120, + "router_z_loss_clip": 0.96923828, + "router_z_loss_mlp": 0.19042969, + "step": 3747, + "time_per_iteration": 2.5038959980010986 + }, + { + "auxiliary_loss_clip": 0.01143077, + "auxiliary_loss_mlp": 0.01054991, + "balance_loss_clip": 1.05348253, + "balance_loss_mlp": 1.03681207, + "epoch": 0.10875747199814288, + "flos": 35873502128640.0, + "grad_norm": 2.7470928547779794, + "language_loss": 0.87118679, + "learning_rate": 3.935294253660289e-06, + "loss": 0.8931675, + "num_input_tokens_seen": 105472985, + "router_z_loss_clip": 0.89648438, + "router_z_loss_mlp": 0.18182373, + "step": 3748, + "time_per_iteration": 2.623619794845581 + }, + { + "auxiliary_loss_clip": 0.01146189, + "auxiliary_loss_mlp": 0.01051409, + "balance_loss_clip": 1.05259061, + "balance_loss_mlp": 1.03337216, + "epoch": 0.10878648946665893, + "flos": 20844636036480.0, + "grad_norm": 2.4680491500358577, + "language_loss": 0.80275154, + "learning_rate": 3.935246820968652e-06, + "loss": 0.82472754, + "num_input_tokens_seen": 105487805, + "router_z_loss_clip": 0.93505859, + "router_z_loss_mlp": 0.18048096, + "step": 3749, + "time_per_iteration": 2.5176026821136475 + }, + { + "auxiliary_loss_clip": 0.01140177, + "auxiliary_loss_mlp": 0.01044899, + "balance_loss_clip": 1.04976511, + "balance_loss_mlp": 1.02713692, + "epoch": 0.10881550693517497, + "flos": 10041305990400.0, + "grad_norm": 3.274626419787398, + "language_loss": 1.11402428, + "learning_rate": 3.935199371184131e-06, + "loss": 1.13587499, + "num_input_tokens_seen": 105498220, + "router_z_loss_clip": 0.90478516, + "router_z_loss_mlp": 0.17773438, + "step": 3750, + "time_per_iteration": 2.518602132797241 + }, + { + "auxiliary_loss_clip": 0.01137844, + "auxiliary_loss_mlp": 0.01042637, + "balance_loss_clip": 1.0526104, + "balance_loss_mlp": 1.02468419, + "epoch": 0.10884452440369102, + "flos": 16611917270400.0, + "grad_norm": 3.3441850555298926, + "language_loss": 0.67158496, + "learning_rate": 3.935151904307148e-06, + "loss": 0.69338977, + "num_input_tokens_seen": 105517035, + "router_z_loss_clip": 0.85205078, + "router_z_loss_mlp": 0.17956543, + "step": 3751, + "time_per_iteration": 2.640094041824341 + }, + { + "auxiliary_loss_clip": 0.01139779, + "auxiliary_loss_mlp": 0.01045715, + "balance_loss_clip": 1.05224609, + "balance_loss_mlp": 1.02890658, + "epoch": 0.10887354187220707, + "flos": 25659358871040.0, + "grad_norm": 2.055268488349508, + "language_loss": 0.72078025, + "learning_rate": 3.935104420338118e-06, + "loss": 0.74263525, + "num_input_tokens_seen": 105532425, + "router_z_loss_clip": 0.87548828, + "router_z_loss_mlp": 0.16802979, + "step": 3752, + "time_per_iteration": 2.5816431045532227 + }, + { + "auxiliary_loss_clip": 0.01033728, + "auxiliary_loss_mlp": 0.01000379, + "balance_loss_clip": 1.01036894, + "balance_loss_mlp": 0.99907923, + "epoch": 0.10890255934072311, + "flos": 56970987955200.0, + "grad_norm": 0.6581616565997396, + "language_loss": 0.48220205, + "learning_rate": 3.935056919277464e-06, + "loss": 0.50254315, + "num_input_tokens_seen": 105596165, + "router_z_loss_clip": 0.23339844, + "router_z_loss_mlp": 0.01300049, + "step": 3753, + "time_per_iteration": 3.2721831798553467 + }, + { + "auxiliary_loss_clip": 0.01034099, + "auxiliary_loss_mlp": 0.01003331, + "balance_loss_clip": 1.01047099, + "balance_loss_mlp": 1.00197172, + "epoch": 0.10893157680923916, + "flos": 74777867907840.0, + "grad_norm": 0.6486993071269631, + "language_loss": 0.48857367, + "learning_rate": 3.935009401125604e-06, + "loss": 0.50894797, + "num_input_tokens_seen": 105661025, + "router_z_loss_clip": 0.23632812, + "router_z_loss_mlp": 0.01361084, + "step": 3754, + "time_per_iteration": 3.2517526149749756 + }, + { + "auxiliary_loss_clip": 0.01141964, + "auxiliary_loss_mlp": 0.01039021, + "balance_loss_clip": 1.05285299, + "balance_loss_mlp": 1.02198553, + "epoch": 0.10896059427775522, + "flos": 38250537488640.0, + "grad_norm": 1.5895377679434175, + "language_loss": 0.73530447, + "learning_rate": 3.934961865882959e-06, + "loss": 0.75711429, + "num_input_tokens_seen": 105684205, + "router_z_loss_clip": 0.89208984, + "router_z_loss_mlp": 0.17047119, + "step": 3755, + "time_per_iteration": 2.682354688644409 + }, + { + "auxiliary_loss_clip": 0.01145931, + "auxiliary_loss_mlp": 0.01044717, + "balance_loss_clip": 1.05532873, + "balance_loss_mlp": 1.02770019, + "epoch": 0.10898961174627125, + "flos": 12415683744000.0, + "grad_norm": 3.9783795199429637, + "language_loss": 0.76808947, + "learning_rate": 3.934914313549946e-06, + "loss": 0.78999597, + "num_input_tokens_seen": 105697765, + "router_z_loss_clip": 0.90673828, + "router_z_loss_mlp": 0.17016602, + "step": 3756, + "time_per_iteration": 2.4869191646575928 + }, + { + "auxiliary_loss_clip": 0.01032421, + "auxiliary_loss_mlp": 0.01008658, + "balance_loss_clip": 1.00905156, + "balance_loss_mlp": 1.0074656, + "epoch": 0.1090186292147873, + "flos": 71453724687360.0, + "grad_norm": 0.68165726807573, + "language_loss": 0.48793447, + "learning_rate": 3.934866744126988e-06, + "loss": 0.50834525, + "num_input_tokens_seen": 105755825, + "router_z_loss_clip": 0.234375, + "router_z_loss_mlp": 0.01190186, + "step": 3757, + "time_per_iteration": 3.0891525745391846 + }, + { + "auxiliary_loss_clip": 0.01031905, + "auxiliary_loss_mlp": 0.01007863, + "balance_loss_clip": 1.00865805, + "balance_loss_mlp": 1.00665307, + "epoch": 0.10904764668330334, + "flos": 56972604067200.0, + "grad_norm": 0.6788163129302722, + "language_loss": 0.50952482, + "learning_rate": 3.934819157614504e-06, + "loss": 0.52992254, + "num_input_tokens_seen": 105817065, + "router_z_loss_clip": 0.23242188, + "router_z_loss_mlp": 0.01208496, + "step": 3758, + "time_per_iteration": 3.121248960494995 + }, + { + "auxiliary_loss_clip": 0.0114325, + "auxiliary_loss_mlp": 0.01041046, + "balance_loss_clip": 1.05346704, + "balance_loss_mlp": 1.0233196, + "epoch": 0.1090766641518194, + "flos": 16938273265920.0, + "grad_norm": 2.294682278987424, + "language_loss": 0.70371783, + "learning_rate": 3.934771554012913e-06, + "loss": 0.72556078, + "num_input_tokens_seen": 105832095, + "router_z_loss_clip": 0.89697266, + "router_z_loss_mlp": 0.17724609, + "step": 3759, + "time_per_iteration": 2.5118043422698975 + }, + { + "auxiliary_loss_clip": 0.01152834, + "auxiliary_loss_mlp": 0.010582, + "balance_loss_clip": 1.05817854, + "balance_loss_mlp": 1.03963923, + "epoch": 0.10910568162033545, + "flos": 13397552991360.0, + "grad_norm": 4.202935916887581, + "language_loss": 0.90429366, + "learning_rate": 3.9347239333226375e-06, + "loss": 0.926404, + "num_input_tokens_seen": 105843880, + "router_z_loss_clip": 0.94726562, + "router_z_loss_mlp": 0.18566895, + "step": 3760, + "time_per_iteration": 2.5218489170074463 + }, + { + "auxiliary_loss_clip": 0.01145709, + "auxiliary_loss_mlp": 0.01042523, + "balance_loss_clip": 1.05203199, + "balance_loss_mlp": 1.02445126, + "epoch": 0.10913469908885148, + "flos": 23871007509120.0, + "grad_norm": 2.3480055824450847, + "language_loss": 0.78335106, + "learning_rate": 3.934676295544098e-06, + "loss": 0.80523336, + "num_input_tokens_seen": 105859485, + "router_z_loss_clip": 0.93652344, + "router_z_loss_mlp": 0.18078613, + "step": 3761, + "time_per_iteration": 2.5760602951049805 + }, + { + "auxiliary_loss_clip": 0.01033365, + "auxiliary_loss_mlp": 0.01001569, + "balance_loss_clip": 1.00999165, + "balance_loss_mlp": 1.00038934, + "epoch": 0.10916371655736753, + "flos": 70825143657600.0, + "grad_norm": 0.6739208757845196, + "language_loss": 0.49991715, + "learning_rate": 3.934628640677714e-06, + "loss": 0.52026647, + "num_input_tokens_seen": 105917145, + "router_z_loss_clip": 0.234375, + "router_z_loss_mlp": 0.01177979, + "step": 3762, + "time_per_iteration": 3.0262486934661865 + }, + { + "auxiliary_loss_clip": 0.01138914, + "auxiliary_loss_mlp": 0.01038288, + "balance_loss_clip": 1.05097151, + "balance_loss_mlp": 1.02174163, + "epoch": 0.10919273402588359, + "flos": 13729475594880.0, + "grad_norm": 2.255078897843176, + "language_loss": 0.66890788, + "learning_rate": 3.9345809687239065e-06, + "loss": 0.69067985, + "num_input_tokens_seen": 105930425, + "router_z_loss_clip": 0.87890625, + "router_z_loss_mlp": 0.16552734, + "step": 3763, + "time_per_iteration": 2.4533004760742188 + }, + { + "auxiliary_loss_clip": 0.01149844, + "auxiliary_loss_mlp": 0.01048509, + "balance_loss_clip": 1.05449724, + "balance_loss_mlp": 1.0298295, + "epoch": 0.10922175149439962, + "flos": 74735310291840.0, + "grad_norm": 3.863505729254019, + "language_loss": 0.89395702, + "learning_rate": 3.934533279683098e-06, + "loss": 0.91594052, + "num_input_tokens_seen": 105952955, + "router_z_loss_clip": 0.95361328, + "router_z_loss_mlp": 0.18676758, + "step": 3764, + "time_per_iteration": 2.9409024715423584 + }, + { + "auxiliary_loss_clip": 0.01147998, + "auxiliary_loss_mlp": 0.01045972, + "balance_loss_clip": 1.05349207, + "balance_loss_mlp": 1.02746451, + "epoch": 0.10925076896291568, + "flos": 28541728719360.0, + "grad_norm": 2.475275945325802, + "language_loss": 0.83783704, + "learning_rate": 3.934485573555708e-06, + "loss": 0.85977668, + "num_input_tokens_seen": 105966815, + "router_z_loss_clip": 0.94482422, + "router_z_loss_mlp": 0.18518066, + "step": 3765, + "time_per_iteration": 2.5638015270233154 + }, + { + "auxiliary_loss_clip": 0.01132666, + "auxiliary_loss_mlp": 0.01036859, + "balance_loss_clip": 1.05270672, + "balance_loss_mlp": 1.02122438, + "epoch": 0.10927978643143173, + "flos": 23687396161920.0, + "grad_norm": 1.9514752266251512, + "language_loss": 0.75447404, + "learning_rate": 3.934437850342159e-06, + "loss": 0.7761693, + "num_input_tokens_seen": 105980520, + "router_z_loss_clip": 0.79980469, + "router_z_loss_mlp": 0.15637207, + "step": 3766, + "time_per_iteration": 2.5343236923217773 + }, + { + "auxiliary_loss_clip": 0.01034635, + "auxiliary_loss_mlp": 0.01001137, + "balance_loss_clip": 1.01084781, + "balance_loss_mlp": 0.99972451, + "epoch": 0.10930880389994777, + "flos": 73766085609600.0, + "grad_norm": 0.6659417006505943, + "language_loss": 0.49333781, + "learning_rate": 3.934390110042872e-06, + "loss": 0.51369554, + "num_input_tokens_seen": 106044065, + "router_z_loss_clip": 0.23828125, + "router_z_loss_mlp": 0.01409912, + "step": 3767, + "time_per_iteration": 3.212406635284424 + }, + { + "auxiliary_loss_clip": 0.0114592, + "auxiliary_loss_mlp": 0.010453, + "balance_loss_clip": 1.05607188, + "balance_loss_mlp": 1.02760923, + "epoch": 0.10933782136846382, + "flos": 19274944717440.0, + "grad_norm": 3.074687104796963, + "language_loss": 0.84027016, + "learning_rate": 3.934342352658268e-06, + "loss": 0.86218238, + "num_input_tokens_seen": 106057580, + "router_z_loss_clip": 0.8984375, + "router_z_loss_mlp": 0.17700195, + "step": 3768, + "time_per_iteration": 2.4827070236206055 + }, + { + "auxiliary_loss_clip": 0.0103398, + "auxiliary_loss_mlp": 0.00999882, + "balance_loss_clip": 1.01005411, + "balance_loss_mlp": 0.99856454, + "epoch": 0.10936683883697987, + "flos": 71159687953920.0, + "grad_norm": 0.6388943433446521, + "language_loss": 0.49019393, + "learning_rate": 3.934294578188771e-06, + "loss": 0.51053262, + "num_input_tokens_seen": 106123215, + "router_z_loss_clip": 0.23925781, + "router_z_loss_mlp": 0.01318359, + "step": 3769, + "time_per_iteration": 3.143472194671631 + }, + { + "auxiliary_loss_clip": 0.01136278, + "auxiliary_loss_mlp": 0.01038323, + "balance_loss_clip": 1.05309105, + "balance_loss_mlp": 1.02224731, + "epoch": 0.1093958563054959, + "flos": 15771032904960.0, + "grad_norm": 2.8202027182457754, + "language_loss": 0.82191861, + "learning_rate": 3.934246786634801e-06, + "loss": 0.84366453, + "num_input_tokens_seen": 106134085, + "router_z_loss_clip": 0.83251953, + "router_z_loss_mlp": 0.1607666, + "step": 3770, + "time_per_iteration": 5.135645389556885 + }, + { + "auxiliary_loss_clip": 0.01135963, + "auxiliary_loss_mlp": 0.01043101, + "balance_loss_clip": 1.05192304, + "balance_loss_mlp": 1.02682948, + "epoch": 0.10942487377401196, + "flos": 19018506545280.0, + "grad_norm": 3.9613399968348157, + "language_loss": 0.91684121, + "learning_rate": 3.93419897799678e-06, + "loss": 0.93863189, + "num_input_tokens_seen": 106144795, + "router_z_loss_clip": 0.84082031, + "router_z_loss_mlp": 0.16271973, + "step": 3771, + "time_per_iteration": 4.927319288253784 + }, + { + "auxiliary_loss_clip": 0.01141747, + "auxiliary_loss_mlp": 0.01044168, + "balance_loss_clip": 1.05468535, + "balance_loss_mlp": 1.02819955, + "epoch": 0.109453891242528, + "flos": 21901594665600.0, + "grad_norm": 1.9096392188206786, + "language_loss": 0.68853843, + "learning_rate": 3.934151152275132e-06, + "loss": 0.71039754, + "num_input_tokens_seen": 106160585, + "router_z_loss_clip": 0.87060547, + "router_z_loss_mlp": 0.15960693, + "step": 3772, + "time_per_iteration": 4.965832471847534 + }, + { + "auxiliary_loss_clip": 0.01033517, + "auxiliary_loss_mlp": 0.01012652, + "balance_loss_clip": 1.00946331, + "balance_loss_mlp": 1.01140046, + "epoch": 0.10948290871104405, + "flos": 64999608192000.0, + "grad_norm": 0.6504609478310819, + "language_loss": 0.47453579, + "learning_rate": 3.934103309470278e-06, + "loss": 0.4949975, + "num_input_tokens_seen": 106226915, + "router_z_loss_clip": 0.24023438, + "router_z_loss_mlp": 0.01251221, + "step": 3773, + "time_per_iteration": 5.521310567855835 + }, + { + "auxiliary_loss_clip": 0.01154135, + "auxiliary_loss_mlp": 0.01073225, + "balance_loss_clip": 1.05700445, + "balance_loss_mlp": 1.05171406, + "epoch": 0.1095119261795601, + "flos": 40984596080640.0, + "grad_norm": 2.4893033048097126, + "language_loss": 0.8140527, + "learning_rate": 3.934055449582641e-06, + "loss": 0.83632624, + "num_input_tokens_seen": 106243620, + "router_z_loss_clip": 0.97167969, + "router_z_loss_mlp": 0.21508789, + "step": 3774, + "time_per_iteration": 2.6661620140075684 + }, + { + "auxiliary_loss_clip": 0.01033097, + "auxiliary_loss_mlp": 0.01011579, + "balance_loss_clip": 1.00892246, + "balance_loss_mlp": 1.01033962, + "epoch": 0.10954094364807614, + "flos": 67705980376320.0, + "grad_norm": 0.6604366654463657, + "language_loss": 0.48945868, + "learning_rate": 3.934007572612643e-06, + "loss": 0.50990546, + "num_input_tokens_seen": 106311970, + "router_z_loss_clip": 0.2421875, + "router_z_loss_mlp": 0.01239014, + "step": 3775, + "time_per_iteration": 3.2581913471221924 + }, + { + "auxiliary_loss_clip": 0.01140603, + "auxiliary_loss_mlp": 0.01048216, + "balance_loss_clip": 1.05363536, + "balance_loss_mlp": 1.03132415, + "epoch": 0.10956996111659219, + "flos": 12231497779200.0, + "grad_norm": 3.6990960446122014, + "language_loss": 0.80238128, + "learning_rate": 3.9339596785607074e-06, + "loss": 0.82426941, + "num_input_tokens_seen": 106322875, + "router_z_loss_clip": 0.87011719, + "router_z_loss_mlp": 0.16906738, + "step": 3776, + "time_per_iteration": 2.5464277267456055 + }, + { + "auxiliary_loss_clip": 0.0103358, + "auxiliary_loss_mlp": 0.01005018, + "balance_loss_clip": 1.00934529, + "balance_loss_mlp": 1.00362957, + "epoch": 0.10959897858510824, + "flos": 59920689847680.0, + "grad_norm": 0.697449104708453, + "language_loss": 0.52202988, + "learning_rate": 3.933911767427258e-06, + "loss": 0.54241586, + "num_input_tokens_seen": 106382930, + "router_z_loss_clip": 0.2421875, + "router_z_loss_mlp": 0.01391602, + "step": 3777, + "time_per_iteration": 3.156233787536621 + }, + { + "auxiliary_loss_clip": 0.01146746, + "auxiliary_loss_mlp": 0.01049274, + "balance_loss_clip": 1.0550859, + "balance_loss_mlp": 1.03220356, + "epoch": 0.10962799605362428, + "flos": 30694429687680.0, + "grad_norm": 2.8019210888385344, + "language_loss": 0.97044063, + "learning_rate": 3.9338638392127174e-06, + "loss": 0.99240077, + "num_input_tokens_seen": 106399915, + "router_z_loss_clip": 0.91748047, + "router_z_loss_mlp": 0.1706543, + "step": 3778, + "time_per_iteration": 2.6100029945373535 + }, + { + "auxiliary_loss_clip": 0.01033236, + "auxiliary_loss_mlp": 0.01000504, + "balance_loss_clip": 1.00928378, + "balance_loss_mlp": 0.9990555, + "epoch": 0.10965701352214033, + "flos": 74773809671040.0, + "grad_norm": 0.6525816295733505, + "language_loss": 0.47513899, + "learning_rate": 3.933815893917509e-06, + "loss": 0.49547639, + "num_input_tokens_seen": 106464465, + "router_z_loss_clip": 0.24023438, + "router_z_loss_mlp": 0.01446533, + "step": 3779, + "time_per_iteration": 3.32651686668396 + }, + { + "auxiliary_loss_clip": 0.01033781, + "auxiliary_loss_mlp": 0.00999975, + "balance_loss_clip": 1.00957572, + "balance_loss_mlp": 0.99853879, + "epoch": 0.10968603099065638, + "flos": 57477758987520.0, + "grad_norm": 0.7229759848768389, + "language_loss": 0.49318033, + "learning_rate": 3.9337679315420555e-06, + "loss": 0.51351792, + "num_input_tokens_seen": 106514820, + "router_z_loss_clip": 0.2421875, + "router_z_loss_mlp": 0.01434326, + "step": 3780, + "time_per_iteration": 2.894822359085083 + }, + { + "auxiliary_loss_clip": 0.0114089, + "auxiliary_loss_mlp": 0.01041674, + "balance_loss_clip": 1.05320501, + "balance_loss_mlp": 1.02597427, + "epoch": 0.10971504845917242, + "flos": 12671295903360.0, + "grad_norm": 2.6581015846071496, + "language_loss": 0.84738052, + "learning_rate": 3.9337199520867816e-06, + "loss": 0.86920619, + "num_input_tokens_seen": 106525610, + "router_z_loss_clip": 0.87646484, + "router_z_loss_mlp": 0.15698242, + "step": 3781, + "time_per_iteration": 2.5351247787475586 + }, + { + "auxiliary_loss_clip": 0.01145416, + "auxiliary_loss_mlp": 0.01041339, + "balance_loss_clip": 1.05615664, + "balance_loss_mlp": 1.02399373, + "epoch": 0.10974406592768847, + "flos": 14493869948160.0, + "grad_norm": 2.096454244980676, + "language_loss": 0.82002908, + "learning_rate": 3.93367195555211e-06, + "loss": 0.84189665, + "num_input_tokens_seen": 106537545, + "router_z_loss_clip": 0.89208984, + "router_z_loss_mlp": 0.17346191, + "step": 3782, + "time_per_iteration": 2.5456106662750244 + }, + { + "auxiliary_loss_clip": 0.01142137, + "auxiliary_loss_mlp": 0.01050294, + "balance_loss_clip": 1.05553114, + "balance_loss_mlp": 1.03322363, + "epoch": 0.10977308339620452, + "flos": 10481714645760.0, + "grad_norm": 3.0139972443138596, + "language_loss": 0.94852501, + "learning_rate": 3.933623941938465e-06, + "loss": 0.97044927, + "num_input_tokens_seen": 106549970, + "router_z_loss_clip": 0.86669922, + "router_z_loss_mlp": 0.17089844, + "step": 3783, + "time_per_iteration": 2.502121925354004 + }, + { + "auxiliary_loss_clip": 0.01137791, + "auxiliary_loss_mlp": 0.01040229, + "balance_loss_clip": 1.05298507, + "balance_loss_mlp": 1.02260947, + "epoch": 0.10980210086472056, + "flos": 29308996160640.0, + "grad_norm": 2.29005417966095, + "language_loss": 0.97370607, + "learning_rate": 3.933575911246272e-06, + "loss": 0.99548626, + "num_input_tokens_seen": 106565935, + "router_z_loss_clip": 0.84765625, + "router_z_loss_mlp": 0.1763916, + "step": 3784, + "time_per_iteration": 2.57431697845459 + }, + { + "auxiliary_loss_clip": 0.01142609, + "auxiliary_loss_mlp": 0.01047882, + "balance_loss_clip": 1.05094147, + "balance_loss_mlp": 1.03023314, + "epoch": 0.10983111833323661, + "flos": 42298244277120.0, + "grad_norm": 2.1306321687040852, + "language_loss": 0.89370918, + "learning_rate": 3.933527863475953e-06, + "loss": 0.91561407, + "num_input_tokens_seen": 106586710, + "router_z_loss_clip": 0.91699219, + "router_z_loss_mlp": 0.17657471, + "step": 3785, + "time_per_iteration": 2.6896181106567383 + }, + { + "auxiliary_loss_clip": 0.01148538, + "auxiliary_loss_mlp": 0.01058983, + "balance_loss_clip": 1.05586648, + "balance_loss_mlp": 1.0404582, + "epoch": 0.10986013580175266, + "flos": 12048209654400.0, + "grad_norm": 2.9919932834649003, + "language_loss": 0.96057659, + "learning_rate": 3.933479798627935e-06, + "loss": 0.98265177, + "num_input_tokens_seen": 106596275, + "router_z_loss_clip": 0.92626953, + "router_z_loss_mlp": 0.18530273, + "step": 3786, + "time_per_iteration": 2.5117580890655518 + }, + { + "auxiliary_loss_clip": 0.01156864, + "auxiliary_loss_mlp": 0.01058081, + "balance_loss_clip": 1.06158817, + "balance_loss_mlp": 1.03912711, + "epoch": 0.1098891532702687, + "flos": 14458857166080.0, + "grad_norm": 3.0890317331221198, + "language_loss": 0.87223387, + "learning_rate": 3.933431716702639e-06, + "loss": 0.89438325, + "num_input_tokens_seen": 106608435, + "router_z_loss_clip": 0.953125, + "router_z_loss_mlp": 0.1895752, + "step": 3787, + "time_per_iteration": 2.46828293800354 + }, + { + "auxiliary_loss_clip": 0.01142883, + "auxiliary_loss_mlp": 0.01039199, + "balance_loss_clip": 1.056777, + "balance_loss_mlp": 1.0240413, + "epoch": 0.10991817073878475, + "flos": 22667353735680.0, + "grad_norm": 2.128220284285545, + "language_loss": 0.58678275, + "learning_rate": 3.933383617700493e-06, + "loss": 0.6086036, + "num_input_tokens_seen": 106623665, + "router_z_loss_clip": 0.86132812, + "router_z_loss_mlp": 0.15155029, + "step": 3788, + "time_per_iteration": 2.4783546924591064 + }, + { + "auxiliary_loss_clip": 0.01143545, + "auxiliary_loss_mlp": 0.01049997, + "balance_loss_clip": 1.05575705, + "balance_loss_mlp": 1.03196096, + "epoch": 0.10994718820730079, + "flos": 22375615472640.0, + "grad_norm": 2.060186092987433, + "language_loss": 0.92588669, + "learning_rate": 3.93333550162192e-06, + "loss": 0.94782215, + "num_input_tokens_seen": 106639960, + "router_z_loss_clip": 0.87792969, + "router_z_loss_mlp": 0.18048096, + "step": 3789, + "time_per_iteration": 2.501284599304199 + }, + { + "auxiliary_loss_clip": 0.01144316, + "auxiliary_loss_mlp": 0.01039709, + "balance_loss_clip": 1.05685139, + "balance_loss_mlp": 1.02425933, + "epoch": 0.10997620567581684, + "flos": 23544471945600.0, + "grad_norm": 1.9044714240720346, + "language_loss": 0.72482586, + "learning_rate": 3.933287368467346e-06, + "loss": 0.74666619, + "num_input_tokens_seen": 106656115, + "router_z_loss_clip": 0.87353516, + "router_z_loss_mlp": 0.15441895, + "step": 3790, + "time_per_iteration": 2.558030843734741 + }, + { + "auxiliary_loss_clip": 0.01147603, + "auxiliary_loss_mlp": 0.01044012, + "balance_loss_clip": 1.05792224, + "balance_loss_mlp": 1.0273881, + "epoch": 0.11000522314433289, + "flos": 31492687587840.0, + "grad_norm": 2.73335028035687, + "language_loss": 0.94439447, + "learning_rate": 3.933239218237196e-06, + "loss": 0.96631062, + "num_input_tokens_seen": 106670460, + "router_z_loss_clip": 0.89746094, + "router_z_loss_mlp": 0.16650391, + "step": 3791, + "time_per_iteration": 2.599045991897583 + }, + { + "auxiliary_loss_clip": 0.01042429, + "auxiliary_loss_mlp": 0.01001302, + "balance_loss_clip": 1.01853251, + "balance_loss_mlp": 0.99999714, + "epoch": 0.11003424061284893, + "flos": 59154032937600.0, + "grad_norm": 0.7362302641080142, + "language_loss": 0.50325263, + "learning_rate": 3.933191050931894e-06, + "loss": 0.52368999, + "num_input_tokens_seen": 106728915, + "router_z_loss_clip": 0.23925781, + "router_z_loss_mlp": 0.01306152, + "step": 3792, + "time_per_iteration": 2.9760406017303467 + }, + { + "auxiliary_loss_clip": 0.01040035, + "auxiliary_loss_mlp": 0.01008625, + "balance_loss_clip": 1.01623642, + "balance_loss_mlp": 1.00745046, + "epoch": 0.11006325808136498, + "flos": 71043731873280.0, + "grad_norm": 0.7061029937252314, + "language_loss": 0.46094885, + "learning_rate": 3.9331428665518665e-06, + "loss": 0.48143545, + "num_input_tokens_seen": 106787720, + "router_z_loss_clip": 0.23828125, + "router_z_loss_mlp": 0.01171875, + "step": 3793, + "time_per_iteration": 3.121397018432617 + }, + { + "auxiliary_loss_clip": 0.01037957, + "auxiliary_loss_mlp": 0.01005381, + "balance_loss_clip": 1.01459134, + "balance_loss_mlp": 1.00419521, + "epoch": 0.11009227554988103, + "flos": 74768135322240.0, + "grad_norm": 0.8557360917522009, + "language_loss": 0.53775215, + "learning_rate": 3.933094665097539e-06, + "loss": 0.55818546, + "num_input_tokens_seen": 106848365, + "router_z_loss_clip": 0.23339844, + "router_z_loss_mlp": 0.01184082, + "step": 3794, + "time_per_iteration": 3.1094460487365723 + }, + { + "auxiliary_loss_clip": 0.01149094, + "auxiliary_loss_mlp": 0.01051223, + "balance_loss_clip": 1.05796885, + "balance_loss_mlp": 1.03338945, + "epoch": 0.11012129301839707, + "flos": 16756062549120.0, + "grad_norm": 3.1410446608427836, + "language_loss": 0.73942232, + "learning_rate": 3.933046446569338e-06, + "loss": 0.7614255, + "num_input_tokens_seen": 106860410, + "router_z_loss_clip": 0.91113281, + "router_z_loss_mlp": 0.17822266, + "step": 3795, + "time_per_iteration": 2.5121097564697266 + }, + { + "auxiliary_loss_clip": 0.01034727, + "auxiliary_loss_mlp": 0.01001959, + "balance_loss_clip": 1.01150119, + "balance_loss_mlp": 1.00090981, + "epoch": 0.11015031048691312, + "flos": 51450906769920.0, + "grad_norm": 0.6864327506399436, + "language_loss": 0.50926983, + "learning_rate": 3.932998210967687e-06, + "loss": 0.52963674, + "num_input_tokens_seen": 106910970, + "router_z_loss_clip": 0.23242188, + "router_z_loss_mlp": 0.01049805, + "step": 3796, + "time_per_iteration": 2.873098134994507 + }, + { + "auxiliary_loss_clip": 0.01140788, + "auxiliary_loss_mlp": 0.01039516, + "balance_loss_clip": 1.05332351, + "balance_loss_mlp": 1.02177167, + "epoch": 0.11017932795542917, + "flos": 32007646920960.0, + "grad_norm": 3.11328357573544, + "language_loss": 0.92861468, + "learning_rate": 3.932949958293015e-06, + "loss": 0.9504177, + "num_input_tokens_seen": 106928265, + "router_z_loss_clip": 0.87451172, + "router_z_loss_mlp": 0.17730713, + "step": 3797, + "time_per_iteration": 2.6301381587982178 + }, + { + "auxiliary_loss_clip": 0.01143068, + "auxiliary_loss_mlp": 0.01042613, + "balance_loss_clip": 1.05857563, + "balance_loss_mlp": 1.02567959, + "epoch": 0.11020834542394521, + "flos": 54993994364160.0, + "grad_norm": 2.171799164418923, + "language_loss": 0.8665005, + "learning_rate": 3.932901688545746e-06, + "loss": 0.88835734, + "num_input_tokens_seen": 106947835, + "router_z_loss_clip": 0.84619141, + "router_z_loss_mlp": 0.16918945, + "step": 3798, + "time_per_iteration": 2.732759714126587 + }, + { + "auxiliary_loss_clip": 0.01155189, + "auxiliary_loss_mlp": 0.01045171, + "balance_loss_clip": 1.0591346, + "balance_loss_mlp": 1.02678895, + "epoch": 0.11023736289246126, + "flos": 23986209404160.0, + "grad_norm": 2.6289167413108654, + "language_loss": 0.84863859, + "learning_rate": 3.932853401726308e-06, + "loss": 0.87064219, + "num_input_tokens_seen": 106963900, + "router_z_loss_clip": 0.96142578, + "router_z_loss_mlp": 0.18383789, + "step": 3799, + "time_per_iteration": 2.5483624935150146 + }, + { + "auxiliary_loss_clip": 0.01136763, + "auxiliary_loss_mlp": 0.01044417, + "balance_loss_clip": 1.05503714, + "balance_loss_mlp": 1.02905691, + "epoch": 0.11026638036097731, + "flos": 30474656323200.0, + "grad_norm": 2.159347701722172, + "language_loss": 0.85787308, + "learning_rate": 3.932805097835125e-06, + "loss": 0.87968493, + "num_input_tokens_seen": 106978790, + "router_z_loss_clip": 0.81689453, + "router_z_loss_mlp": 0.15356445, + "step": 3800, + "time_per_iteration": 2.562178134918213 + }, + { + "auxiliary_loss_clip": 0.01140719, + "auxiliary_loss_mlp": 0.01047374, + "balance_loss_clip": 1.05626798, + "balance_loss_mlp": 1.03259206, + "epoch": 0.11029539782949335, + "flos": 49993613107200.0, + "grad_norm": 2.6054281031161644, + "language_loss": 0.90705633, + "learning_rate": 3.932756776872627e-06, + "loss": 0.92893732, + "num_input_tokens_seen": 106995460, + "router_z_loss_clip": 0.84472656, + "router_z_loss_mlp": 0.14782715, + "step": 3801, + "time_per_iteration": 2.7738466262817383 + }, + { + "auxiliary_loss_clip": 0.01143377, + "auxiliary_loss_mlp": 0.0104059, + "balance_loss_clip": 1.05625725, + "balance_loss_mlp": 1.02504516, + "epoch": 0.1103244152980094, + "flos": 19055997365760.0, + "grad_norm": 2.807186342929469, + "language_loss": 0.78218246, + "learning_rate": 3.9327084388392385e-06, + "loss": 0.80402213, + "num_input_tokens_seen": 107011535, + "router_z_loss_clip": 0.87109375, + "router_z_loss_mlp": 0.15539551, + "step": 3802, + "time_per_iteration": 2.515368938446045 + }, + { + "auxiliary_loss_clip": 0.01043424, + "auxiliary_loss_mlp": 0.01017523, + "balance_loss_clip": 1.01955187, + "balance_loss_mlp": 1.01631868, + "epoch": 0.11035343276652544, + "flos": 68574802544640.0, + "grad_norm": 0.6482600141155502, + "language_loss": 0.50123674, + "learning_rate": 3.932660083735387e-06, + "loss": 0.52184623, + "num_input_tokens_seen": 107077805, + "router_z_loss_clip": 0.23828125, + "router_z_loss_mlp": 0.01202393, + "step": 3803, + "time_per_iteration": 3.184495449066162 + }, + { + "auxiliary_loss_clip": 0.01043991, + "auxiliary_loss_mlp": 0.01005619, + "balance_loss_clip": 1.02006757, + "balance_loss_mlp": 1.00442719, + "epoch": 0.1103824502350415, + "flos": 74774312461440.0, + "grad_norm": 0.6929530343916193, + "language_loss": 0.50656128, + "learning_rate": 3.932611711561499e-06, + "loss": 0.52705741, + "num_input_tokens_seen": 107138635, + "router_z_loss_clip": 0.23925781, + "router_z_loss_mlp": 0.01190186, + "step": 3804, + "time_per_iteration": 3.1589155197143555 + }, + { + "auxiliary_loss_clip": 0.0104289, + "auxiliary_loss_mlp": 0.01001486, + "balance_loss_clip": 1.0192337, + "balance_loss_mlp": 1.00042462, + "epoch": 0.11041146770355754, + "flos": 74777688339840.0, + "grad_norm": 0.6545333057971017, + "language_loss": 0.45186812, + "learning_rate": 3.932563322318002e-06, + "loss": 0.47231185, + "num_input_tokens_seen": 107202275, + "router_z_loss_clip": 0.23632812, + "router_z_loss_mlp": 0.01062012, + "step": 3805, + "time_per_iteration": 3.1627204418182373 + }, + { + "auxiliary_loss_clip": 0.01041009, + "auxiliary_loss_mlp": 0.01005514, + "balance_loss_clip": 1.01698697, + "balance_loss_mlp": 1.00438714, + "epoch": 0.11044048517207358, + "flos": 71227055911680.0, + "grad_norm": 0.6887318029999199, + "language_loss": 0.49722809, + "learning_rate": 3.932514916005325e-06, + "loss": 0.51769328, + "num_input_tokens_seen": 107265980, + "router_z_loss_clip": 0.24023438, + "router_z_loss_mlp": 0.0112915, + "step": 3806, + "time_per_iteration": 3.282170534133911 + }, + { + "auxiliary_loss_clip": 0.01136715, + "auxiliary_loss_mlp": 0.01044651, + "balance_loss_clip": 1.05513406, + "balance_loss_mlp": 1.02846813, + "epoch": 0.11046950264058963, + "flos": 16390491880320.0, + "grad_norm": 2.530686479961799, + "language_loss": 0.82049489, + "learning_rate": 3.932466492623894e-06, + "loss": 0.84230858, + "num_input_tokens_seen": 107277875, + "router_z_loss_clip": 0.81591797, + "router_z_loss_mlp": 0.1618042, + "step": 3807, + "time_per_iteration": 2.488529682159424 + }, + { + "auxiliary_loss_clip": 0.01144656, + "auxiliary_loss_mlp": 0.0105316, + "balance_loss_clip": 1.05765831, + "balance_loss_mlp": 1.03682792, + "epoch": 0.11049852010910569, + "flos": 27889627259520.0, + "grad_norm": 2.2314993123612887, + "language_loss": 0.79156727, + "learning_rate": 3.932418052174136e-06, + "loss": 0.81354547, + "num_input_tokens_seen": 107297210, + "router_z_loss_clip": 0.87060547, + "router_z_loss_mlp": 0.16333008, + "step": 3808, + "time_per_iteration": 2.6716840267181396 + }, + { + "auxiliary_loss_clip": 0.01149262, + "auxiliary_loss_mlp": 0.01054928, + "balance_loss_clip": 1.06115305, + "balance_loss_mlp": 1.039639, + "epoch": 0.11052753757762172, + "flos": 13070514637440.0, + "grad_norm": 2.8789425206721377, + "language_loss": 0.74209321, + "learning_rate": 3.9323695946564805e-06, + "loss": 0.76413512, + "num_input_tokens_seen": 107308680, + "router_z_loss_clip": 0.88037109, + "router_z_loss_mlp": 0.15289307, + "step": 3809, + "time_per_iteration": 2.4959466457366943 + }, + { + "auxiliary_loss_clip": 0.01157304, + "auxiliary_loss_mlp": 0.01054073, + "balance_loss_clip": 1.06179643, + "balance_loss_mlp": 1.03655541, + "epoch": 0.11055655504613777, + "flos": 38655394657920.0, + "grad_norm": 1.9859823325624444, + "language_loss": 0.80819291, + "learning_rate": 3.932321120071355e-06, + "loss": 0.83030665, + "num_input_tokens_seen": 107330750, + "router_z_loss_clip": 0.95703125, + "router_z_loss_mlp": 0.17510986, + "step": 3810, + "time_per_iteration": 2.6959424018859863 + }, + { + "auxiliary_loss_clip": 0.01155957, + "auxiliary_loss_mlp": 0.01055452, + "balance_loss_clip": 1.06134272, + "balance_loss_mlp": 1.03881013, + "epoch": 0.11058557251465383, + "flos": 20920048640640.0, + "grad_norm": 2.180370211743574, + "language_loss": 0.83423537, + "learning_rate": 3.932272628419187e-06, + "loss": 0.85634947, + "num_input_tokens_seen": 107344410, + "router_z_loss_clip": 0.94677734, + "router_z_loss_mlp": 0.16619873, + "step": 3811, + "time_per_iteration": 2.4912681579589844 + }, + { + "auxiliary_loss_clip": 0.01042907, + "auxiliary_loss_mlp": 0.01056641, + "balance_loss_clip": 1.01907766, + "balance_loss_mlp": 1.0555619, + "epoch": 0.11061458998316986, + "flos": 67440707458560.0, + "grad_norm": 0.7025000794772363, + "language_loss": 0.45844468, + "learning_rate": 3.932224119700406e-06, + "loss": 0.47944015, + "num_input_tokens_seen": 107408730, + "router_z_loss_clip": 0.23828125, + "router_z_loss_mlp": 0.01080322, + "step": 3812, + "time_per_iteration": 3.171304702758789 + }, + { + "auxiliary_loss_clip": 0.01153333, + "auxiliary_loss_mlp": 0.01042167, + "balance_loss_clip": 1.06100678, + "balance_loss_mlp": 1.02554893, + "epoch": 0.11064360745168592, + "flos": 28396793341440.0, + "grad_norm": 3.0449638418691922, + "language_loss": 0.64310288, + "learning_rate": 3.932175593915439e-06, + "loss": 0.6650579, + "num_input_tokens_seen": 107435865, + "router_z_loss_clip": 0.92333984, + "router_z_loss_mlp": 0.16638184, + "step": 3813, + "time_per_iteration": 2.944981098175049 + }, + { + "auxiliary_loss_clip": 0.01143601, + "auxiliary_loss_mlp": 0.01039995, + "balance_loss_clip": 1.05714238, + "balance_loss_mlp": 1.02453971, + "epoch": 0.11067262492020197, + "flos": 27117762877440.0, + "grad_norm": 3.9132151378723115, + "language_loss": 1.0580231, + "learning_rate": 3.932127051064714e-06, + "loss": 1.07985902, + "num_input_tokens_seen": 107449955, + "router_z_loss_clip": 0.86376953, + "router_z_loss_mlp": 0.15466309, + "step": 3814, + "time_per_iteration": 2.5355401039123535 + }, + { + "auxiliary_loss_clip": 0.01037682, + "auxiliary_loss_mlp": 0.01012205, + "balance_loss_clip": 1.01421523, + "balance_loss_mlp": 1.01117945, + "epoch": 0.110701642388718, + "flos": 66673260449280.0, + "grad_norm": 0.8080467100001192, + "language_loss": 0.50386173, + "learning_rate": 3.932078491148663e-06, + "loss": 0.52436066, + "num_input_tokens_seen": 107505585, + "router_z_loss_clip": 0.234375, + "router_z_loss_mlp": 0.01025391, + "step": 3815, + "time_per_iteration": 3.0111119747161865 + }, + { + "auxiliary_loss_clip": 0.01036826, + "auxiliary_loss_mlp": 0.01004159, + "balance_loss_clip": 1.01331043, + "balance_loss_mlp": 1.00306273, + "epoch": 0.11073065985723406, + "flos": 65979250796160.0, + "grad_norm": 0.7276039047672943, + "language_loss": 0.50344479, + "learning_rate": 3.932029914167712e-06, + "loss": 0.52385461, + "num_input_tokens_seen": 107566995, + "router_z_loss_clip": 0.234375, + "router_z_loss_mlp": 0.01098633, + "step": 3816, + "time_per_iteration": 3.2211687564849854 + }, + { + "auxiliary_loss_clip": 0.01155204, + "auxiliary_loss_mlp": 0.01048147, + "balance_loss_clip": 1.06323814, + "balance_loss_mlp": 1.03176785, + "epoch": 0.11075967732575011, + "flos": 34708739806080.0, + "grad_norm": 2.3707722497510657, + "language_loss": 0.83144689, + "learning_rate": 3.931981320122292e-06, + "loss": 0.85348046, + "num_input_tokens_seen": 107582890, + "router_z_loss_clip": 0.91894531, + "router_z_loss_mlp": 0.16387939, + "step": 3817, + "time_per_iteration": 2.6212968826293945 + }, + { + "auxiliary_loss_clip": 0.0113955, + "auxiliary_loss_mlp": 0.01038949, + "balance_loss_clip": 1.05503452, + "balance_loss_mlp": 1.02512074, + "epoch": 0.11078869479426615, + "flos": 23694973931520.0, + "grad_norm": 1.9673217928066804, + "language_loss": 0.70969337, + "learning_rate": 3.93193270901283e-06, + "loss": 0.73147833, + "num_input_tokens_seen": 107597725, + "router_z_loss_clip": 0.84570312, + "router_z_loss_mlp": 0.13830566, + "step": 3818, + "time_per_iteration": 2.514636516571045 + }, + { + "auxiliary_loss_clip": 0.01147606, + "auxiliary_loss_mlp": 0.01044395, + "balance_loss_clip": 1.05736732, + "balance_loss_mlp": 1.02823019, + "epoch": 0.1108177122627822, + "flos": 10042562966400.0, + "grad_norm": 2.586632544861249, + "language_loss": 0.92927563, + "learning_rate": 3.931884080839757e-06, + "loss": 0.95119572, + "num_input_tokens_seen": 107607315, + "router_z_loss_clip": 0.90185547, + "router_z_loss_mlp": 0.16174316, + "step": 3819, + "time_per_iteration": 2.4735379219055176 + }, + { + "auxiliary_loss_clip": 0.01037348, + "auxiliary_loss_mlp": 0.01010275, + "balance_loss_clip": 1.01376212, + "balance_loss_mlp": 1.00921369, + "epoch": 0.11084672973129824, + "flos": 58979579558400.0, + "grad_norm": 0.757711753011222, + "language_loss": 0.52072573, + "learning_rate": 3.931835435603502e-06, + "loss": 0.54120195, + "num_input_tokens_seen": 107664805, + "router_z_loss_clip": 0.23632812, + "router_z_loss_mlp": 0.01062012, + "step": 3820, + "time_per_iteration": 3.003289222717285 + }, + { + "auxiliary_loss_clip": 0.01152886, + "auxiliary_loss_mlp": 0.01045542, + "balance_loss_clip": 1.05717719, + "balance_loss_mlp": 1.02829838, + "epoch": 0.11087574719981429, + "flos": 23433364200960.0, + "grad_norm": 5.242147344706524, + "language_loss": 0.94351298, + "learning_rate": 3.931786773304494e-06, + "loss": 0.96549726, + "num_input_tokens_seen": 107678655, + "router_z_loss_clip": 0.95605469, + "router_z_loss_mlp": 0.17236328, + "step": 3821, + "time_per_iteration": 2.4973881244659424 + }, + { + "auxiliary_loss_clip": 0.0103883, + "auxiliary_loss_mlp": 0.01006541, + "balance_loss_clip": 1.01531744, + "balance_loss_mlp": 1.00537872, + "epoch": 0.11090476466833034, + "flos": 66783542181120.0, + "grad_norm": 0.6449345184163009, + "language_loss": 0.4616105, + "learning_rate": 3.931738093943165e-06, + "loss": 0.48206419, + "num_input_tokens_seen": 107735380, + "router_z_loss_clip": 0.23535156, + "router_z_loss_mlp": 0.01159668, + "step": 3822, + "time_per_iteration": 3.055738925933838 + }, + { + "auxiliary_loss_clip": 0.0113534, + "auxiliary_loss_mlp": 0.01033222, + "balance_loss_clip": 1.05439925, + "balance_loss_mlp": 1.01921487, + "epoch": 0.11093378213684638, + "flos": 21171710304000.0, + "grad_norm": 2.500295436593212, + "language_loss": 0.89792097, + "learning_rate": 3.931689397519943e-06, + "loss": 0.91960657, + "num_input_tokens_seen": 107748560, + "router_z_loss_clip": 0.81005859, + "router_z_loss_mlp": 0.14019775, + "step": 3823, + "time_per_iteration": 2.5686357021331787 + }, + { + "auxiliary_loss_clip": 0.0114897, + "auxiliary_loss_mlp": 0.01044046, + "balance_loss_clip": 1.0583179, + "balance_loss_mlp": 1.02676058, + "epoch": 0.11096279960536243, + "flos": 21098488429440.0, + "grad_norm": 2.7983539599903544, + "language_loss": 0.9160831, + "learning_rate": 3.931640684035258e-06, + "loss": 0.93801326, + "num_input_tokens_seen": 107761190, + "router_z_loss_clip": 0.90673828, + "router_z_loss_mlp": 0.17297363, + "step": 3824, + "time_per_iteration": 2.5796663761138916 + }, + { + "auxiliary_loss_clip": 0.01153675, + "auxiliary_loss_mlp": 0.01052042, + "balance_loss_clip": 1.06287813, + "balance_loss_mlp": 1.03515553, + "epoch": 0.11099181707387848, + "flos": 22704521333760.0, + "grad_norm": 2.9151779589064843, + "language_loss": 0.76895607, + "learning_rate": 3.9315919534895415e-06, + "loss": 0.79101324, + "num_input_tokens_seen": 107776280, + "router_z_loss_clip": 0.90820312, + "router_z_loss_mlp": 0.16894531, + "step": 3825, + "time_per_iteration": 2.53623628616333 + }, + { + "auxiliary_loss_clip": 0.01147578, + "auxiliary_loss_mlp": 0.01047007, + "balance_loss_clip": 1.05800641, + "balance_loss_mlp": 1.03145635, + "epoch": 0.11102083454239452, + "flos": 16611953184000.0, + "grad_norm": 2.9100960747975315, + "language_loss": 0.77399993, + "learning_rate": 3.931543205883223e-06, + "loss": 0.79594582, + "num_input_tokens_seen": 107790805, + "router_z_loss_clip": 0.89697266, + "router_z_loss_mlp": 0.15545654, + "step": 3826, + "time_per_iteration": 2.4987640380859375 + }, + { + "auxiliary_loss_clip": 0.01152605, + "auxiliary_loss_mlp": 0.01064671, + "balance_loss_clip": 1.05945754, + "balance_loss_mlp": 1.04771328, + "epoch": 0.11104985201091057, + "flos": 16251338592000.0, + "grad_norm": 1.9617316337204718, + "language_loss": 0.73328841, + "learning_rate": 3.931494441216733e-06, + "loss": 0.7554611, + "num_input_tokens_seen": 107805655, + "router_z_loss_clip": 0.93164062, + "router_z_loss_mlp": 0.1696167, + "step": 3827, + "time_per_iteration": 2.4369919300079346 + }, + { + "auxiliary_loss_clip": 0.01145001, + "auxiliary_loss_mlp": 0.01051553, + "balance_loss_clip": 1.05494416, + "balance_loss_mlp": 1.03274214, + "epoch": 0.11107886947942662, + "flos": 26361844133760.0, + "grad_norm": 2.041575644802075, + "language_loss": 0.79530501, + "learning_rate": 3.931445659490502e-06, + "loss": 0.81727052, + "num_input_tokens_seen": 107825105, + "router_z_loss_clip": 0.90087891, + "router_z_loss_mlp": 0.18811035, + "step": 3828, + "time_per_iteration": 2.557520866394043 + }, + { + "auxiliary_loss_clip": 0.01134666, + "auxiliary_loss_mlp": 0.0104852, + "balance_loss_clip": 1.05309629, + "balance_loss_mlp": 1.03277826, + "epoch": 0.11110788694794266, + "flos": 34707554657280.0, + "grad_norm": 2.9525061156178145, + "language_loss": 0.80174029, + "learning_rate": 3.931396860704963e-06, + "loss": 0.8235721, + "num_input_tokens_seen": 107842550, + "router_z_loss_clip": 0.81542969, + "router_z_loss_mlp": 0.1572876, + "step": 3829, + "time_per_iteration": 2.6084773540496826 + }, + { + "auxiliary_loss_clip": 0.01149166, + "auxiliary_loss_mlp": 0.0105875, + "balance_loss_clip": 1.05869925, + "balance_loss_mlp": 1.04235828, + "epoch": 0.11113690441645871, + "flos": 36129186115200.0, + "grad_norm": 11.401834994841561, + "language_loss": 0.88464069, + "learning_rate": 3.931348044860544e-06, + "loss": 0.90671992, + "num_input_tokens_seen": 107858815, + "router_z_loss_clip": 0.90478516, + "router_z_loss_mlp": 0.16394043, + "step": 3830, + "time_per_iteration": 2.6583433151245117 + }, + { + "auxiliary_loss_clip": 0.01143964, + "auxiliary_loss_mlp": 0.01058489, + "balance_loss_clip": 1.05519569, + "balance_loss_mlp": 1.04211557, + "epoch": 0.11116592188497476, + "flos": 33215323017600.0, + "grad_norm": 2.4194965955380563, + "language_loss": 0.79902488, + "learning_rate": 3.931299211957678e-06, + "loss": 0.82104945, + "num_input_tokens_seen": 107878770, + "router_z_loss_clip": 0.88818359, + "router_z_loss_mlp": 0.16369629, + "step": 3831, + "time_per_iteration": 2.6022841930389404 + }, + { + "auxiliary_loss_clip": 0.01040368, + "auxiliary_loss_mlp": 0.01053038, + "balance_loss_clip": 1.01561975, + "balance_loss_mlp": 1.05178595, + "epoch": 0.1111949393534908, + "flos": 63178327036800.0, + "grad_norm": 0.6845925602341323, + "language_loss": 0.51640272, + "learning_rate": 3.931250361996796e-06, + "loss": 0.53733677, + "num_input_tokens_seen": 107940600, + "router_z_loss_clip": 0.24804688, + "router_z_loss_mlp": 0.01251221, + "step": 3832, + "time_per_iteration": 3.0765202045440674 + }, + { + "auxiliary_loss_clip": 0.01037981, + "auxiliary_loss_mlp": 0.0103289, + "balance_loss_clip": 1.01403904, + "balance_loss_mlp": 1.03177571, + "epoch": 0.11122395682200685, + "flos": 55987430768640.0, + "grad_norm": 0.7570166076583394, + "language_loss": 0.57340652, + "learning_rate": 3.931201494978329e-06, + "loss": 0.5941152, + "num_input_tokens_seen": 107997445, + "router_z_loss_clip": 0.24023438, + "router_z_loss_mlp": 0.01116943, + "step": 3833, + "time_per_iteration": 2.974076271057129 + }, + { + "auxiliary_loss_clip": 0.01035982, + "auxiliary_loss_mlp": 0.01013096, + "balance_loss_clip": 1.01244259, + "balance_loss_mlp": 1.01202893, + "epoch": 0.11125297429052289, + "flos": 64803606652800.0, + "grad_norm": 0.7129118395328099, + "language_loss": 0.53338325, + "learning_rate": 3.931152610902709e-06, + "loss": 0.55387402, + "num_input_tokens_seen": 108054745, + "router_z_loss_clip": 0.23632812, + "router_z_loss_mlp": 0.01068115, + "step": 3834, + "time_per_iteration": 3.0063886642456055 + }, + { + "auxiliary_loss_clip": 0.01136252, + "auxiliary_loss_mlp": 0.01040214, + "balance_loss_clip": 1.05378807, + "balance_loss_mlp": 1.02522326, + "epoch": 0.11128199175903894, + "flos": 38135766556800.0, + "grad_norm": 2.345594233577177, + "language_loss": 0.75447249, + "learning_rate": 3.931103709770367e-06, + "loss": 0.77623713, + "num_input_tokens_seen": 108069225, + "router_z_loss_clip": 0.82519531, + "router_z_loss_mlp": 0.14978027, + "step": 3835, + "time_per_iteration": 2.6058006286621094 + }, + { + "auxiliary_loss_clip": 0.01036361, + "auxiliary_loss_mlp": 0.0101285, + "balance_loss_clip": 1.01247072, + "balance_loss_mlp": 1.01164556, + "epoch": 0.11131100922755499, + "flos": 68064942942720.0, + "grad_norm": 0.6930369268900518, + "language_loss": 0.52569979, + "learning_rate": 3.931054791581737e-06, + "loss": 0.54619187, + "num_input_tokens_seen": 108131370, + "router_z_loss_clip": 0.23828125, + "router_z_loss_mlp": 0.01202393, + "step": 3836, + "time_per_iteration": 3.133822202682495 + }, + { + "auxiliary_loss_clip": 0.01149546, + "auxiliary_loss_mlp": 0.01046074, + "balance_loss_clip": 1.05898952, + "balance_loss_mlp": 1.02996898, + "epoch": 0.11134002669607103, + "flos": 16795025827200.0, + "grad_norm": 2.2769273178272247, + "language_loss": 0.82958329, + "learning_rate": 3.931005856337249e-06, + "loss": 0.85153949, + "num_input_tokens_seen": 108144760, + "router_z_loss_clip": 0.90673828, + "router_z_loss_mlp": 0.16107178, + "step": 3837, + "time_per_iteration": 2.4895777702331543 + }, + { + "auxiliary_loss_clip": 0.01152094, + "auxiliary_loss_mlp": 0.01050132, + "balance_loss_clip": 1.05694294, + "balance_loss_mlp": 1.03172016, + "epoch": 0.11136904416458708, + "flos": 20588377432320.0, + "grad_norm": 2.3031909005036884, + "language_loss": 0.91493076, + "learning_rate": 3.930956904037335e-06, + "loss": 0.93695307, + "num_input_tokens_seen": 108159845, + "router_z_loss_clip": 0.95117188, + "router_z_loss_mlp": 0.18414307, + "step": 3838, + "time_per_iteration": 2.5408902168273926 + }, + { + "auxiliary_loss_clip": 0.01038315, + "auxiliary_loss_mlp": 0.0104197, + "balance_loss_clip": 1.01378775, + "balance_loss_mlp": 1.04083788, + "epoch": 0.11139806163310313, + "flos": 74769607779840.0, + "grad_norm": 0.681050556480699, + "language_loss": 0.4582836, + "learning_rate": 3.930907934682429e-06, + "loss": 0.47908646, + "num_input_tokens_seen": 108221375, + "router_z_loss_clip": 0.24511719, + "router_z_loss_mlp": 0.01135254, + "step": 3839, + "time_per_iteration": 3.1071653366088867 + }, + { + "auxiliary_loss_clip": 0.01038044, + "auxiliary_loss_mlp": 0.01022313, + "balance_loss_clip": 1.01403594, + "balance_loss_mlp": 1.0213232, + "epoch": 0.11142707910161917, + "flos": 61459175226240.0, + "grad_norm": 0.7078090543026407, + "language_loss": 0.48250031, + "learning_rate": 3.930858948272964e-06, + "loss": 0.50310385, + "num_input_tokens_seen": 108285490, + "router_z_loss_clip": 0.24023438, + "router_z_loss_mlp": 0.0098877, + "step": 3840, + "time_per_iteration": 3.173125743865967 + }, + { + "auxiliary_loss_clip": 0.01152242, + "auxiliary_loss_mlp": 0.01038066, + "balance_loss_clip": 1.06026626, + "balance_loss_mlp": 1.0205065, + "epoch": 0.11145609657013522, + "flos": 36970609184640.0, + "grad_norm": 2.250732369831768, + "language_loss": 0.72753084, + "learning_rate": 3.93080994480937e-06, + "loss": 0.74943388, + "num_input_tokens_seen": 108302125, + "router_z_loss_clip": 0.92041016, + "router_z_loss_mlp": 0.17553711, + "step": 3841, + "time_per_iteration": 2.7570323944091797 + }, + { + "auxiliary_loss_clip": 0.01135028, + "auxiliary_loss_mlp": 0.01036438, + "balance_loss_clip": 1.05572581, + "balance_loss_mlp": 1.02263999, + "epoch": 0.11148511403865127, + "flos": 9238451149440.0, + "grad_norm": 4.35940089664654, + "language_loss": 0.93071115, + "learning_rate": 3.930760924292081e-06, + "loss": 0.95242578, + "num_input_tokens_seen": 108312200, + "router_z_loss_clip": 0.79345703, + "router_z_loss_mlp": 0.13806152, + "step": 3842, + "time_per_iteration": 5.045713424682617 + }, + { + "auxiliary_loss_clip": 0.0103707, + "auxiliary_loss_mlp": 0.01029212, + "balance_loss_clip": 1.01382387, + "balance_loss_mlp": 1.028229, + "epoch": 0.11151413150716731, + "flos": 65470684083840.0, + "grad_norm": 0.6478726705036879, + "language_loss": 0.4734031, + "learning_rate": 3.930711886721531e-06, + "loss": 0.49406588, + "num_input_tokens_seen": 108375655, + "router_z_loss_clip": 0.23242188, + "router_z_loss_mlp": 0.00982666, + "step": 3843, + "time_per_iteration": 5.477735996246338 + }, + { + "auxiliary_loss_clip": 0.01163642, + "auxiliary_loss_mlp": 0.01053709, + "balance_loss_clip": 1.06346965, + "balance_loss_mlp": 1.03531504, + "epoch": 0.11154314897568336, + "flos": 53095899974400.0, + "grad_norm": 3.733542819042933, + "language_loss": 0.99978966, + "learning_rate": 3.930662832098153e-06, + "loss": 1.02196312, + "num_input_tokens_seen": 108398380, + "router_z_loss_clip": 1.00195312, + "router_z_loss_mlp": 0.18408203, + "step": 3844, + "time_per_iteration": 5.24269700050354 + }, + { + "auxiliary_loss_clip": 0.01039732, + "auxiliary_loss_mlp": 0.01052786, + "balance_loss_clip": 1.01591206, + "balance_loss_mlp": 1.05186844, + "epoch": 0.11157216644419941, + "flos": 69594988625280.0, + "grad_norm": 0.6543697854436673, + "language_loss": 0.45266396, + "learning_rate": 3.930613760422378e-06, + "loss": 0.47358912, + "num_input_tokens_seen": 108456785, + "router_z_loss_clip": 0.23828125, + "router_z_loss_mlp": 0.00915527, + "step": 3845, + "time_per_iteration": 3.037222146987915 + }, + { + "auxiliary_loss_clip": 0.01038469, + "auxiliary_loss_mlp": 0.01044108, + "balance_loss_clip": 1.01466775, + "balance_loss_mlp": 1.04308915, + "epoch": 0.11160118391271545, + "flos": 61824207191040.0, + "grad_norm": 0.6556204033695266, + "language_loss": 0.41476452, + "learning_rate": 3.930564671694641e-06, + "loss": 0.43559027, + "num_input_tokens_seen": 108519430, + "router_z_loss_clip": 0.23828125, + "router_z_loss_mlp": 0.01019287, + "step": 3846, + "time_per_iteration": 3.022716760635376 + }, + { + "auxiliary_loss_clip": 0.01152217, + "auxiliary_loss_mlp": 0.01046625, + "balance_loss_clip": 1.06244361, + "balance_loss_mlp": 1.02995932, + "epoch": 0.1116302013812315, + "flos": 16246490256000.0, + "grad_norm": 2.661581220036234, + "language_loss": 0.73283279, + "learning_rate": 3.930515565915377e-06, + "loss": 0.75482118, + "num_input_tokens_seen": 108531290, + "router_z_loss_clip": 0.89746094, + "router_z_loss_mlp": 0.16674805, + "step": 3847, + "time_per_iteration": 2.487637519836426 + }, + { + "auxiliary_loss_clip": 0.0115916, + "auxiliary_loss_mlp": 0.01054788, + "balance_loss_clip": 1.0598073, + "balance_loss_mlp": 1.03594661, + "epoch": 0.11165921884974755, + "flos": 22813151040000.0, + "grad_norm": 2.424723227794914, + "language_loss": 1.01491666, + "learning_rate": 3.930466443085018e-06, + "loss": 1.03705621, + "num_input_tokens_seen": 108545640, + "router_z_loss_clip": 0.99267578, + "router_z_loss_mlp": 0.18835449, + "step": 3848, + "time_per_iteration": 2.536129951477051 + }, + { + "auxiliary_loss_clip": 0.01154868, + "auxiliary_loss_mlp": 0.01049552, + "balance_loss_clip": 1.06158245, + "balance_loss_mlp": 1.03165889, + "epoch": 0.11168823631826359, + "flos": 20411697409920.0, + "grad_norm": 3.3592592363199607, + "language_loss": 0.81539077, + "learning_rate": 3.930417303203997e-06, + "loss": 0.83743501, + "num_input_tokens_seen": 108560995, + "router_z_loss_clip": 0.93261719, + "router_z_loss_mlp": 0.17907715, + "step": 3849, + "time_per_iteration": 2.532238483428955 + }, + { + "auxiliary_loss_clip": 0.0114853, + "auxiliary_loss_mlp": 0.01046249, + "balance_loss_clip": 1.05848873, + "balance_loss_mlp": 1.02799809, + "epoch": 0.11171725378677964, + "flos": 19931032586880.0, + "grad_norm": 3.5267637088285197, + "language_loss": 0.90017867, + "learning_rate": 3.9303681462727505e-06, + "loss": 0.92212641, + "num_input_tokens_seen": 108574765, + "router_z_loss_clip": 0.90039062, + "router_z_loss_mlp": 0.18249512, + "step": 3850, + "time_per_iteration": 2.504912853240967 + }, + { + "auxiliary_loss_clip": 0.01037433, + "auxiliary_loss_mlp": 0.01004981, + "balance_loss_clip": 1.01327074, + "balance_loss_mlp": 1.00406921, + "epoch": 0.11174627125529568, + "flos": 61042717964160.0, + "grad_norm": 1.0190737082219026, + "language_loss": 0.49624276, + "learning_rate": 3.9303189722917115e-06, + "loss": 0.51666689, + "num_input_tokens_seen": 108625990, + "router_z_loss_clip": 0.2421875, + "router_z_loss_mlp": 0.00909424, + "step": 3851, + "time_per_iteration": 2.854987621307373 + }, + { + "auxiliary_loss_clip": 0.01149109, + "auxiliary_loss_mlp": 0.01055441, + "balance_loss_clip": 1.05624008, + "balance_loss_mlp": 1.03760159, + "epoch": 0.11177528872381173, + "flos": 25622765890560.0, + "grad_norm": 2.539974138355442, + "language_loss": 0.86383796, + "learning_rate": 3.930269781261313e-06, + "loss": 0.88588345, + "num_input_tokens_seen": 108644670, + "router_z_loss_clip": 0.92919922, + "router_z_loss_mlp": 0.17858887, + "step": 3852, + "time_per_iteration": 2.7696738243103027 + }, + { + "auxiliary_loss_clip": 0.01038331, + "auxiliary_loss_mlp": 0.01008057, + "balance_loss_clip": 1.01400971, + "balance_loss_mlp": 1.00689507, + "epoch": 0.11180430619232778, + "flos": 52287301935360.0, + "grad_norm": 0.6934342198198593, + "language_loss": 0.48589814, + "learning_rate": 3.930220573181992e-06, + "loss": 0.50636202, + "num_input_tokens_seen": 108693975, + "router_z_loss_clip": 0.24414062, + "router_z_loss_mlp": 0.01159668, + "step": 3853, + "time_per_iteration": 2.811234951019287 + }, + { + "auxiliary_loss_clip": 0.01038664, + "auxiliary_loss_mlp": 0.01009564, + "balance_loss_clip": 1.01446676, + "balance_loss_mlp": 1.00842607, + "epoch": 0.11183332366084382, + "flos": 64617912316800.0, + "grad_norm": 0.6824917159710471, + "language_loss": 0.46874055, + "learning_rate": 3.930171348054181e-06, + "loss": 0.48922285, + "num_input_tokens_seen": 108754200, + "router_z_loss_clip": 0.2421875, + "router_z_loss_mlp": 0.01141357, + "step": 3854, + "time_per_iteration": 3.1259424686431885 + }, + { + "auxiliary_loss_clip": 0.01037679, + "auxiliary_loss_mlp": 0.01002556, + "balance_loss_clip": 1.01363373, + "balance_loss_mlp": 1.0014832, + "epoch": 0.11186234112935987, + "flos": 73934972380800.0, + "grad_norm": 0.6478834129351891, + "language_loss": 0.50042093, + "learning_rate": 3.9301221058783155e-06, + "loss": 0.5208233, + "num_input_tokens_seen": 108817245, + "router_z_loss_clip": 0.24023438, + "router_z_loss_mlp": 0.01074219, + "step": 3855, + "time_per_iteration": 3.1005218029022217 + }, + { + "auxiliary_loss_clip": 0.01151199, + "auxiliary_loss_mlp": 0.01054322, + "balance_loss_clip": 1.0583384, + "balance_loss_mlp": 1.03378177, + "epoch": 0.11189135859787593, + "flos": 20369106858240.0, + "grad_norm": 2.3614344673847554, + "language_loss": 0.92950118, + "learning_rate": 3.930072846654831e-06, + "loss": 0.95155638, + "num_input_tokens_seen": 108830430, + "router_z_loss_clip": 0.92919922, + "router_z_loss_mlp": 0.20544434, + "step": 3856, + "time_per_iteration": 2.4586539268493652 + }, + { + "auxiliary_loss_clip": 0.01133557, + "auxiliary_loss_mlp": 0.01043963, + "balance_loss_clip": 1.0514853, + "balance_loss_mlp": 1.02971721, + "epoch": 0.11192037606639196, + "flos": 34049994330240.0, + "grad_norm": 2.1034732678819794, + "language_loss": 0.8300243, + "learning_rate": 3.930023570384162e-06, + "loss": 0.85179955, + "num_input_tokens_seen": 108847140, + "router_z_loss_clip": 0.8203125, + "router_z_loss_mlp": 0.14245605, + "step": 3857, + "time_per_iteration": 2.5962722301483154 + }, + { + "auxiliary_loss_clip": 0.01037225, + "auxiliary_loss_mlp": 0.01016293, + "balance_loss_clip": 1.01337469, + "balance_loss_mlp": 1.01524377, + "epoch": 0.11194939353490801, + "flos": 74768602199040.0, + "grad_norm": 0.6199717158893335, + "language_loss": 0.47399968, + "learning_rate": 3.929974277066744e-06, + "loss": 0.49453479, + "num_input_tokens_seen": 108914345, + "router_z_loss_clip": 0.23828125, + "router_z_loss_mlp": 0.01049805, + "step": 3858, + "time_per_iteration": 3.340089797973633 + }, + { + "auxiliary_loss_clip": 0.01037586, + "auxiliary_loss_mlp": 0.01022578, + "balance_loss_clip": 1.01335764, + "balance_loss_mlp": 1.02153516, + "epoch": 0.11197841100342407, + "flos": 63357736492800.0, + "grad_norm": 0.6394139982511077, + "language_loss": 0.48042583, + "learning_rate": 3.9299249667030115e-06, + "loss": 0.50102746, + "num_input_tokens_seen": 108975070, + "router_z_loss_clip": 0.2421875, + "router_z_loss_mlp": 0.01043701, + "step": 3859, + "time_per_iteration": 3.0308406352996826 + }, + { + "auxiliary_loss_clip": 0.01141628, + "auxiliary_loss_mlp": 0.01046035, + "balance_loss_clip": 1.05446553, + "balance_loss_mlp": 1.02968514, + "epoch": 0.1120074284719401, + "flos": 21575238670080.0, + "grad_norm": 1.922115241443213, + "language_loss": 0.87046599, + "learning_rate": 3.929875639293401e-06, + "loss": 0.89234263, + "num_input_tokens_seen": 108992945, + "router_z_loss_clip": 0.87158203, + "router_z_loss_mlp": 0.16345215, + "step": 3860, + "time_per_iteration": 2.5242373943328857 + }, + { + "auxiliary_loss_clip": 0.01145765, + "auxiliary_loss_mlp": 0.01042339, + "balance_loss_clip": 1.05556059, + "balance_loss_mlp": 1.02591157, + "epoch": 0.11203644594045616, + "flos": 29676721645440.0, + "grad_norm": 1.9659401732290942, + "language_loss": 0.71766186, + "learning_rate": 3.929826294838348e-06, + "loss": 0.7395429, + "num_input_tokens_seen": 109010140, + "router_z_loss_clip": 0.90234375, + "router_z_loss_mlp": 0.16430664, + "step": 3861, + "time_per_iteration": 2.5729124546051025 + }, + { + "auxiliary_loss_clip": 0.01037402, + "auxiliary_loss_mlp": 0.01006888, + "balance_loss_clip": 1.01340699, + "balance_loss_mlp": 1.00576711, + "epoch": 0.11206546340897221, + "flos": 58816974689280.0, + "grad_norm": 0.68995550413306, + "language_loss": 0.47717676, + "learning_rate": 3.929776933338289e-06, + "loss": 0.49761963, + "num_input_tokens_seen": 109065545, + "router_z_loss_clip": 0.24023438, + "router_z_loss_mlp": 0.01123047, + "step": 3862, + "time_per_iteration": 2.9528310298919678 + }, + { + "auxiliary_loss_clip": 0.01147344, + "auxiliary_loss_mlp": 0.01056939, + "balance_loss_clip": 1.05699885, + "balance_loss_mlp": 1.03904533, + "epoch": 0.11209448087748825, + "flos": 29089043228160.0, + "grad_norm": 2.853720246553567, + "language_loss": 0.91862929, + "learning_rate": 3.9297275547936585e-06, + "loss": 0.94067204, + "num_input_tokens_seen": 109079885, + "router_z_loss_clip": 0.90283203, + "router_z_loss_mlp": 0.17877197, + "step": 3863, + "time_per_iteration": 2.5791735649108887 + }, + { + "auxiliary_loss_clip": 0.01034942, + "auxiliary_loss_mlp": 0.01000933, + "balance_loss_clip": 1.01091897, + "balance_loss_mlp": 0.99971151, + "epoch": 0.1121234983460043, + "flos": 70983546635520.0, + "grad_norm": 0.6285838556485406, + "language_loss": 0.48337996, + "learning_rate": 3.929678159204894e-06, + "loss": 0.5037387, + "num_input_tokens_seen": 109150060, + "router_z_loss_clip": 0.24023438, + "router_z_loss_mlp": 0.01220703, + "step": 3864, + "time_per_iteration": 3.195329189300537 + }, + { + "auxiliary_loss_clip": 0.01143787, + "auxiliary_loss_mlp": 0.01040003, + "balance_loss_clip": 1.05644035, + "balance_loss_mlp": 1.02389193, + "epoch": 0.11215251581452033, + "flos": 33905346261120.0, + "grad_norm": 2.375883142509824, + "language_loss": 0.88994718, + "learning_rate": 3.9296287465724294e-06, + "loss": 0.91178513, + "num_input_tokens_seen": 109165405, + "router_z_loss_clip": 0.87304688, + "router_z_loss_mlp": 0.16113281, + "step": 3865, + "time_per_iteration": 2.6459407806396484 + }, + { + "auxiliary_loss_clip": 0.01036203, + "auxiliary_loss_mlp": 0.01001965, + "balance_loss_clip": 1.01213014, + "balance_loss_mlp": 1.00074279, + "epoch": 0.11218153328303639, + "flos": 73392721689600.0, + "grad_norm": 0.8084014308322233, + "language_loss": 0.46977812, + "learning_rate": 3.929579316896705e-06, + "loss": 0.49015981, + "num_input_tokens_seen": 109225100, + "router_z_loss_clip": 0.24023438, + "router_z_loss_mlp": 0.01220703, + "step": 3866, + "time_per_iteration": 3.12395977973938 + }, + { + "auxiliary_loss_clip": 0.01146774, + "auxiliary_loss_mlp": 0.01046071, + "balance_loss_clip": 1.05598664, + "balance_loss_mlp": 1.02918482, + "epoch": 0.11221055075155244, + "flos": 36565536533760.0, + "grad_norm": 3.11801942732693, + "language_loss": 0.86088729, + "learning_rate": 3.9295298701781534e-06, + "loss": 0.88281572, + "num_input_tokens_seen": 109245515, + "router_z_loss_clip": 0.90820312, + "router_z_loss_mlp": 0.16876221, + "step": 3867, + "time_per_iteration": 2.756791353225708 + }, + { + "auxiliary_loss_clip": 0.01144028, + "auxiliary_loss_mlp": 0.0104218, + "balance_loss_clip": 1.05894756, + "balance_loss_mlp": 1.0262239, + "epoch": 0.11223956822006848, + "flos": 12085987783680.0, + "grad_norm": 2.5828730637357724, + "language_loss": 0.8098104, + "learning_rate": 3.929480406417215e-06, + "loss": 0.83167243, + "num_input_tokens_seen": 109258195, + "router_z_loss_clip": 0.85058594, + "router_z_loss_mlp": 0.159729, + "step": 3868, + "time_per_iteration": 2.559720516204834 + }, + { + "auxiliary_loss_clip": 0.01145429, + "auxiliary_loss_mlp": 0.01045289, + "balance_loss_clip": 1.05574751, + "balance_loss_mlp": 1.02919555, + "epoch": 0.11226858568858453, + "flos": 16316587647360.0, + "grad_norm": 3.2678500445795473, + "language_loss": 0.7675674, + "learning_rate": 3.929430925614324e-06, + "loss": 0.78947455, + "num_input_tokens_seen": 109271110, + "router_z_loss_clip": 0.89746094, + "router_z_loss_mlp": 0.16064453, + "step": 3869, + "time_per_iteration": 2.451009511947632 + }, + { + "auxiliary_loss_clip": 0.01139817, + "auxiliary_loss_mlp": 0.01045378, + "balance_loss_clip": 1.05357504, + "balance_loss_mlp": 1.02937436, + "epoch": 0.11229760315710058, + "flos": 14093322410880.0, + "grad_norm": 4.804793660861111, + "language_loss": 0.74744833, + "learning_rate": 3.929381427769918e-06, + "loss": 0.76930028, + "num_input_tokens_seen": 109282785, + "router_z_loss_clip": 0.86230469, + "router_z_loss_mlp": 0.16003418, + "step": 3870, + "time_per_iteration": 2.5230512619018555 + }, + { + "auxiliary_loss_clip": 0.01156633, + "auxiliary_loss_mlp": 0.01044511, + "balance_loss_clip": 1.05879343, + "balance_loss_mlp": 1.02717757, + "epoch": 0.11232662062561662, + "flos": 35108748639360.0, + "grad_norm": 2.011816150389342, + "language_loss": 0.73434436, + "learning_rate": 3.929331912884435e-06, + "loss": 0.75635582, + "num_input_tokens_seen": 109306165, + "router_z_loss_clip": 0.97900391, + "router_z_loss_mlp": 0.17321777, + "step": 3871, + "time_per_iteration": 2.613847494125366 + }, + { + "auxiliary_loss_clip": 0.01034961, + "auxiliary_loss_mlp": 0.01008448, + "balance_loss_clip": 1.01093864, + "balance_loss_mlp": 1.00713623, + "epoch": 0.11235563809413267, + "flos": 64047867477120.0, + "grad_norm": 0.6984517768021419, + "language_loss": 0.51461214, + "learning_rate": 3.929282380958311e-06, + "loss": 0.53504622, + "num_input_tokens_seen": 109368290, + "router_z_loss_clip": 0.24023438, + "router_z_loss_mlp": 0.01312256, + "step": 3872, + "time_per_iteration": 3.1111321449279785 + }, + { + "auxiliary_loss_clip": 0.01035176, + "auxiliary_loss_mlp": 0.01003379, + "balance_loss_clip": 1.01104045, + "balance_loss_mlp": 1.00205541, + "epoch": 0.11238465556264872, + "flos": 71820372764160.0, + "grad_norm": 0.6777546004193243, + "language_loss": 0.50895602, + "learning_rate": 3.9292328319919855e-06, + "loss": 0.52934158, + "num_input_tokens_seen": 109434930, + "router_z_loss_clip": 0.2421875, + "router_z_loss_mlp": 0.01324463, + "step": 3873, + "time_per_iteration": 3.141552686691284 + }, + { + "auxiliary_loss_clip": 0.01150308, + "auxiliary_loss_mlp": 0.01055502, + "balance_loss_clip": 1.05561709, + "balance_loss_mlp": 1.03733468, + "epoch": 0.11241367303116476, + "flos": 38903213566080.0, + "grad_norm": 2.133628108062549, + "language_loss": 1.00485206, + "learning_rate": 3.929183265985894e-06, + "loss": 1.02691007, + "num_input_tokens_seen": 109452645, + "router_z_loss_clip": 0.94824219, + "router_z_loss_mlp": 0.1817627, + "step": 3874, + "time_per_iteration": 2.6624152660369873 + }, + { + "auxiliary_loss_clip": 0.01140356, + "auxiliary_loss_mlp": 0.01036098, + "balance_loss_clip": 1.05379093, + "balance_loss_mlp": 1.02009428, + "epoch": 0.11244269049968081, + "flos": 30219906090240.0, + "grad_norm": 1.8575118562985915, + "language_loss": 0.67884624, + "learning_rate": 3.929133682940476e-06, + "loss": 0.70061082, + "num_input_tokens_seen": 109473015, + "router_z_loss_clip": 0.86572266, + "router_z_loss_mlp": 0.16009521, + "step": 3875, + "time_per_iteration": 2.708402633666992 + }, + { + "auxiliary_loss_clip": 0.0103475, + "auxiliary_loss_mlp": 0.01005224, + "balance_loss_clip": 1.0106405, + "balance_loss_mlp": 1.003901, + "epoch": 0.11247170796819686, + "flos": 62515523324160.0, + "grad_norm": 1.5787253256581253, + "language_loss": 0.4749077, + "learning_rate": 3.929084082856167e-06, + "loss": 0.49530745, + "num_input_tokens_seen": 109536095, + "router_z_loss_clip": 0.24121094, + "router_z_loss_mlp": 0.01324463, + "step": 3876, + "time_per_iteration": 3.0135931968688965 + }, + { + "auxiliary_loss_clip": 0.01145132, + "auxiliary_loss_mlp": 0.01050429, + "balance_loss_clip": 1.05427694, + "balance_loss_mlp": 1.03270292, + "epoch": 0.1125007254367129, + "flos": 29563243603200.0, + "grad_norm": 2.3577452615736707, + "language_loss": 0.77227151, + "learning_rate": 3.9290344657334085e-06, + "loss": 0.79422718, + "num_input_tokens_seen": 109554585, + "router_z_loss_clip": 0.90917969, + "router_z_loss_mlp": 0.17730713, + "step": 3877, + "time_per_iteration": 2.622126817703247 + }, + { + "auxiliary_loss_clip": 0.01145518, + "auxiliary_loss_mlp": 0.01050832, + "balance_loss_clip": 1.05309248, + "balance_loss_mlp": 1.03256965, + "epoch": 0.11252974290522895, + "flos": 32554638207360.0, + "grad_norm": 1.8790187172445472, + "language_loss": 0.75404835, + "learning_rate": 3.928984831572637e-06, + "loss": 0.77601182, + "num_input_tokens_seen": 109572990, + "router_z_loss_clip": 0.92382812, + "router_z_loss_mlp": 0.18261719, + "step": 3878, + "time_per_iteration": 2.5918262004852295 + }, + { + "auxiliary_loss_clip": 0.0113942, + "auxiliary_loss_mlp": 0.01042375, + "balance_loss_clip": 1.05397201, + "balance_loss_mlp": 1.02625751, + "epoch": 0.112558760373745, + "flos": 33284881704960.0, + "grad_norm": 2.970881469342003, + "language_loss": 0.68742728, + "learning_rate": 3.92893518037429e-06, + "loss": 0.7092452, + "num_input_tokens_seen": 109589855, + "router_z_loss_clip": 0.85449219, + "router_z_loss_mlp": 0.16107178, + "step": 3879, + "time_per_iteration": 2.601323366165161 + }, + { + "auxiliary_loss_clip": 0.01143143, + "auxiliary_loss_mlp": 0.01047316, + "balance_loss_clip": 1.05328417, + "balance_loss_mlp": 1.03047764, + "epoch": 0.11258777784226104, + "flos": 17854893457920.0, + "grad_norm": 2.4553817037741235, + "language_loss": 0.86796576, + "learning_rate": 3.928885512138808e-06, + "loss": 0.88987035, + "num_input_tokens_seen": 109611180, + "router_z_loss_clip": 0.8984375, + "router_z_loss_mlp": 0.168396, + "step": 3880, + "time_per_iteration": 2.6751394271850586 + }, + { + "auxiliary_loss_clip": 0.01033384, + "auxiliary_loss_mlp": 0.01019704, + "balance_loss_clip": 1.00961709, + "balance_loss_mlp": 1.0186789, + "epoch": 0.11261679531077709, + "flos": 46062799130880.0, + "grad_norm": 0.6452485678997878, + "language_loss": 0.40844873, + "learning_rate": 3.928835826866628e-06, + "loss": 0.42897961, + "num_input_tokens_seen": 109655680, + "router_z_loss_clip": 0.23828125, + "router_z_loss_mlp": 0.01025391, + "step": 3881, + "time_per_iteration": 2.8755526542663574 + }, + { + "auxiliary_loss_clip": 0.01129016, + "auxiliary_loss_mlp": 0.01041185, + "balance_loss_clip": 1.04991293, + "balance_loss_mlp": 1.02692747, + "epoch": 0.11264581277929313, + "flos": 39961824220800.0, + "grad_norm": 2.1554954616006357, + "language_loss": 0.85522413, + "learning_rate": 3.928786124558189e-06, + "loss": 0.87692618, + "num_input_tokens_seen": 109673585, + "router_z_loss_clip": 0.79052734, + "router_z_loss_mlp": 0.14239502, + "step": 3882, + "time_per_iteration": 2.699791669845581 + }, + { + "auxiliary_loss_clip": 0.01033097, + "auxiliary_loss_mlp": 0.01017362, + "balance_loss_clip": 1.00913382, + "balance_loss_mlp": 1.01640856, + "epoch": 0.11267483024780918, + "flos": 74768817680640.0, + "grad_norm": 0.7044918912203171, + "language_loss": 0.48168865, + "learning_rate": 3.928736405213931e-06, + "loss": 0.50219321, + "num_input_tokens_seen": 109736060, + "router_z_loss_clip": 0.24023438, + "router_z_loss_mlp": 0.00952148, + "step": 3883, + "time_per_iteration": 3.1842329502105713 + }, + { + "auxiliary_loss_clip": 0.01033522, + "auxiliary_loss_mlp": 0.01008987, + "balance_loss_clip": 1.00956964, + "balance_loss_mlp": 1.00790811, + "epoch": 0.11270384771632523, + "flos": 53135117625600.0, + "grad_norm": 0.725904536625596, + "language_loss": 0.534437, + "learning_rate": 3.928686668834292e-06, + "loss": 0.55486208, + "num_input_tokens_seen": 109793745, + "router_z_loss_clip": 0.24023438, + "router_z_loss_mlp": 0.01080322, + "step": 3884, + "time_per_iteration": 3.0243101119995117 + }, + { + "auxiliary_loss_clip": 0.01146051, + "auxiliary_loss_mlp": 0.01045445, + "balance_loss_clip": 1.0511483, + "balance_loss_mlp": 1.02749789, + "epoch": 0.11273286518484127, + "flos": 36241730403840.0, + "grad_norm": 1.7893949743144146, + "language_loss": 0.83332217, + "learning_rate": 3.928636915419713e-06, + "loss": 0.85523713, + "num_input_tokens_seen": 109817305, + "router_z_loss_clip": 0.94921875, + "router_z_loss_mlp": 0.17950439, + "step": 3885, + "time_per_iteration": 2.6763460636138916 + }, + { + "auxiliary_loss_clip": 0.01136624, + "auxiliary_loss_mlp": 0.01040046, + "balance_loss_clip": 1.05077791, + "balance_loss_mlp": 1.02414346, + "epoch": 0.11276188265335732, + "flos": 17231914949760.0, + "grad_norm": 2.809321647155744, + "language_loss": 0.7853657, + "learning_rate": 3.928587144970631e-06, + "loss": 0.80713242, + "num_input_tokens_seen": 109830035, + "router_z_loss_clip": 0.85791016, + "router_z_loss_mlp": 0.15917969, + "step": 3886, + "time_per_iteration": 2.477959394454956 + }, + { + "auxiliary_loss_clip": 0.01138326, + "auxiliary_loss_mlp": 0.01050185, + "balance_loss_clip": 1.04827309, + "balance_loss_mlp": 1.03314984, + "epoch": 0.11279090012187337, + "flos": 74732473117440.0, + "grad_norm": 2.740706072668094, + "language_loss": 0.85398144, + "learning_rate": 3.928537357487487e-06, + "loss": 0.87586659, + "num_input_tokens_seen": 109851310, + "router_z_loss_clip": 0.90087891, + "router_z_loss_mlp": 0.17047119, + "step": 3887, + "time_per_iteration": 2.9423017501831055 + }, + { + "auxiliary_loss_clip": 0.01137763, + "auxiliary_loss_mlp": 0.01050211, + "balance_loss_clip": 1.04728675, + "balance_loss_mlp": 1.03296793, + "epoch": 0.11281991759038941, + "flos": 34013437263360.0, + "grad_norm": 2.3262204592408713, + "language_loss": 0.82085371, + "learning_rate": 3.928487552970722e-06, + "loss": 0.8427335, + "num_input_tokens_seen": 109866615, + "router_z_loss_clip": 0.90429688, + "router_z_loss_mlp": 0.17254639, + "step": 3888, + "time_per_iteration": 2.5825066566467285 + }, + { + "auxiliary_loss_clip": 0.01135328, + "auxiliary_loss_mlp": 0.01055138, + "balance_loss_clip": 1.04760075, + "balance_loss_mlp": 1.03732824, + "epoch": 0.11284893505890546, + "flos": 12450121908480.0, + "grad_norm": 4.1204486201506425, + "language_loss": 0.98545885, + "learning_rate": 3.928437731420774e-06, + "loss": 1.00736356, + "num_input_tokens_seen": 109877135, + "router_z_loss_clip": 0.87744141, + "router_z_loss_mlp": 0.17797852, + "step": 3889, + "time_per_iteration": 2.4460256099700928 + }, + { + "auxiliary_loss_clip": 0.01137487, + "auxiliary_loss_mlp": 0.01056122, + "balance_loss_clip": 1.05282784, + "balance_loss_mlp": 1.04048169, + "epoch": 0.11287795252742151, + "flos": 26108853667200.0, + "grad_norm": 2.469713116728352, + "language_loss": 0.72038913, + "learning_rate": 3.928387892838083e-06, + "loss": 0.74232519, + "num_input_tokens_seen": 109893545, + "router_z_loss_clip": 0.84667969, + "router_z_loss_mlp": 0.15649414, + "step": 3890, + "time_per_iteration": 2.4730870723724365 + }, + { + "auxiliary_loss_clip": 0.01037588, + "auxiliary_loss_mlp": 0.01076259, + "balance_loss_clip": 1.011554, + "balance_loss_mlp": 1.07482278, + "epoch": 0.11290696999593755, + "flos": 63652958375040.0, + "grad_norm": 0.6901937136902634, + "language_loss": 0.49169308, + "learning_rate": 3.928338037223091e-06, + "loss": 0.51283157, + "num_input_tokens_seen": 109959035, + "router_z_loss_clip": 0.26171875, + "router_z_loss_mlp": 0.01434326, + "step": 3891, + "time_per_iteration": 3.117751359939575 + }, + { + "auxiliary_loss_clip": 0.01141162, + "auxiliary_loss_mlp": 0.01055221, + "balance_loss_clip": 1.05129027, + "balance_loss_mlp": 1.03739369, + "epoch": 0.1129359874644536, + "flos": 58534786465920.0, + "grad_norm": 2.0182987301907733, + "language_loss": 0.79137635, + "learning_rate": 3.9282881645762365e-06, + "loss": 0.81334019, + "num_input_tokens_seen": 109981830, + "router_z_loss_clip": 0.89794922, + "router_z_loss_mlp": 0.17822266, + "step": 3892, + "time_per_iteration": 2.7419943809509277 + }, + { + "auxiliary_loss_clip": 0.01037647, + "auxiliary_loss_mlp": 0.01012932, + "balance_loss_clip": 1.01228833, + "balance_loss_mlp": 1.01169801, + "epoch": 0.11296500493296965, + "flos": 74791694643840.0, + "grad_norm": 0.6492158371245422, + "language_loss": 0.50180471, + "learning_rate": 3.9282382748979604e-06, + "loss": 0.5223105, + "num_input_tokens_seen": 110052370, + "router_z_loss_clip": 0.25390625, + "router_z_loss_mlp": 0.0123291, + "step": 3893, + "time_per_iteration": 3.296980857849121 + }, + { + "auxiliary_loss_clip": 0.01142497, + "auxiliary_loss_mlp": 0.01045601, + "balance_loss_clip": 1.05171955, + "balance_loss_mlp": 1.02729034, + "epoch": 0.11299402240148569, + "flos": 22272300979200.0, + "grad_norm": 2.7094746453085814, + "language_loss": 0.76893425, + "learning_rate": 3.928188368188704e-06, + "loss": 0.79081523, + "num_input_tokens_seen": 110068625, + "router_z_loss_clip": 0.90771484, + "router_z_loss_mlp": 0.1829834, + "step": 3894, + "time_per_iteration": 2.5084338188171387 + }, + { + "auxiliary_loss_clip": 0.01159462, + "auxiliary_loss_mlp": 0.01059836, + "balance_loss_clip": 1.05633283, + "balance_loss_mlp": 1.03800893, + "epoch": 0.11302303987000174, + "flos": 22484424746880.0, + "grad_norm": 2.756969593646576, + "language_loss": 0.96892595, + "learning_rate": 3.928138444448906e-06, + "loss": 0.99111891, + "num_input_tokens_seen": 110083775, + "router_z_loss_clip": 1.03222656, + "router_z_loss_mlp": 0.21838379, + "step": 3895, + "time_per_iteration": 2.544015884399414 + }, + { + "auxiliary_loss_clip": 0.0103711, + "auxiliary_loss_mlp": 0.01047143, + "balance_loss_clip": 1.01137328, + "balance_loss_mlp": 1.04598033, + "epoch": 0.1130520573385178, + "flos": 61960810613760.0, + "grad_norm": 0.7101955542579919, + "language_loss": 0.50646597, + "learning_rate": 3.928088503679011e-06, + "loss": 0.52730846, + "num_input_tokens_seen": 110139160, + "router_z_loss_clip": 0.2578125, + "router_z_loss_mlp": 0.01159668, + "step": 3896, + "time_per_iteration": 2.915818691253662 + }, + { + "auxiliary_loss_clip": 0.01139665, + "auxiliary_loss_mlp": 0.01064309, + "balance_loss_clip": 1.05271161, + "balance_loss_mlp": 1.04685044, + "epoch": 0.11308107480703383, + "flos": 19565749226880.0, + "grad_norm": 3.6453280114634965, + "language_loss": 0.88702714, + "learning_rate": 3.928038545879457e-06, + "loss": 0.90906692, + "num_input_tokens_seen": 110152525, + "router_z_loss_clip": 0.86962891, + "router_z_loss_mlp": 0.17474365, + "step": 3897, + "time_per_iteration": 2.4974923133850098 + }, + { + "auxiliary_loss_clip": 0.01038556, + "auxiliary_loss_mlp": 0.01050643, + "balance_loss_clip": 1.01304722, + "balance_loss_mlp": 1.04933786, + "epoch": 0.11311009227554988, + "flos": 60335926047360.0, + "grad_norm": 0.6610595766568051, + "language_loss": 0.50698352, + "learning_rate": 3.927988571050688e-06, + "loss": 0.52787554, + "num_input_tokens_seen": 110215930, + "router_z_loss_clip": 0.25390625, + "router_z_loss_mlp": 0.01306152, + "step": 3898, + "time_per_iteration": 3.075894594192505 + }, + { + "auxiliary_loss_clip": 0.01142902, + "auxiliary_loss_mlp": 0.01058817, + "balance_loss_clip": 1.05205822, + "balance_loss_mlp": 1.04352224, + "epoch": 0.11313910974406592, + "flos": 19602988652160.0, + "grad_norm": 2.3407469574957647, + "language_loss": 0.94464004, + "learning_rate": 3.927938579193142e-06, + "loss": 0.96665728, + "num_input_tokens_seen": 110230465, + "router_z_loss_clip": 0.90771484, + "router_z_loss_mlp": 0.1529541, + "step": 3899, + "time_per_iteration": 2.548323392868042 + }, + { + "auxiliary_loss_clip": 0.01147283, + "auxiliary_loss_mlp": 0.01054678, + "balance_loss_clip": 1.05246401, + "balance_loss_mlp": 1.03829265, + "epoch": 0.11316812721258197, + "flos": 45765706763520.0, + "grad_norm": 2.3921016487155464, + "language_loss": 0.83187675, + "learning_rate": 3.927888570307263e-06, + "loss": 0.85389638, + "num_input_tokens_seen": 110247270, + "router_z_loss_clip": 0.94775391, + "router_z_loss_mlp": 0.16381836, + "step": 3900, + "time_per_iteration": 2.704684257507324 + }, + { + "auxiliary_loss_clip": 0.01034512, + "auxiliary_loss_mlp": 0.01042308, + "balance_loss_clip": 1.0103159, + "balance_loss_mlp": 1.0410738, + "epoch": 0.11319714468109802, + "flos": 67254223023360.0, + "grad_norm": 0.684795207526034, + "language_loss": 0.4671475, + "learning_rate": 3.927838544393492e-06, + "loss": 0.48791569, + "num_input_tokens_seen": 110301960, + "router_z_loss_clip": 0.2421875, + "router_z_loss_mlp": 0.0123291, + "step": 3901, + "time_per_iteration": 2.9802539348602295 + }, + { + "auxiliary_loss_clip": 0.01135796, + "auxiliary_loss_mlp": 0.01054837, + "balance_loss_clip": 1.04988897, + "balance_loss_mlp": 1.03897035, + "epoch": 0.11322616214961406, + "flos": 36719378484480.0, + "grad_norm": 2.303587772373818, + "language_loss": 0.92379624, + "learning_rate": 3.92778850145227e-06, + "loss": 0.94570267, + "num_input_tokens_seen": 110318455, + "router_z_loss_clip": 0.85888672, + "router_z_loss_mlp": 0.15863037, + "step": 3902, + "time_per_iteration": 2.5559744834899902 + }, + { + "auxiliary_loss_clip": 0.01143111, + "auxiliary_loss_mlp": 0.01041409, + "balance_loss_clip": 1.0536108, + "balance_loss_mlp": 1.02395701, + "epoch": 0.11325517961813011, + "flos": 26767455488640.0, + "grad_norm": 2.525586025332367, + "language_loss": 0.80688989, + "learning_rate": 3.927738441484042e-06, + "loss": 0.82873511, + "num_input_tokens_seen": 110334150, + "router_z_loss_clip": 0.89501953, + "router_z_loss_mlp": 0.17431641, + "step": 3903, + "time_per_iteration": 2.5663561820983887 + }, + { + "auxiliary_loss_clip": 0.0115496, + "auxiliary_loss_mlp": 0.01048291, + "balance_loss_clip": 1.05692506, + "balance_loss_mlp": 1.03092813, + "epoch": 0.11328419708664617, + "flos": 27487427696640.0, + "grad_norm": 2.4698802935518946, + "language_loss": 0.82262158, + "learning_rate": 3.927688364489246e-06, + "loss": 0.84465408, + "num_input_tokens_seen": 110355810, + "router_z_loss_clip": 0.98046875, + "router_z_loss_mlp": 0.17370605, + "step": 3904, + "time_per_iteration": 2.8100335597991943 + }, + { + "auxiliary_loss_clip": 0.01144917, + "auxiliary_loss_mlp": 0.010492, + "balance_loss_clip": 1.05352509, + "balance_loss_mlp": 1.0317955, + "epoch": 0.1133132145551622, + "flos": 12596960707200.0, + "grad_norm": 3.0894376362492606, + "language_loss": 0.8607949, + "learning_rate": 3.927638270468327e-06, + "loss": 0.88273609, + "num_input_tokens_seen": 110366885, + "router_z_loss_clip": 0.9140625, + "router_z_loss_mlp": 0.17401123, + "step": 3905, + "time_per_iteration": 2.472412109375 + }, + { + "auxiliary_loss_clip": 0.01142732, + "auxiliary_loss_mlp": 0.01039559, + "balance_loss_clip": 1.05157542, + "balance_loss_mlp": 1.02215481, + "epoch": 0.11334223202367825, + "flos": 33541679013120.0, + "grad_norm": 4.447069581017422, + "language_loss": 0.81990993, + "learning_rate": 3.927588159421727e-06, + "loss": 0.8417328, + "num_input_tokens_seen": 110385580, + "router_z_loss_clip": 0.91113281, + "router_z_loss_mlp": 0.17419434, + "step": 3906, + "time_per_iteration": 2.6281373500823975 + }, + { + "auxiliary_loss_clip": 0.01140368, + "auxiliary_loss_mlp": 0.01052817, + "balance_loss_clip": 1.05269504, + "balance_loss_mlp": 1.03540611, + "epoch": 0.1133712494921943, + "flos": 28576741501440.0, + "grad_norm": 3.373610674068046, + "language_loss": 0.89015383, + "learning_rate": 3.927538031349888e-06, + "loss": 0.91208565, + "num_input_tokens_seen": 110399305, + "router_z_loss_clip": 0.87695312, + "router_z_loss_mlp": 0.17388916, + "step": 3907, + "time_per_iteration": 2.539947509765625 + }, + { + "auxiliary_loss_clip": 0.01134189, + "auxiliary_loss_mlp": 0.01043938, + "balance_loss_clip": 1.05060399, + "balance_loss_mlp": 1.02827966, + "epoch": 0.11340026696071034, + "flos": 30329828686080.0, + "grad_norm": 2.549272236367455, + "language_loss": 0.84351599, + "learning_rate": 3.927487886253253e-06, + "loss": 0.86529732, + "num_input_tokens_seen": 110413070, + "router_z_loss_clip": 0.8359375, + "router_z_loss_mlp": 0.15655518, + "step": 3908, + "time_per_iteration": 2.5940277576446533 + }, + { + "auxiliary_loss_clip": 0.01150304, + "auxiliary_loss_mlp": 0.01043336, + "balance_loss_clip": 1.05702209, + "balance_loss_mlp": 1.02585387, + "epoch": 0.1134292844292264, + "flos": 74733335043840.0, + "grad_norm": 3.0523867323249023, + "language_loss": 0.73712456, + "learning_rate": 3.927437724132265e-06, + "loss": 0.75906098, + "num_input_tokens_seen": 110434435, + "router_z_loss_clip": 0.93115234, + "router_z_loss_mlp": 0.17486572, + "step": 3909, + "time_per_iteration": 2.888244152069092 + }, + { + "auxiliary_loss_clip": 0.01146725, + "auxiliary_loss_mlp": 0.01053215, + "balance_loss_clip": 1.05341983, + "balance_loss_mlp": 1.0356735, + "epoch": 0.11345830189774245, + "flos": 20368388586240.0, + "grad_norm": 4.262321913531248, + "language_loss": 0.93425226, + "learning_rate": 3.927387544987367e-06, + "loss": 0.95625162, + "num_input_tokens_seen": 110446750, + "router_z_loss_clip": 0.93359375, + "router_z_loss_mlp": 0.17553711, + "step": 3910, + "time_per_iteration": 2.553415536880493 + }, + { + "auxiliary_loss_clip": 0.01040861, + "auxiliary_loss_mlp": 0.01077153, + "balance_loss_clip": 1.0159874, + "balance_loss_mlp": 1.07594264, + "epoch": 0.11348731936625849, + "flos": 65194029532800.0, + "grad_norm": 0.6899893058470238, + "language_loss": 0.49568519, + "learning_rate": 3.927337348819003e-06, + "loss": 0.51686531, + "num_input_tokens_seen": 110513395, + "router_z_loss_clip": 0.24804688, + "router_z_loss_mlp": 0.01208496, + "step": 3911, + "time_per_iteration": 3.2754099369049072 + }, + { + "auxiliary_loss_clip": 0.01042883, + "auxiliary_loss_mlp": 0.01038794, + "balance_loss_clip": 1.01871717, + "balance_loss_mlp": 1.03760803, + "epoch": 0.11351633683477454, + "flos": 71133869053440.0, + "grad_norm": 0.6274780769360578, + "language_loss": 0.48814997, + "learning_rate": 3.927287135627615e-06, + "loss": 0.50896674, + "num_input_tokens_seen": 110582165, + "router_z_loss_clip": 0.2421875, + "router_z_loss_mlp": 0.01184082, + "step": 3912, + "time_per_iteration": 5.781883478164673 + }, + { + "auxiliary_loss_clip": 0.01041765, + "auxiliary_loss_mlp": 0.01004854, + "balance_loss_clip": 1.01831484, + "balance_loss_mlp": 1.00367379, + "epoch": 0.11354535430329057, + "flos": 61899224745600.0, + "grad_norm": 0.678046979612434, + "language_loss": 0.50576186, + "learning_rate": 3.9272369054136475e-06, + "loss": 0.52622807, + "num_input_tokens_seen": 110645815, + "router_z_loss_clip": 0.234375, + "router_z_loss_mlp": 0.01177979, + "step": 3913, + "time_per_iteration": 3.102032423019409 + }, + { + "auxiliary_loss_clip": 0.01138194, + "auxiliary_loss_mlp": 0.01048616, + "balance_loss_clip": 1.05234504, + "balance_loss_mlp": 1.03240907, + "epoch": 0.11357437177180663, + "flos": 20185962387840.0, + "grad_norm": 2.6795862978376976, + "language_loss": 0.71247005, + "learning_rate": 3.927186658177544e-06, + "loss": 0.7343381, + "num_input_tokens_seen": 110660280, + "router_z_loss_clip": 0.85888672, + "router_z_loss_mlp": 0.16217041, + "step": 3914, + "time_per_iteration": 4.87189507484436 + }, + { + "auxiliary_loss_clip": 0.01044019, + "auxiliary_loss_mlp": 0.01040315, + "balance_loss_clip": 1.02092695, + "balance_loss_mlp": 1.03936696, + "epoch": 0.11360338924032268, + "flos": 65324563557120.0, + "grad_norm": 0.6665882596145151, + "language_loss": 0.52168763, + "learning_rate": 3.927136393919748e-06, + "loss": 0.54253089, + "num_input_tokens_seen": 110723815, + "router_z_loss_clip": 0.23046875, + "router_z_loss_mlp": 0.00946045, + "step": 3915, + "time_per_iteration": 5.321086406707764 + }, + { + "auxiliary_loss_clip": 0.01159708, + "auxiliary_loss_mlp": 0.01079443, + "balance_loss_clip": 1.06073689, + "balance_loss_mlp": 1.06144214, + "epoch": 0.11363240670883872, + "flos": 15881637859200.0, + "grad_norm": 2.793389055613129, + "language_loss": 1.03215945, + "learning_rate": 3.927086112640703e-06, + "loss": 1.05455089, + "num_input_tokens_seen": 110735890, + "router_z_loss_clip": 0.98876953, + "router_z_loss_mlp": 0.18017578, + "step": 3916, + "time_per_iteration": 2.485124111175537 + }, + { + "auxiliary_loss_clip": 0.01140337, + "auxiliary_loss_mlp": 0.0108801, + "balance_loss_clip": 1.0542959, + "balance_loss_mlp": 1.07318068, + "epoch": 0.11366142417735477, + "flos": 20224674270720.0, + "grad_norm": 2.068598821551333, + "language_loss": 0.76956141, + "learning_rate": 3.927035814340854e-06, + "loss": 0.79184484, + "num_input_tokens_seen": 110750810, + "router_z_loss_clip": 0.86035156, + "router_z_loss_mlp": 0.14825439, + "step": 3917, + "time_per_iteration": 2.4963736534118652 + }, + { + "auxiliary_loss_clip": 0.01147812, + "auxiliary_loss_mlp": 0.01082014, + "balance_loss_clip": 1.05813396, + "balance_loss_mlp": 1.0656352, + "epoch": 0.11369044164587082, + "flos": 30584614832640.0, + "grad_norm": 3.9805439427582465, + "language_loss": 0.88070548, + "learning_rate": 3.926985499020645e-06, + "loss": 0.90300369, + "num_input_tokens_seen": 110765030, + "router_z_loss_clip": 0.89599609, + "router_z_loss_mlp": 0.16394043, + "step": 3918, + "time_per_iteration": 2.609971046447754 + }, + { + "auxiliary_loss_clip": 0.01146633, + "auxiliary_loss_mlp": 0.01090742, + "balance_loss_clip": 1.05634093, + "balance_loss_mlp": 1.07474446, + "epoch": 0.11371945911438686, + "flos": 27921264163200.0, + "grad_norm": 2.2567162450282905, + "language_loss": 0.90965402, + "learning_rate": 3.926935166680519e-06, + "loss": 0.93202782, + "num_input_tokens_seen": 110781500, + "router_z_loss_clip": 0.90283203, + "router_z_loss_mlp": 0.16003418, + "step": 3919, + "time_per_iteration": 2.6873040199279785 + }, + { + "auxiliary_loss_clip": 0.01145754, + "auxiliary_loss_mlp": 0.01087898, + "balance_loss_clip": 1.0567807, + "balance_loss_mlp": 1.07057667, + "epoch": 0.11374847658290291, + "flos": 36714135098880.0, + "grad_norm": 2.5294784974812417, + "language_loss": 0.85773778, + "learning_rate": 3.926884817320924e-06, + "loss": 0.88007426, + "num_input_tokens_seen": 110798115, + "router_z_loss_clip": 0.88964844, + "router_z_loss_mlp": 0.17340088, + "step": 3920, + "time_per_iteration": 2.6913509368896484 + }, + { + "auxiliary_loss_clip": 0.01154611, + "auxiliary_loss_mlp": 0.01088295, + "balance_loss_clip": 1.05909324, + "balance_loss_mlp": 1.06905484, + "epoch": 0.11377749405141896, + "flos": 10261223009280.0, + "grad_norm": 4.520583370944699, + "language_loss": 0.79258466, + "learning_rate": 3.926834450942301e-06, + "loss": 0.81501377, + "num_input_tokens_seen": 110810035, + "router_z_loss_clip": 0.95458984, + "router_z_loss_mlp": 0.19250488, + "step": 3921, + "time_per_iteration": 2.47404146194458 + }, + { + "auxiliary_loss_clip": 0.01050355, + "auxiliary_loss_mlp": 0.01134253, + "balance_loss_clip": 1.02538788, + "balance_loss_mlp": 1.13304317, + "epoch": 0.113806511519935, + "flos": 65030311342080.0, + "grad_norm": 0.7170096371263505, + "language_loss": 0.47796968, + "learning_rate": 3.926784067545097e-06, + "loss": 0.49981576, + "num_input_tokens_seen": 110867155, + "router_z_loss_clip": 0.25, + "router_z_loss_mlp": 0.01208496, + "step": 3922, + "time_per_iteration": 3.0438239574432373 + }, + { + "auxiliary_loss_clip": 0.01145398, + "auxiliary_loss_mlp": 0.01090479, + "balance_loss_clip": 1.05521119, + "balance_loss_mlp": 1.07334268, + "epoch": 0.11383552898845105, + "flos": 11726773822080.0, + "grad_norm": 2.6856566795866015, + "language_loss": 0.96753812, + "learning_rate": 3.926733667129756e-06, + "loss": 0.98989683, + "num_input_tokens_seen": 110882750, + "router_z_loss_clip": 0.90185547, + "router_z_loss_mlp": 0.17144775, + "step": 3923, + "time_per_iteration": 2.5122857093811035 + }, + { + "auxiliary_loss_clip": 0.01140322, + "auxiliary_loss_mlp": 0.01062726, + "balance_loss_clip": 1.05608749, + "balance_loss_mlp": 1.04631734, + "epoch": 0.1138645464569671, + "flos": 18440488886400.0, + "grad_norm": 2.6696569739458313, + "language_loss": 0.69181973, + "learning_rate": 3.926683249696724e-06, + "loss": 0.71385026, + "num_input_tokens_seen": 110899195, + "router_z_loss_clip": 0.84277344, + "router_z_loss_mlp": 0.1640625, + "step": 3924, + "time_per_iteration": 2.511350393295288 + }, + { + "auxiliary_loss_clip": 0.01041219, + "auxiliary_loss_mlp": 0.01091356, + "balance_loss_clip": 1.01834917, + "balance_loss_mlp": 1.09027672, + "epoch": 0.11389356392548314, + "flos": 63091781216640.0, + "grad_norm": 0.6964674699100922, + "language_loss": 0.45367539, + "learning_rate": 3.926632815246446e-06, + "loss": 0.47500116, + "num_input_tokens_seen": 110959765, + "router_z_loss_clip": 0.22851562, + "router_z_loss_mlp": 0.01080322, + "step": 3925, + "time_per_iteration": 3.0124731063842773 + }, + { + "auxiliary_loss_clip": 0.01141375, + "auxiliary_loss_mlp": 0.01055655, + "balance_loss_clip": 1.05381393, + "balance_loss_mlp": 1.03851318, + "epoch": 0.11392258139399919, + "flos": 28686556356480.0, + "grad_norm": 3.628417304101072, + "language_loss": 0.85212636, + "learning_rate": 3.926582363779367e-06, + "loss": 0.87409669, + "num_input_tokens_seen": 110974035, + "router_z_loss_clip": 0.87597656, + "router_z_loss_mlp": 0.17132568, + "step": 3926, + "time_per_iteration": 2.5728626251220703 + }, + { + "auxiliary_loss_clip": 0.01144447, + "auxiliary_loss_mlp": 0.01039721, + "balance_loss_clip": 1.05350471, + "balance_loss_mlp": 1.02225661, + "epoch": 0.11395159886251524, + "flos": 26098726032000.0, + "grad_norm": 2.2581552042363717, + "language_loss": 0.90172458, + "learning_rate": 3.926531895295934e-06, + "loss": 0.92356634, + "num_input_tokens_seen": 110989295, + "router_z_loss_clip": 0.91015625, + "router_z_loss_mlp": 0.17474365, + "step": 3927, + "time_per_iteration": 2.583765745162964 + }, + { + "auxiliary_loss_clip": 0.01038445, + "auxiliary_loss_mlp": 0.01025466, + "balance_loss_clip": 1.01524007, + "balance_loss_mlp": 1.02446508, + "epoch": 0.11398061633103128, + "flos": 65440088674560.0, + "grad_norm": 0.7265958998993254, + "language_loss": 0.51990521, + "learning_rate": 3.92648140979659e-06, + "loss": 0.54054439, + "num_input_tokens_seen": 111056260, + "router_z_loss_clip": 0.23242188, + "router_z_loss_mlp": 0.01000977, + "step": 3928, + "time_per_iteration": 3.134683609008789 + }, + { + "auxiliary_loss_clip": 0.0114244, + "auxiliary_loss_mlp": 0.01047637, + "balance_loss_clip": 1.05293453, + "balance_loss_mlp": 1.03204417, + "epoch": 0.11400963379954733, + "flos": 16467017806080.0, + "grad_norm": 2.7090872795586467, + "language_loss": 0.78614658, + "learning_rate": 3.926430907281784e-06, + "loss": 0.80804741, + "num_input_tokens_seen": 111070430, + "router_z_loss_clip": 0.89501953, + "router_z_loss_mlp": 0.15600586, + "step": 3929, + "time_per_iteration": 2.5185654163360596 + }, + { + "auxiliary_loss_clip": 0.01137341, + "auxiliary_loss_mlp": 0.0104399, + "balance_loss_clip": 1.0527004, + "balance_loss_mlp": 1.02871943, + "epoch": 0.11403865126806337, + "flos": 30044447130240.0, + "grad_norm": 1.8678620112906243, + "language_loss": 0.80971169, + "learning_rate": 3.92638038775196e-06, + "loss": 0.83152497, + "num_input_tokens_seen": 111090160, + "router_z_loss_clip": 0.84619141, + "router_z_loss_mlp": 0.15270996, + "step": 3930, + "time_per_iteration": 2.603541374206543 + }, + { + "auxiliary_loss_clip": 0.01158407, + "auxiliary_loss_mlp": 0.01063657, + "balance_loss_clip": 1.05885029, + "balance_loss_mlp": 1.04215217, + "epoch": 0.11406766873657942, + "flos": 16180846151040.0, + "grad_norm": 5.605012276041756, + "language_loss": 0.84776157, + "learning_rate": 3.926329851207565e-06, + "loss": 0.86998224, + "num_input_tokens_seen": 111104025, + "router_z_loss_clip": 0.99560547, + "router_z_loss_mlp": 0.21520996, + "step": 3931, + "time_per_iteration": 2.4621224403381348 + }, + { + "auxiliary_loss_clip": 0.01153405, + "auxiliary_loss_mlp": 0.01050851, + "balance_loss_clip": 1.0567081, + "balance_loss_mlp": 1.03286839, + "epoch": 0.11409668620509547, + "flos": 45619047532800.0, + "grad_norm": 2.5390349193988486, + "language_loss": 0.8382163, + "learning_rate": 3.9262792976490455e-06, + "loss": 0.86025888, + "num_input_tokens_seen": 111127170, + "router_z_loss_clip": 0.96533203, + "router_z_loss_mlp": 0.17980957, + "step": 3932, + "time_per_iteration": 2.6994781494140625 + }, + { + "auxiliary_loss_clip": 0.01141111, + "auxiliary_loss_mlp": 0.01062325, + "balance_loss_clip": 1.05390716, + "balance_loss_mlp": 1.04700637, + "epoch": 0.11412570367361151, + "flos": 29236312990080.0, + "grad_norm": 1.9329114606213884, + "language_loss": 0.64713907, + "learning_rate": 3.926228727076847e-06, + "loss": 0.66917348, + "num_input_tokens_seen": 111148405, + "router_z_loss_clip": 0.87353516, + "router_z_loss_mlp": 0.15307617, + "step": 3933, + "time_per_iteration": 2.7010905742645264 + }, + { + "auxiliary_loss_clip": 0.01143766, + "auxiliary_loss_mlp": 0.01070927, + "balance_loss_clip": 1.0559963, + "balance_loss_mlp": 1.05399942, + "epoch": 0.11415472114212756, + "flos": 11904710820480.0, + "grad_norm": 3.205093920663672, + "language_loss": 0.94955301, + "learning_rate": 3.926178139491418e-06, + "loss": 0.97169983, + "num_input_tokens_seen": 111161070, + "router_z_loss_clip": 0.87841797, + "router_z_loss_mlp": 0.16906738, + "step": 3934, + "time_per_iteration": 2.493347644805908 + }, + { + "auxiliary_loss_clip": 0.01041201, + "auxiliary_loss_mlp": 0.01086607, + "balance_loss_clip": 1.01611066, + "balance_loss_mlp": 1.08546853, + "epoch": 0.11418373861064361, + "flos": 61526399529600.0, + "grad_norm": 0.7175998429848357, + "language_loss": 0.53064108, + "learning_rate": 3.9261275348932036e-06, + "loss": 0.55191916, + "num_input_tokens_seen": 111225735, + "router_z_loss_clip": 0.25, + "router_z_loss_mlp": 0.01141357, + "step": 3935, + "time_per_iteration": 3.2568483352661133 + }, + { + "auxiliary_loss_clip": 0.01040407, + "auxiliary_loss_mlp": 0.01033609, + "balance_loss_clip": 1.0162009, + "balance_loss_mlp": 1.03263175, + "epoch": 0.11421275607915965, + "flos": 67415786398080.0, + "grad_norm": 0.6777045835341277, + "language_loss": 0.49376529, + "learning_rate": 3.9260769132826515e-06, + "loss": 0.51450551, + "num_input_tokens_seen": 111290520, + "router_z_loss_clip": 0.2421875, + "router_z_loss_mlp": 0.00976562, + "step": 3936, + "time_per_iteration": 3.1473865509033203 + }, + { + "auxiliary_loss_clip": 0.01149862, + "auxiliary_loss_mlp": 0.01040694, + "balance_loss_clip": 1.05658865, + "balance_loss_mlp": 1.02419591, + "epoch": 0.1142417735476757, + "flos": 12234227212800.0, + "grad_norm": 1.9409807786952498, + "language_loss": 0.68406641, + "learning_rate": 3.926026274660208e-06, + "loss": 0.70597196, + "num_input_tokens_seen": 111304770, + "router_z_loss_clip": 0.93359375, + "router_z_loss_mlp": 0.16485596, + "step": 3937, + "time_per_iteration": 2.4721806049346924 + }, + { + "auxiliary_loss_clip": 0.01141592, + "auxiliary_loss_mlp": 0.01043104, + "balance_loss_clip": 1.0554018, + "balance_loss_mlp": 1.02724349, + "epoch": 0.11427079101619175, + "flos": 20515586520960.0, + "grad_norm": 2.116257660594714, + "language_loss": 0.69963557, + "learning_rate": 3.925975619026322e-06, + "loss": 0.72148252, + "num_input_tokens_seen": 111319740, + "router_z_loss_clip": 0.86083984, + "router_z_loss_mlp": 0.1585083, + "step": 3938, + "time_per_iteration": 2.4666802883148193 + }, + { + "auxiliary_loss_clip": 0.01038346, + "auxiliary_loss_mlp": 0.01060226, + "balance_loss_clip": 1.01477838, + "balance_loss_mlp": 1.05927849, + "epoch": 0.11429980848470779, + "flos": 56938704606720.0, + "grad_norm": 0.7144161481064638, + "language_loss": 0.48670834, + "learning_rate": 3.9259249463814406e-06, + "loss": 0.50769407, + "num_input_tokens_seen": 111376500, + "router_z_loss_clip": 0.23632812, + "router_z_loss_mlp": 0.00946045, + "step": 3939, + "time_per_iteration": 2.971487283706665 + }, + { + "auxiliary_loss_clip": 0.0103663, + "auxiliary_loss_mlp": 0.01071915, + "balance_loss_clip": 1.01319194, + "balance_loss_mlp": 1.07096183, + "epoch": 0.11432882595322384, + "flos": 68938254311040.0, + "grad_norm": 0.6669397543790155, + "language_loss": 0.50625795, + "learning_rate": 3.9258742567260095e-06, + "loss": 0.52734345, + "num_input_tokens_seen": 111440920, + "router_z_loss_clip": 0.234375, + "router_z_loss_mlp": 0.00952148, + "step": 3940, + "time_per_iteration": 3.2515296936035156 + }, + { + "auxiliary_loss_clip": 0.01140931, + "auxiliary_loss_mlp": 0.01054209, + "balance_loss_clip": 1.05350399, + "balance_loss_mlp": 1.03866363, + "epoch": 0.1143578434217399, + "flos": 35444011207680.0, + "grad_norm": 2.3508052154676506, + "language_loss": 0.88593376, + "learning_rate": 3.925823550060478e-06, + "loss": 0.90788519, + "num_input_tokens_seen": 111459425, + "router_z_loss_clip": 0.87402344, + "router_z_loss_mlp": 0.15527344, + "step": 3941, + "time_per_iteration": 2.6251323223114014 + }, + { + "auxiliary_loss_clip": 0.01135626, + "auxiliary_loss_mlp": 0.01054602, + "balance_loss_clip": 1.05171752, + "balance_loss_mlp": 1.03924751, + "epoch": 0.11438686089025593, + "flos": 18507497708160.0, + "grad_norm": 2.6831915657610104, + "language_loss": 0.7670908, + "learning_rate": 3.925772826385293e-06, + "loss": 0.78899312, + "num_input_tokens_seen": 111472385, + "router_z_loss_clip": 0.83886719, + "router_z_loss_mlp": 0.15368652, + "step": 3942, + "time_per_iteration": 2.505624294281006 + }, + { + "auxiliary_loss_clip": 0.01133078, + "auxiliary_loss_mlp": 0.01050976, + "balance_loss_clip": 1.05202508, + "balance_loss_mlp": 1.0353061, + "epoch": 0.11441587835877198, + "flos": 10774135267200.0, + "grad_norm": 2.776639189255586, + "language_loss": 0.6589849, + "learning_rate": 3.9257220857009044e-06, + "loss": 0.68082541, + "num_input_tokens_seen": 111483810, + "router_z_loss_clip": 0.81054688, + "router_z_loss_mlp": 0.15673828, + "step": 3943, + "time_per_iteration": 2.4732918739318848 + }, + { + "auxiliary_loss_clip": 0.0103498, + "auxiliary_loss_mlp": 0.01050634, + "balance_loss_clip": 1.01221728, + "balance_loss_mlp": 1.04952562, + "epoch": 0.11444489582728802, + "flos": 58830621857280.0, + "grad_norm": 0.7211243869437747, + "language_loss": 0.54473531, + "learning_rate": 3.9256713280077585e-06, + "loss": 0.56559145, + "num_input_tokens_seen": 111538990, + "router_z_loss_clip": 0.22851562, + "router_z_loss_mlp": 0.0111084, + "step": 3944, + "time_per_iteration": 2.9751856327056885 + }, + { + "auxiliary_loss_clip": 0.01034194, + "auxiliary_loss_mlp": 0.01045001, + "balance_loss_clip": 1.0111928, + "balance_loss_mlp": 1.04378462, + "epoch": 0.11447391329580407, + "flos": 69674064416640.0, + "grad_norm": 0.6692355530931481, + "language_loss": 0.46552825, + "learning_rate": 3.925620553306304e-06, + "loss": 0.4863202, + "num_input_tokens_seen": 111603210, + "router_z_loss_clip": 0.23046875, + "router_z_loss_mlp": 0.012146, + "step": 3945, + "time_per_iteration": 3.1427841186523438 + }, + { + "auxiliary_loss_clip": 0.01133027, + "auxiliary_loss_mlp": 0.01045305, + "balance_loss_clip": 1.05073798, + "balance_loss_mlp": 1.02825236, + "epoch": 0.11450293076432012, + "flos": 16468993054080.0, + "grad_norm": 3.353444847520234, + "language_loss": 0.74727368, + "learning_rate": 3.92556976159699e-06, + "loss": 0.76905704, + "num_input_tokens_seen": 111617580, + "router_z_loss_clip": 0.82226562, + "router_z_loss_mlp": 0.17059326, + "step": 3946, + "time_per_iteration": 2.4970433712005615 + }, + { + "auxiliary_loss_clip": 0.01146248, + "auxiliary_loss_mlp": 0.01047731, + "balance_loss_clip": 1.05339098, + "balance_loss_mlp": 1.02961123, + "epoch": 0.11453194823283616, + "flos": 29308313802240.0, + "grad_norm": 1.9587553643504638, + "language_loss": 0.78959441, + "learning_rate": 3.925518952880264e-06, + "loss": 0.81153417, + "num_input_tokens_seen": 111634320, + "router_z_loss_clip": 0.92773438, + "router_z_loss_mlp": 0.18115234, + "step": 3947, + "time_per_iteration": 2.5651676654815674 + }, + { + "auxiliary_loss_clip": 0.01140464, + "auxiliary_loss_mlp": 0.01042201, + "balance_loss_clip": 1.05326569, + "balance_loss_mlp": 1.02559543, + "epoch": 0.11456096570135221, + "flos": 45365338794240.0, + "grad_norm": 1.5666242559485992, + "language_loss": 0.8736729, + "learning_rate": 3.925468127156576e-06, + "loss": 0.89549959, + "num_input_tokens_seen": 111657025, + "router_z_loss_clip": 0.87207031, + "router_z_loss_mlp": 0.16607666, + "step": 3948, + "time_per_iteration": 2.7222647666931152 + }, + { + "auxiliary_loss_clip": 0.01140125, + "auxiliary_loss_mlp": 0.01043612, + "balance_loss_clip": 1.05755901, + "balance_loss_mlp": 1.02890754, + "epoch": 0.11458998316986826, + "flos": 11248227901440.0, + "grad_norm": 2.498084471144575, + "language_loss": 0.69666332, + "learning_rate": 3.9254172844263745e-06, + "loss": 0.71850073, + "num_input_tokens_seen": 111667980, + "router_z_loss_clip": 0.82568359, + "router_z_loss_mlp": 0.14697266, + "step": 3949, + "time_per_iteration": 2.5095505714416504 + }, + { + "auxiliary_loss_clip": 0.01033449, + "auxiliary_loss_mlp": 0.01014753, + "balance_loss_clip": 1.01056743, + "balance_loss_mlp": 1.01348329, + "epoch": 0.1146190006383843, + "flos": 63542425248000.0, + "grad_norm": 0.6624562382886471, + "language_loss": 0.48379472, + "learning_rate": 3.925366424690107e-06, + "loss": 0.50427675, + "num_input_tokens_seen": 111736400, + "router_z_loss_clip": 0.22851562, + "router_z_loss_mlp": 0.01269531, + "step": 3950, + "time_per_iteration": 3.3089420795440674 + }, + { + "auxiliary_loss_clip": 0.01034046, + "auxiliary_loss_mlp": 0.01015201, + "balance_loss_clip": 1.01111293, + "balance_loss_mlp": 1.01388347, + "epoch": 0.11464801810690035, + "flos": 64307537873280.0, + "grad_norm": 0.7076643535898539, + "language_loss": 0.53416282, + "learning_rate": 3.9253155479482255e-06, + "loss": 0.55465525, + "num_input_tokens_seen": 111797420, + "router_z_loss_clip": 0.22949219, + "router_z_loss_mlp": 0.01318359, + "step": 3951, + "time_per_iteration": 3.0662548542022705 + }, + { + "auxiliary_loss_clip": 0.01141746, + "auxiliary_loss_mlp": 0.01047605, + "balance_loss_clip": 1.05349541, + "balance_loss_mlp": 1.0281738, + "epoch": 0.1146770355754164, + "flos": 16136495832960.0, + "grad_norm": 3.0392083094950118, + "language_loss": 0.9169777, + "learning_rate": 3.925264654201178e-06, + "loss": 0.93887115, + "num_input_tokens_seen": 111811490, + "router_z_loss_clip": 0.88232422, + "router_z_loss_mlp": 0.19433594, + "step": 3952, + "time_per_iteration": 2.4952950477600098 + }, + { + "auxiliary_loss_clip": 0.01145593, + "auxiliary_loss_mlp": 0.01040379, + "balance_loss_clip": 1.06046546, + "balance_loss_mlp": 1.02631211, + "epoch": 0.11470605304393244, + "flos": 32228425866240.0, + "grad_norm": 2.0639409649128453, + "language_loss": 0.76485193, + "learning_rate": 3.925213743449413e-06, + "loss": 0.78671163, + "num_input_tokens_seen": 111827825, + "router_z_loss_clip": 0.8515625, + "router_z_loss_mlp": 0.140625, + "step": 3953, + "time_per_iteration": 2.606956720352173 + }, + { + "auxiliary_loss_clip": 0.01144999, + "auxiliary_loss_mlp": 0.01050195, + "balance_loss_clip": 1.05603886, + "balance_loss_mlp": 1.03109193, + "epoch": 0.1147350705124485, + "flos": 20806857907200.0, + "grad_norm": 5.1470056301219556, + "language_loss": 1.02561188, + "learning_rate": 3.9251628156933825e-06, + "loss": 1.04756379, + "num_input_tokens_seen": 111840895, + "router_z_loss_clip": 0.88867188, + "router_z_loss_mlp": 0.19116211, + "step": 3954, + "time_per_iteration": 2.5273735523223877 + }, + { + "auxiliary_loss_clip": 0.01033805, + "auxiliary_loss_mlp": 0.01025902, + "balance_loss_clip": 1.01087236, + "balance_loss_mlp": 1.02462661, + "epoch": 0.11476408798096455, + "flos": 59510158329600.0, + "grad_norm": 0.682348146954294, + "language_loss": 0.48884365, + "learning_rate": 3.9251118709335345e-06, + "loss": 0.50944072, + "num_input_tokens_seen": 111893540, + "router_z_loss_clip": 0.22851562, + "router_z_loss_mlp": 0.01275635, + "step": 3955, + "time_per_iteration": 2.9016530513763428 + }, + { + "auxiliary_loss_clip": 0.01134348, + "auxiliary_loss_mlp": 0.01049636, + "balance_loss_clip": 1.05197561, + "balance_loss_mlp": 1.03283381, + "epoch": 0.11479310544948058, + "flos": 15809313824640.0, + "grad_norm": 7.328232654401788, + "language_loss": 0.58923852, + "learning_rate": 3.925060909170318e-06, + "loss": 0.61107838, + "num_input_tokens_seen": 111908995, + "router_z_loss_clip": 0.82324219, + "router_z_loss_mlp": 0.16796875, + "step": 3956, + "time_per_iteration": 2.4634265899658203 + }, + { + "auxiliary_loss_clip": 0.01033708, + "auxiliary_loss_mlp": 0.01017285, + "balance_loss_clip": 1.01075184, + "balance_loss_mlp": 1.01602101, + "epoch": 0.11482212291799664, + "flos": 62808985440000.0, + "grad_norm": 0.678107577205086, + "language_loss": 0.48596859, + "learning_rate": 3.925009930404186e-06, + "loss": 0.50647855, + "num_input_tokens_seen": 111970470, + "router_z_loss_clip": 0.22949219, + "router_z_loss_mlp": 0.01263428, + "step": 3957, + "time_per_iteration": 3.060950994491577 + }, + { + "auxiliary_loss_clip": 0.01150508, + "auxiliary_loss_mlp": 0.01055803, + "balance_loss_clip": 1.05473566, + "balance_loss_mlp": 1.03696823, + "epoch": 0.11485114038651269, + "flos": 18838809780480.0, + "grad_norm": 5.8021279917476125, + "language_loss": 0.81698191, + "learning_rate": 3.924958934635587e-06, + "loss": 0.83904499, + "num_input_tokens_seen": 111985070, + "router_z_loss_clip": 0.95849609, + "router_z_loss_mlp": 0.18847656, + "step": 3958, + "time_per_iteration": 2.501558542251587 + }, + { + "auxiliary_loss_clip": 0.0114215, + "auxiliary_loss_mlp": 0.0104634, + "balance_loss_clip": 1.05426502, + "balance_loss_mlp": 1.02957332, + "epoch": 0.11488015785502873, + "flos": 12926082049920.0, + "grad_norm": 3.020509245290731, + "language_loss": 0.7913518, + "learning_rate": 3.924907921864973e-06, + "loss": 0.81323671, + "num_input_tokens_seen": 111999115, + "router_z_loss_clip": 0.87841797, + "router_z_loss_mlp": 0.16772461, + "step": 3959, + "time_per_iteration": 2.4822444915771484 + }, + { + "auxiliary_loss_clip": 0.01146726, + "auxiliary_loss_mlp": 0.01042486, + "balance_loss_clip": 1.05676603, + "balance_loss_mlp": 1.02496195, + "epoch": 0.11490917532354478, + "flos": 45476266970880.0, + "grad_norm": 1.5319138433393407, + "language_loss": 0.63828069, + "learning_rate": 3.924856892092792e-06, + "loss": 0.66017282, + "num_input_tokens_seen": 112022605, + "router_z_loss_clip": 0.8984375, + "router_z_loss_mlp": 0.1751709, + "step": 3960, + "time_per_iteration": 2.750028610229492 + }, + { + "auxiliary_loss_clip": 0.01140389, + "auxiliary_loss_mlp": 0.01048572, + "balance_loss_clip": 1.05420053, + "balance_loss_mlp": 1.03214478, + "epoch": 0.11493819279206081, + "flos": 16100980260480.0, + "grad_norm": 2.956345760606684, + "language_loss": 0.80332327, + "learning_rate": 3.924805845319496e-06, + "loss": 0.8252129, + "num_input_tokens_seen": 112035750, + "router_z_loss_clip": 0.86279297, + "router_z_loss_mlp": 0.16418457, + "step": 3961, + "time_per_iteration": 2.4975266456604004 + }, + { + "auxiliary_loss_clip": 0.01034627, + "auxiliary_loss_mlp": 0.01004009, + "balance_loss_clip": 1.01159537, + "balance_loss_mlp": 1.00285292, + "epoch": 0.11496721026057687, + "flos": 59545817556480.0, + "grad_norm": 0.7303277965504005, + "language_loss": 0.49210978, + "learning_rate": 3.924754781545536e-06, + "loss": 0.51249617, + "num_input_tokens_seen": 112085960, + "router_z_loss_clip": 0.23046875, + "router_z_loss_mlp": 0.01153564, + "step": 3962, + "time_per_iteration": 2.890348196029663 + }, + { + "auxiliary_loss_clip": 0.01144171, + "auxiliary_loss_mlp": 0.01050599, + "balance_loss_clip": 1.05523527, + "balance_loss_mlp": 1.03249145, + "epoch": 0.11499622772909292, + "flos": 16684887749760.0, + "grad_norm": 4.177700583809226, + "language_loss": 0.81631851, + "learning_rate": 3.9247037007713634e-06, + "loss": 0.83826619, + "num_input_tokens_seen": 112099350, + "router_z_loss_clip": 0.88867188, + "router_z_loss_mlp": 0.18109131, + "step": 3963, + "time_per_iteration": 2.482640504837036 + }, + { + "auxiliary_loss_clip": 0.01035015, + "auxiliary_loss_mlp": 0.00999937, + "balance_loss_clip": 1.01182961, + "balance_loss_mlp": 0.99881697, + "epoch": 0.11502524519760896, + "flos": 61275958928640.0, + "grad_norm": 0.6942927761860435, + "language_loss": 0.48003167, + "learning_rate": 3.924652602997428e-06, + "loss": 0.50038123, + "num_input_tokens_seen": 112159430, + "router_z_loss_clip": 0.23242188, + "router_z_loss_mlp": 0.01123047, + "step": 3964, + "time_per_iteration": 3.023630380630493 + }, + { + "auxiliary_loss_clip": 0.01151059, + "auxiliary_loss_mlp": 0.01047007, + "balance_loss_clip": 1.05691433, + "balance_loss_mlp": 1.0287081, + "epoch": 0.115054262666125, + "flos": 15516965030400.0, + "grad_norm": 2.5333543493951756, + "language_loss": 0.86811066, + "learning_rate": 3.9246014882241825e-06, + "loss": 0.8900913, + "num_input_tokens_seen": 112172395, + "router_z_loss_clip": 0.94189453, + "router_z_loss_mlp": 0.1829834, + "step": 3965, + "time_per_iteration": 2.4995229244232178 + }, + { + "auxiliary_loss_clip": 0.01147849, + "auxiliary_loss_mlp": 0.01047051, + "balance_loss_clip": 1.05670822, + "balance_loss_mlp": 1.0299449, + "epoch": 0.11508328013464106, + "flos": 24310805633280.0, + "grad_norm": 2.3432254996626813, + "language_loss": 1.03094935, + "learning_rate": 3.924550356452078e-06, + "loss": 1.05289829, + "num_input_tokens_seen": 112189715, + "router_z_loss_clip": 0.91162109, + "router_z_loss_mlp": 0.17102051, + "step": 3966, + "time_per_iteration": 2.5697121620178223 + }, + { + "auxiliary_loss_clip": 0.01033842, + "auxiliary_loss_mlp": 0.01002214, + "balance_loss_clip": 1.01094937, + "balance_loss_mlp": 1.00118256, + "epoch": 0.1151122976031571, + "flos": 68082037816320.0, + "grad_norm": 0.6981003908945234, + "language_loss": 0.46693996, + "learning_rate": 3.9244992076815655e-06, + "loss": 0.48730052, + "num_input_tokens_seen": 112253870, + "router_z_loss_clip": 0.22851562, + "router_z_loss_mlp": 0.01031494, + "step": 3967, + "time_per_iteration": 3.2952706813812256 + }, + { + "auxiliary_loss_clip": 0.01144732, + "auxiliary_loss_mlp": 0.01038721, + "balance_loss_clip": 1.05679274, + "balance_loss_mlp": 1.02315199, + "epoch": 0.11514131507167315, + "flos": 15230541980160.0, + "grad_norm": 3.9861887056258416, + "language_loss": 0.52134705, + "learning_rate": 3.9244480419130974e-06, + "loss": 0.54318154, + "num_input_tokens_seen": 112270460, + "router_z_loss_clip": 0.87939453, + "router_z_loss_mlp": 0.15576172, + "step": 3968, + "time_per_iteration": 2.493828296661377 + }, + { + "auxiliary_loss_clip": 0.01144206, + "auxiliary_loss_mlp": 0.01040999, + "balance_loss_clip": 1.05410063, + "balance_loss_mlp": 1.02299893, + "epoch": 0.1151703325401892, + "flos": 25549148966400.0, + "grad_norm": 7.101038891205278, + "language_loss": 0.96356356, + "learning_rate": 3.924396859147126e-06, + "loss": 0.98541558, + "num_input_tokens_seen": 112283700, + "router_z_loss_clip": 0.89941406, + "router_z_loss_mlp": 0.18005371, + "step": 3969, + "time_per_iteration": 2.5799813270568848 + }, + { + "auxiliary_loss_clip": 0.01135592, + "auxiliary_loss_mlp": 0.01043319, + "balance_loss_clip": 1.0539149, + "balance_loss_mlp": 1.02721393, + "epoch": 0.11519935000870524, + "flos": 26213604704640.0, + "grad_norm": 2.2934016282543133, + "language_loss": 0.8747434, + "learning_rate": 3.924345659384103e-06, + "loss": 0.89653248, + "num_input_tokens_seen": 112299215, + "router_z_loss_clip": 0.81689453, + "router_z_loss_mlp": 0.16101074, + "step": 3970, + "time_per_iteration": 2.5649337768554688 + }, + { + "auxiliary_loss_clip": 0.01155874, + "auxiliary_loss_mlp": 0.01052399, + "balance_loss_clip": 1.06038368, + "balance_loss_mlp": 1.03483915, + "epoch": 0.11522836747722129, + "flos": 31789920631680.0, + "grad_norm": 2.5817812488871104, + "language_loss": 0.92658532, + "learning_rate": 3.924294442624479e-06, + "loss": 0.94866806, + "num_input_tokens_seen": 112320055, + "router_z_loss_clip": 0.95556641, + "router_z_loss_mlp": 0.17547607, + "step": 3971, + "time_per_iteration": 2.603536367416382 + }, + { + "auxiliary_loss_clip": 0.01035034, + "auxiliary_loss_mlp": 0.01009388, + "balance_loss_clip": 1.01184094, + "balance_loss_mlp": 1.00829685, + "epoch": 0.11525738494573734, + "flos": 69271002927360.0, + "grad_norm": 0.8728497291040075, + "language_loss": 0.48229563, + "learning_rate": 3.924243208868708e-06, + "loss": 0.50273985, + "num_input_tokens_seen": 112381190, + "router_z_loss_clip": 0.23242188, + "router_z_loss_mlp": 0.01092529, + "step": 3972, + "time_per_iteration": 3.124624013900757 + }, + { + "auxiliary_loss_clip": 0.01153299, + "auxiliary_loss_mlp": 0.01047395, + "balance_loss_clip": 1.05934954, + "balance_loss_mlp": 1.030527, + "epoch": 0.11528640241425338, + "flos": 24492872695680.0, + "grad_norm": 2.4670875620698953, + "language_loss": 0.75224769, + "learning_rate": 3.924191958117243e-06, + "loss": 0.77425468, + "num_input_tokens_seen": 112398815, + "router_z_loss_clip": 0.94042969, + "router_z_loss_mlp": 0.16876221, + "step": 3973, + "time_per_iteration": 2.550657272338867 + }, + { + "auxiliary_loss_clip": 0.01143889, + "auxiliary_loss_mlp": 0.01035946, + "balance_loss_clip": 1.05476391, + "balance_loss_mlp": 1.01950729, + "epoch": 0.11531541988276943, + "flos": 29782406436480.0, + "grad_norm": 1.87128770611559, + "language_loss": 0.71601886, + "learning_rate": 3.9241406903705365e-06, + "loss": 0.73781723, + "num_input_tokens_seen": 112418150, + "router_z_loss_clip": 0.89111328, + "router_z_loss_mlp": 0.16436768, + "step": 3974, + "time_per_iteration": 2.622735023498535 + }, + { + "auxiliary_loss_clip": 0.01132517, + "auxiliary_loss_mlp": 0.0103563, + "balance_loss_clip": 1.05333638, + "balance_loss_mlp": 1.02115786, + "epoch": 0.11534443735128547, + "flos": 16611378566400.0, + "grad_norm": 2.1473728137654176, + "language_loss": 0.6829226, + "learning_rate": 3.9240894056290395e-06, + "loss": 0.70460403, + "num_input_tokens_seen": 112432280, + "router_z_loss_clip": 0.79199219, + "router_z_loss_mlp": 0.14471436, + "step": 3975, + "time_per_iteration": 2.5246810913085938 + }, + { + "auxiliary_loss_clip": 0.01147975, + "auxiliary_loss_mlp": 0.01048758, + "balance_loss_clip": 1.05757582, + "balance_loss_mlp": 1.03174639, + "epoch": 0.11537345481980152, + "flos": 22921529351040.0, + "grad_norm": 4.020610833503902, + "language_loss": 0.77983826, + "learning_rate": 3.924038103893208e-06, + "loss": 0.80180562, + "num_input_tokens_seen": 112445850, + "router_z_loss_clip": 0.90332031, + "router_z_loss_mlp": 0.17004395, + "step": 3976, + "time_per_iteration": 2.5090794563293457 + }, + { + "auxiliary_loss_clip": 0.01147262, + "auxiliary_loss_mlp": 0.01043536, + "balance_loss_clip": 1.05527472, + "balance_loss_mlp": 1.02630377, + "epoch": 0.11540247228831757, + "flos": 42622804592640.0, + "grad_norm": 4.893747461950467, + "language_loss": 0.72919822, + "learning_rate": 3.9239867851634925e-06, + "loss": 0.75110626, + "num_input_tokens_seen": 112462520, + "router_z_loss_clip": 0.92041016, + "router_z_loss_mlp": 0.17230225, + "step": 3977, + "time_per_iteration": 2.594872236251831 + }, + { + "auxiliary_loss_clip": 0.01135874, + "auxiliary_loss_mlp": 0.0103888, + "balance_loss_clip": 1.0524708, + "balance_loss_mlp": 1.0218451, + "epoch": 0.11543148975683361, + "flos": 46894091587200.0, + "grad_norm": 2.0715663724155644, + "language_loss": 0.75494874, + "learning_rate": 3.923935449440347e-06, + "loss": 0.77669632, + "num_input_tokens_seen": 112480655, + "router_z_loss_clip": 0.83398438, + "router_z_loss_mlp": 0.17028809, + "step": 3978, + "time_per_iteration": 2.729597806930542 + }, + { + "auxiliary_loss_clip": 0.01139817, + "auxiliary_loss_mlp": 0.0104821, + "balance_loss_clip": 1.05224013, + "balance_loss_mlp": 1.030949, + "epoch": 0.11546050722534966, + "flos": 15260634599040.0, + "grad_norm": 3.5707945811441815, + "language_loss": 0.94178331, + "learning_rate": 3.923884096724225e-06, + "loss": 0.96366358, + "num_input_tokens_seen": 112492690, + "router_z_loss_clip": 0.87646484, + "router_z_loss_mlp": 0.17260742, + "step": 3979, + "time_per_iteration": 2.5064892768859863 + }, + { + "auxiliary_loss_clip": 0.01036438, + "auxiliary_loss_mlp": 0.01002682, + "balance_loss_clip": 1.01338148, + "balance_loss_mlp": 1.0015254, + "epoch": 0.11548952469386571, + "flos": 74780848736640.0, + "grad_norm": 0.6235608189762695, + "language_loss": 0.53859901, + "learning_rate": 3.92383272701558e-06, + "loss": 0.55899018, + "num_input_tokens_seen": 112557940, + "router_z_loss_clip": 0.23046875, + "router_z_loss_mlp": 0.01153564, + "step": 3980, + "time_per_iteration": 3.219947338104248 + }, + { + "auxiliary_loss_clip": 0.01145875, + "auxiliary_loss_mlp": 0.01047077, + "balance_loss_clip": 1.05574632, + "balance_loss_mlp": 1.02842641, + "epoch": 0.11551854216238175, + "flos": 44668096917120.0, + "grad_norm": 2.2354508505153383, + "language_loss": 0.92751133, + "learning_rate": 3.923781340314866e-06, + "loss": 0.94944084, + "num_input_tokens_seen": 112576120, + "router_z_loss_clip": 0.90136719, + "router_z_loss_mlp": 0.1864624, + "step": 3981, + "time_per_iteration": 2.6839282512664795 + }, + { + "auxiliary_loss_clip": 0.01137512, + "auxiliary_loss_mlp": 0.01041102, + "balance_loss_clip": 1.05026722, + "balance_loss_mlp": 1.02446651, + "epoch": 0.1155475596308978, + "flos": 24274140825600.0, + "grad_norm": 2.519482332524265, + "language_loss": 1.02551043, + "learning_rate": 3.923729936622537e-06, + "loss": 1.04729652, + "num_input_tokens_seen": 112591455, + "router_z_loss_clip": 0.87207031, + "router_z_loss_mlp": 0.16638184, + "step": 3982, + "time_per_iteration": 2.512855052947998 + }, + { + "auxiliary_loss_clip": 0.01034734, + "auxiliary_loss_mlp": 0.01000776, + "balance_loss_clip": 1.0117625, + "balance_loss_mlp": 0.99964923, + "epoch": 0.11557657709941385, + "flos": 57800846845440.0, + "grad_norm": 0.7134909115247876, + "language_loss": 0.45790762, + "learning_rate": 3.9236785159390465e-06, + "loss": 0.47826272, + "num_input_tokens_seen": 112649195, + "router_z_loss_clip": 0.23046875, + "router_z_loss_mlp": 0.0112915, + "step": 3983, + "time_per_iteration": 5.370785236358643 + }, + { + "auxiliary_loss_clip": 0.01132738, + "auxiliary_loss_mlp": 0.01038199, + "balance_loss_clip": 1.05077839, + "balance_loss_mlp": 1.02389395, + "epoch": 0.11560559456792989, + "flos": 15041579506560.0, + "grad_norm": 5.764201309969225, + "language_loss": 0.76184785, + "learning_rate": 3.92362707826485e-06, + "loss": 0.78355718, + "num_input_tokens_seen": 112663830, + "router_z_loss_clip": 0.81884766, + "router_z_loss_mlp": 0.14300537, + "step": 3984, + "time_per_iteration": 2.501728057861328 + }, + { + "auxiliary_loss_clip": 0.01033299, + "auxiliary_loss_mlp": 0.0100258, + "balance_loss_clip": 1.01038134, + "balance_loss_mlp": 1.00145328, + "epoch": 0.11563461203644594, + "flos": 52182945947520.0, + "grad_norm": 0.7310841538103118, + "language_loss": 0.55310082, + "learning_rate": 3.923575623600399e-06, + "loss": 0.57345963, + "num_input_tokens_seen": 112720170, + "router_z_loss_clip": 0.22851562, + "router_z_loss_mlp": 0.0112915, + "step": 3985, + "time_per_iteration": 7.659862041473389 + }, + { + "auxiliary_loss_clip": 0.01034196, + "auxiliary_loss_mlp": 0.01002042, + "balance_loss_clip": 1.01132953, + "balance_loss_mlp": 1.00082052, + "epoch": 0.11566362950496199, + "flos": 74775066647040.0, + "grad_norm": 0.7697987102280331, + "language_loss": 0.4500677, + "learning_rate": 3.92352415194615e-06, + "loss": 0.47043008, + "num_input_tokens_seen": 112785350, + "router_z_loss_clip": 0.22851562, + "router_z_loss_mlp": 0.01220703, + "step": 3986, + "time_per_iteration": 3.115372657775879 + }, + { + "auxiliary_loss_clip": 0.01143959, + "auxiliary_loss_mlp": 0.01038418, + "balance_loss_clip": 1.0565573, + "balance_loss_mlp": 1.02188396, + "epoch": 0.11569264697347803, + "flos": 16247495836800.0, + "grad_norm": 2.7473061661434333, + "language_loss": 0.74956638, + "learning_rate": 3.923472663302558e-06, + "loss": 0.77139008, + "num_input_tokens_seen": 112797010, + "router_z_loss_clip": 0.875, + "router_z_loss_mlp": 0.16534424, + "step": 3987, + "time_per_iteration": 4.940680980682373 + }, + { + "auxiliary_loss_clip": 0.01030643, + "auxiliary_loss_mlp": 0.0100026, + "balance_loss_clip": 1.00795901, + "balance_loss_mlp": 0.99914497, + "epoch": 0.11572166444199408, + "flos": 55326063600000.0, + "grad_norm": 0.6679619501636758, + "language_loss": 0.42422599, + "learning_rate": 3.923421157670077e-06, + "loss": 0.44453502, + "num_input_tokens_seen": 112854955, + "router_z_loss_clip": 0.2265625, + "router_z_loss_mlp": 0.01116943, + "step": 3988, + "time_per_iteration": 3.1310672760009766 + }, + { + "auxiliary_loss_clip": 0.01031379, + "auxiliary_loss_mlp": 0.01002813, + "balance_loss_clip": 1.0086453, + "balance_loss_mlp": 1.00172198, + "epoch": 0.11575068191051013, + "flos": 63885909052800.0, + "grad_norm": 0.722712784035103, + "language_loss": 0.49924308, + "learning_rate": 3.9233696350491614e-06, + "loss": 0.51958501, + "num_input_tokens_seen": 112905210, + "router_z_loss_clip": 0.22753906, + "router_z_loss_mlp": 0.01092529, + "step": 3989, + "time_per_iteration": 2.912242889404297 + }, + { + "auxiliary_loss_clip": 0.01144163, + "auxiliary_loss_mlp": 0.01053655, + "balance_loss_clip": 1.05433428, + "balance_loss_mlp": 1.03579164, + "epoch": 0.11577969937902617, + "flos": 54374212166400.0, + "grad_norm": 5.618075043665732, + "language_loss": 0.87743092, + "learning_rate": 3.923318095440268e-06, + "loss": 0.89940906, + "num_input_tokens_seen": 112926635, + "router_z_loss_clip": 0.8984375, + "router_z_loss_mlp": 0.17871094, + "step": 3990, + "time_per_iteration": 2.793804407119751 + }, + { + "auxiliary_loss_clip": 0.01137525, + "auxiliary_loss_mlp": 0.01054468, + "balance_loss_clip": 1.05129218, + "balance_loss_mlp": 1.03574026, + "epoch": 0.11580871684754222, + "flos": 17048734565760.0, + "grad_norm": 2.0945779793961083, + "language_loss": 0.84113872, + "learning_rate": 3.92326653884385e-06, + "loss": 0.86305863, + "num_input_tokens_seen": 112940215, + "router_z_loss_clip": 0.86279297, + "router_z_loss_mlp": 0.18725586, + "step": 3991, + "time_per_iteration": 2.4682586193084717 + }, + { + "auxiliary_loss_clip": 0.0113473, + "auxiliary_loss_mlp": 0.01037523, + "balance_loss_clip": 1.05221558, + "balance_loss_mlp": 1.02297997, + "epoch": 0.11583773431605826, + "flos": 19971647890560.0, + "grad_norm": 2.6947662750809807, + "language_loss": 0.7922256, + "learning_rate": 3.9232149652603635e-06, + "loss": 0.81394804, + "num_input_tokens_seen": 112956740, + "router_z_loss_clip": 0.82519531, + "router_z_loss_mlp": 0.14538574, + "step": 3992, + "time_per_iteration": 2.5394554138183594 + }, + { + "auxiliary_loss_clip": 0.01135797, + "auxiliary_loss_mlp": 0.01042318, + "balance_loss_clip": 1.05115926, + "balance_loss_mlp": 1.02635562, + "epoch": 0.11586675178457431, + "flos": 53244283057920.0, + "grad_norm": 2.6418285968842126, + "language_loss": 0.79148382, + "learning_rate": 3.923163374690265e-06, + "loss": 0.81326497, + "num_input_tokens_seen": 112975445, + "router_z_loss_clip": 0.84570312, + "router_z_loss_mlp": 0.15966797, + "step": 3993, + "time_per_iteration": 2.771817207336426 + }, + { + "auxiliary_loss_clip": 0.01142254, + "auxiliary_loss_mlp": 0.01043629, + "balance_loss_clip": 1.05530095, + "balance_loss_mlp": 1.02645087, + "epoch": 0.11589576925309036, + "flos": 11503983715200.0, + "grad_norm": 2.6060548402139543, + "language_loss": 0.74090141, + "learning_rate": 3.923111767134009e-06, + "loss": 0.76276028, + "num_input_tokens_seen": 112987485, + "router_z_loss_clip": 0.87011719, + "router_z_loss_mlp": 0.17175293, + "step": 3994, + "time_per_iteration": 2.4985361099243164 + }, + { + "auxiliary_loss_clip": 0.01032948, + "auxiliary_loss_mlp": 0.01007289, + "balance_loss_clip": 1.01023316, + "balance_loss_mlp": 1.00628746, + "epoch": 0.1159247867216064, + "flos": 61126282955520.0, + "grad_norm": 0.6781235602778782, + "language_loss": 0.49256316, + "learning_rate": 3.923060142592052e-06, + "loss": 0.51296556, + "num_input_tokens_seen": 113043450, + "router_z_loss_clip": 0.2265625, + "router_z_loss_mlp": 0.01000977, + "step": 3995, + "time_per_iteration": 3.0047507286071777 + }, + { + "auxiliary_loss_clip": 0.01141188, + "auxiliary_loss_mlp": 0.01049796, + "balance_loss_clip": 1.05587459, + "balance_loss_mlp": 1.03384614, + "epoch": 0.11595380419012245, + "flos": 52554403468800.0, + "grad_norm": 1.9186307410481147, + "language_loss": 0.94295275, + "learning_rate": 3.9230085010648495e-06, + "loss": 0.96486259, + "num_input_tokens_seen": 113069505, + "router_z_loss_clip": 0.85302734, + "router_z_loss_mlp": 0.15930176, + "step": 3996, + "time_per_iteration": 2.7886366844177246 + }, + { + "auxiliary_loss_clip": 0.01148879, + "auxiliary_loss_mlp": 0.01044406, + "balance_loss_clip": 1.05559027, + "balance_loss_mlp": 1.0246588, + "epoch": 0.1159828216586385, + "flos": 74737501021440.0, + "grad_norm": 2.009693070530208, + "language_loss": 0.87145424, + "learning_rate": 3.922956842552857e-06, + "loss": 0.89338708, + "num_input_tokens_seen": 113095235, + "router_z_loss_clip": 0.93310547, + "router_z_loss_mlp": 0.1973877, + "step": 3997, + "time_per_iteration": 2.9151673316955566 + }, + { + "auxiliary_loss_clip": 0.01149141, + "auxiliary_loss_mlp": 0.01038789, + "balance_loss_clip": 1.05727673, + "balance_loss_mlp": 1.02249265, + "epoch": 0.11601183912715454, + "flos": 30444384136320.0, + "grad_norm": 2.1743198285109657, + "language_loss": 1.02782404, + "learning_rate": 3.922905167056532e-06, + "loss": 1.04970336, + "num_input_tokens_seen": 113115130, + "router_z_loss_clip": 0.91992188, + "router_z_loss_mlp": 0.16296387, + "step": 3998, + "time_per_iteration": 2.564587354660034 + }, + { + "auxiliary_loss_clip": 0.01147403, + "auxiliary_loss_mlp": 0.01051423, + "balance_loss_clip": 1.0562942, + "balance_loss_mlp": 1.03341043, + "epoch": 0.1160408565956706, + "flos": 12341959079040.0, + "grad_norm": 2.698080621823278, + "language_loss": 0.80129391, + "learning_rate": 3.92285347457633e-06, + "loss": 0.82328212, + "num_input_tokens_seen": 113126570, + "router_z_loss_clip": 0.91064453, + "router_z_loss_mlp": 0.18011475, + "step": 3999, + "time_per_iteration": 2.4626054763793945 + }, + { + "auxiliary_loss_clip": 0.01033716, + "auxiliary_loss_mlp": 0.01006304, + "balance_loss_clip": 1.01091886, + "balance_loss_mlp": 1.00531411, + "epoch": 0.11606987406418665, + "flos": 64659245892480.0, + "grad_norm": 0.6976891786771657, + "language_loss": 0.48766044, + "learning_rate": 3.922801765112709e-06, + "loss": 0.50806063, + "num_input_tokens_seen": 113181885, + "router_z_loss_clip": 0.22851562, + "router_z_loss_mlp": 0.0098877, + "step": 4000, + "time_per_iteration": 3.021000862121582 + }, + { + "auxiliary_loss_clip": 0.0114099, + "auxiliary_loss_mlp": 0.01041139, + "balance_loss_clip": 1.05270112, + "balance_loss_mlp": 1.02263236, + "epoch": 0.11609889153270268, + "flos": 30771745712640.0, + "grad_norm": 2.1319883056549638, + "language_loss": 0.87336993, + "learning_rate": 3.922750038666124e-06, + "loss": 0.89519125, + "num_input_tokens_seen": 113201595, + "router_z_loss_clip": 0.8828125, + "router_z_loss_mlp": 0.18511963, + "step": 4001, + "time_per_iteration": 2.5849907398223877 + }, + { + "auxiliary_loss_clip": 0.01141737, + "auxiliary_loss_mlp": 0.01043912, + "balance_loss_clip": 1.05344915, + "balance_loss_mlp": 1.02516007, + "epoch": 0.11612790900121873, + "flos": 23614605250560.0, + "grad_norm": 2.397908657062735, + "language_loss": 1.07806087, + "learning_rate": 3.922698295237034e-06, + "loss": 1.09991729, + "num_input_tokens_seen": 113215800, + "router_z_loss_clip": 0.88134766, + "router_z_loss_mlp": 0.18737793, + "step": 4002, + "time_per_iteration": 2.524991035461426 + }, + { + "auxiliary_loss_clip": 0.01034071, + "auxiliary_loss_mlp": 0.0100799, + "balance_loss_clip": 1.01148701, + "balance_loss_mlp": 1.00695312, + "epoch": 0.11615692646973479, + "flos": 63244761240960.0, + "grad_norm": 0.6821145781513466, + "language_loss": 0.53758836, + "learning_rate": 3.922646534825893e-06, + "loss": 0.55800891, + "num_input_tokens_seen": 113278060, + "router_z_loss_clip": 0.2265625, + "router_z_loss_mlp": 0.01037598, + "step": 4003, + "time_per_iteration": 3.037281036376953 + }, + { + "auxiliary_loss_clip": 0.01149162, + "auxiliary_loss_mlp": 0.01052063, + "balance_loss_clip": 1.05747938, + "balance_loss_mlp": 1.03348458, + "epoch": 0.11618594393825082, + "flos": 17488281294720.0, + "grad_norm": 2.5704367522981886, + "language_loss": 0.93577027, + "learning_rate": 3.92259475743316e-06, + "loss": 0.95778251, + "num_input_tokens_seen": 113297275, + "router_z_loss_clip": 0.91503906, + "router_z_loss_mlp": 0.18579102, + "step": 4004, + "time_per_iteration": 2.625534772872925 + }, + { + "auxiliary_loss_clip": 0.01034206, + "auxiliary_loss_mlp": 0.01005752, + "balance_loss_clip": 1.01147819, + "balance_loss_mlp": 1.00476289, + "epoch": 0.11621496140676688, + "flos": 60724191133440.0, + "grad_norm": 0.7209270856712086, + "language_loss": 0.48606485, + "learning_rate": 3.9225429630592925e-06, + "loss": 0.50646442, + "num_input_tokens_seen": 113352460, + "router_z_loss_clip": 0.2265625, + "router_z_loss_mlp": 0.0098877, + "step": 4005, + "time_per_iteration": 3.0161454677581787 + }, + { + "auxiliary_loss_clip": 0.01143655, + "auxiliary_loss_mlp": 0.01051235, + "balance_loss_clip": 1.05482292, + "balance_loss_mlp": 1.03408647, + "epoch": 0.11624397887528291, + "flos": 46821875293440.0, + "grad_norm": 2.045869296417913, + "language_loss": 0.69856191, + "learning_rate": 3.922491151704747e-06, + "loss": 0.72051084, + "num_input_tokens_seen": 113375690, + "router_z_loss_clip": 0.88916016, + "router_z_loss_mlp": 0.17138672, + "step": 4006, + "time_per_iteration": 2.87270450592041 + }, + { + "auxiliary_loss_clip": 0.01033314, + "auxiliary_loss_mlp": 0.01000804, + "balance_loss_clip": 1.01079535, + "balance_loss_mlp": 0.99976724, + "epoch": 0.11627299634379896, + "flos": 74771798509440.0, + "grad_norm": 0.6754486174776921, + "language_loss": 0.51337767, + "learning_rate": 3.922439323369983e-06, + "loss": 0.53371882, + "num_input_tokens_seen": 113442120, + "router_z_loss_clip": 0.22558594, + "router_z_loss_mlp": 0.01037598, + "step": 4007, + "time_per_iteration": 3.1443893909454346 + }, + { + "auxiliary_loss_clip": 0.01032114, + "auxiliary_loss_mlp": 0.01001166, + "balance_loss_clip": 1.00966859, + "balance_loss_mlp": 1.00008702, + "epoch": 0.11630201381231502, + "flos": 67630388204160.0, + "grad_norm": 0.6367169956608166, + "language_loss": 0.48938364, + "learning_rate": 3.922387478055456e-06, + "loss": 0.50971639, + "num_input_tokens_seen": 113503990, + "router_z_loss_clip": 0.22460938, + "router_z_loss_mlp": 0.01080322, + "step": 4008, + "time_per_iteration": 3.1829679012298584 + }, + { + "auxiliary_loss_clip": 0.01145367, + "auxiliary_loss_mlp": 0.01048055, + "balance_loss_clip": 1.05680323, + "balance_loss_mlp": 1.02969086, + "epoch": 0.11633103128083105, + "flos": 38758601410560.0, + "grad_norm": 2.3099751496027716, + "language_loss": 0.95635545, + "learning_rate": 3.922335615761625e-06, + "loss": 0.9782896, + "num_input_tokens_seen": 113520880, + "router_z_loss_clip": 0.88574219, + "router_z_loss_mlp": 0.18353271, + "step": 4009, + "time_per_iteration": 2.7244393825531006 + }, + { + "auxiliary_loss_clip": 0.01137049, + "auxiliary_loss_mlp": 0.01045153, + "balance_loss_clip": 1.05220783, + "balance_loss_mlp": 1.02911377, + "epoch": 0.1163600487493471, + "flos": 16537977123840.0, + "grad_norm": 2.174468148046988, + "language_loss": 0.77999341, + "learning_rate": 3.922283736488947e-06, + "loss": 0.80181539, + "num_input_tokens_seen": 113535160, + "router_z_loss_clip": 0.84814453, + "router_z_loss_mlp": 0.16040039, + "step": 4010, + "time_per_iteration": 2.572213888168335 + }, + { + "auxiliary_loss_clip": 0.01139397, + "auxiliary_loss_mlp": 0.01039237, + "balance_loss_clip": 1.05131936, + "balance_loss_mlp": 1.02288723, + "epoch": 0.11638906621786316, + "flos": 18252352425600.0, + "grad_norm": 2.052711518289479, + "language_loss": 0.6312319, + "learning_rate": 3.922231840237883e-06, + "loss": 0.65301824, + "num_input_tokens_seen": 113549810, + "router_z_loss_clip": 0.87988281, + "router_z_loss_mlp": 0.16345215, + "step": 4011, + "time_per_iteration": 2.670825958251953 + }, + { + "auxiliary_loss_clip": 0.01142868, + "auxiliary_loss_mlp": 0.01036113, + "balance_loss_clip": 1.05131817, + "balance_loss_mlp": 1.01857102, + "epoch": 0.1164180836863792, + "flos": 18690893573760.0, + "grad_norm": 2.3342121152431665, + "language_loss": 0.83476692, + "learning_rate": 3.922179927008888e-06, + "loss": 0.85655677, + "num_input_tokens_seen": 113562975, + "router_z_loss_clip": 0.91455078, + "router_z_loss_mlp": 0.17529297, + "step": 4012, + "time_per_iteration": 2.5117135047912598 + }, + { + "auxiliary_loss_clip": 0.01148448, + "auxiliary_loss_mlp": 0.01051649, + "balance_loss_clip": 1.05519748, + "balance_loss_mlp": 1.03221846, + "epoch": 0.11644710115489525, + "flos": 25844981379840.0, + "grad_norm": 1.8452684487075879, + "language_loss": 0.86929303, + "learning_rate": 3.9221279968024236e-06, + "loss": 0.891294, + "num_input_tokens_seen": 113582995, + "router_z_loss_clip": 0.93212891, + "router_z_loss_mlp": 0.19433594, + "step": 4013, + "time_per_iteration": 2.607166290283203 + }, + { + "auxiliary_loss_clip": 0.0114601, + "auxiliary_loss_mlp": 0.01051381, + "balance_loss_clip": 1.05284882, + "balance_loss_mlp": 1.0322957, + "epoch": 0.1164761186234113, + "flos": 24927463347840.0, + "grad_norm": 2.5002240219259897, + "language_loss": 0.93379319, + "learning_rate": 3.9220760496189455e-06, + "loss": 0.9557671, + "num_input_tokens_seen": 113599260, + "router_z_loss_clip": 0.93066406, + "router_z_loss_mlp": 0.19042969, + "step": 4014, + "time_per_iteration": 2.530541181564331 + }, + { + "auxiliary_loss_clip": 0.0113841, + "auxiliary_loss_mlp": 0.01040669, + "balance_loss_clip": 1.05303597, + "balance_loss_mlp": 1.02377725, + "epoch": 0.11650513609192734, + "flos": 30268817435520.0, + "grad_norm": 2.4167537615421897, + "language_loss": 0.88313365, + "learning_rate": 3.922024085458915e-06, + "loss": 0.90492451, + "num_input_tokens_seen": 113618710, + "router_z_loss_clip": 0.85400391, + "router_z_loss_mlp": 0.16906738, + "step": 4015, + "time_per_iteration": 2.569000482559204 + }, + { + "auxiliary_loss_clip": 0.01144202, + "auxiliary_loss_mlp": 0.01046454, + "balance_loss_clip": 1.05353498, + "balance_loss_mlp": 1.02595019, + "epoch": 0.11653415356044339, + "flos": 29928742444800.0, + "grad_norm": 2.560717961115464, + "language_loss": 0.92469621, + "learning_rate": 3.9219721043227885e-06, + "loss": 0.94660282, + "num_input_tokens_seen": 113633685, + "router_z_loss_clip": 0.90869141, + "router_z_loss_mlp": 0.20495605, + "step": 4016, + "time_per_iteration": 2.559622287750244 + }, + { + "auxiliary_loss_clip": 0.0115076, + "auxiliary_loss_mlp": 0.01048484, + "balance_loss_clip": 1.0584836, + "balance_loss_mlp": 1.02975035, + "epoch": 0.11656317102895944, + "flos": 33325281527040.0, + "grad_norm": 1.9492069077141683, + "language_loss": 0.87924433, + "learning_rate": 3.9219201062110285e-06, + "loss": 0.90123677, + "num_input_tokens_seen": 113656670, + "router_z_loss_clip": 0.92236328, + "router_z_loss_mlp": 0.18731689, + "step": 4017, + "time_per_iteration": 2.746617555618286 + }, + { + "auxiliary_loss_clip": 0.01138256, + "auxiliary_loss_mlp": 0.01049576, + "balance_loss_clip": 1.05487442, + "balance_loss_mlp": 1.03370881, + "epoch": 0.11659218849747548, + "flos": 27009348652800.0, + "grad_norm": 2.176526891070075, + "language_loss": 0.70226282, + "learning_rate": 3.921868091124091e-06, + "loss": 0.72414112, + "num_input_tokens_seen": 113671255, + "router_z_loss_clip": 0.83251953, + "router_z_loss_mlp": 0.15869141, + "step": 4018, + "time_per_iteration": 2.5187971591949463 + }, + { + "auxiliary_loss_clip": 0.01134251, + "auxiliary_loss_mlp": 0.0104256, + "balance_loss_clip": 1.05038762, + "balance_loss_mlp": 1.02574015, + "epoch": 0.11662120596599153, + "flos": 22411346526720.0, + "grad_norm": 4.688259830386436, + "language_loss": 0.91363841, + "learning_rate": 3.9218160590624376e-06, + "loss": 0.93540651, + "num_input_tokens_seen": 113685305, + "router_z_loss_clip": 0.83789062, + "router_z_loss_mlp": 0.16821289, + "step": 4019, + "time_per_iteration": 2.5202741622924805 + }, + { + "auxiliary_loss_clip": 0.01041847, + "auxiliary_loss_mlp": 0.01007494, + "balance_loss_clip": 1.01920414, + "balance_loss_mlp": 1.00647497, + "epoch": 0.11665022343450758, + "flos": 58204626606720.0, + "grad_norm": 0.8098963336472653, + "language_loss": 0.52774286, + "learning_rate": 3.9217640100265265e-06, + "loss": 0.54823625, + "num_input_tokens_seen": 113750560, + "router_z_loss_clip": 0.2265625, + "router_z_loss_mlp": 0.01019287, + "step": 4020, + "time_per_iteration": 3.159081220626831 + }, + { + "auxiliary_loss_clip": 0.01040453, + "auxiliary_loss_mlp": 0.01007167, + "balance_loss_clip": 1.01783395, + "balance_loss_mlp": 1.00615358, + "epoch": 0.11667924090302362, + "flos": 71016440515200.0, + "grad_norm": 0.6760168005863584, + "language_loss": 0.49358922, + "learning_rate": 3.921711944016819e-06, + "loss": 0.51406538, + "num_input_tokens_seen": 113813025, + "router_z_loss_clip": 0.2265625, + "router_z_loss_mlp": 0.01013184, + "step": 4021, + "time_per_iteration": 3.090071201324463 + }, + { + "auxiliary_loss_clip": 0.01145839, + "auxiliary_loss_mlp": 0.01054246, + "balance_loss_clip": 1.05664086, + "balance_loss_mlp": 1.03504157, + "epoch": 0.11670825837153967, + "flos": 42185628161280.0, + "grad_norm": 2.599984352042368, + "language_loss": 0.71580219, + "learning_rate": 3.921659861033773e-06, + "loss": 0.73780298, + "num_input_tokens_seen": 113829925, + "router_z_loss_clip": 0.89160156, + "router_z_loss_mlp": 0.19189453, + "step": 4022, + "time_per_iteration": 2.7244606018066406 + }, + { + "auxiliary_loss_clip": 0.01160709, + "auxiliary_loss_mlp": 0.01068798, + "balance_loss_clip": 1.06138539, + "balance_loss_mlp": 1.04836512, + "epoch": 0.11673727584005571, + "flos": 12378767541120.0, + "grad_norm": 3.4041927821271543, + "language_loss": 0.85881698, + "learning_rate": 3.92160776107785e-06, + "loss": 0.8811121, + "num_input_tokens_seen": 113840845, + "router_z_loss_clip": 0.99316406, + "router_z_loss_mlp": 0.2043457, + "step": 4023, + "time_per_iteration": 2.5052907466888428 + }, + { + "auxiliary_loss_clip": 0.01143427, + "auxiliary_loss_mlp": 0.01050935, + "balance_loss_clip": 1.05540144, + "balance_loss_mlp": 1.03419852, + "epoch": 0.11676629330857176, + "flos": 29819394466560.0, + "grad_norm": 2.0810441398669117, + "language_loss": 0.91685367, + "learning_rate": 3.921555644149509e-06, + "loss": 0.9387973, + "num_input_tokens_seen": 113859055, + "router_z_loss_clip": 0.88134766, + "router_z_loss_mlp": 0.16729736, + "step": 4024, + "time_per_iteration": 2.573836326599121 + }, + { + "auxiliary_loss_clip": 0.01035234, + "auxiliary_loss_mlp": 0.01017593, + "balance_loss_clip": 1.01260471, + "balance_loss_mlp": 1.01667488, + "epoch": 0.11679531077708781, + "flos": 69553690963200.0, + "grad_norm": 0.6324236182312034, + "language_loss": 0.45482349, + "learning_rate": 3.921503510249212e-06, + "loss": 0.47535175, + "num_input_tokens_seen": 113917345, + "router_z_loss_clip": 0.2265625, + "router_z_loss_mlp": 0.00915527, + "step": 4025, + "time_per_iteration": 3.0244085788726807 + }, + { + "auxiliary_loss_clip": 0.01145324, + "auxiliary_loss_mlp": 0.0105311, + "balance_loss_clip": 1.05570841, + "balance_loss_mlp": 1.03478789, + "epoch": 0.11682432824560385, + "flos": 26535399672960.0, + "grad_norm": 2.0896059380168097, + "language_loss": 0.94546312, + "learning_rate": 3.9214513593774175e-06, + "loss": 0.96744746, + "num_input_tokens_seen": 113936020, + "router_z_loss_clip": 0.89550781, + "router_z_loss_mlp": 0.18322754, + "step": 4026, + "time_per_iteration": 2.5671095848083496 + }, + { + "auxiliary_loss_clip": 0.01138918, + "auxiliary_loss_mlp": 0.01055107, + "balance_loss_clip": 1.05336452, + "balance_loss_mlp": 1.03931201, + "epoch": 0.1168533457141199, + "flos": 22412100712320.0, + "grad_norm": 2.289387816612587, + "language_loss": 0.87057567, + "learning_rate": 3.921399191534588e-06, + "loss": 0.89251602, + "num_input_tokens_seen": 113951040, + "router_z_loss_clip": 0.85546875, + "router_z_loss_mlp": 0.15795898, + "step": 4027, + "time_per_iteration": 2.5033581256866455 + }, + { + "auxiliary_loss_clip": 0.01034957, + "auxiliary_loss_mlp": 0.01034352, + "balance_loss_clip": 1.0123899, + "balance_loss_mlp": 1.03338003, + "epoch": 0.11688236318263595, + "flos": 58691504482560.0, + "grad_norm": 0.6983532634407914, + "language_loss": 0.51054651, + "learning_rate": 3.921347006721182e-06, + "loss": 0.53123951, + "num_input_tokens_seen": 114012100, + "router_z_loss_clip": 0.22558594, + "router_z_loss_mlp": 0.00970459, + "step": 4028, + "time_per_iteration": 3.0292768478393555 + }, + { + "auxiliary_loss_clip": 0.01033806, + "auxiliary_loss_mlp": 0.01019761, + "balance_loss_clip": 1.01131284, + "balance_loss_mlp": 1.01877117, + "epoch": 0.11691138065115199, + "flos": 60832497617280.0, + "grad_norm": 0.7145583921370849, + "language_loss": 0.49359888, + "learning_rate": 3.921294804937663e-06, + "loss": 0.51413453, + "num_input_tokens_seen": 114069340, + "router_z_loss_clip": 0.22460938, + "router_z_loss_mlp": 0.0098877, + "step": 4029, + "time_per_iteration": 2.9797730445861816 + }, + { + "auxiliary_loss_clip": 0.01132927, + "auxiliary_loss_mlp": 0.01040808, + "balance_loss_clip": 1.05301213, + "balance_loss_mlp": 1.02461982, + "epoch": 0.11694039811966804, + "flos": 47586808350720.0, + "grad_norm": 2.5057780684212734, + "language_loss": 0.85651237, + "learning_rate": 3.9212425861844905e-06, + "loss": 0.87824976, + "num_input_tokens_seen": 114086090, + "router_z_loss_clip": 0.79931641, + "router_z_loss_mlp": 0.16186523, + "step": 4030, + "time_per_iteration": 2.7494242191314697 + }, + { + "auxiliary_loss_clip": 0.01032619, + "auxiliary_loss_mlp": 0.0100187, + "balance_loss_clip": 1.01011872, + "balance_loss_mlp": 1.00090396, + "epoch": 0.11696941558818409, + "flos": 66595441633920.0, + "grad_norm": 0.7031607518747255, + "language_loss": 0.51933205, + "learning_rate": 3.921190350462126e-06, + "loss": 0.53967702, + "num_input_tokens_seen": 114137590, + "router_z_loss_clip": 0.22460938, + "router_z_loss_mlp": 0.00964355, + "step": 4031, + "time_per_iteration": 2.955827236175537 + }, + { + "auxiliary_loss_clip": 0.01032258, + "auxiliary_loss_mlp": 0.00999551, + "balance_loss_clip": 1.00961208, + "balance_loss_mlp": 0.99853796, + "epoch": 0.11699843305670013, + "flos": 67989638079360.0, + "grad_norm": 0.8218792304317352, + "language_loss": 0.52780187, + "learning_rate": 3.921138097771031e-06, + "loss": 0.54812002, + "num_input_tokens_seen": 114197110, + "router_z_loss_clip": 0.2265625, + "router_z_loss_mlp": 0.01013184, + "step": 4032, + "time_per_iteration": 3.033565044403076 + }, + { + "auxiliary_loss_clip": 0.01032182, + "auxiliary_loss_mlp": 0.01000941, + "balance_loss_clip": 1.0095005, + "balance_loss_mlp": 0.99992764, + "epoch": 0.11702745052521618, + "flos": 65474706407040.0, + "grad_norm": 0.6765233548262848, + "language_loss": 0.50919455, + "learning_rate": 3.921085828111667e-06, + "loss": 0.52952576, + "num_input_tokens_seen": 114259645, + "router_z_loss_clip": 0.2265625, + "router_z_loss_mlp": 0.01013184, + "step": 4033, + "time_per_iteration": 3.0410919189453125 + }, + { + "auxiliary_loss_clip": 0.0103166, + "auxiliary_loss_mlp": 0.01002834, + "balance_loss_clip": 1.00901604, + "balance_loss_mlp": 1.00181437, + "epoch": 0.11705646799373223, + "flos": 58244236329600.0, + "grad_norm": 0.706877136054339, + "language_loss": 0.48268631, + "learning_rate": 3.921033541484495e-06, + "loss": 0.50303125, + "num_input_tokens_seen": 114315160, + "router_z_loss_clip": 0.2265625, + "router_z_loss_mlp": 0.01019287, + "step": 4034, + "time_per_iteration": 2.901695489883423 + }, + { + "auxiliary_loss_clip": 0.01144991, + "auxiliary_loss_mlp": 0.01051005, + "balance_loss_clip": 1.05386591, + "balance_loss_mlp": 1.03422678, + "epoch": 0.11708548546224827, + "flos": 40109201723520.0, + "grad_norm": 4.226668926266322, + "language_loss": 0.95650274, + "learning_rate": 3.920981237889978e-06, + "loss": 0.9784627, + "num_input_tokens_seen": 114335045, + "router_z_loss_clip": 0.91162109, + "router_z_loss_mlp": 0.16790771, + "step": 4035, + "time_per_iteration": 2.68888258934021 + }, + { + "auxiliary_loss_clip": 0.01149067, + "auxiliary_loss_mlp": 0.0104993, + "balance_loss_clip": 1.05421913, + "balance_loss_mlp": 1.03001046, + "epoch": 0.11711450293076432, + "flos": 29938762339200.0, + "grad_norm": 2.7453174361353625, + "language_loss": 0.90636593, + "learning_rate": 3.9209289173285766e-06, + "loss": 0.92835593, + "num_input_tokens_seen": 114354260, + "router_z_loss_clip": 0.94873047, + "router_z_loss_mlp": 0.19909668, + "step": 4036, + "time_per_iteration": 2.5667831897735596 + }, + { + "auxiliary_loss_clip": 0.01032863, + "auxiliary_loss_mlp": 0.01005197, + "balance_loss_clip": 1.01020348, + "balance_loss_mlp": 1.00422561, + "epoch": 0.11714352039928037, + "flos": 56679106037760.0, + "grad_norm": 0.7219629364352679, + "language_loss": 0.53616172, + "learning_rate": 3.920876579800754e-06, + "loss": 0.55654234, + "num_input_tokens_seen": 114416515, + "router_z_loss_clip": 0.2265625, + "router_z_loss_mlp": 0.00970459, + "step": 4037, + "time_per_iteration": 3.0283217430114746 + }, + { + "auxiliary_loss_clip": 0.01032924, + "auxiliary_loss_mlp": 0.01006824, + "balance_loss_clip": 1.01043236, + "balance_loss_mlp": 1.00586438, + "epoch": 0.11717253786779641, + "flos": 74782823984640.0, + "grad_norm": 0.6434654622956896, + "language_loss": 0.54005849, + "learning_rate": 3.920824225306973e-06, + "loss": 0.56045592, + "num_input_tokens_seen": 114486560, + "router_z_loss_clip": 0.22460938, + "router_z_loss_mlp": 0.00958252, + "step": 4038, + "time_per_iteration": 3.21056866645813 + }, + { + "auxiliary_loss_clip": 0.01133145, + "auxiliary_loss_mlp": 0.01046565, + "balance_loss_clip": 1.05087161, + "balance_loss_mlp": 1.02970862, + "epoch": 0.11720155533631246, + "flos": 15370521281280.0, + "grad_norm": 2.42561547796754, + "language_loss": 0.71158028, + "learning_rate": 3.9207718538476946e-06, + "loss": 0.73337734, + "num_input_tokens_seen": 114502860, + "router_z_loss_clip": 0.82226562, + "router_z_loss_mlp": 0.16851807, + "step": 4039, + "time_per_iteration": 2.616105794906616 + }, + { + "auxiliary_loss_clip": 0.0114151, + "auxiliary_loss_mlp": 0.01042569, + "balance_loss_clip": 1.05378389, + "balance_loss_mlp": 1.02595687, + "epoch": 0.1172305728048285, + "flos": 14823637735680.0, + "grad_norm": 2.038357954044783, + "language_loss": 0.87741661, + "learning_rate": 3.920719465423381e-06, + "loss": 0.89925736, + "num_input_tokens_seen": 114517000, + "router_z_loss_clip": 0.87792969, + "router_z_loss_mlp": 0.16619873, + "step": 4040, + "time_per_iteration": 2.4933021068573 + }, + { + "auxiliary_loss_clip": 0.01031924, + "auxiliary_loss_mlp": 0.01006797, + "balance_loss_clip": 1.00947607, + "balance_loss_mlp": 1.00583744, + "epoch": 0.11725959027334455, + "flos": 74775533523840.0, + "grad_norm": 0.7062167980944001, + "language_loss": 0.54368365, + "learning_rate": 3.920667060034497e-06, + "loss": 0.56407088, + "num_input_tokens_seen": 114578505, + "router_z_loss_clip": 0.22460938, + "router_z_loss_mlp": 0.00958252, + "step": 4041, + "time_per_iteration": 3.127915620803833 + }, + { + "auxiliary_loss_clip": 0.01133424, + "auxiliary_loss_mlp": 0.01045826, + "balance_loss_clip": 1.04862475, + "balance_loss_mlp": 1.02843368, + "epoch": 0.1172886077418606, + "flos": 38029399407360.0, + "grad_norm": 1.8251122295091962, + "language_loss": 0.73901224, + "learning_rate": 3.920614637681505e-06, + "loss": 0.76080471, + "num_input_tokens_seen": 114597185, + "router_z_loss_clip": 0.84667969, + "router_z_loss_mlp": 0.1739502, + "step": 4042, + "time_per_iteration": 2.6679601669311523 + }, + { + "auxiliary_loss_clip": 0.01131232, + "auxiliary_loss_mlp": 0.01044797, + "balance_loss_clip": 1.04810107, + "balance_loss_mlp": 1.02890635, + "epoch": 0.11731762521037664, + "flos": 28618326472320.0, + "grad_norm": 2.4291409234925387, + "language_loss": 0.67781073, + "learning_rate": 3.920562198364866e-06, + "loss": 0.69957101, + "num_input_tokens_seen": 114611725, + "router_z_loss_clip": 0.83056641, + "router_z_loss_mlp": 0.15893555, + "step": 4043, + "time_per_iteration": 2.4894983768463135 + }, + { + "auxiliary_loss_clip": 0.01137293, + "auxiliary_loss_mlp": 0.01041555, + "balance_loss_clip": 1.05221629, + "balance_loss_mlp": 1.02459788, + "epoch": 0.1173466426788927, + "flos": 15517216425600.0, + "grad_norm": 2.4020831690515165, + "language_loss": 0.90239966, + "learning_rate": 3.920509742085045e-06, + "loss": 0.9241882, + "num_input_tokens_seen": 114624900, + "router_z_loss_clip": 0.85107422, + "router_z_loss_mlp": 0.16967773, + "step": 4044, + "time_per_iteration": 2.4886057376861572 + }, + { + "auxiliary_loss_clip": 0.01031438, + "auxiliary_loss_mlp": 0.01004163, + "balance_loss_clip": 1.00899029, + "balance_loss_mlp": 1.00312579, + "epoch": 0.11737566014740874, + "flos": 74782788071040.0, + "grad_norm": 0.6124150594405616, + "language_loss": 0.50151241, + "learning_rate": 3.920457268842504e-06, + "loss": 0.52186847, + "num_input_tokens_seen": 114693545, + "router_z_loss_clip": 0.22460938, + "router_z_loss_mlp": 0.01037598, + "step": 4045, + "time_per_iteration": 3.2192578315734863 + }, + { + "auxiliary_loss_clip": 0.01139325, + "auxiliary_loss_mlp": 0.01049665, + "balance_loss_clip": 1.05207705, + "balance_loss_mlp": 1.03299379, + "epoch": 0.11740467761592478, + "flos": 33913354993920.0, + "grad_norm": 1.9454981272732144, + "language_loss": 0.92704427, + "learning_rate": 3.920404778637708e-06, + "loss": 0.9489342, + "num_input_tokens_seen": 114715810, + "router_z_loss_clip": 0.87207031, + "router_z_loss_mlp": 0.16662598, + "step": 4046, + "time_per_iteration": 2.655641555786133 + }, + { + "auxiliary_loss_clip": 0.01137133, + "auxiliary_loss_mlp": 0.01049614, + "balance_loss_clip": 1.04857183, + "balance_loss_mlp": 1.03207862, + "epoch": 0.11743369508444083, + "flos": 13401575314560.0, + "grad_norm": 2.5060346044159654, + "language_loss": 0.86704975, + "learning_rate": 3.92035227147112e-06, + "loss": 0.88891721, + "num_input_tokens_seen": 114730260, + "router_z_loss_clip": 0.88525391, + "router_z_loss_mlp": 0.1751709, + "step": 4047, + "time_per_iteration": 2.45412540435791 + }, + { + "auxiliary_loss_clip": 0.01141467, + "auxiliary_loss_mlp": 0.01043882, + "balance_loss_clip": 1.05416179, + "balance_loss_mlp": 1.02741957, + "epoch": 0.11746271255295689, + "flos": 20040919269120.0, + "grad_norm": 2.216998760223084, + "language_loss": 0.86257243, + "learning_rate": 3.920299747343203e-06, + "loss": 0.88442588, + "num_input_tokens_seen": 114743855, + "router_z_loss_clip": 0.87353516, + "router_z_loss_mlp": 0.16455078, + "step": 4048, + "time_per_iteration": 2.501819610595703 + }, + { + "auxiliary_loss_clip": 0.01132133, + "auxiliary_loss_mlp": 0.01037311, + "balance_loss_clip": 1.04966235, + "balance_loss_mlp": 1.0222851, + "epoch": 0.11749173002147292, + "flos": 24783353982720.0, + "grad_norm": 2.2280349734245855, + "language_loss": 0.69601166, + "learning_rate": 3.920247206254422e-06, + "loss": 0.71770608, + "num_input_tokens_seen": 114759735, + "router_z_loss_clip": 0.82519531, + "router_z_loss_mlp": 0.15032959, + "step": 4049, + "time_per_iteration": 2.5504825115203857 + }, + { + "auxiliary_loss_clip": 0.01148436, + "auxiliary_loss_mlp": 0.01052548, + "balance_loss_clip": 1.05608356, + "balance_loss_mlp": 1.03360534, + "epoch": 0.11752074748998897, + "flos": 27190553788800.0, + "grad_norm": 2.891887589358331, + "language_loss": 0.94415379, + "learning_rate": 3.9201946482052406e-06, + "loss": 0.96616364, + "num_input_tokens_seen": 114776035, + "router_z_loss_clip": 0.92236328, + "router_z_loss_mlp": 0.1895752, + "step": 4050, + "time_per_iteration": 2.6422061920166016 + }, + { + "auxiliary_loss_clip": 0.01030367, + "auxiliary_loss_mlp": 0.01003243, + "balance_loss_clip": 1.00798643, + "balance_loss_mlp": 1.00225377, + "epoch": 0.11754976495850503, + "flos": 74191985170560.0, + "grad_norm": 0.6108077937100379, + "language_loss": 0.47448903, + "learning_rate": 3.920142073196123e-06, + "loss": 0.49482515, + "num_input_tokens_seen": 114839130, + "router_z_loss_clip": 0.22363281, + "router_z_loss_mlp": 0.0098877, + "step": 4051, + "time_per_iteration": 3.199873685836792 + }, + { + "auxiliary_loss_clip": 0.01150487, + "auxiliary_loss_mlp": 0.01045276, + "balance_loss_clip": 1.05739748, + "balance_loss_mlp": 1.02799678, + "epoch": 0.11757878242702106, + "flos": 20552323155840.0, + "grad_norm": 2.058190246145636, + "language_loss": 0.8390969, + "learning_rate": 3.920089481227534e-06, + "loss": 0.86105454, + "num_input_tokens_seen": 114856485, + "router_z_loss_clip": 0.93115234, + "router_z_loss_mlp": 0.17285156, + "step": 4052, + "time_per_iteration": 2.479330062866211 + }, + { + "auxiliary_loss_clip": 0.01030321, + "auxiliary_loss_mlp": 0.01005902, + "balance_loss_clip": 1.00805521, + "balance_loss_mlp": 1.00481725, + "epoch": 0.11760779989553712, + "flos": 56480734200960.0, + "grad_norm": 0.7436582729259681, + "language_loss": 0.49958035, + "learning_rate": 3.920036872299937e-06, + "loss": 0.51994252, + "num_input_tokens_seen": 114909740, + "router_z_loss_clip": 0.22265625, + "router_z_loss_mlp": 0.01086426, + "step": 4053, + "time_per_iteration": 2.932349443435669 + }, + { + "auxiliary_loss_clip": 0.01144642, + "auxiliary_loss_mlp": 0.01047133, + "balance_loss_clip": 1.0533185, + "balance_loss_mlp": 1.02758288, + "epoch": 0.11763681736405315, + "flos": 19675887304320.0, + "grad_norm": 2.167386496685054, + "language_loss": 0.82085222, + "learning_rate": 3.919984246413798e-06, + "loss": 0.84276998, + "num_input_tokens_seen": 114925680, + "router_z_loss_clip": 0.91308594, + "router_z_loss_mlp": 0.19555664, + "step": 4054, + "time_per_iteration": 2.52101993560791 + }, + { + "auxiliary_loss_clip": 0.01140269, + "auxiliary_loss_mlp": 0.01047712, + "balance_loss_clip": 1.04997945, + "balance_loss_mlp": 1.02804828, + "epoch": 0.1176658348325692, + "flos": 27122647127040.0, + "grad_norm": 2.7171532531949696, + "language_loss": 1.04771566, + "learning_rate": 3.919931603569582e-06, + "loss": 1.06959534, + "num_input_tokens_seen": 114944515, + "router_z_loss_clip": 0.90283203, + "router_z_loss_mlp": 0.19659424, + "step": 4055, + "time_per_iteration": 4.860634803771973 + }, + { + "auxiliary_loss_clip": 0.01030766, + "auxiliary_loss_mlp": 0.01004095, + "balance_loss_clip": 1.00854182, + "balance_loss_mlp": 1.00293303, + "epoch": 0.11769485230108526, + "flos": 74777724253440.0, + "grad_norm": 0.66916134943685, + "language_loss": 0.51897258, + "learning_rate": 3.919878943767751e-06, + "loss": 0.53932118, + "num_input_tokens_seen": 115013190, + "router_z_loss_clip": 0.22265625, + "router_z_loss_mlp": 0.01159668, + "step": 4056, + "time_per_iteration": 7.874709367752075 + }, + { + "auxiliary_loss_clip": 0.01135535, + "auxiliary_loss_mlp": 0.01038294, + "balance_loss_clip": 1.05004036, + "balance_loss_mlp": 1.02240372, + "epoch": 0.1177238697696013, + "flos": 34121887401600.0, + "grad_norm": 2.045225641683143, + "language_loss": 0.85705101, + "learning_rate": 3.919826267008774e-06, + "loss": 0.87878937, + "num_input_tokens_seen": 115031705, + "router_z_loss_clip": 0.85498047, + "router_z_loss_mlp": 0.15881348, + "step": 4057, + "time_per_iteration": 5.0999205112457275 + }, + { + "auxiliary_loss_clip": 0.01147166, + "auxiliary_loss_mlp": 0.01048184, + "balance_loss_clip": 1.0549624, + "balance_loss_mlp": 1.02916408, + "epoch": 0.11775288723811735, + "flos": 40640178522240.0, + "grad_norm": 2.131239009205459, + "language_loss": 0.9760986, + "learning_rate": 3.919773573293114e-06, + "loss": 0.99805212, + "num_input_tokens_seen": 115053335, + "router_z_loss_clip": 0.921875, + "router_z_loss_mlp": 0.19000244, + "step": 4058, + "time_per_iteration": 2.706716537475586 + }, + { + "auxiliary_loss_clip": 0.01130985, + "auxiliary_loss_mlp": 0.01039992, + "balance_loss_clip": 1.04613936, + "balance_loss_mlp": 1.02305841, + "epoch": 0.1177819047066334, + "flos": 38173580599680.0, + "grad_norm": 2.9179664841288178, + "language_loss": 0.73989177, + "learning_rate": 3.919720862621237e-06, + "loss": 0.76160157, + "num_input_tokens_seen": 115069255, + "router_z_loss_clip": 0.84863281, + "router_z_loss_mlp": 0.16949463, + "step": 4059, + "time_per_iteration": 2.663788080215454 + }, + { + "auxiliary_loss_clip": 0.01142757, + "auxiliary_loss_mlp": 0.01041836, + "balance_loss_clip": 1.05593646, + "balance_loss_mlp": 1.025087, + "epoch": 0.11781092217514944, + "flos": 22995577238400.0, + "grad_norm": 2.1819154709436117, + "language_loss": 0.83130825, + "learning_rate": 3.919668134993608e-06, + "loss": 0.85315412, + "num_input_tokens_seen": 115085900, + "router_z_loss_clip": 0.86865234, + "router_z_loss_mlp": 0.16760254, + "step": 4060, + "time_per_iteration": 2.4943675994873047 + }, + { + "auxiliary_loss_clip": 0.01032689, + "auxiliary_loss_mlp": 0.00998888, + "balance_loss_clip": 1.01044893, + "balance_loss_mlp": 0.99782068, + "epoch": 0.11783993964366549, + "flos": 64270729411200.0, + "grad_norm": 0.6039941233902374, + "language_loss": 0.50109911, + "learning_rate": 3.919615390410694e-06, + "loss": 0.52141488, + "num_input_tokens_seen": 115149450, + "router_z_loss_clip": 0.22265625, + "router_z_loss_mlp": 0.01068115, + "step": 4061, + "time_per_iteration": 3.127570867538452 + }, + { + "auxiliary_loss_clip": 0.01032589, + "auxiliary_loss_mlp": 0.00999065, + "balance_loss_clip": 1.01040983, + "balance_loss_mlp": 0.9979561, + "epoch": 0.11786895711218154, + "flos": 69450412383360.0, + "grad_norm": 0.6869711791512173, + "language_loss": 0.51211369, + "learning_rate": 3.91956262887296e-06, + "loss": 0.53243023, + "num_input_tokens_seen": 115209255, + "router_z_loss_clip": 0.22167969, + "router_z_loss_mlp": 0.0111084, + "step": 4062, + "time_per_iteration": 3.109809160232544 + }, + { + "auxiliary_loss_clip": 0.0103314, + "auxiliary_loss_mlp": 0.01001321, + "balance_loss_clip": 1.01102304, + "balance_loss_mlp": 1.00031996, + "epoch": 0.11789797458069758, + "flos": 65212881194880.0, + "grad_norm": 0.6570165850790961, + "language_loss": 0.54132825, + "learning_rate": 3.919509850380872e-06, + "loss": 0.56167287, + "num_input_tokens_seen": 115270680, + "router_z_loss_clip": 0.22070312, + "router_z_loss_mlp": 0.01000977, + "step": 4063, + "time_per_iteration": 3.080317258834839 + }, + { + "auxiliary_loss_clip": 0.01032956, + "auxiliary_loss_mlp": 0.01005219, + "balance_loss_clip": 1.01054835, + "balance_loss_mlp": 1.00414622, + "epoch": 0.11792699204921363, + "flos": 70796272101120.0, + "grad_norm": 0.6254931108074, + "language_loss": 0.47514012, + "learning_rate": 3.919457054934896e-06, + "loss": 0.49552187, + "num_input_tokens_seen": 115333050, + "router_z_loss_clip": 0.22363281, + "router_z_loss_mlp": 0.01074219, + "step": 4064, + "time_per_iteration": 3.0822675228118896 + }, + { + "auxiliary_loss_clip": 0.01138041, + "auxiliary_loss_mlp": 0.01044176, + "balance_loss_clip": 1.0508523, + "balance_loss_mlp": 1.02810073, + "epoch": 0.11795600951772968, + "flos": 38211322815360.0, + "grad_norm": 1.85817690682363, + "language_loss": 0.72478926, + "learning_rate": 3.9194042425354985e-06, + "loss": 0.74661148, + "num_input_tokens_seen": 115351685, + "router_z_loss_clip": 0.87158203, + "router_z_loss_mlp": 0.16070557, + "step": 4065, + "time_per_iteration": 2.7030766010284424 + }, + { + "auxiliary_loss_clip": 0.01030922, + "auxiliary_loss_mlp": 0.01002584, + "balance_loss_clip": 1.00885367, + "balance_loss_mlp": 1.00155282, + "epoch": 0.11798502698624572, + "flos": 74778729834240.0, + "grad_norm": 0.6409800091594328, + "language_loss": 0.50251335, + "learning_rate": 3.919351413183146e-06, + "loss": 0.52284843, + "num_input_tokens_seen": 115418440, + "router_z_loss_clip": 0.22070312, + "router_z_loss_mlp": 0.01031494, + "step": 4066, + "time_per_iteration": 3.192162275314331 + }, + { + "auxiliary_loss_clip": 0.01146937, + "auxiliary_loss_mlp": 0.01048467, + "balance_loss_clip": 1.05234265, + "balance_loss_mlp": 1.03039491, + "epoch": 0.11801404445476177, + "flos": 26133415591680.0, + "grad_norm": 2.4293082056285433, + "language_loss": 0.96248114, + "learning_rate": 3.919298566878306e-06, + "loss": 0.9844352, + "num_input_tokens_seen": 115432880, + "router_z_loss_clip": 0.9453125, + "router_z_loss_mlp": 0.1809082, + "step": 4067, + "time_per_iteration": 2.556821823120117 + }, + { + "auxiliary_loss_clip": 0.01149157, + "auxiliary_loss_mlp": 0.01051783, + "balance_loss_clip": 1.0563314, + "balance_loss_mlp": 1.03436017, + "epoch": 0.11804306192327782, + "flos": 50724826272000.0, + "grad_norm": 2.003280732643895, + "language_loss": 0.85591662, + "learning_rate": 3.9192457036214435e-06, + "loss": 0.87792599, + "num_input_tokens_seen": 115451015, + "router_z_loss_clip": 0.92871094, + "router_z_loss_mlp": 0.17419434, + "step": 4068, + "time_per_iteration": 2.8353431224823 + }, + { + "auxiliary_loss_clip": 0.01134154, + "auxiliary_loss_mlp": 0.01045149, + "balance_loss_clip": 1.04949498, + "balance_loss_mlp": 1.02805495, + "epoch": 0.11807207939179386, + "flos": 15989297898240.0, + "grad_norm": 3.692559893025631, + "language_loss": 0.90614611, + "learning_rate": 3.919192823413026e-06, + "loss": 0.92793912, + "num_input_tokens_seen": 115461350, + "router_z_loss_clip": 0.84570312, + "router_z_loss_mlp": 0.17089844, + "step": 4069, + "time_per_iteration": 2.571084499359131 + }, + { + "auxiliary_loss_clip": 0.01031357, + "auxiliary_loss_mlp": 0.0100544, + "balance_loss_clip": 1.00922489, + "balance_loss_mlp": 1.00430202, + "epoch": 0.11810109686030991, + "flos": 66721920399360.0, + "grad_norm": 0.6192254154628515, + "language_loss": 0.48000556, + "learning_rate": 3.919139926253522e-06, + "loss": 0.50037348, + "num_input_tokens_seen": 115527770, + "router_z_loss_clip": 0.22070312, + "router_z_loss_mlp": 0.01141357, + "step": 4070, + "time_per_iteration": 3.1598923206329346 + }, + { + "auxiliary_loss_clip": 0.01031735, + "auxiliary_loss_mlp": 0.01002764, + "balance_loss_clip": 1.0095911, + "balance_loss_mlp": 1.00160742, + "epoch": 0.11813011432882595, + "flos": 48866775546240.0, + "grad_norm": 0.7029156190312187, + "language_loss": 0.48740143, + "learning_rate": 3.919087012143398e-06, + "loss": 0.5077464, + "num_input_tokens_seen": 115582695, + "router_z_loss_clip": 0.22070312, + "router_z_loss_mlp": 0.01153564, + "step": 4071, + "time_per_iteration": 2.9716031551361084 + }, + { + "auxiliary_loss_clip": 0.01127333, + "auxiliary_loss_mlp": 0.01048018, + "balance_loss_clip": 1.04729962, + "balance_loss_mlp": 1.03407025, + "epoch": 0.118159131797342, + "flos": 12490306248960.0, + "grad_norm": 2.9469461291557284, + "language_loss": 0.84911919, + "learning_rate": 3.919034081083119e-06, + "loss": 0.87087268, + "num_input_tokens_seen": 115595475, + "router_z_loss_clip": 0.80029297, + "router_z_loss_mlp": 0.13952637, + "step": 4072, + "time_per_iteration": 2.4606215953826904 + }, + { + "auxiliary_loss_clip": 0.01141096, + "auxiliary_loss_mlp": 0.01043342, + "balance_loss_clip": 1.05131698, + "balance_loss_mlp": 1.02597284, + "epoch": 0.11818814926585805, + "flos": 74738398861440.0, + "grad_norm": 3.910770983140083, + "language_loss": 0.69803399, + "learning_rate": 3.918981133073156e-06, + "loss": 0.71987832, + "num_input_tokens_seen": 115620695, + "router_z_loss_clip": 0.89794922, + "router_z_loss_mlp": 0.17370605, + "step": 4073, + "time_per_iteration": 2.918802499771118 + }, + { + "auxiliary_loss_clip": 0.01134124, + "auxiliary_loss_mlp": 0.01049407, + "balance_loss_clip": 1.04978621, + "balance_loss_mlp": 1.03295577, + "epoch": 0.11821716673437409, + "flos": 29455260341760.0, + "grad_norm": 2.6872645256528003, + "language_loss": 0.74387848, + "learning_rate": 3.918928168113974e-06, + "loss": 0.76571381, + "num_input_tokens_seen": 115641470, + "router_z_loss_clip": 0.84423828, + "router_z_loss_mlp": 0.16448975, + "step": 4074, + "time_per_iteration": 2.584794282913208 + }, + { + "auxiliary_loss_clip": 0.01127211, + "auxiliary_loss_mlp": 0.01041762, + "balance_loss_clip": 1.04879093, + "balance_loss_mlp": 1.02785623, + "epoch": 0.11824618420289014, + "flos": 15187197242880.0, + "grad_norm": 4.369560572760023, + "language_loss": 0.77230263, + "learning_rate": 3.918875186206042e-06, + "loss": 0.7939924, + "num_input_tokens_seen": 115653870, + "router_z_loss_clip": 0.78417969, + "router_z_loss_mlp": 0.13903809, + "step": 4075, + "time_per_iteration": 2.514674425125122 + }, + { + "auxiliary_loss_clip": 0.01031653, + "auxiliary_loss_mlp": 0.00999812, + "balance_loss_clip": 1.00935137, + "balance_loss_mlp": 0.99864942, + "epoch": 0.11827520167140619, + "flos": 63504862600320.0, + "grad_norm": 0.7253445288396696, + "language_loss": 0.51248908, + "learning_rate": 3.918822187349829e-06, + "loss": 0.53280377, + "num_input_tokens_seen": 115713885, + "router_z_loss_clip": 0.22265625, + "router_z_loss_mlp": 0.01159668, + "step": 4076, + "time_per_iteration": 3.038609027862549 + }, + { + "auxiliary_loss_clip": 0.01030397, + "auxiliary_loss_mlp": 0.01000775, + "balance_loss_clip": 1.00829875, + "balance_loss_mlp": 0.99949402, + "epoch": 0.11830421913992223, + "flos": 67357250208000.0, + "grad_norm": 0.620074178999357, + "language_loss": 0.4631508, + "learning_rate": 3.918769171545801e-06, + "loss": 0.48346254, + "num_input_tokens_seen": 115772080, + "router_z_loss_clip": 0.22070312, + "router_z_loss_mlp": 0.01281738, + "step": 4077, + "time_per_iteration": 3.0880050659179688 + }, + { + "auxiliary_loss_clip": 0.01137053, + "auxiliary_loss_mlp": 0.01034359, + "balance_loss_clip": 1.0500927, + "balance_loss_mlp": 1.01849806, + "epoch": 0.11833323660843828, + "flos": 16574857413120.0, + "grad_norm": 3.0995608520783295, + "language_loss": 0.77085245, + "learning_rate": 3.918716138794427e-06, + "loss": 0.79256654, + "num_input_tokens_seen": 115784590, + "router_z_loss_clip": 0.86914062, + "router_z_loss_mlp": 0.15863037, + "step": 4078, + "time_per_iteration": 2.4753150939941406 + }, + { + "auxiliary_loss_clip": 0.01133923, + "auxiliary_loss_mlp": 0.01041053, + "balance_loss_clip": 1.05068099, + "balance_loss_mlp": 1.02535343, + "epoch": 0.11836225407695433, + "flos": 18691144968960.0, + "grad_norm": 2.386879320351878, + "language_loss": 0.79166269, + "learning_rate": 3.918663089096177e-06, + "loss": 0.81341243, + "num_input_tokens_seen": 115798510, + "router_z_loss_clip": 0.83105469, + "router_z_loss_mlp": 0.15704346, + "step": 4079, + "time_per_iteration": 2.630234718322754 + }, + { + "auxiliary_loss_clip": 0.0103041, + "auxiliary_loss_mlp": 0.01002911, + "balance_loss_clip": 1.00828671, + "balance_loss_mlp": 1.00169492, + "epoch": 0.11839127154547037, + "flos": 52065017596800.0, + "grad_norm": 0.6988392594273657, + "language_loss": 0.49057484, + "learning_rate": 3.918610022451517e-06, + "loss": 0.51090801, + "num_input_tokens_seen": 115858975, + "router_z_loss_clip": 0.22070312, + "router_z_loss_mlp": 0.012146, + "step": 4080, + "time_per_iteration": 3.1862986087799072 + }, + { + "auxiliary_loss_clip": 0.01138144, + "auxiliary_loss_mlp": 0.01045862, + "balance_loss_clip": 1.05175734, + "balance_loss_mlp": 1.02894032, + "epoch": 0.11842028901398642, + "flos": 29964940375680.0, + "grad_norm": 1.8199854930621218, + "language_loss": 0.83787203, + "learning_rate": 3.918556938860917e-06, + "loss": 0.85971212, + "num_input_tokens_seen": 115886140, + "router_z_loss_clip": 0.86328125, + "router_z_loss_mlp": 0.16918945, + "step": 4081, + "time_per_iteration": 2.962738037109375 + }, + { + "auxiliary_loss_clip": 0.01146694, + "auxiliary_loss_mlp": 0.01052315, + "balance_loss_clip": 1.05703497, + "balance_loss_mlp": 1.03350985, + "epoch": 0.11844930648250247, + "flos": 13622246519040.0, + "grad_norm": 2.5846688736619963, + "language_loss": 0.83071733, + "learning_rate": 3.918503838324846e-06, + "loss": 0.85270733, + "num_input_tokens_seen": 115899685, + "router_z_loss_clip": 0.89648438, + "router_z_loss_mlp": 0.18798828, + "step": 4082, + "time_per_iteration": 2.520705461502075 + }, + { + "auxiliary_loss_clip": 0.01143908, + "auxiliary_loss_mlp": 0.01046955, + "balance_loss_clip": 1.05537128, + "balance_loss_mlp": 1.03003299, + "epoch": 0.11847832395101851, + "flos": 14312808466560.0, + "grad_norm": 2.8283200938813264, + "language_loss": 0.86609149, + "learning_rate": 3.9184507208437725e-06, + "loss": 0.88800013, + "num_input_tokens_seen": 115911165, + "router_z_loss_clip": 0.88476562, + "router_z_loss_mlp": 0.16900635, + "step": 4083, + "time_per_iteration": 2.4932873249053955 + }, + { + "auxiliary_loss_clip": 0.01139211, + "auxiliary_loss_mlp": 0.01040178, + "balance_loss_clip": 1.05251384, + "balance_loss_mlp": 1.02255273, + "epoch": 0.11850734141953456, + "flos": 21613016799360.0, + "grad_norm": 3.9183700899424188, + "language_loss": 0.92064512, + "learning_rate": 3.918397586418167e-06, + "loss": 0.94243896, + "num_input_tokens_seen": 115928845, + "router_z_loss_clip": 0.8671875, + "router_z_loss_mlp": 0.17626953, + "step": 4084, + "time_per_iteration": 2.5573158264160156 + }, + { + "auxiliary_loss_clip": 0.01137356, + "auxiliary_loss_mlp": 0.01044411, + "balance_loss_clip": 1.05223846, + "balance_loss_mlp": 1.02755475, + "epoch": 0.1185363588880506, + "flos": 20772922533120.0, + "grad_norm": 4.920393613085586, + "language_loss": 0.82678598, + "learning_rate": 3.918344435048496e-06, + "loss": 0.84860361, + "num_input_tokens_seen": 115943210, + "router_z_loss_clip": 0.8515625, + "router_z_loss_mlp": 0.16845703, + "step": 4085, + "time_per_iteration": 2.550111770629883 + }, + { + "auxiliary_loss_clip": 0.01151179, + "auxiliary_loss_mlp": 0.0104169, + "balance_loss_clip": 1.05846238, + "balance_loss_mlp": 1.02298558, + "epoch": 0.11856537635656665, + "flos": 47441980713600.0, + "grad_norm": 2.897153777860445, + "language_loss": 0.82369608, + "learning_rate": 3.918291266735232e-06, + "loss": 0.8456248, + "num_input_tokens_seen": 115962070, + "router_z_loss_clip": 0.92724609, + "router_z_loss_mlp": 0.18725586, + "step": 4086, + "time_per_iteration": 2.7377943992614746 + }, + { + "auxiliary_loss_clip": 0.01033365, + "auxiliary_loss_mlp": 0.01002092, + "balance_loss_clip": 1.01117611, + "balance_loss_mlp": 1.00094163, + "epoch": 0.1185943938250827, + "flos": 74779699501440.0, + "grad_norm": 0.6609878595188712, + "language_loss": 0.50747716, + "learning_rate": 3.9182380814788425e-06, + "loss": 0.52783167, + "num_input_tokens_seen": 116027130, + "router_z_loss_clip": 0.22265625, + "router_z_loss_mlp": 0.01147461, + "step": 4087, + "time_per_iteration": 3.2306289672851562 + }, + { + "auxiliary_loss_clip": 0.01135205, + "auxiliary_loss_mlp": 0.01035383, + "balance_loss_clip": 1.05227089, + "balance_loss_mlp": 1.01938534, + "epoch": 0.11862341129359874, + "flos": 35261082218880.0, + "grad_norm": 6.318648646476465, + "language_loss": 0.76898289, + "learning_rate": 3.918184879279799e-06, + "loss": 0.79068875, + "num_input_tokens_seen": 116050625, + "router_z_loss_clip": 0.83007812, + "router_z_loss_mlp": 0.15991211, + "step": 4088, + "time_per_iteration": 2.6305816173553467 + }, + { + "auxiliary_loss_clip": 0.01142469, + "auxiliary_loss_mlp": 0.01043122, + "balance_loss_clip": 1.05375218, + "balance_loss_mlp": 1.02597392, + "epoch": 0.11865242876211479, + "flos": 34648123605120.0, + "grad_norm": 1.844967181365028, + "language_loss": 0.67396593, + "learning_rate": 3.918131660138569e-06, + "loss": 0.69582188, + "num_input_tokens_seen": 116070080, + "router_z_loss_clip": 0.88623047, + "router_z_loss_mlp": 0.17144775, + "step": 4089, + "time_per_iteration": 2.6882076263427734 + }, + { + "auxiliary_loss_clip": 0.01154994, + "auxiliary_loss_mlp": 0.01048395, + "balance_loss_clip": 1.06021237, + "balance_loss_mlp": 1.0310086, + "epoch": 0.11868144623063084, + "flos": 35291174837760.0, + "grad_norm": 2.4693770991531894, + "language_loss": 0.80695701, + "learning_rate": 3.918078424055626e-06, + "loss": 0.828991, + "num_input_tokens_seen": 116084980, + "router_z_loss_clip": 0.94775391, + "router_z_loss_mlp": 0.17382812, + "step": 4090, + "time_per_iteration": 2.657290458679199 + }, + { + "auxiliary_loss_clip": 0.01143241, + "auxiliary_loss_mlp": 0.01043873, + "balance_loss_clip": 1.05346096, + "balance_loss_mlp": 1.02767253, + "epoch": 0.11871046369914688, + "flos": 19821505040640.0, + "grad_norm": 2.5721501237647546, + "language_loss": 0.67449576, + "learning_rate": 3.918025171031436e-06, + "loss": 0.69636691, + "num_input_tokens_seen": 116098505, + "router_z_loss_clip": 0.89892578, + "router_z_loss_mlp": 0.16204834, + "step": 4091, + "time_per_iteration": 2.5006275177001953 + }, + { + "auxiliary_loss_clip": 0.01036189, + "auxiliary_loss_mlp": 0.01004714, + "balance_loss_clip": 1.01385188, + "balance_loss_mlp": 1.00354004, + "epoch": 0.11873948116766293, + "flos": 74774312461440.0, + "grad_norm": 0.6884851447484021, + "language_loss": 0.49978578, + "learning_rate": 3.917971901066473e-06, + "loss": 0.52019477, + "num_input_tokens_seen": 116162680, + "router_z_loss_clip": 0.22265625, + "router_z_loss_mlp": 0.01171875, + "step": 4092, + "time_per_iteration": 3.1308321952819824 + }, + { + "auxiliary_loss_clip": 0.01147414, + "auxiliary_loss_mlp": 0.01045851, + "balance_loss_clip": 1.05596519, + "balance_loss_mlp": 1.02690303, + "epoch": 0.11876849863617898, + "flos": 14129089378560.0, + "grad_norm": 3.1507343751185077, + "language_loss": 0.90842557, + "learning_rate": 3.917918614161206e-06, + "loss": 0.93035823, + "num_input_tokens_seen": 116174190, + "router_z_loss_clip": 0.91455078, + "router_z_loss_mlp": 0.18951416, + "step": 4093, + "time_per_iteration": 2.5215141773223877 + }, + { + "auxiliary_loss_clip": 0.01032308, + "auxiliary_loss_mlp": 0.0100349, + "balance_loss_clip": 1.01020432, + "balance_loss_mlp": 1.00225627, + "epoch": 0.11879751610469502, + "flos": 67040447230080.0, + "grad_norm": 0.6884215655568823, + "language_loss": 0.49629712, + "learning_rate": 3.917865310316105e-06, + "loss": 0.51665509, + "num_input_tokens_seen": 116237345, + "router_z_loss_clip": 0.22070312, + "router_z_loss_mlp": 0.0123291, + "step": 4094, + "time_per_iteration": 3.2357981204986572 + }, + { + "auxiliary_loss_clip": 0.01134357, + "auxiliary_loss_mlp": 0.01039988, + "balance_loss_clip": 1.05095732, + "balance_loss_mlp": 1.02379298, + "epoch": 0.11882653357321107, + "flos": 35401061520000.0, + "grad_norm": 2.197484716875849, + "language_loss": 0.7867856, + "learning_rate": 3.917811989531642e-06, + "loss": 0.80852914, + "num_input_tokens_seen": 116253210, + "router_z_loss_clip": 0.83447266, + "router_z_loss_mlp": 0.16186523, + "step": 4095, + "time_per_iteration": 2.6519551277160645 + }, + { + "auxiliary_loss_clip": 0.01030227, + "auxiliary_loss_mlp": 0.01002445, + "balance_loss_clip": 1.00811386, + "balance_loss_mlp": 1.00127685, + "epoch": 0.11885555104172713, + "flos": 53354424113280.0, + "grad_norm": 0.7456228221971556, + "language_loss": 0.52016985, + "learning_rate": 3.917758651808287e-06, + "loss": 0.54049653, + "num_input_tokens_seen": 116308440, + "router_z_loss_clip": 0.22070312, + "router_z_loss_mlp": 0.01165771, + "step": 4096, + "time_per_iteration": 2.95619535446167 + }, + { + "auxiliary_loss_clip": 0.01156442, + "auxiliary_loss_mlp": 0.01047759, + "balance_loss_clip": 1.05851698, + "balance_loss_mlp": 1.02787459, + "epoch": 0.11888456851024316, + "flos": 46719350899200.0, + "grad_norm": 2.263460720144725, + "language_loss": 0.86498356, + "learning_rate": 3.917705297146511e-06, + "loss": 0.88702554, + "num_input_tokens_seen": 116329735, + "router_z_loss_clip": 0.98046875, + "router_z_loss_mlp": 0.19873047, + "step": 4097, + "time_per_iteration": 2.7307350635528564 + }, + { + "auxiliary_loss_clip": 0.01030099, + "auxiliary_loss_mlp": 0.01000544, + "balance_loss_clip": 1.00783181, + "balance_loss_mlp": 0.99929827, + "epoch": 0.11891358597875921, + "flos": 70790382270720.0, + "grad_norm": 0.6813185455251783, + "language_loss": 0.46533114, + "learning_rate": 3.9176519255467875e-06, + "loss": 0.48563758, + "num_input_tokens_seen": 116385180, + "router_z_loss_clip": 0.22265625, + "router_z_loss_mlp": 0.01245117, + "step": 4098, + "time_per_iteration": 3.0768115520477295 + }, + { + "auxiliary_loss_clip": 0.01140374, + "auxiliary_loss_mlp": 0.01045785, + "balance_loss_clip": 1.05309188, + "balance_loss_mlp": 1.02945375, + "epoch": 0.11894260344727527, + "flos": 17486341960320.0, + "grad_norm": 2.748715183193145, + "language_loss": 0.99642885, + "learning_rate": 3.917598537009585e-06, + "loss": 1.0182904, + "num_input_tokens_seen": 116400840, + "router_z_loss_clip": 0.87158203, + "router_z_loss_mlp": 0.16326904, + "step": 4099, + "time_per_iteration": 2.471446990966797 + }, + { + "auxiliary_loss_clip": 0.01159546, + "auxiliary_loss_mlp": 0.01056372, + "balance_loss_clip": 1.0601722, + "balance_loss_mlp": 1.03698814, + "epoch": 0.1189716209157913, + "flos": 14208344737920.0, + "grad_norm": 3.4350348241168076, + "language_loss": 0.85206616, + "learning_rate": 3.917545131535377e-06, + "loss": 0.87422526, + "num_input_tokens_seen": 116414015, + "router_z_loss_clip": 0.99414062, + "router_z_loss_mlp": 0.19384766, + "step": 4100, + "time_per_iteration": 2.4676945209503174 + }, + { + "auxiliary_loss_clip": 0.01137226, + "auxiliary_loss_mlp": 0.01044803, + "balance_loss_clip": 1.05214548, + "balance_loss_mlp": 1.02753568, + "epoch": 0.11900063838430736, + "flos": 23763598865280.0, + "grad_norm": 2.2657202094608997, + "language_loss": 1.08502364, + "learning_rate": 3.917491709124634e-06, + "loss": 1.10684395, + "num_input_tokens_seen": 116430320, + "router_z_loss_clip": 0.85058594, + "router_z_loss_mlp": 0.17272949, + "step": 4101, + "time_per_iteration": 2.5251522064208984 + }, + { + "auxiliary_loss_clip": 0.01130235, + "auxiliary_loss_mlp": 0.01037863, + "balance_loss_clip": 1.05083323, + "balance_loss_mlp": 1.02169824, + "epoch": 0.1190296558528234, + "flos": 36715427988480.0, + "grad_norm": 1.7803246479628179, + "language_loss": 0.65656567, + "learning_rate": 3.9174382697778284e-06, + "loss": 0.67824668, + "num_input_tokens_seen": 116448875, + "router_z_loss_clip": 0.79296875, + "router_z_loss_mlp": 0.16162109, + "step": 4102, + "time_per_iteration": 2.653597593307495 + }, + { + "auxiliary_loss_clip": 0.01154084, + "auxiliary_loss_mlp": 0.01058493, + "balance_loss_clip": 1.05630338, + "balance_loss_mlp": 1.03868067, + "epoch": 0.11905867332133944, + "flos": 30441295566720.0, + "grad_norm": 1.8827358033477866, + "language_loss": 0.81909919, + "learning_rate": 3.917384813495431e-06, + "loss": 0.84122491, + "num_input_tokens_seen": 116472115, + "router_z_loss_clip": 0.97802734, + "router_z_loss_mlp": 0.19812012, + "step": 4103, + "time_per_iteration": 2.7908761501312256 + }, + { + "auxiliary_loss_clip": 0.01157895, + "auxiliary_loss_mlp": 0.01046117, + "balance_loss_clip": 1.06249416, + "balance_loss_mlp": 1.0269959, + "epoch": 0.1190876907898555, + "flos": 11647410721920.0, + "grad_norm": 2.457244415231392, + "language_loss": 0.8854863, + "learning_rate": 3.917331340277917e-06, + "loss": 0.90752637, + "num_input_tokens_seen": 116483385, + "router_z_loss_clip": 0.95361328, + "router_z_loss_mlp": 0.19116211, + "step": 4104, + "time_per_iteration": 2.4507672786712646 + }, + { + "auxiliary_loss_clip": 0.01139566, + "auxiliary_loss_mlp": 0.01053964, + "balance_loss_clip": 1.05337965, + "balance_loss_mlp": 1.03717971, + "epoch": 0.11911670825837153, + "flos": 21352340822400.0, + "grad_norm": 2.023665000029481, + "language_loss": 0.84747791, + "learning_rate": 3.917277850125755e-06, + "loss": 0.8694132, + "num_input_tokens_seen": 116498020, + "router_z_loss_clip": 0.86181641, + "router_z_loss_mlp": 0.16784668, + "step": 4105, + "time_per_iteration": 2.493955373764038 + }, + { + "auxiliary_loss_clip": 0.01140548, + "auxiliary_loss_mlp": 0.01047338, + "balance_loss_clip": 1.05337846, + "balance_loss_mlp": 1.03018427, + "epoch": 0.11914572572688759, + "flos": 17229508738560.0, + "grad_norm": 2.9522587471674973, + "language_loss": 0.95243269, + "learning_rate": 3.91722434303942e-06, + "loss": 0.97431159, + "num_input_tokens_seen": 116510550, + "router_z_loss_clip": 0.87304688, + "router_z_loss_mlp": 0.17144775, + "step": 4106, + "time_per_iteration": 2.5184247493743896 + }, + { + "auxiliary_loss_clip": 0.01031383, + "auxiliary_loss_mlp": 0.01016881, + "balance_loss_clip": 1.00835204, + "balance_loss_mlp": 1.01574254, + "epoch": 0.11917474319540364, + "flos": 65902545302400.0, + "grad_norm": 0.7315158778933908, + "language_loss": 0.50334364, + "learning_rate": 3.917170819019384e-06, + "loss": 0.52382624, + "num_input_tokens_seen": 116573130, + "router_z_loss_clip": 0.23046875, + "router_z_loss_mlp": 0.01141357, + "step": 4107, + "time_per_iteration": 3.0835959911346436 + }, + { + "auxiliary_loss_clip": 0.010324, + "auxiliary_loss_mlp": 0.01004203, + "balance_loss_clip": 1.0096823, + "balance_loss_mlp": 1.00304115, + "epoch": 0.11920376066391968, + "flos": 74760162503040.0, + "grad_norm": 0.6675512468848752, + "language_loss": 0.49576339, + "learning_rate": 3.91711727806612e-06, + "loss": 0.51612943, + "num_input_tokens_seen": 116627850, + "router_z_loss_clip": 0.2265625, + "router_z_loss_mlp": 0.01159668, + "step": 4108, + "time_per_iteration": 3.0214781761169434 + }, + { + "auxiliary_loss_clip": 0.01132572, + "auxiliary_loss_mlp": 0.01044147, + "balance_loss_clip": 1.04882097, + "balance_loss_mlp": 1.02670622, + "epoch": 0.11923277813243573, + "flos": 11975562397440.0, + "grad_norm": 4.005293155885132, + "language_loss": 1.00295389, + "learning_rate": 3.917063720180099e-06, + "loss": 1.02472103, + "num_input_tokens_seen": 116638420, + "router_z_loss_clip": 0.83740234, + "router_z_loss_mlp": 0.17443848, + "step": 4109, + "time_per_iteration": 2.4321320056915283 + }, + { + "auxiliary_loss_clip": 0.01136903, + "auxiliary_loss_mlp": 0.01046673, + "balance_loss_clip": 1.05227947, + "balance_loss_mlp": 1.0292325, + "epoch": 0.11926179560095178, + "flos": 28870203617280.0, + "grad_norm": 3.921634064683359, + "language_loss": 0.9544549, + "learning_rate": 3.917010145361796e-06, + "loss": 0.97629064, + "num_input_tokens_seen": 116655610, + "router_z_loss_clip": 0.84667969, + "router_z_loss_mlp": 0.17443848, + "step": 4110, + "time_per_iteration": 2.5709259510040283 + }, + { + "auxiliary_loss_clip": 0.01147414, + "auxiliary_loss_mlp": 0.01049127, + "balance_loss_clip": 1.05195308, + "balance_loss_mlp": 1.02944005, + "epoch": 0.11929081306946782, + "flos": 17380262119680.0, + "grad_norm": 2.4062005127630726, + "language_loss": 0.73585719, + "learning_rate": 3.916956553611684e-06, + "loss": 0.75782257, + "num_input_tokens_seen": 116669440, + "router_z_loss_clip": 0.95458984, + "router_z_loss_mlp": 0.19683838, + "step": 4111, + "time_per_iteration": 2.539125919342041 + }, + { + "auxiliary_loss_clip": 0.01144077, + "auxiliary_loss_mlp": 0.01047651, + "balance_loss_clip": 1.05369663, + "balance_loss_mlp": 1.02874482, + "epoch": 0.11931983053798387, + "flos": 35766237139200.0, + "grad_norm": 1.994409083805022, + "language_loss": 0.63342488, + "learning_rate": 3.9169029449302355e-06, + "loss": 0.65534216, + "num_input_tokens_seen": 116685695, + "router_z_loss_clip": 0.90332031, + "router_z_loss_mlp": 0.18908691, + "step": 4112, + "time_per_iteration": 2.5950851440429688 + }, + { + "auxiliary_loss_clip": 0.01029536, + "auxiliary_loss_mlp": 0.0100129, + "balance_loss_clip": 1.00659096, + "balance_loss_mlp": 1.00009787, + "epoch": 0.11934884800649992, + "flos": 58608801417600.0, + "grad_norm": 0.6454655933370963, + "language_loss": 0.47723475, + "learning_rate": 3.9168493193179256e-06, + "loss": 0.49754298, + "num_input_tokens_seen": 116743190, + "router_z_loss_clip": 0.22851562, + "router_z_loss_mlp": 0.01190186, + "step": 4113, + "time_per_iteration": 2.9898648262023926 + }, + { + "auxiliary_loss_clip": 0.01126164, + "auxiliary_loss_mlp": 0.01039994, + "balance_loss_clip": 1.04887557, + "balance_loss_mlp": 1.02481866, + "epoch": 0.11937786547501596, + "flos": 20915918576640.0, + "grad_norm": 2.3467942116064986, + "language_loss": 0.79280442, + "learning_rate": 3.916795676775225e-06, + "loss": 0.814466, + "num_input_tokens_seen": 116756015, + "router_z_loss_clip": 0.77246094, + "router_z_loss_mlp": 0.15167236, + "step": 4114, + "time_per_iteration": 2.5571346282958984 + }, + { + "auxiliary_loss_clip": 0.01149855, + "auxiliary_loss_mlp": 0.01055615, + "balance_loss_clip": 1.0552156, + "balance_loss_mlp": 1.03266692, + "epoch": 0.11940688294353201, + "flos": 29161223608320.0, + "grad_norm": 2.568490245928989, + "language_loss": 0.98345554, + "learning_rate": 3.9167420173026105e-06, + "loss": 1.00551021, + "num_input_tokens_seen": 116769995, + "router_z_loss_clip": 0.94580078, + "router_z_loss_mlp": 0.22949219, + "step": 4115, + "time_per_iteration": 2.5761468410491943 + }, + { + "auxiliary_loss_clip": 0.01142828, + "auxiliary_loss_mlp": 0.01041603, + "balance_loss_clip": 1.05280471, + "balance_loss_mlp": 1.0247643, + "epoch": 0.11943590041204805, + "flos": 29455332168960.0, + "grad_norm": 2.920795411510157, + "language_loss": 0.63236117, + "learning_rate": 3.916688340900555e-06, + "loss": 0.6542055, + "num_input_tokens_seen": 116785950, + "router_z_loss_clip": 0.90087891, + "router_z_loss_mlp": 0.168396, + "step": 4116, + "time_per_iteration": 2.490121603012085 + }, + { + "auxiliary_loss_clip": 0.01135369, + "auxiliary_loss_mlp": 0.01044749, + "balance_loss_clip": 1.05242777, + "balance_loss_mlp": 1.02892971, + "epoch": 0.1194649178805641, + "flos": 16100836606080.0, + "grad_norm": 2.923581997753726, + "language_loss": 0.72794902, + "learning_rate": 3.916634647569533e-06, + "loss": 0.74975026, + "num_input_tokens_seen": 116800110, + "router_z_loss_clip": 0.82958984, + "router_z_loss_mlp": 0.15826416, + "step": 4117, + "time_per_iteration": 2.4824533462524414 + }, + { + "auxiliary_loss_clip": 0.0113182, + "auxiliary_loss_mlp": 0.01035521, + "balance_loss_clip": 1.04799426, + "balance_loss_mlp": 1.01987433, + "epoch": 0.11949393534908015, + "flos": 16829643559680.0, + "grad_norm": 3.2100790532437182, + "language_loss": 0.63141191, + "learning_rate": 3.916580937310017e-06, + "loss": 0.65308529, + "num_input_tokens_seen": 116816185, + "router_z_loss_clip": 0.83837891, + "router_z_loss_mlp": 0.15631104, + "step": 4118, + "time_per_iteration": 2.630998134613037 + }, + { + "auxiliary_loss_clip": 0.01140165, + "auxiliary_loss_mlp": 0.01042646, + "balance_loss_clip": 1.05585861, + "balance_loss_mlp": 1.02752399, + "epoch": 0.11952295281759619, + "flos": 29818065663360.0, + "grad_norm": 2.399696803245033, + "language_loss": 0.83081263, + "learning_rate": 3.9165272101224834e-06, + "loss": 0.85264075, + "num_input_tokens_seen": 116831840, + "router_z_loss_clip": 0.84228516, + "router_z_loss_mlp": 0.15136719, + "step": 4119, + "time_per_iteration": 2.5771846771240234 + }, + { + "auxiliary_loss_clip": 0.01129087, + "auxiliary_loss_mlp": 0.01040372, + "balance_loss_clip": 1.04885316, + "balance_loss_mlp": 1.02460098, + "epoch": 0.11955197028611224, + "flos": 31898622165120.0, + "grad_norm": 1.7313295619610867, + "language_loss": 0.77826345, + "learning_rate": 3.916473466007405e-06, + "loss": 0.79995799, + "num_input_tokens_seen": 116853015, + "router_z_loss_clip": 0.80224609, + "router_z_loss_mlp": 0.15765381, + "step": 4120, + "time_per_iteration": 2.6430203914642334 + }, + { + "auxiliary_loss_clip": 0.01133391, + "auxiliary_loss_mlp": 0.01043554, + "balance_loss_clip": 1.0490433, + "balance_loss_mlp": 1.02676392, + "epoch": 0.11958098775462829, + "flos": 44668743361920.0, + "grad_norm": 2.330889988919473, + "language_loss": 0.89737421, + "learning_rate": 3.916419704965259e-06, + "loss": 0.91914362, + "num_input_tokens_seen": 116873985, + "router_z_loss_clip": 0.84375, + "router_z_loss_mlp": 0.16784668, + "step": 4121, + "time_per_iteration": 2.752878189086914 + }, + { + "auxiliary_loss_clip": 0.01132663, + "auxiliary_loss_mlp": 0.01048542, + "balance_loss_clip": 1.05120897, + "balance_loss_mlp": 1.03327084, + "epoch": 0.11961000522314433, + "flos": 21938295386880.0, + "grad_norm": 11.07627682133912, + "language_loss": 0.89178008, + "learning_rate": 3.916365926996517e-06, + "loss": 0.9135921, + "num_input_tokens_seen": 116890795, + "router_z_loss_clip": 0.81396484, + "router_z_loss_mlp": 0.15258789, + "step": 4122, + "time_per_iteration": 2.5304758548736572 + }, + { + "auxiliary_loss_clip": 0.01039347, + "auxiliary_loss_mlp": 0.0100436, + "balance_loss_clip": 1.01692486, + "balance_loss_mlp": 1.00338829, + "epoch": 0.11963902269166038, + "flos": 74792879792640.0, + "grad_norm": 0.6476254532045737, + "language_loss": 0.50007743, + "learning_rate": 3.916312132101657e-06, + "loss": 0.52051449, + "num_input_tokens_seen": 116961340, + "router_z_loss_clip": 0.22363281, + "router_z_loss_mlp": 0.00970459, + "step": 4123, + "time_per_iteration": 3.400902032852173 + }, + { + "auxiliary_loss_clip": 0.01039254, + "auxiliary_loss_mlp": 0.01007047, + "balance_loss_clip": 1.01690745, + "balance_loss_mlp": 1.00602233, + "epoch": 0.11966804016017643, + "flos": 74785912554240.0, + "grad_norm": 0.6656039457958232, + "language_loss": 0.48818976, + "learning_rate": 3.916258320281152e-06, + "loss": 0.50865281, + "num_input_tokens_seen": 117028615, + "router_z_loss_clip": 0.22265625, + "router_z_loss_mlp": 0.01025391, + "step": 4124, + "time_per_iteration": 3.2746989727020264 + }, + { + "auxiliary_loss_clip": 0.01037899, + "auxiliary_loss_mlp": 0.01004796, + "balance_loss_clip": 1.01561141, + "balance_loss_mlp": 1.00377107, + "epoch": 0.11969705762869247, + "flos": 61706850480000.0, + "grad_norm": 0.7098813200966908, + "language_loss": 0.53670752, + "learning_rate": 3.916204491535478e-06, + "loss": 0.55713451, + "num_input_tokens_seen": 117083115, + "router_z_loss_clip": 0.22265625, + "router_z_loss_mlp": 0.01025391, + "step": 4125, + "time_per_iteration": 2.906679630279541 + }, + { + "auxiliary_loss_clip": 0.01144211, + "auxiliary_loss_mlp": 0.01044882, + "balance_loss_clip": 1.0537852, + "balance_loss_mlp": 1.02702451, + "epoch": 0.11972607509720852, + "flos": 21573263422080.0, + "grad_norm": 2.476768814336538, + "language_loss": 0.75402451, + "learning_rate": 3.9161506458651115e-06, + "loss": 0.7759155, + "num_input_tokens_seen": 117099230, + "router_z_loss_clip": 0.90332031, + "router_z_loss_mlp": 0.1784668, + "step": 4126, + "time_per_iteration": 5.04464054107666 + }, + { + "auxiliary_loss_clip": 0.01140889, + "auxiliary_loss_mlp": 0.01041694, + "balance_loss_clip": 1.05094004, + "balance_loss_mlp": 1.02428961, + "epoch": 0.11975509256572457, + "flos": 14857644936960.0, + "grad_norm": 3.998797692789188, + "language_loss": 0.97942704, + "learning_rate": 3.916096783270526e-06, + "loss": 1.00125289, + "num_input_tokens_seen": 117109955, + "router_z_loss_clip": 0.90087891, + "router_z_loss_mlp": 0.17419434, + "step": 4127, + "time_per_iteration": 7.179277181625366 + }, + { + "auxiliary_loss_clip": 0.01034285, + "auxiliary_loss_mlp": 0.01002059, + "balance_loss_clip": 1.01204181, + "balance_loss_mlp": 1.00100994, + "epoch": 0.11978411003424061, + "flos": 48980002193280.0, + "grad_norm": 0.7563204750484689, + "language_loss": 0.50104117, + "learning_rate": 3.916042903752199e-06, + "loss": 0.52140462, + "num_input_tokens_seen": 117167285, + "router_z_loss_clip": 0.22265625, + "router_z_loss_mlp": 0.01049805, + "step": 4128, + "time_per_iteration": 5.470261573791504 + }, + { + "auxiliary_loss_clip": 0.01133621, + "auxiliary_loss_mlp": 0.01055362, + "balance_loss_clip": 1.05143106, + "balance_loss_mlp": 1.03853536, + "epoch": 0.11981312750275666, + "flos": 32632708417920.0, + "grad_norm": 2.0242159007536067, + "language_loss": 0.92479432, + "learning_rate": 3.915989007310605e-06, + "loss": 0.94668418, + "num_input_tokens_seen": 117187545, + "router_z_loss_clip": 0.82128906, + "router_z_loss_mlp": 0.16827393, + "step": 4129, + "time_per_iteration": 2.627098798751831 + }, + { + "auxiliary_loss_clip": 0.01140994, + "auxiliary_loss_mlp": 0.01045531, + "balance_loss_clip": 1.05288768, + "balance_loss_mlp": 1.02757788, + "epoch": 0.11984214497127271, + "flos": 32569003647360.0, + "grad_norm": 2.2957718467540498, + "language_loss": 0.89075845, + "learning_rate": 3.9159350939462216e-06, + "loss": 0.9126237, + "num_input_tokens_seen": 117207415, + "router_z_loss_clip": 0.88085938, + "router_z_loss_mlp": 0.17944336, + "step": 4130, + "time_per_iteration": 2.63078236579895 + }, + { + "auxiliary_loss_clip": 0.01133258, + "auxiliary_loss_mlp": 0.01041578, + "balance_loss_clip": 1.04912543, + "balance_loss_mlp": 1.02473402, + "epoch": 0.11987116243978875, + "flos": 30694429687680.0, + "grad_norm": 2.180381013002161, + "language_loss": 0.81135285, + "learning_rate": 3.915881163659524e-06, + "loss": 0.83310127, + "num_input_tokens_seen": 117223735, + "router_z_loss_clip": 0.84082031, + "router_z_loss_mlp": 0.16821289, + "step": 4131, + "time_per_iteration": 2.6037840843200684 + }, + { + "auxiliary_loss_clip": 0.01030752, + "auxiliary_loss_mlp": 0.01007453, + "balance_loss_clip": 1.0084765, + "balance_loss_mlp": 1.0063864, + "epoch": 0.1199001799083048, + "flos": 56825980750080.0, + "grad_norm": 0.6725541125286286, + "language_loss": 0.49982715, + "learning_rate": 3.915827216450989e-06, + "loss": 0.52020925, + "num_input_tokens_seen": 117285645, + "router_z_loss_clip": 0.22265625, + "router_z_loss_mlp": 0.01068115, + "step": 4132, + "time_per_iteration": 3.145064353942871 + }, + { + "auxiliary_loss_clip": 0.01137342, + "auxiliary_loss_mlp": 0.01041977, + "balance_loss_clip": 1.05049109, + "balance_loss_mlp": 1.02617586, + "epoch": 0.11992919737682084, + "flos": 31902321265920.0, + "grad_norm": 1.8341087973435437, + "language_loss": 0.83313322, + "learning_rate": 3.915773252321091e-06, + "loss": 0.85492647, + "num_input_tokens_seen": 117304535, + "router_z_loss_clip": 0.86914062, + "router_z_loss_mlp": 0.15795898, + "step": 4133, + "time_per_iteration": 2.604069709777832 + }, + { + "auxiliary_loss_clip": 0.01144098, + "auxiliary_loss_mlp": 0.01062511, + "balance_loss_clip": 1.05196476, + "balance_loss_mlp": 1.04356909, + "epoch": 0.11995821484533689, + "flos": 48023625646080.0, + "grad_norm": 3.4630295176687262, + "language_loss": 0.94388843, + "learning_rate": 3.91571927127031e-06, + "loss": 0.96595448, + "num_input_tokens_seen": 117319740, + "router_z_loss_clip": 0.92138672, + "router_z_loss_mlp": 0.18945312, + "step": 4134, + "time_per_iteration": 2.6712443828582764 + }, + { + "auxiliary_loss_clip": 0.01136341, + "auxiliary_loss_mlp": 0.01051507, + "balance_loss_clip": 1.05144501, + "balance_loss_mlp": 1.03363705, + "epoch": 0.11998723231385294, + "flos": 36094496555520.0, + "grad_norm": 2.8203074950757396, + "language_loss": 1.00893188, + "learning_rate": 3.915665273299121e-06, + "loss": 1.03081036, + "num_input_tokens_seen": 117338995, + "router_z_loss_clip": 0.84912109, + "router_z_loss_mlp": 0.17858887, + "step": 4135, + "time_per_iteration": 2.668154001235962 + }, + { + "auxiliary_loss_clip": 0.01142281, + "auxiliary_loss_mlp": 0.01051759, + "balance_loss_clip": 1.05212963, + "balance_loss_mlp": 1.03393769, + "epoch": 0.12001624978236898, + "flos": 27452379000960.0, + "grad_norm": 2.3101480977029354, + "language_loss": 0.89816201, + "learning_rate": 3.915611258408002e-06, + "loss": 0.92010248, + "num_input_tokens_seen": 117354265, + "router_z_loss_clip": 0.90087891, + "router_z_loss_mlp": 0.17834473, + "step": 4136, + "time_per_iteration": 2.584280490875244 + }, + { + "auxiliary_loss_clip": 0.01138571, + "auxiliary_loss_mlp": 0.0106539, + "balance_loss_clip": 1.05032361, + "balance_loss_mlp": 1.04619718, + "epoch": 0.12004526725088503, + "flos": 18036098593920.0, + "grad_norm": 2.704624552601433, + "language_loss": 0.85528547, + "learning_rate": 3.9155572265974275e-06, + "loss": 0.87732512, + "num_input_tokens_seen": 117368480, + "router_z_loss_clip": 0.88183594, + "router_z_loss_mlp": 0.1920166, + "step": 4137, + "time_per_iteration": 2.4968695640563965 + }, + { + "auxiliary_loss_clip": 0.01128525, + "auxiliary_loss_mlp": 0.01050373, + "balance_loss_clip": 1.04783118, + "balance_loss_mlp": 1.03386211, + "epoch": 0.12007428471940108, + "flos": 39086824913280.0, + "grad_norm": 2.368559634685581, + "language_loss": 0.72266084, + "learning_rate": 3.915503177867877e-06, + "loss": 0.74444979, + "num_input_tokens_seen": 117387655, + "router_z_loss_clip": 0.80664062, + "router_z_loss_mlp": 0.16497803, + "step": 4138, + "time_per_iteration": 2.572253465652466 + }, + { + "auxiliary_loss_clip": 0.01133996, + "auxiliary_loss_mlp": 0.01044745, + "balance_loss_clip": 1.05170298, + "balance_loss_mlp": 1.02853251, + "epoch": 0.12010330218791712, + "flos": 30952807194240.0, + "grad_norm": 1.928440129754137, + "language_loss": 0.8038727, + "learning_rate": 3.915449112219828e-06, + "loss": 0.82566017, + "num_input_tokens_seen": 117407030, + "router_z_loss_clip": 0.82275391, + "router_z_loss_mlp": 0.16229248, + "step": 4139, + "time_per_iteration": 2.615096092224121 + }, + { + "auxiliary_loss_clip": 0.01139681, + "auxiliary_loss_mlp": 0.0105407, + "balance_loss_clip": 1.0529952, + "balance_loss_mlp": 1.03635538, + "epoch": 0.12013231965643317, + "flos": 28688854826880.0, + "grad_norm": 2.703888872522108, + "language_loss": 0.86689055, + "learning_rate": 3.9153950296537564e-06, + "loss": 0.88882804, + "num_input_tokens_seen": 117422445, + "router_z_loss_clip": 0.86621094, + "router_z_loss_mlp": 0.17718506, + "step": 4140, + "time_per_iteration": 2.5930023193359375 + }, + { + "auxiliary_loss_clip": 0.01139632, + "auxiliary_loss_mlp": 0.01058141, + "balance_loss_clip": 1.05297256, + "balance_loss_mlp": 1.0394969, + "epoch": 0.12016133712494922, + "flos": 39524611875840.0, + "grad_norm": 1.9227207726627156, + "language_loss": 0.8311857, + "learning_rate": 3.9153409301701414e-06, + "loss": 0.85316336, + "num_input_tokens_seen": 117443810, + "router_z_loss_clip": 0.8671875, + "router_z_loss_mlp": 0.18640137, + "step": 4141, + "time_per_iteration": 2.740307569503784 + }, + { + "auxiliary_loss_clip": 0.01141027, + "auxiliary_loss_mlp": 0.01048207, + "balance_loss_clip": 1.05443001, + "balance_loss_mlp": 1.03038502, + "epoch": 0.12019035459346526, + "flos": 12814902478080.0, + "grad_norm": 2.8721392461697706, + "language_loss": 0.55157137, + "learning_rate": 3.91528681376946e-06, + "loss": 0.57346368, + "num_input_tokens_seen": 117456665, + "router_z_loss_clip": 0.86523438, + "router_z_loss_mlp": 0.17822266, + "step": 4142, + "time_per_iteration": 2.65305757522583 + }, + { + "auxiliary_loss_clip": 0.01148656, + "auxiliary_loss_mlp": 0.01051365, + "balance_loss_clip": 1.05557275, + "balance_loss_mlp": 1.03230309, + "epoch": 0.12021937206198131, + "flos": 16464072890880.0, + "grad_norm": 4.216395438071982, + "language_loss": 0.76264739, + "learning_rate": 3.91523268045219e-06, + "loss": 0.78464764, + "num_input_tokens_seen": 117468690, + "router_z_loss_clip": 0.9296875, + "router_z_loss_mlp": 0.19049072, + "step": 4143, + "time_per_iteration": 2.540407180786133 + }, + { + "auxiliary_loss_clip": 0.01128121, + "auxiliary_loss_mlp": 0.01045025, + "balance_loss_clip": 1.048908, + "balance_loss_mlp": 1.0307374, + "epoch": 0.12024838953049737, + "flos": 17523653212800.0, + "grad_norm": 2.320973549528966, + "language_loss": 0.75719345, + "learning_rate": 3.915178530218811e-06, + "loss": 0.77892482, + "num_input_tokens_seen": 117482890, + "router_z_loss_clip": 0.79150391, + "router_z_loss_mlp": 0.1428833, + "step": 4144, + "time_per_iteration": 2.5794479846954346 + }, + { + "auxiliary_loss_clip": 0.01033508, + "auxiliary_loss_mlp": 0.01002953, + "balance_loss_clip": 1.0111413, + "balance_loss_mlp": 1.00186813, + "epoch": 0.1202774069990134, + "flos": 68710544040960.0, + "grad_norm": 0.6756526223093402, + "language_loss": 0.48559296, + "learning_rate": 3.915124363069799e-06, + "loss": 0.50595754, + "num_input_tokens_seen": 117541705, + "router_z_loss_clip": 0.22460938, + "router_z_loss_mlp": 0.01086426, + "step": 4145, + "time_per_iteration": 3.1215553283691406 + }, + { + "auxiliary_loss_clip": 0.01140064, + "auxiliary_loss_mlp": 0.01050605, + "balance_loss_clip": 1.04894078, + "balance_loss_mlp": 1.03060818, + "epoch": 0.12030642446752945, + "flos": 18179058723840.0, + "grad_norm": 5.16059501055413, + "language_loss": 0.78553891, + "learning_rate": 3.915070179005635e-06, + "loss": 0.80744559, + "num_input_tokens_seen": 117556605, + "router_z_loss_clip": 0.91113281, + "router_z_loss_mlp": 0.19989014, + "step": 4146, + "time_per_iteration": 2.4573476314544678 + }, + { + "auxiliary_loss_clip": 0.01134843, + "auxiliary_loss_mlp": 0.01050369, + "balance_loss_clip": 1.05222285, + "balance_loss_mlp": 1.03363228, + "epoch": 0.1203354419360455, + "flos": 23469957181440.0, + "grad_norm": 2.454111595543812, + "language_loss": 0.9333148, + "learning_rate": 3.915015978026795e-06, + "loss": 0.95516688, + "num_input_tokens_seen": 117570440, + "router_z_loss_clip": 0.82568359, + "router_z_loss_mlp": 0.16729736, + "step": 4147, + "time_per_iteration": 2.6032347679138184 + }, + { + "auxiliary_loss_clip": 0.0114251, + "auxiliary_loss_mlp": 0.01045307, + "balance_loss_clip": 1.05310071, + "balance_loss_mlp": 1.02595925, + "epoch": 0.12036445940456154, + "flos": 15663193297920.0, + "grad_norm": 3.1617585722592545, + "language_loss": 0.99434376, + "learning_rate": 3.91496176013376e-06, + "loss": 1.01622188, + "num_input_tokens_seen": 117587570, + "router_z_loss_clip": 0.89355469, + "router_z_loss_mlp": 0.19360352, + "step": 4148, + "time_per_iteration": 2.7182328701019287 + }, + { + "auxiliary_loss_clip": 0.01144569, + "auxiliary_loss_mlp": 0.01054807, + "balance_loss_clip": 1.05286241, + "balance_loss_mlp": 1.03604341, + "epoch": 0.1203934768730776, + "flos": 19637570471040.0, + "grad_norm": 3.255862285725235, + "language_loss": 0.8972339, + "learning_rate": 3.914907525327007e-06, + "loss": 0.91922772, + "num_input_tokens_seen": 117598485, + "router_z_loss_clip": 0.91699219, + "router_z_loss_mlp": 0.18774414, + "step": 4149, + "time_per_iteration": 2.539043426513672 + }, + { + "auxiliary_loss_clip": 0.01143877, + "auxiliary_loss_mlp": 0.010533, + "balance_loss_clip": 1.05498958, + "balance_loss_mlp": 1.03674221, + "epoch": 0.12042249434159363, + "flos": 13510420502400.0, + "grad_norm": 2.125653686550592, + "language_loss": 0.9415012, + "learning_rate": 3.914853273607017e-06, + "loss": 0.96347302, + "num_input_tokens_seen": 117613075, + "router_z_loss_clip": 0.88769531, + "router_z_loss_mlp": 0.16558838, + "step": 4150, + "time_per_iteration": 2.5489063262939453 + }, + { + "auxiliary_loss_clip": 0.01139052, + "auxiliary_loss_mlp": 0.01044991, + "balance_loss_clip": 1.04927588, + "balance_loss_mlp": 1.02683568, + "epoch": 0.12045151181010968, + "flos": 32189821724160.0, + "grad_norm": 2.5497787112756645, + "language_loss": 0.89016342, + "learning_rate": 3.914799004974266e-06, + "loss": 0.91200387, + "num_input_tokens_seen": 117628400, + "router_z_loss_clip": 0.89697266, + "router_z_loss_mlp": 0.18164062, + "step": 4151, + "time_per_iteration": 2.6167383193969727 + }, + { + "auxiliary_loss_clip": 0.01138547, + "auxiliary_loss_mlp": 0.01049281, + "balance_loss_clip": 1.05322826, + "balance_loss_mlp": 1.03144121, + "epoch": 0.12048052927862574, + "flos": 16660397652480.0, + "grad_norm": 2.5571703647556387, + "language_loss": 0.70564127, + "learning_rate": 3.9147447194292374e-06, + "loss": 0.72751957, + "num_input_tokens_seen": 117643370, + "router_z_loss_clip": 0.85351562, + "router_z_loss_mlp": 0.17840576, + "step": 4152, + "time_per_iteration": 2.511885166168213 + }, + { + "auxiliary_loss_clip": 0.01139352, + "auxiliary_loss_mlp": 0.01041074, + "balance_loss_clip": 1.05347753, + "balance_loss_mlp": 1.02323413, + "epoch": 0.12050954674714177, + "flos": 27338505909120.0, + "grad_norm": 2.415341738659785, + "language_loss": 0.61514664, + "learning_rate": 3.914690416972408e-06, + "loss": 0.63695085, + "num_input_tokens_seen": 117659040, + "router_z_loss_clip": 0.85791016, + "router_z_loss_mlp": 0.17822266, + "step": 4153, + "time_per_iteration": 2.4933130741119385 + }, + { + "auxiliary_loss_clip": 0.01038097, + "auxiliary_loss_mlp": 0.01002479, + "balance_loss_clip": 1.01569128, + "balance_loss_mlp": 1.00138807, + "epoch": 0.12053856421565783, + "flos": 73116964978560.0, + "grad_norm": 0.7147726755289695, + "language_loss": 0.51744115, + "learning_rate": 3.914636097604258e-06, + "loss": 0.53784692, + "num_input_tokens_seen": 117726060, + "router_z_loss_clip": 0.22460938, + "router_z_loss_mlp": 0.01092529, + "step": 4154, + "time_per_iteration": 3.302461624145508 + }, + { + "auxiliary_loss_clip": 0.01038537, + "auxiliary_loss_mlp": 0.00998873, + "balance_loss_clip": 1.01604939, + "balance_loss_mlp": 0.99771708, + "epoch": 0.12056758168417388, + "flos": 59626904509440.0, + "grad_norm": 0.7023252822645304, + "language_loss": 0.50090343, + "learning_rate": 3.9145817613252666e-06, + "loss": 0.52127755, + "num_input_tokens_seen": 117784295, + "router_z_loss_clip": 0.22460938, + "router_z_loss_mlp": 0.01153564, + "step": 4155, + "time_per_iteration": 2.9546635150909424 + }, + { + "auxiliary_loss_clip": 0.01145339, + "auxiliary_loss_mlp": 0.0105474, + "balance_loss_clip": 1.05512369, + "balance_loss_mlp": 1.03527308, + "epoch": 0.12059659915268992, + "flos": 16717781629440.0, + "grad_norm": 2.766504913239025, + "language_loss": 0.78919691, + "learning_rate": 3.914527408135915e-06, + "loss": 0.8111977, + "num_input_tokens_seen": 117796050, + "router_z_loss_clip": 0.90136719, + "router_z_loss_mlp": 0.19470215, + "step": 4156, + "time_per_iteration": 2.4976813793182373 + }, + { + "auxiliary_loss_clip": 0.01149194, + "auxiliary_loss_mlp": 0.01050021, + "balance_loss_clip": 1.05776358, + "balance_loss_mlp": 1.03130543, + "epoch": 0.12062561662120597, + "flos": 18107417047680.0, + "grad_norm": 2.755518988467834, + "language_loss": 0.97819567, + "learning_rate": 3.914473038036682e-06, + "loss": 1.00018775, + "num_input_tokens_seen": 117808785, + "router_z_loss_clip": 0.9140625, + "router_z_loss_mlp": 0.18725586, + "step": 4157, + "time_per_iteration": 2.5126352310180664 + }, + { + "auxiliary_loss_clip": 0.01140807, + "auxiliary_loss_mlp": 0.01047631, + "balance_loss_clip": 1.05101049, + "balance_loss_mlp": 1.02831912, + "epoch": 0.12065463408972202, + "flos": 25698932680320.0, + "grad_norm": 2.589127340696898, + "language_loss": 0.77089661, + "learning_rate": 3.914418651028049e-06, + "loss": 0.792781, + "num_input_tokens_seen": 117824955, + "router_z_loss_clip": 0.89892578, + "router_z_loss_mlp": 0.19311523, + "step": 4158, + "time_per_iteration": 2.558753728866577 + }, + { + "auxiliary_loss_clip": 0.0103706, + "auxiliary_loss_mlp": 0.01007945, + "balance_loss_clip": 1.01452541, + "balance_loss_mlp": 1.00669312, + "epoch": 0.12068365155823806, + "flos": 68755720371840.0, + "grad_norm": 0.6634195926420651, + "language_loss": 0.48207086, + "learning_rate": 3.914364247110495e-06, + "loss": 0.50252092, + "num_input_tokens_seen": 117882130, + "router_z_loss_clip": 0.22460938, + "router_z_loss_mlp": 0.01251221, + "step": 4159, + "time_per_iteration": 3.066455125808716 + }, + { + "auxiliary_loss_clip": 0.01037055, + "auxiliary_loss_mlp": 0.01013985, + "balance_loss_clip": 1.01464415, + "balance_loss_mlp": 1.01273346, + "epoch": 0.12071266902675411, + "flos": 64433513848320.0, + "grad_norm": 0.6775357098456485, + "language_loss": 0.49946451, + "learning_rate": 3.914309826284502e-06, + "loss": 0.51997495, + "num_input_tokens_seen": 117943075, + "router_z_loss_clip": 0.22460938, + "router_z_loss_mlp": 0.01251221, + "step": 4160, + "time_per_iteration": 3.202369451522827 + }, + { + "auxiliary_loss_clip": 0.0103621, + "auxiliary_loss_mlp": 0.01008429, + "balance_loss_clip": 1.01393938, + "balance_loss_mlp": 1.00726128, + "epoch": 0.12074168649527016, + "flos": 63352133971200.0, + "grad_norm": 0.7268879194453994, + "language_loss": 0.45455676, + "learning_rate": 3.91425538855055e-06, + "loss": 0.47500312, + "num_input_tokens_seen": 117998880, + "router_z_loss_clip": 0.22265625, + "router_z_loss_mlp": 0.01165771, + "step": 4161, + "time_per_iteration": 3.042677402496338 + }, + { + "auxiliary_loss_clip": 0.01141804, + "auxiliary_loss_mlp": 0.0104776, + "balance_loss_clip": 1.05537486, + "balance_loss_mlp": 1.03011751, + "epoch": 0.1207707039637862, + "flos": 56058351194880.0, + "grad_norm": 2.1524847703243952, + "language_loss": 0.76193476, + "learning_rate": 3.91420093390912e-06, + "loss": 0.78383034, + "num_input_tokens_seen": 118022025, + "router_z_loss_clip": 0.86572266, + "router_z_loss_mlp": 0.17626953, + "step": 4162, + "time_per_iteration": 2.8646881580352783 + }, + { + "auxiliary_loss_clip": 0.01150967, + "auxiliary_loss_mlp": 0.01060343, + "balance_loss_clip": 1.05470288, + "balance_loss_mlp": 1.03803909, + "epoch": 0.12079972143230225, + "flos": 19493497019520.0, + "grad_norm": 2.49654109683778, + "language_loss": 0.89690554, + "learning_rate": 3.914146462360693e-06, + "loss": 0.91901863, + "num_input_tokens_seen": 118037895, + "router_z_loss_clip": 0.96191406, + "router_z_loss_mlp": 0.22314453, + "step": 4163, + "time_per_iteration": 2.6118948459625244 + }, + { + "auxiliary_loss_clip": 0.01143557, + "auxiliary_loss_mlp": 0.01046044, + "balance_loss_clip": 1.05630744, + "balance_loss_mlp": 1.0289135, + "epoch": 0.12082873890081829, + "flos": 74737608762240.0, + "grad_norm": 1.9743175117310399, + "language_loss": 0.88371205, + "learning_rate": 3.914091973905748e-06, + "loss": 0.905608, + "num_input_tokens_seen": 118070220, + "router_z_loss_clip": 0.87255859, + "router_z_loss_mlp": 0.17126465, + "step": 4164, + "time_per_iteration": 3.007540464401245 + }, + { + "auxiliary_loss_clip": 0.01140181, + "auxiliary_loss_mlp": 0.0104724, + "balance_loss_clip": 1.0516448, + "balance_loss_mlp": 1.02734435, + "epoch": 0.12085775636933434, + "flos": 20880079781760.0, + "grad_norm": 2.7725260105372787, + "language_loss": 0.86611414, + "learning_rate": 3.91403746854477e-06, + "loss": 0.88798833, + "num_input_tokens_seen": 118085060, + "router_z_loss_clip": 0.88574219, + "router_z_loss_mlp": 0.19909668, + "step": 4165, + "time_per_iteration": 2.6098783016204834 + }, + { + "auxiliary_loss_clip": 0.01140522, + "auxiliary_loss_mlp": 0.01038726, + "balance_loss_clip": 1.05489182, + "balance_loss_mlp": 1.02138138, + "epoch": 0.12088677383785039, + "flos": 25406512058880.0, + "grad_norm": 2.7308101418937363, + "language_loss": 0.80224961, + "learning_rate": 3.9139829462782375e-06, + "loss": 0.82404214, + "num_input_tokens_seen": 118102780, + "router_z_loss_clip": 0.85693359, + "router_z_loss_mlp": 0.17364502, + "step": 4166, + "time_per_iteration": 2.650912284851074 + }, + { + "auxiliary_loss_clip": 0.0103547, + "auxiliary_loss_mlp": 0.01006824, + "balance_loss_clip": 1.01279831, + "balance_loss_mlp": 1.00534558, + "epoch": 0.12091579130636643, + "flos": 62292876871680.0, + "grad_norm": 0.7178112474106935, + "language_loss": 0.54553628, + "learning_rate": 3.913928407106634e-06, + "loss": 0.56595922, + "num_input_tokens_seen": 118164150, + "router_z_loss_clip": 0.2265625, + "router_z_loss_mlp": 0.01477051, + "step": 4167, + "time_per_iteration": 3.257437229156494 + }, + { + "auxiliary_loss_clip": 0.01141752, + "auxiliary_loss_mlp": 0.01052496, + "balance_loss_clip": 1.05467844, + "balance_loss_mlp": 1.03416777, + "epoch": 0.12094480877488248, + "flos": 36897638705280.0, + "grad_norm": 2.088796991514473, + "language_loss": 0.95738602, + "learning_rate": 3.913873851030441e-06, + "loss": 0.97932851, + "num_input_tokens_seen": 118183600, + "router_z_loss_clip": 0.87109375, + "router_z_loss_mlp": 0.18347168, + "step": 4168, + "time_per_iteration": 2.758305549621582 + }, + { + "auxiliary_loss_clip": 0.01033897, + "auxiliary_loss_mlp": 0.0100377, + "balance_loss_clip": 1.0117681, + "balance_loss_mlp": 1.00241125, + "epoch": 0.12097382624339853, + "flos": 74776575018240.0, + "grad_norm": 0.6815060352000049, + "language_loss": 0.50882977, + "learning_rate": 3.913819278050138e-06, + "loss": 0.52920645, + "num_input_tokens_seen": 118248060, + "router_z_loss_clip": 0.22167969, + "router_z_loss_mlp": 0.01361084, + "step": 4169, + "time_per_iteration": 3.2134170532226562 + }, + { + "auxiliary_loss_clip": 0.01146222, + "auxiliary_loss_mlp": 0.01050582, + "balance_loss_clip": 1.05441594, + "balance_loss_mlp": 1.03150868, + "epoch": 0.12100284371191457, + "flos": 17159124038400.0, + "grad_norm": 2.6427745111689394, + "language_loss": 0.85380912, + "learning_rate": 3.9137646881662085e-06, + "loss": 0.87577718, + "num_input_tokens_seen": 118260825, + "router_z_loss_clip": 0.91748047, + "router_z_loss_mlp": 0.1907959, + "step": 4170, + "time_per_iteration": 2.5374929904937744 + }, + { + "auxiliary_loss_clip": 0.01139021, + "auxiliary_loss_mlp": 0.01048045, + "balance_loss_clip": 1.05017126, + "balance_loss_mlp": 1.02975869, + "epoch": 0.12103186118043062, + "flos": 20296387774080.0, + "grad_norm": 3.0473261491838803, + "language_loss": 0.7840687, + "learning_rate": 3.913710081379136e-06, + "loss": 0.80593944, + "num_input_tokens_seen": 118274650, + "router_z_loss_clip": 0.88867188, + "router_z_loss_mlp": 0.18273926, + "step": 4171, + "time_per_iteration": 2.6478872299194336 + }, + { + "auxiliary_loss_clip": 0.01135655, + "auxiliary_loss_mlp": 0.0104991, + "balance_loss_clip": 1.05327129, + "balance_loss_mlp": 1.03369164, + "epoch": 0.12106087864894667, + "flos": 22740036906240.0, + "grad_norm": 2.2720170860920157, + "language_loss": 0.8084625, + "learning_rate": 3.913655457689401e-06, + "loss": 0.83031815, + "num_input_tokens_seen": 118294680, + "router_z_loss_clip": 0.82470703, + "router_z_loss_mlp": 0.16241455, + "step": 4172, + "time_per_iteration": 2.779343605041504 + }, + { + "auxiliary_loss_clip": 0.01143415, + "auxiliary_loss_mlp": 0.01053394, + "balance_loss_clip": 1.05694818, + "balance_loss_mlp": 1.0351789, + "epoch": 0.12108989611746271, + "flos": 25220997290880.0, + "grad_norm": 2.9593018354734104, + "language_loss": 0.90094924, + "learning_rate": 3.913600817097487e-06, + "loss": 0.92291737, + "num_input_tokens_seen": 118310490, + "router_z_loss_clip": 0.86474609, + "router_z_loss_mlp": 0.18206787, + "step": 4173, + "time_per_iteration": 2.627572774887085 + }, + { + "auxiliary_loss_clip": 0.01035635, + "auxiliary_loss_mlp": 0.01013828, + "balance_loss_clip": 1.01392424, + "balance_loss_mlp": 1.0125885, + "epoch": 0.12111891358597876, + "flos": 74767596618240.0, + "grad_norm": 0.650176763562251, + "language_loss": 0.44250059, + "learning_rate": 3.913546159603877e-06, + "loss": 0.46299517, + "num_input_tokens_seen": 118370190, + "router_z_loss_clip": 0.21679688, + "router_z_loss_mlp": 0.01239014, + "step": 4174, + "time_per_iteration": 3.1938247680664062 + }, + { + "auxiliary_loss_clip": 0.01138243, + "auxiliary_loss_mlp": 0.01058358, + "balance_loss_clip": 1.05433106, + "balance_loss_mlp": 1.04116249, + "epoch": 0.12114793105449481, + "flos": 19457370915840.0, + "grad_norm": 2.601660115830612, + "language_loss": 0.82395542, + "learning_rate": 3.913491485209052e-06, + "loss": 0.8459214, + "num_input_tokens_seen": 118384550, + "router_z_loss_clip": 0.83984375, + "router_z_loss_mlp": 0.17205811, + "step": 4175, + "time_per_iteration": 2.562427282333374 + }, + { + "auxiliary_loss_clip": 0.01036002, + "auxiliary_loss_mlp": 0.01011956, + "balance_loss_clip": 1.01411295, + "balance_loss_mlp": 1.01078165, + "epoch": 0.12117694852301085, + "flos": 60413277985920.0, + "grad_norm": 0.6844281160406674, + "language_loss": 0.48506826, + "learning_rate": 3.913436793913496e-06, + "loss": 0.50554782, + "num_input_tokens_seen": 118448535, + "router_z_loss_clip": 0.21875, + "router_z_loss_mlp": 0.01171875, + "step": 4176, + "time_per_iteration": 3.223500967025757 + }, + { + "auxiliary_loss_clip": 0.01035427, + "auxiliary_loss_mlp": 0.01007163, + "balance_loss_clip": 1.01375973, + "balance_loss_mlp": 1.00612593, + "epoch": 0.1212059659915269, + "flos": 52400169446400.0, + "grad_norm": 0.7691293906448289, + "language_loss": 0.52310061, + "learning_rate": 3.913382085717692e-06, + "loss": 0.54352653, + "num_input_tokens_seen": 118498555, + "router_z_loss_clip": 0.21679688, + "router_z_loss_mlp": 0.01037598, + "step": 4177, + "time_per_iteration": 2.935232162475586 + }, + { + "auxiliary_loss_clip": 0.01140629, + "auxiliary_loss_mlp": 0.01049628, + "balance_loss_clip": 1.05405569, + "balance_loss_mlp": 1.03105474, + "epoch": 0.12123498346004295, + "flos": 25915617475200.0, + "grad_norm": 8.370126634622846, + "language_loss": 0.77111763, + "learning_rate": 3.913327360622123e-06, + "loss": 0.79302013, + "num_input_tokens_seen": 118516975, + "router_z_loss_clip": 0.86669922, + "router_z_loss_mlp": 0.18566895, + "step": 4178, + "time_per_iteration": 2.626877784729004 + }, + { + "auxiliary_loss_clip": 0.01129574, + "auxiliary_loss_mlp": 0.01037176, + "balance_loss_clip": 1.04863524, + "balance_loss_mlp": 1.02048719, + "epoch": 0.12126400092855899, + "flos": 22120362449280.0, + "grad_norm": 2.3628050091653754, + "language_loss": 0.95188737, + "learning_rate": 3.913272618627273e-06, + "loss": 0.97355485, + "num_input_tokens_seen": 118531775, + "router_z_loss_clip": 0.80859375, + "router_z_loss_mlp": 0.16687012, + "step": 4179, + "time_per_iteration": 2.602823495864868 + }, + { + "auxiliary_loss_clip": 0.01129051, + "auxiliary_loss_mlp": 0.01044109, + "balance_loss_clip": 1.05395269, + "balance_loss_mlp": 1.0296905, + "epoch": 0.12129301839707504, + "flos": 17487024318720.0, + "grad_norm": 2.449653501465674, + "language_loss": 0.58598709, + "learning_rate": 3.913217859733624e-06, + "loss": 0.60771871, + "num_input_tokens_seen": 118545025, + "router_z_loss_clip": 0.75097656, + "router_z_loss_mlp": 0.14422607, + "step": 4180, + "time_per_iteration": 2.5949292182922363 + }, + { + "auxiliary_loss_clip": 0.01143171, + "auxiliary_loss_mlp": 0.01048277, + "balance_loss_clip": 1.05486131, + "balance_loss_mlp": 1.03024697, + "epoch": 0.12132203586559108, + "flos": 21099960887040.0, + "grad_norm": 2.448781774250267, + "language_loss": 0.78251308, + "learning_rate": 3.913163083941661e-06, + "loss": 0.8044275, + "num_input_tokens_seen": 118559500, + "router_z_loss_clip": 0.8828125, + "router_z_loss_mlp": 0.18048096, + "step": 4181, + "time_per_iteration": 2.5575826168060303 + }, + { + "auxiliary_loss_clip": 0.01147857, + "auxiliary_loss_mlp": 0.01049969, + "balance_loss_clip": 1.06025803, + "balance_loss_mlp": 1.029966, + "epoch": 0.12135105333410713, + "flos": 16759222945920.0, + "grad_norm": 2.7296965005772043, + "language_loss": 0.82128417, + "learning_rate": 3.913108291251868e-06, + "loss": 0.84326249, + "num_input_tokens_seen": 118574475, + "router_z_loss_clip": 0.87597656, + "router_z_loss_mlp": 0.19995117, + "step": 4182, + "time_per_iteration": 2.5486819744110107 + }, + { + "auxiliary_loss_clip": 0.01133449, + "auxiliary_loss_mlp": 0.01044535, + "balance_loss_clip": 1.0548929, + "balance_loss_mlp": 1.02797675, + "epoch": 0.12138007080262318, + "flos": 43829367367680.0, + "grad_norm": 1.859651854538475, + "language_loss": 0.73222244, + "learning_rate": 3.9130534816647286e-06, + "loss": 0.75400221, + "num_input_tokens_seen": 118591820, + "router_z_loss_clip": 0.78564453, + "router_z_loss_mlp": 0.16564941, + "step": 4183, + "time_per_iteration": 2.793689250946045 + }, + { + "auxiliary_loss_clip": 0.01136358, + "auxiliary_loss_mlp": 0.01041071, + "balance_loss_clip": 1.05418205, + "balance_loss_mlp": 1.02555013, + "epoch": 0.12140908827113922, + "flos": 22014390349440.0, + "grad_norm": 2.1329095447109268, + "language_loss": 0.76824987, + "learning_rate": 3.912998655180727e-06, + "loss": 0.79002416, + "num_input_tokens_seen": 118609555, + "router_z_loss_clip": 0.82226562, + "router_z_loss_mlp": 0.15509033, + "step": 4184, + "time_per_iteration": 2.589200258255005 + }, + { + "auxiliary_loss_clip": 0.0114372, + "auxiliary_loss_mlp": 0.01047789, + "balance_loss_clip": 1.05646682, + "balance_loss_mlp": 1.02873921, + "epoch": 0.12143810573965527, + "flos": 32375731541760.0, + "grad_norm": 2.3734635958727024, + "language_loss": 0.85828751, + "learning_rate": 3.912943811800346e-06, + "loss": 0.88020259, + "num_input_tokens_seen": 118627890, + "router_z_loss_clip": 0.87304688, + "router_z_loss_mlp": 0.19042969, + "step": 4185, + "time_per_iteration": 2.677546977996826 + }, + { + "auxiliary_loss_clip": 0.01037071, + "auxiliary_loss_mlp": 0.01007432, + "balance_loss_clip": 1.01519656, + "balance_loss_mlp": 1.00604951, + "epoch": 0.12146712320817132, + "flos": 59411297122560.0, + "grad_norm": 0.6879000737201921, + "language_loss": 0.49846265, + "learning_rate": 3.912888951524072e-06, + "loss": 0.51890767, + "num_input_tokens_seen": 118689050, + "router_z_loss_clip": 0.21875, + "router_z_loss_mlp": 0.01385498, + "step": 4186, + "time_per_iteration": 3.089836359024048 + }, + { + "auxiliary_loss_clip": 0.01141527, + "auxiliary_loss_mlp": 0.01043011, + "balance_loss_clip": 1.05524755, + "balance_loss_mlp": 1.02477813, + "epoch": 0.12149614067668736, + "flos": 22851539700480.0, + "grad_norm": 2.975604414979452, + "language_loss": 1.05022633, + "learning_rate": 3.912834074352388e-06, + "loss": 1.07207167, + "num_input_tokens_seen": 118705460, + "router_z_loss_clip": 0.86181641, + "router_z_loss_mlp": 0.18237305, + "step": 4187, + "time_per_iteration": 2.608905792236328 + }, + { + "auxiliary_loss_clip": 0.01129006, + "auxiliary_loss_mlp": 0.01041384, + "balance_loss_clip": 1.05141377, + "balance_loss_mlp": 1.02536869, + "epoch": 0.12152515814520341, + "flos": 17231017109760.0, + "grad_norm": 2.6695909920582657, + "language_loss": 0.96938187, + "learning_rate": 3.912779180285779e-06, + "loss": 0.99108583, + "num_input_tokens_seen": 118717210, + "router_z_loss_clip": 0.77636719, + "router_z_loss_mlp": 0.16003418, + "step": 4188, + "time_per_iteration": 2.6426644325256348 + }, + { + "auxiliary_loss_clip": 0.01036728, + "auxiliary_loss_mlp": 0.01003053, + "balance_loss_clip": 1.01529193, + "balance_loss_mlp": 1.00182521, + "epoch": 0.12155417561371946, + "flos": 61044010853760.0, + "grad_norm": 0.6550154440596174, + "language_loss": 0.47339854, + "learning_rate": 3.912724269324732e-06, + "loss": 0.49379635, + "num_input_tokens_seen": 118777045, + "router_z_loss_clip": 0.21484375, + "router_z_loss_mlp": 0.01226807, + "step": 4189, + "time_per_iteration": 3.127739191055298 + }, + { + "auxiliary_loss_clip": 0.01161229, + "auxiliary_loss_mlp": 0.01061679, + "balance_loss_clip": 1.0667398, + "balance_loss_mlp": 1.04107976, + "epoch": 0.1215831930822355, + "flos": 74731431623040.0, + "grad_norm": 3.580335033900296, + "language_loss": 0.81302679, + "learning_rate": 3.912669341469729e-06, + "loss": 0.83525574, + "num_input_tokens_seen": 118802345, + "router_z_loss_clip": 0.94482422, + "router_z_loss_mlp": 0.20593262, + "step": 4190, + "time_per_iteration": 3.000934362411499 + }, + { + "auxiliary_loss_clip": 0.01037199, + "auxiliary_loss_mlp": 0.01001152, + "balance_loss_clip": 1.0156517, + "balance_loss_mlp": 0.99996626, + "epoch": 0.12161221055075155, + "flos": 74773414621440.0, + "grad_norm": 0.6679437278178054, + "language_loss": 0.4932313, + "learning_rate": 3.912614396721257e-06, + "loss": 0.51361483, + "num_input_tokens_seen": 118862585, + "router_z_loss_clip": 0.21582031, + "router_z_loss_mlp": 0.01184082, + "step": 4191, + "time_per_iteration": 3.1082606315612793 + }, + { + "auxiliary_loss_clip": 0.01143822, + "auxiliary_loss_mlp": 0.01061033, + "balance_loss_clip": 1.05551636, + "balance_loss_mlp": 1.04166126, + "epoch": 0.1216412280192676, + "flos": 12742542529920.0, + "grad_norm": 2.7923916312184804, + "language_loss": 0.93433011, + "learning_rate": 3.912559435079801e-06, + "loss": 0.9563787, + "num_input_tokens_seen": 118873805, + "router_z_loss_clip": 0.88183594, + "router_z_loss_mlp": 0.19372559, + "step": 4192, + "time_per_iteration": 2.574923038482666 + }, + { + "auxiliary_loss_clip": 0.01148289, + "auxiliary_loss_mlp": 0.01050517, + "balance_loss_clip": 1.05599082, + "balance_loss_mlp": 1.03113341, + "epoch": 0.12167024548778364, + "flos": 34744003983360.0, + "grad_norm": 2.692482536183962, + "language_loss": 0.93589902, + "learning_rate": 3.9125044565458444e-06, + "loss": 0.95788705, + "num_input_tokens_seen": 118889295, + "router_z_loss_clip": 0.92333984, + "router_z_loss_mlp": 0.1940918, + "step": 4193, + "time_per_iteration": 2.6596193313598633 + }, + { + "auxiliary_loss_clip": 0.01037631, + "auxiliary_loss_mlp": 0.01012732, + "balance_loss_clip": 1.01560569, + "balance_loss_mlp": 1.01159942, + "epoch": 0.1216992629562997, + "flos": 74232600474240.0, + "grad_norm": 0.6099351592251198, + "language_loss": 0.51808923, + "learning_rate": 3.912449461119876e-06, + "loss": 0.53859288, + "num_input_tokens_seen": 118959685, + "router_z_loss_clip": 0.22070312, + "router_z_loss_mlp": 0.01135254, + "step": 4194, + "time_per_iteration": 3.4059598445892334 + }, + { + "auxiliary_loss_clip": 0.01154014, + "auxiliary_loss_mlp": 0.01063626, + "balance_loss_clip": 1.0621562, + "balance_loss_mlp": 1.04392016, + "epoch": 0.12172828042481573, + "flos": 43899285191040.0, + "grad_norm": 2.5384003689127104, + "language_loss": 0.77267563, + "learning_rate": 3.91239444880238e-06, + "loss": 0.79485196, + "num_input_tokens_seen": 118975815, + "router_z_loss_clip": 0.91796875, + "router_z_loss_mlp": 0.19702148, + "step": 4195, + "time_per_iteration": 2.742734909057617 + }, + { + "auxiliary_loss_clip": 0.01035224, + "auxiliary_loss_mlp": 0.01012611, + "balance_loss_clip": 1.01328766, + "balance_loss_mlp": 1.01150823, + "epoch": 0.12175729789333178, + "flos": 68871640538880.0, + "grad_norm": 0.6639130022361153, + "language_loss": 0.50013709, + "learning_rate": 3.912339419593843e-06, + "loss": 0.52061546, + "num_input_tokens_seen": 119038055, + "router_z_loss_clip": 0.21875, + "router_z_loss_mlp": 0.01104736, + "step": 4196, + "time_per_iteration": 3.103721857070923 + }, + { + "auxiliary_loss_clip": 0.01034343, + "auxiliary_loss_mlp": 0.010068, + "balance_loss_clip": 1.01254487, + "balance_loss_mlp": 1.00564337, + "epoch": 0.12178631536184784, + "flos": 66242332984320.0, + "grad_norm": 0.6930947639005034, + "language_loss": 0.52842796, + "learning_rate": 3.912284373494748e-06, + "loss": 0.54883939, + "num_input_tokens_seen": 119102215, + "router_z_loss_clip": 0.21777344, + "router_z_loss_mlp": 0.01153564, + "step": 4197, + "time_per_iteration": 5.30478310585022 + }, + { + "auxiliary_loss_clip": 0.0113787, + "auxiliary_loss_mlp": 0.01045212, + "balance_loss_clip": 1.05508816, + "balance_loss_mlp": 1.0284512, + "epoch": 0.12181533283036387, + "flos": 17267466435840.0, + "grad_norm": 1.9343645795771502, + "language_loss": 0.63433933, + "learning_rate": 3.912229310505586e-06, + "loss": 0.65617019, + "num_input_tokens_seen": 119115375, + "router_z_loss_clip": 0.828125, + "router_z_loss_mlp": 0.16766357, + "step": 4198, + "time_per_iteration": 7.185467720031738 + }, + { + "auxiliary_loss_clip": 0.01144623, + "auxiliary_loss_mlp": 0.01055145, + "balance_loss_clip": 1.05778873, + "balance_loss_mlp": 1.03720963, + "epoch": 0.12184435029887992, + "flos": 39632810618880.0, + "grad_norm": 2.462301411359834, + "language_loss": 0.8992545, + "learning_rate": 3.91217423062684e-06, + "loss": 0.92125219, + "num_input_tokens_seen": 119133080, + "router_z_loss_clip": 0.8671875, + "router_z_loss_mlp": 0.17938232, + "step": 4199, + "time_per_iteration": 2.6889102458953857 + }, + { + "auxiliary_loss_clip": 0.0115209, + "auxiliary_loss_mlp": 0.01044005, + "balance_loss_clip": 1.06120729, + "balance_loss_mlp": 1.02505112, + "epoch": 0.12187336776739598, + "flos": 15407293829760.0, + "grad_norm": 2.4066918250979326, + "language_loss": 0.73459667, + "learning_rate": 3.912119133858997e-06, + "loss": 0.75655758, + "num_input_tokens_seen": 119145950, + "router_z_loss_clip": 0.90869141, + "router_z_loss_mlp": 0.18963623, + "step": 4200, + "time_per_iteration": 4.994617462158203 + }, + { + "auxiliary_loss_clip": 0.01125001, + "auxiliary_loss_mlp": 0.01039246, + "balance_loss_clip": 1.05013227, + "balance_loss_mlp": 1.02467906, + "epoch": 0.12190238523591201, + "flos": 23106110365440.0, + "grad_norm": 2.1448029931931525, + "language_loss": 0.97054839, + "learning_rate": 3.912064020202545e-06, + "loss": 0.99219084, + "num_input_tokens_seen": 119162405, + "router_z_loss_clip": 0.74804688, + "router_z_loss_mlp": 0.14575195, + "step": 4201, + "time_per_iteration": 2.5619826316833496 + }, + { + "auxiliary_loss_clip": 0.01144146, + "auxiliary_loss_mlp": 0.01040658, + "balance_loss_clip": 1.0566299, + "balance_loss_mlp": 1.02280641, + "epoch": 0.12193140270442807, + "flos": 28687992900480.0, + "grad_norm": 2.614927498723724, + "language_loss": 0.82657456, + "learning_rate": 3.91200888965797e-06, + "loss": 0.84842265, + "num_input_tokens_seen": 119182360, + "router_z_loss_clip": 0.87548828, + "router_z_loss_mlp": 0.17840576, + "step": 4202, + "time_per_iteration": 2.6781322956085205 + }, + { + "auxiliary_loss_clip": 0.01148379, + "auxiliary_loss_mlp": 0.01046583, + "balance_loss_clip": 1.05669188, + "balance_loss_mlp": 1.02793908, + "epoch": 0.12196042017294412, + "flos": 16793158320000.0, + "grad_norm": 3.2279297693672118, + "language_loss": 0.78254646, + "learning_rate": 3.911953742225757e-06, + "loss": 0.80449605, + "num_input_tokens_seen": 119196845, + "router_z_loss_clip": 0.91748047, + "router_z_loss_mlp": 0.18640137, + "step": 4203, + "time_per_iteration": 2.5726826190948486 + }, + { + "auxiliary_loss_clip": 0.01142686, + "auxiliary_loss_mlp": 0.01053718, + "balance_loss_clip": 1.05669236, + "balance_loss_mlp": 1.03493118, + "epoch": 0.12198943764146016, + "flos": 36495834192000.0, + "grad_norm": 2.571007258695634, + "language_loss": 0.88011873, + "learning_rate": 3.911898577906396e-06, + "loss": 0.90208286, + "num_input_tokens_seen": 119213065, + "router_z_loss_clip": 0.85986328, + "router_z_loss_mlp": 0.18798828, + "step": 4204, + "time_per_iteration": 2.666748523712158 + }, + { + "auxiliary_loss_clip": 0.0104194, + "auxiliary_loss_mlp": 0.01012411, + "balance_loss_clip": 1.01991105, + "balance_loss_mlp": 1.01126039, + "epoch": 0.1220184551099762, + "flos": 74631208677120.0, + "grad_norm": 0.6043782401450657, + "language_loss": 0.46414798, + "learning_rate": 3.911843396700373e-06, + "loss": 0.48469153, + "num_input_tokens_seen": 119283680, + "router_z_loss_clip": 0.22070312, + "router_z_loss_mlp": 0.01147461, + "step": 4205, + "time_per_iteration": 3.304131507873535 + }, + { + "auxiliary_loss_clip": 0.01150092, + "auxiliary_loss_mlp": 0.01048589, + "balance_loss_clip": 1.05799103, + "balance_loss_mlp": 1.03080297, + "epoch": 0.12204747257849226, + "flos": 22199474154240.0, + "grad_norm": 2.181534610838569, + "language_loss": 0.73625791, + "learning_rate": 3.911788198608176e-06, + "loss": 0.75824475, + "num_input_tokens_seen": 119297805, + "router_z_loss_clip": 0.92138672, + "router_z_loss_mlp": 0.17773438, + "step": 4206, + "time_per_iteration": 2.5596320629119873 + }, + { + "auxiliary_loss_clip": 0.01037408, + "auxiliary_loss_mlp": 0.01000323, + "balance_loss_clip": 1.01558685, + "balance_loss_mlp": 0.99923247, + "epoch": 0.1220764900470083, + "flos": 65619354476160.0, + "grad_norm": 0.6334503352006176, + "language_loss": 0.49885517, + "learning_rate": 3.911732983630292e-06, + "loss": 0.51923245, + "num_input_tokens_seen": 119364465, + "router_z_loss_clip": 0.21875, + "router_z_loss_mlp": 0.01092529, + "step": 4207, + "time_per_iteration": 3.19760799407959 + }, + { + "auxiliary_loss_clip": 0.01033446, + "auxiliary_loss_mlp": 0.00999509, + "balance_loss_clip": 1.01175165, + "balance_loss_mlp": 0.99837703, + "epoch": 0.12210550751552435, + "flos": 60986914185600.0, + "grad_norm": 0.6785236374918264, + "language_loss": 0.49126554, + "learning_rate": 3.911677751767208e-06, + "loss": 0.51159507, + "num_input_tokens_seen": 119429640, + "router_z_loss_clip": 0.21679688, + "router_z_loss_mlp": 0.01135254, + "step": 4208, + "time_per_iteration": 3.155013084411621 + }, + { + "auxiliary_loss_clip": 0.01031545, + "auxiliary_loss_mlp": 0.01001695, + "balance_loss_clip": 1.00989819, + "balance_loss_mlp": 1.00064015, + "epoch": 0.1221345249840404, + "flos": 51058475706240.0, + "grad_norm": 0.7204793312057479, + "language_loss": 0.52455497, + "learning_rate": 3.911622503019413e-06, + "loss": 0.54488742, + "num_input_tokens_seen": 119482875, + "router_z_loss_clip": 0.21679688, + "router_z_loss_mlp": 0.01055908, + "step": 4209, + "time_per_iteration": 2.944559097290039 + }, + { + "auxiliary_loss_clip": 0.01029752, + "auxiliary_loss_mlp": 0.01007668, + "balance_loss_clip": 1.00810981, + "balance_loss_mlp": 1.00658309, + "epoch": 0.12216354245255644, + "flos": 74771942163840.0, + "grad_norm": 0.6843317599168807, + "language_loss": 0.52045178, + "learning_rate": 3.911567237387394e-06, + "loss": 0.54082596, + "num_input_tokens_seen": 119547265, + "router_z_loss_clip": 0.21679688, + "router_z_loss_mlp": 0.01086426, + "step": 4210, + "time_per_iteration": 3.127988815307617 + }, + { + "auxiliary_loss_clip": 0.01137529, + "auxiliary_loss_mlp": 0.01056365, + "balance_loss_clip": 1.05079293, + "balance_loss_mlp": 1.03783393, + "epoch": 0.12219255992107249, + "flos": 33033112300800.0, + "grad_norm": 2.025239540599884, + "language_loss": 0.77686298, + "learning_rate": 3.91151195487164e-06, + "loss": 0.7988019, + "num_input_tokens_seen": 119567220, + "router_z_loss_clip": 0.86865234, + "router_z_loss_mlp": 0.18536377, + "step": 4211, + "time_per_iteration": 2.6160268783569336 + }, + { + "auxiliary_loss_clip": 0.01152711, + "auxiliary_loss_mlp": 0.01053595, + "balance_loss_clip": 1.05937243, + "balance_loss_mlp": 1.03385353, + "epoch": 0.12222157738958853, + "flos": 28285721510400.0, + "grad_norm": 3.4037158377494494, + "language_loss": 0.85315728, + "learning_rate": 3.911456655472639e-06, + "loss": 0.87522036, + "num_input_tokens_seen": 119580765, + "router_z_loss_clip": 0.93408203, + "router_z_loss_mlp": 0.19750977, + "step": 4212, + "time_per_iteration": 2.4909474849700928 + }, + { + "auxiliary_loss_clip": 0.01138809, + "auxiliary_loss_mlp": 0.01054925, + "balance_loss_clip": 1.05257428, + "balance_loss_mlp": 1.03684139, + "epoch": 0.12225059485810458, + "flos": 34452122065920.0, + "grad_norm": 2.7357768441310344, + "language_loss": 0.88354468, + "learning_rate": 3.911401339190879e-06, + "loss": 0.90548205, + "num_input_tokens_seen": 119594915, + "router_z_loss_clip": 0.86181641, + "router_z_loss_mlp": 0.1807251, + "step": 4213, + "time_per_iteration": 2.6486456394195557 + }, + { + "auxiliary_loss_clip": 0.01035442, + "auxiliary_loss_mlp": 0.01048379, + "balance_loss_clip": 1.01343751, + "balance_loss_mlp": 1.04732418, + "epoch": 0.12227961232662063, + "flos": 74782716243840.0, + "grad_norm": 0.6884766193427606, + "language_loss": 0.48204201, + "learning_rate": 3.911346006026849e-06, + "loss": 0.50288022, + "num_input_tokens_seen": 119663540, + "router_z_loss_clip": 0.22070312, + "router_z_loss_mlp": 0.01055908, + "step": 4214, + "time_per_iteration": 3.264340400695801 + }, + { + "auxiliary_loss_clip": 0.01036251, + "auxiliary_loss_mlp": 0.01034704, + "balance_loss_clip": 1.01413226, + "balance_loss_mlp": 1.03359556, + "epoch": 0.12230862979513667, + "flos": 56182857690240.0, + "grad_norm": 0.7267692246110179, + "language_loss": 0.45991552, + "learning_rate": 3.911290655981038e-06, + "loss": 0.48062506, + "num_input_tokens_seen": 119722285, + "router_z_loss_clip": 0.22070312, + "router_z_loss_mlp": 0.0111084, + "step": 4215, + "time_per_iteration": 3.132028341293335 + }, + { + "auxiliary_loss_clip": 0.01141928, + "auxiliary_loss_mlp": 0.01058105, + "balance_loss_clip": 1.05213618, + "balance_loss_mlp": 1.0376606, + "epoch": 0.12233764726365272, + "flos": 30515630762880.0, + "grad_norm": 1.873783637037144, + "language_loss": 0.96476406, + "learning_rate": 3.911235289053934e-06, + "loss": 0.98676437, + "num_input_tokens_seen": 119744700, + "router_z_loss_clip": 0.89746094, + "router_z_loss_mlp": 0.2043457, + "step": 4216, + "time_per_iteration": 2.7087411880493164 + }, + { + "auxiliary_loss_clip": 0.01032977, + "auxiliary_loss_mlp": 0.01006246, + "balance_loss_clip": 1.0115248, + "balance_loss_mlp": 1.00514305, + "epoch": 0.12236666473216877, + "flos": 74759911107840.0, + "grad_norm": 0.6809599789368103, + "language_loss": 0.47760019, + "learning_rate": 3.911179905246027e-06, + "loss": 0.49799243, + "num_input_tokens_seen": 119798410, + "router_z_loss_clip": 0.21484375, + "router_z_loss_mlp": 0.01104736, + "step": 4217, + "time_per_iteration": 3.01696515083313 + }, + { + "auxiliary_loss_clip": 0.01149608, + "auxiliary_loss_mlp": 0.01056525, + "balance_loss_clip": 1.05705786, + "balance_loss_mlp": 1.03542554, + "epoch": 0.12239568220068481, + "flos": 22995433584000.0, + "grad_norm": 2.5773184865324095, + "language_loss": 0.73010945, + "learning_rate": 3.911124504557806e-06, + "loss": 0.7521708, + "num_input_tokens_seen": 119813855, + "router_z_loss_clip": 0.92578125, + "router_z_loss_mlp": 0.21105957, + "step": 4218, + "time_per_iteration": 2.500793933868408 + }, + { + "auxiliary_loss_clip": 0.01129957, + "auxiliary_loss_mlp": 0.01039979, + "balance_loss_clip": 1.04740846, + "balance_loss_mlp": 1.02376044, + "epoch": 0.12242469966920086, + "flos": 31096449682560.0, + "grad_norm": 2.1809358296812595, + "language_loss": 0.86252558, + "learning_rate": 3.9110690869897584e-06, + "loss": 0.88422489, + "num_input_tokens_seen": 119831595, + "router_z_loss_clip": 0.82568359, + "router_z_loss_mlp": 0.16223145, + "step": 4219, + "time_per_iteration": 2.5897793769836426 + }, + { + "auxiliary_loss_clip": 0.01141421, + "auxiliary_loss_mlp": 0.01054196, + "balance_loss_clip": 1.05510592, + "balance_loss_mlp": 1.03640985, + "epoch": 0.12245371713771691, + "flos": 20376576887040.0, + "grad_norm": 3.935469004940176, + "language_loss": 0.60153216, + "learning_rate": 3.911013652542377e-06, + "loss": 0.62348837, + "num_input_tokens_seen": 119845415, + "router_z_loss_clip": 0.86279297, + "router_z_loss_mlp": 0.17785645, + "step": 4220, + "time_per_iteration": 2.5650007724761963 + }, + { + "auxiliary_loss_clip": 0.01033263, + "auxiliary_loss_mlp": 0.01011158, + "balance_loss_clip": 1.01124322, + "balance_loss_mlp": 1.00995946, + "epoch": 0.12248273460623295, + "flos": 74779304451840.0, + "grad_norm": 0.6280318110295243, + "language_loss": 0.49286062, + "learning_rate": 3.910958201216149e-06, + "loss": 0.51330483, + "num_input_tokens_seen": 119911025, + "router_z_loss_clip": 0.22070312, + "router_z_loss_mlp": 0.01196289, + "step": 4221, + "time_per_iteration": 3.1559677124023438 + }, + { + "auxiliary_loss_clip": 0.01142946, + "auxiliary_loss_mlp": 0.01049617, + "balance_loss_clip": 1.05208659, + "balance_loss_mlp": 1.0297091, + "epoch": 0.122511752074749, + "flos": 28365120524160.0, + "grad_norm": 1.9369021579250487, + "language_loss": 0.74278069, + "learning_rate": 3.910902733011565e-06, + "loss": 0.76470637, + "num_input_tokens_seen": 119937475, + "router_z_loss_clip": 0.90917969, + "router_z_loss_mlp": 0.19909668, + "step": 4222, + "time_per_iteration": 2.9323980808258057 + }, + { + "auxiliary_loss_clip": 0.01136504, + "auxiliary_loss_mlp": 0.01039053, + "balance_loss_clip": 1.05106044, + "balance_loss_mlp": 1.02146935, + "epoch": 0.12254076954326505, + "flos": 15078387968640.0, + "grad_norm": 3.6041560194911835, + "language_loss": 0.89084101, + "learning_rate": 3.9108472479291145e-06, + "loss": 0.91259664, + "num_input_tokens_seen": 119950245, + "router_z_loss_clip": 0.85498047, + "router_z_loss_mlp": 0.17590332, + "step": 4223, + "time_per_iteration": 2.473815441131592 + }, + { + "auxiliary_loss_clip": 0.01140613, + "auxiliary_loss_mlp": 0.01048374, + "balance_loss_clip": 1.05117357, + "balance_loss_mlp": 1.02988458, + "epoch": 0.12256978701178109, + "flos": 33102383679360.0, + "grad_norm": 2.772904973065541, + "language_loss": 0.80769348, + "learning_rate": 3.9107917459692885e-06, + "loss": 0.82958335, + "num_input_tokens_seen": 119969630, + "router_z_loss_clip": 0.89453125, + "router_z_loss_mlp": 0.18505859, + "step": 4224, + "time_per_iteration": 2.705397129058838 + }, + { + "auxiliary_loss_clip": 0.01128179, + "auxiliary_loss_mlp": 0.01042452, + "balance_loss_clip": 1.04658771, + "balance_loss_mlp": 1.02646029, + "epoch": 0.12259880448029714, + "flos": 11064006023040.0, + "grad_norm": 2.0930776255209342, + "language_loss": 0.78460866, + "learning_rate": 3.910736227132577e-06, + "loss": 0.80631495, + "num_input_tokens_seen": 119981565, + "router_z_loss_clip": 0.81494141, + "router_z_loss_mlp": 0.16015625, + "step": 4225, + "time_per_iteration": 2.492654800415039 + }, + { + "auxiliary_loss_clip": 0.01139241, + "auxiliary_loss_mlp": 0.01061708, + "balance_loss_clip": 1.05171812, + "balance_loss_mlp": 1.04237819, + "epoch": 0.12262782194881318, + "flos": 30806004309120.0, + "grad_norm": 3.2122655051391376, + "language_loss": 0.94755632, + "learning_rate": 3.9106806914194685e-06, + "loss": 0.96956581, + "num_input_tokens_seen": 119999950, + "router_z_loss_clip": 0.87548828, + "router_z_loss_mlp": 0.1932373, + "step": 4226, + "time_per_iteration": 2.6054110527038574 + }, + { + "auxiliary_loss_clip": 0.01031369, + "auxiliary_loss_mlp": 0.00997351, + "balance_loss_clip": 1.00999868, + "balance_loss_mlp": 0.99620038, + "epoch": 0.12265683941732923, + "flos": 59695134393600.0, + "grad_norm": 0.6791215879776834, + "language_loss": 0.46161777, + "learning_rate": 3.9106251388304555e-06, + "loss": 0.48190492, + "num_input_tokens_seen": 120054615, + "router_z_loss_clip": 0.21386719, + "router_z_loss_mlp": 0.01147461, + "step": 4227, + "time_per_iteration": 2.9770212173461914 + }, + { + "auxiliary_loss_clip": 0.01134849, + "auxiliary_loss_mlp": 0.01061922, + "balance_loss_clip": 1.05307436, + "balance_loss_mlp": 1.04336071, + "epoch": 0.12268585688584528, + "flos": 14713140522240.0, + "grad_norm": 3.104372134875791, + "language_loss": 0.8657766, + "learning_rate": 3.910569569366029e-06, + "loss": 0.88774431, + "num_input_tokens_seen": 120066950, + "router_z_loss_clip": 0.81884766, + "router_z_loss_mlp": 0.18579102, + "step": 4228, + "time_per_iteration": 2.6037251949310303 + }, + { + "auxiliary_loss_clip": 0.01123613, + "auxiliary_loss_mlp": 0.01032998, + "balance_loss_clip": 1.04859006, + "balance_loss_mlp": 1.01938462, + "epoch": 0.12271487435436132, + "flos": 41695341471360.0, + "grad_norm": 2.5981089918615368, + "language_loss": 1.03402066, + "learning_rate": 3.910513983026678e-06, + "loss": 1.05558681, + "num_input_tokens_seen": 120086150, + "router_z_loss_clip": 0.75, + "router_z_loss_mlp": 0.13604736, + "step": 4229, + "time_per_iteration": 2.806126117706299 + }, + { + "auxiliary_loss_clip": 0.01134234, + "auxiliary_loss_mlp": 0.01050008, + "balance_loss_clip": 1.04953527, + "balance_loss_mlp": 1.03240085, + "epoch": 0.12274389182287737, + "flos": 15368366465280.0, + "grad_norm": 2.5955945724437886, + "language_loss": 0.71192932, + "learning_rate": 3.910458379812894e-06, + "loss": 0.7337718, + "num_input_tokens_seen": 120099655, + "router_z_loss_clip": 0.84667969, + "router_z_loss_mlp": 0.17633057, + "step": 4230, + "time_per_iteration": 2.4777839183807373 + }, + { + "auxiliary_loss_clip": 0.01145086, + "auxiliary_loss_mlp": 0.01056913, + "balance_loss_clip": 1.05501032, + "balance_loss_mlp": 1.0382328, + "epoch": 0.12277290929139342, + "flos": 34453163560320.0, + "grad_norm": 2.3928647809212524, + "language_loss": 0.84609967, + "learning_rate": 3.910402759725169e-06, + "loss": 0.86811972, + "num_input_tokens_seen": 120119735, + "router_z_loss_clip": 0.90087891, + "router_z_loss_mlp": 0.18676758, + "step": 4231, + "time_per_iteration": 2.6942853927612305 + }, + { + "auxiliary_loss_clip": 0.01131905, + "auxiliary_loss_mlp": 0.01044929, + "balance_loss_clip": 1.05488753, + "balance_loss_mlp": 1.0296104, + "epoch": 0.12280192675990946, + "flos": 15953925980160.0, + "grad_norm": 2.360102608993699, + "language_loss": 0.65856254, + "learning_rate": 3.910347122763994e-06, + "loss": 0.68033081, + "num_input_tokens_seen": 120133105, + "router_z_loss_clip": 0.76953125, + "router_z_loss_mlp": 0.15325928, + "step": 4232, + "time_per_iteration": 2.5407073497772217 + }, + { + "auxiliary_loss_clip": 0.01132571, + "auxiliary_loss_mlp": 0.01052418, + "balance_loss_clip": 1.05019188, + "balance_loss_mlp": 1.03578782, + "epoch": 0.12283094422842551, + "flos": 15771751176960.0, + "grad_norm": 3.185117987918129, + "language_loss": 0.97867376, + "learning_rate": 3.9102914689298605e-06, + "loss": 1.00052357, + "num_input_tokens_seen": 120145225, + "router_z_loss_clip": 0.82324219, + "router_z_loss_mlp": 0.16644287, + "step": 4233, + "time_per_iteration": 2.5603792667388916 + }, + { + "auxiliary_loss_clip": 0.01033501, + "auxiliary_loss_mlp": 0.0102132, + "balance_loss_clip": 1.01233208, + "balance_loss_mlp": 1.02011037, + "epoch": 0.12285996169694156, + "flos": 72286887628800.0, + "grad_norm": 0.6567982031998271, + "language_loss": 0.48056298, + "learning_rate": 3.910235798223259e-06, + "loss": 0.50111115, + "num_input_tokens_seen": 120207180, + "router_z_loss_clip": 0.21191406, + "router_z_loss_mlp": 0.01208496, + "step": 4234, + "time_per_iteration": 3.2333924770355225 + }, + { + "auxiliary_loss_clip": 0.01130223, + "auxiliary_loss_mlp": 0.01040967, + "balance_loss_clip": 1.05021524, + "balance_loss_mlp": 1.02541041, + "epoch": 0.1228889791654576, + "flos": 28870562753280.0, + "grad_norm": 2.475213472567537, + "language_loss": 0.8644408, + "learning_rate": 3.910180110644682e-06, + "loss": 0.88615274, + "num_input_tokens_seen": 120221825, + "router_z_loss_clip": 0.80029297, + "router_z_loss_mlp": 0.15557861, + "step": 4235, + "time_per_iteration": 2.6147005558013916 + }, + { + "auxiliary_loss_clip": 0.01130636, + "auxiliary_loss_mlp": 0.01045602, + "balance_loss_clip": 1.05000734, + "balance_loss_mlp": 1.03058112, + "epoch": 0.12291799663397365, + "flos": 21682719141120.0, + "grad_norm": 1.9412701834404535, + "language_loss": 0.68771207, + "learning_rate": 3.910124406194623e-06, + "loss": 0.7094745, + "num_input_tokens_seen": 120236640, + "router_z_loss_clip": 0.80712891, + "router_z_loss_mlp": 0.15026855, + "step": 4236, + "time_per_iteration": 2.517428159713745 + }, + { + "auxiliary_loss_clip": 0.01136173, + "auxiliary_loss_mlp": 0.01048805, + "balance_loss_clip": 1.05190253, + "balance_loss_mlp": 1.03035092, + "epoch": 0.1229470141024897, + "flos": 16553348144640.0, + "grad_norm": 3.289008610420409, + "language_loss": 0.85379869, + "learning_rate": 3.91006868487357e-06, + "loss": 0.8756485, + "num_input_tokens_seen": 120251915, + "router_z_loss_clip": 0.84277344, + "router_z_loss_mlp": 0.18463135, + "step": 4237, + "time_per_iteration": 2.4976882934570312 + }, + { + "auxiliary_loss_clip": 0.01034932, + "auxiliary_loss_mlp": 0.0100635, + "balance_loss_clip": 1.01394129, + "balance_loss_mlp": 1.00513446, + "epoch": 0.12297603157100574, + "flos": 74777077808640.0, + "grad_norm": 0.6534483121629739, + "language_loss": 0.49866471, + "learning_rate": 3.910012946682018e-06, + "loss": 0.51907754, + "num_input_tokens_seen": 120314075, + "router_z_loss_clip": 0.2109375, + "router_z_loss_mlp": 0.012146, + "step": 4238, + "time_per_iteration": 3.179924249649048 + }, + { + "auxiliary_loss_clip": 0.01144061, + "auxiliary_loss_mlp": 0.01050039, + "balance_loss_clip": 1.05564511, + "balance_loss_mlp": 1.03208661, + "epoch": 0.1230050490395218, + "flos": 27708996741120.0, + "grad_norm": 1.9252962787204475, + "language_loss": 0.88563675, + "learning_rate": 3.909957191620459e-06, + "loss": 0.90757775, + "num_input_tokens_seen": 120332855, + "router_z_loss_clip": 0.8828125, + "router_z_loss_mlp": 0.17956543, + "step": 4239, + "time_per_iteration": 2.6100850105285645 + }, + { + "auxiliary_loss_clip": 0.01128914, + "auxiliary_loss_mlp": 0.0103859, + "balance_loss_clip": 1.05011296, + "balance_loss_mlp": 1.02371323, + "epoch": 0.12303406650803785, + "flos": 28579435021440.0, + "grad_norm": 1.9339267423511752, + "language_loss": 0.66185063, + "learning_rate": 3.9099014196893855e-06, + "loss": 0.68352568, + "num_input_tokens_seen": 120350035, + "router_z_loss_clip": 0.78857422, + "router_z_loss_mlp": 0.14886475, + "step": 4240, + "time_per_iteration": 2.5960798263549805 + }, + { + "auxiliary_loss_clip": 0.01034139, + "auxiliary_loss_mlp": 0.01001103, + "balance_loss_clip": 1.01298153, + "balance_loss_mlp": 0.99982166, + "epoch": 0.12306308397655388, + "flos": 64668044724480.0, + "grad_norm": 0.6612025370402416, + "language_loss": 0.47989342, + "learning_rate": 3.90984563088929e-06, + "loss": 0.50024581, + "num_input_tokens_seen": 120405465, + "router_z_loss_clip": 0.21191406, + "router_z_loss_mlp": 0.01281738, + "step": 4241, + "time_per_iteration": 3.0464813709259033 + }, + { + "auxiliary_loss_clip": 0.01144947, + "auxiliary_loss_mlp": 0.01057521, + "balance_loss_clip": 1.05677807, + "balance_loss_mlp": 1.03800607, + "epoch": 0.12309210144506993, + "flos": 26171696511360.0, + "grad_norm": 5.07197038224271, + "language_loss": 0.81241643, + "learning_rate": 3.909789825220664e-06, + "loss": 0.83444107, + "num_input_tokens_seen": 120419735, + "router_z_loss_clip": 0.88134766, + "router_z_loss_mlp": 0.19519043, + "step": 4242, + "time_per_iteration": 2.5402989387512207 + }, + { + "auxiliary_loss_clip": 0.01134004, + "auxiliary_loss_mlp": 0.01050808, + "balance_loss_clip": 1.05184329, + "balance_loss_mlp": 1.03418434, + "epoch": 0.12312111891358597, + "flos": 26866604004480.0, + "grad_norm": 1.6527592827632067, + "language_loss": 0.78071296, + "learning_rate": 3.909734002684002e-06, + "loss": 0.8025611, + "num_input_tokens_seen": 120440465, + "router_z_loss_clip": 0.8203125, + "router_z_loss_mlp": 0.1663208, + "step": 4243, + "time_per_iteration": 2.7157716751098633 + }, + { + "auxiliary_loss_clip": 0.01141861, + "auxiliary_loss_mlp": 0.01044367, + "balance_loss_clip": 1.05360556, + "balance_loss_mlp": 1.02568638, + "epoch": 0.12315013638210202, + "flos": 21718773417600.0, + "grad_norm": 2.7389444880499827, + "language_loss": 0.90073121, + "learning_rate": 3.909678163279797e-06, + "loss": 0.92259347, + "num_input_tokens_seen": 120453595, + "router_z_loss_clip": 0.8828125, + "router_z_loss_mlp": 0.18688965, + "step": 4244, + "time_per_iteration": 2.5480000972747803 + }, + { + "auxiliary_loss_clip": 0.0114154, + "auxiliary_loss_mlp": 0.0105794, + "balance_loss_clip": 1.05364537, + "balance_loss_mlp": 1.03955805, + "epoch": 0.12317915385061808, + "flos": 27816728607360.0, + "grad_norm": 2.1794260169590904, + "language_loss": 0.96465242, + "learning_rate": 3.909622307008541e-06, + "loss": 0.98664725, + "num_input_tokens_seen": 120474755, + "router_z_loss_clip": 0.87890625, + "router_z_loss_mlp": 0.18383789, + "step": 4245, + "time_per_iteration": 2.617863178253174 + }, + { + "auxiliary_loss_clip": 0.0114023, + "auxiliary_loss_mlp": 0.01045964, + "balance_loss_clip": 1.05508935, + "balance_loss_mlp": 1.02545977, + "epoch": 0.12320817131913411, + "flos": 20514042236160.0, + "grad_norm": 2.288859387717206, + "language_loss": 0.77987248, + "learning_rate": 3.909566433870728e-06, + "loss": 0.80173439, + "num_input_tokens_seen": 120489960, + "router_z_loss_clip": 0.8515625, + "router_z_loss_mlp": 0.20495605, + "step": 4246, + "time_per_iteration": 2.547691583633423 + }, + { + "auxiliary_loss_clip": 0.01133821, + "auxiliary_loss_mlp": 0.01035181, + "balance_loss_clip": 1.05396628, + "balance_loss_mlp": 1.01979136, + "epoch": 0.12323718878765016, + "flos": 24089883033600.0, + "grad_norm": 2.2324313880294087, + "language_loss": 0.76396513, + "learning_rate": 3.909510543866852e-06, + "loss": 0.78565514, + "num_input_tokens_seen": 120504515, + "router_z_loss_clip": 0.79931641, + "router_z_loss_mlp": 0.15393066, + "step": 4247, + "time_per_iteration": 2.5993261337280273 + }, + { + "auxiliary_loss_clip": 0.01157724, + "auxiliary_loss_mlp": 0.01057726, + "balance_loss_clip": 1.06212664, + "balance_loss_mlp": 1.03742456, + "epoch": 0.12326620625616622, + "flos": 48463890647040.0, + "grad_norm": 3.3531994005639425, + "language_loss": 0.82514441, + "learning_rate": 3.909454636997406e-06, + "loss": 0.84729898, + "num_input_tokens_seen": 120522525, + "router_z_loss_clip": 0.95458984, + "router_z_loss_mlp": 0.20300293, + "step": 4248, + "time_per_iteration": 2.78104567527771 + }, + { + "auxiliary_loss_clip": 0.01142199, + "auxiliary_loss_mlp": 0.01043488, + "balance_loss_clip": 1.05497897, + "balance_loss_mlp": 1.02603531, + "epoch": 0.12329522372468225, + "flos": 33103389260160.0, + "grad_norm": 1.9766226983456179, + "language_loss": 0.85438108, + "learning_rate": 3.909398713262884e-06, + "loss": 0.87623799, + "num_input_tokens_seen": 120543140, + "router_z_loss_clip": 0.87109375, + "router_z_loss_mlp": 0.17443848, + "step": 4249, + "time_per_iteration": 2.6548211574554443 + }, + { + "auxiliary_loss_clip": 0.01036859, + "auxiliary_loss_mlp": 0.01005555, + "balance_loss_clip": 1.01600111, + "balance_loss_mlp": 1.0043807, + "epoch": 0.1233242411931983, + "flos": 74773917411840.0, + "grad_norm": 0.6620948008507022, + "language_loss": 0.47803909, + "learning_rate": 3.90934277266378e-06, + "loss": 0.49846321, + "num_input_tokens_seen": 120605625, + "router_z_loss_clip": 0.20898438, + "router_z_loss_mlp": 0.01171875, + "step": 4250, + "time_per_iteration": 3.1957247257232666 + }, + { + "auxiliary_loss_clip": 0.01135156, + "auxiliary_loss_mlp": 0.01040338, + "balance_loss_clip": 1.05320013, + "balance_loss_mlp": 1.02443516, + "epoch": 0.12335325866171436, + "flos": 17195896586880.0, + "grad_norm": 2.1508937067614005, + "language_loss": 0.75793815, + "learning_rate": 3.909286815200588e-06, + "loss": 0.77969313, + "num_input_tokens_seen": 120619175, + "router_z_loss_clip": 0.8203125, + "router_z_loss_mlp": 0.15905762, + "step": 4251, + "time_per_iteration": 2.586705446243286 + }, + { + "auxiliary_loss_clip": 0.01037552, + "auxiliary_loss_mlp": 0.01007924, + "balance_loss_clip": 1.01669025, + "balance_loss_mlp": 1.00674999, + "epoch": 0.1233822761302304, + "flos": 74790796803840.0, + "grad_norm": 0.616950847091151, + "language_loss": 0.47772402, + "learning_rate": 3.909230840873802e-06, + "loss": 0.49817878, + "num_input_tokens_seen": 120692490, + "router_z_loss_clip": 0.20898438, + "router_z_loss_mlp": 0.01171875, + "step": 4252, + "time_per_iteration": 3.349575996398926 + }, + { + "auxiliary_loss_clip": 0.01037121, + "auxiliary_loss_mlp": 0.0100361, + "balance_loss_clip": 1.01628447, + "balance_loss_mlp": 1.0023644, + "epoch": 0.12341129359874645, + "flos": 65219094247680.0, + "grad_norm": 0.6742859110977046, + "language_loss": 0.49204466, + "learning_rate": 3.909174849683917e-06, + "loss": 0.51245201, + "num_input_tokens_seen": 120753605, + "router_z_loss_clip": 0.20898438, + "router_z_loss_mlp": 0.01245117, + "step": 4253, + "time_per_iteration": 3.153822660446167 + }, + { + "auxiliary_loss_clip": 0.01142158, + "auxiliary_loss_mlp": 0.01050952, + "balance_loss_clip": 1.05750906, + "balance_loss_mlp": 1.03382206, + "epoch": 0.1234403110672625, + "flos": 18106411466880.0, + "grad_norm": 2.4007017046575476, + "language_loss": 0.80354106, + "learning_rate": 3.909118841631427e-06, + "loss": 0.82547212, + "num_input_tokens_seen": 120767665, + "router_z_loss_clip": 0.84521484, + "router_z_loss_mlp": 0.17126465, + "step": 4254, + "time_per_iteration": 2.5676021575927734 + }, + { + "auxiliary_loss_clip": 0.01037742, + "auxiliary_loss_mlp": 0.01004512, + "balance_loss_clip": 1.01689291, + "balance_loss_mlp": 1.00327194, + "epoch": 0.12346932853577854, + "flos": 60429400214400.0, + "grad_norm": 0.6276724035173651, + "language_loss": 0.47372308, + "learning_rate": 3.909062816716827e-06, + "loss": 0.49414563, + "num_input_tokens_seen": 120828340, + "router_z_loss_clip": 0.20898438, + "router_z_loss_mlp": 0.01239014, + "step": 4255, + "time_per_iteration": 3.0529112815856934 + }, + { + "auxiliary_loss_clip": 0.01157602, + "auxiliary_loss_mlp": 0.01057033, + "balance_loss_clip": 1.06037509, + "balance_loss_mlp": 1.03733945, + "epoch": 0.12349834600429459, + "flos": 12780751622400.0, + "grad_norm": 2.674276860265611, + "language_loss": 0.73307931, + "learning_rate": 3.909006774940611e-06, + "loss": 0.75522566, + "num_input_tokens_seen": 120839385, + "router_z_loss_clip": 0.97265625, + "router_z_loss_mlp": 0.19689941, + "step": 4256, + "time_per_iteration": 2.4552457332611084 + }, + { + "auxiliary_loss_clip": 0.01150255, + "auxiliary_loss_mlp": 0.01059264, + "balance_loss_clip": 1.05558395, + "balance_loss_mlp": 1.03936839, + "epoch": 0.12352736347281063, + "flos": 27050143524480.0, + "grad_norm": 2.0895356515625854, + "language_loss": 0.85111731, + "learning_rate": 3.908950716303275e-06, + "loss": 0.87321246, + "num_input_tokens_seen": 120857565, + "router_z_loss_clip": 0.94628906, + "router_z_loss_mlp": 0.19909668, + "step": 4257, + "time_per_iteration": 2.550511360168457 + }, + { + "auxiliary_loss_clip": 0.01147063, + "auxiliary_loss_mlp": 0.01051006, + "balance_loss_clip": 1.05637956, + "balance_loss_mlp": 1.03079391, + "epoch": 0.12355638094132668, + "flos": 30293343446400.0, + "grad_norm": 2.255085754308025, + "language_loss": 0.75590128, + "learning_rate": 3.908894640805315e-06, + "loss": 0.77788198, + "num_input_tokens_seen": 120873765, + "router_z_loss_clip": 0.90625, + "router_z_loss_mlp": 0.2020874, + "step": 4258, + "time_per_iteration": 2.5708253383636475 + }, + { + "auxiliary_loss_clip": 0.01137445, + "auxiliary_loss_mlp": 0.01039508, + "balance_loss_clip": 1.05824113, + "balance_loss_mlp": 1.02246737, + "epoch": 0.12358539840984273, + "flos": 19456796298240.0, + "grad_norm": 2.3094577392464384, + "language_loss": 0.7840693, + "learning_rate": 3.9088385484472235e-06, + "loss": 0.80583882, + "num_input_tokens_seen": 120888725, + "router_z_loss_clip": 0.79199219, + "router_z_loss_mlp": 0.17059326, + "step": 4259, + "time_per_iteration": 2.5216715335845947 + }, + { + "auxiliary_loss_clip": 0.01139337, + "auxiliary_loss_mlp": 0.01039309, + "balance_loss_clip": 1.0559752, + "balance_loss_mlp": 1.0223279, + "epoch": 0.12361441587835877, + "flos": 21463592221440.0, + "grad_norm": 2.9526090493082733, + "language_loss": 0.75455463, + "learning_rate": 3.908782439229498e-06, + "loss": 0.77634114, + "num_input_tokens_seen": 120902830, + "router_z_loss_clip": 0.83349609, + "router_z_loss_mlp": 0.16986084, + "step": 4260, + "time_per_iteration": 2.451997756958008 + }, + { + "auxiliary_loss_clip": 0.01040684, + "auxiliary_loss_mlp": 0.0101398, + "balance_loss_clip": 1.01977253, + "balance_loss_mlp": 1.01263273, + "epoch": 0.12364343334687482, + "flos": 64787161201920.0, + "grad_norm": 0.6423131934987267, + "language_loss": 0.50246668, + "learning_rate": 3.908726313152633e-06, + "loss": 0.52301335, + "num_input_tokens_seen": 120970225, + "router_z_loss_clip": 0.20898438, + "router_z_loss_mlp": 0.01348877, + "step": 4261, + "time_per_iteration": 3.2464487552642822 + }, + { + "auxiliary_loss_clip": 0.01140516, + "auxiliary_loss_mlp": 0.01042895, + "balance_loss_clip": 1.05729699, + "balance_loss_mlp": 1.02618206, + "epoch": 0.12367245081539087, + "flos": 55030695085440.0, + "grad_norm": 2.4938062415706885, + "language_loss": 0.86797035, + "learning_rate": 3.908670170217126e-06, + "loss": 0.88980448, + "num_input_tokens_seen": 120988295, + "router_z_loss_clip": 0.83251953, + "router_z_loss_mlp": 0.16723633, + "step": 4262, + "time_per_iteration": 2.7987747192382812 + }, + { + "auxiliary_loss_clip": 0.01041101, + "auxiliary_loss_mlp": 0.01004172, + "balance_loss_clip": 1.02019, + "balance_loss_mlp": 1.00283659, + "epoch": 0.1237014682839069, + "flos": 52843271621760.0, + "grad_norm": 0.6093729503218025, + "language_loss": 0.4223116, + "learning_rate": 3.908614010423471e-06, + "loss": 0.44276434, + "num_input_tokens_seen": 121044190, + "router_z_loss_clip": 0.20898438, + "router_z_loss_mlp": 0.0133667, + "step": 4263, + "time_per_iteration": 2.9418258666992188 + }, + { + "auxiliary_loss_clip": 0.01143813, + "auxiliary_loss_mlp": 0.01054639, + "balance_loss_clip": 1.05420709, + "balance_loss_mlp": 1.03440309, + "epoch": 0.12373048575242296, + "flos": 14749661675520.0, + "grad_norm": 2.1264205434790515, + "language_loss": 0.93305403, + "learning_rate": 3.908557833772165e-06, + "loss": 0.95503849, + "num_input_tokens_seen": 121057500, + "router_z_loss_clip": 0.89599609, + "router_z_loss_mlp": 0.20245361, + "step": 4264, + "time_per_iteration": 2.4910032749176025 + }, + { + "auxiliary_loss_clip": 0.01145556, + "auxiliary_loss_mlp": 0.01051828, + "balance_loss_clip": 1.05819917, + "balance_loss_mlp": 1.03339863, + "epoch": 0.12375950322093901, + "flos": 26096678956800.0, + "grad_norm": 2.3478805791766884, + "language_loss": 0.80594546, + "learning_rate": 3.908501640263704e-06, + "loss": 0.82791924, + "num_input_tokens_seen": 121071570, + "router_z_loss_clip": 0.87451172, + "router_z_loss_mlp": 0.18432617, + "step": 4265, + "time_per_iteration": 2.5624542236328125 + }, + { + "auxiliary_loss_clip": 0.01143346, + "auxiliary_loss_mlp": 0.01054819, + "balance_loss_clip": 1.05708778, + "balance_loss_mlp": 1.03693724, + "epoch": 0.12378852068945505, + "flos": 36386701695360.0, + "grad_norm": 3.1005908283012684, + "language_loss": 0.80839926, + "learning_rate": 3.9084454298985834e-06, + "loss": 0.83038092, + "num_input_tokens_seen": 121088730, + "router_z_loss_clip": 0.86279297, + "router_z_loss_mlp": 0.17883301, + "step": 4266, + "time_per_iteration": 2.5796492099761963 + }, + { + "auxiliary_loss_clip": 0.01148788, + "auxiliary_loss_mlp": 0.01051287, + "balance_loss_clip": 1.05714226, + "balance_loss_mlp": 1.03189182, + "epoch": 0.1238175381579711, + "flos": 23615934053760.0, + "grad_norm": 2.8656103215498994, + "language_loss": 0.88860011, + "learning_rate": 3.908389202677301e-06, + "loss": 0.91060078, + "num_input_tokens_seen": 121102835, + "router_z_loss_clip": 0.91650391, + "router_z_loss_mlp": 0.19396973, + "step": 4267, + "time_per_iteration": 2.541240930557251 + }, + { + "auxiliary_loss_clip": 0.01145636, + "auxiliary_loss_mlp": 0.01052617, + "balance_loss_clip": 1.05613875, + "balance_loss_mlp": 1.03298378, + "epoch": 0.12384655562648715, + "flos": 33723207371520.0, + "grad_norm": 1.9782169738856472, + "language_loss": 0.86330324, + "learning_rate": 3.908332958600353e-06, + "loss": 0.88528574, + "num_input_tokens_seen": 121118780, + "router_z_loss_clip": 0.89453125, + "router_z_loss_mlp": 0.19641113, + "step": 4268, + "time_per_iteration": 5.000117778778076 + }, + { + "auxiliary_loss_clip": 0.01131372, + "auxiliary_loss_mlp": 0.01046602, + "balance_loss_clip": 1.05536199, + "balance_loss_mlp": 1.03029466, + "epoch": 0.12387557309500319, + "flos": 30952555799040.0, + "grad_norm": 2.365389455008548, + "language_loss": 1.02831364, + "learning_rate": 3.908276697668237e-06, + "loss": 1.05009341, + "num_input_tokens_seen": 121134030, + "router_z_loss_clip": 0.75976562, + "router_z_loss_mlp": 0.1630249, + "step": 4269, + "time_per_iteration": 7.326718091964722 + }, + { + "auxiliary_loss_clip": 0.01145152, + "auxiliary_loss_mlp": 0.01055949, + "balance_loss_clip": 1.05746472, + "balance_loss_mlp": 1.03685164, + "epoch": 0.12390459056351924, + "flos": 11610602259840.0, + "grad_norm": 3.219715591219382, + "language_loss": 0.78276539, + "learning_rate": 3.908220419881448e-06, + "loss": 0.80477637, + "num_input_tokens_seen": 121145765, + "router_z_loss_clip": 0.87841797, + "router_z_loss_mlp": 0.19091797, + "step": 4270, + "time_per_iteration": 2.458961009979248 + }, + { + "auxiliary_loss_clip": 0.01036654, + "auxiliary_loss_mlp": 0.01017241, + "balance_loss_clip": 1.01591134, + "balance_loss_mlp": 1.01598918, + "epoch": 0.12393360803203529, + "flos": 71966816513280.0, + "grad_norm": 0.6225046046128696, + "language_loss": 0.44553518, + "learning_rate": 3.908164125240484e-06, + "loss": 0.46607414, + "num_input_tokens_seen": 121209420, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.01251221, + "step": 4271, + "time_per_iteration": 5.588155508041382 + }, + { + "auxiliary_loss_clip": 0.01036475, + "auxiliary_loss_mlp": 0.01011627, + "balance_loss_clip": 1.01567781, + "balance_loss_mlp": 1.0103817, + "epoch": 0.12396262550055133, + "flos": 74777867907840.0, + "grad_norm": 0.6532752571547914, + "language_loss": 0.54016793, + "learning_rate": 3.908107813745842e-06, + "loss": 0.56064892, + "num_input_tokens_seen": 121275380, + "router_z_loss_clip": 0.20898438, + "router_z_loss_mlp": 0.01245117, + "step": 4272, + "time_per_iteration": 3.2069740295410156 + }, + { + "auxiliary_loss_clip": 0.01144392, + "auxiliary_loss_mlp": 0.01051608, + "balance_loss_clip": 1.06039548, + "balance_loss_mlp": 1.0343169, + "epoch": 0.12399164296906738, + "flos": 13107646321920.0, + "grad_norm": 2.3969660761233036, + "language_loss": 0.73468924, + "learning_rate": 3.908051485398021e-06, + "loss": 0.75664926, + "num_input_tokens_seen": 121287060, + "router_z_loss_clip": 0.83935547, + "router_z_loss_mlp": 0.17303467, + "step": 4273, + "time_per_iteration": 2.453673839569092 + }, + { + "auxiliary_loss_clip": 0.01139803, + "auxiliary_loss_mlp": 0.0104123, + "balance_loss_clip": 1.05588603, + "balance_loss_mlp": 1.02346754, + "epoch": 0.12402066043758342, + "flos": 26826958368000.0, + "grad_norm": 4.0713295813383095, + "language_loss": 0.66622925, + "learning_rate": 3.9079951401975165e-06, + "loss": 0.6880396, + "num_input_tokens_seen": 121301820, + "router_z_loss_clip": 0.83886719, + "router_z_loss_mlp": 0.1776123, + "step": 4274, + "time_per_iteration": 2.5549936294555664 + }, + { + "auxiliary_loss_clip": 0.01138399, + "auxiliary_loss_mlp": 0.01038682, + "balance_loss_clip": 1.05716503, + "balance_loss_mlp": 1.02251148, + "epoch": 0.12404967790609947, + "flos": 27301625619840.0, + "grad_norm": 1.9799429978506866, + "language_loss": 0.74264538, + "learning_rate": 3.907938778144827e-06, + "loss": 0.76441622, + "num_input_tokens_seen": 121316325, + "router_z_loss_clip": 0.8125, + "router_z_loss_mlp": 0.16174316, + "step": 4275, + "time_per_iteration": 2.54038667678833 + }, + { + "auxiliary_loss_clip": 0.0114428, + "auxiliary_loss_mlp": 0.01042814, + "balance_loss_clip": 1.05839825, + "balance_loss_mlp": 1.0249027, + "epoch": 0.12407869537461552, + "flos": 31024700265600.0, + "grad_norm": 2.6713389916232835, + "language_loss": 0.84181976, + "learning_rate": 3.9078823992404495e-06, + "loss": 0.86369061, + "num_input_tokens_seen": 121332475, + "router_z_loss_clip": 0.859375, + "router_z_loss_mlp": 0.17907715, + "step": 4276, + "time_per_iteration": 2.6372175216674805 + }, + { + "auxiliary_loss_clip": 0.01036174, + "auxiliary_loss_mlp": 0.01000889, + "balance_loss_clip": 1.01530266, + "balance_loss_mlp": 0.99969715, + "epoch": 0.12410771284313156, + "flos": 68897495352960.0, + "grad_norm": 0.6891326298781562, + "language_loss": 0.45795426, + "learning_rate": 3.907826003484883e-06, + "loss": 0.47832486, + "num_input_tokens_seen": 121393170, + "router_z_loss_clip": 0.20898438, + "router_z_loss_mlp": 0.01190186, + "step": 4277, + "time_per_iteration": 3.1116836071014404 + }, + { + "auxiliary_loss_clip": 0.0113945, + "auxiliary_loss_mlp": 0.01039144, + "balance_loss_clip": 1.05890226, + "balance_loss_mlp": 1.02284861, + "epoch": 0.12413673031164761, + "flos": 15225190853760.0, + "grad_norm": 2.7710964848530963, + "language_loss": 0.59488839, + "learning_rate": 3.907769590878625e-06, + "loss": 0.6166743, + "num_input_tokens_seen": 121407655, + "router_z_loss_clip": 0.80517578, + "router_z_loss_mlp": 0.16296387, + "step": 4278, + "time_per_iteration": 2.486837148666382 + }, + { + "auxiliary_loss_clip": 0.01035577, + "auxiliary_loss_mlp": 0.01007552, + "balance_loss_clip": 1.01452625, + "balance_loss_mlp": 1.00625849, + "epoch": 0.12416574778016366, + "flos": 66342774389760.0, + "grad_norm": 0.7102971448451304, + "language_loss": 0.50092626, + "learning_rate": 3.907713161422174e-06, + "loss": 0.52135754, + "num_input_tokens_seen": 121467400, + "router_z_loss_clip": 0.2109375, + "router_z_loss_mlp": 0.01293945, + "step": 4279, + "time_per_iteration": 2.9995346069335938 + }, + { + "auxiliary_loss_clip": 0.01148671, + "auxiliary_loss_mlp": 0.01039548, + "balance_loss_clip": 1.05963218, + "balance_loss_mlp": 1.0220958, + "epoch": 0.1241947652486797, + "flos": 14057950492800.0, + "grad_norm": 2.882642133130818, + "language_loss": 0.76375842, + "learning_rate": 3.907656715116028e-06, + "loss": 0.7856406, + "num_input_tokens_seen": 121480440, + "router_z_loss_clip": 0.890625, + "router_z_loss_mlp": 0.17456055, + "step": 4280, + "time_per_iteration": 2.4886860847473145 + }, + { + "auxiliary_loss_clip": 0.01140602, + "auxiliary_loss_mlp": 0.01040117, + "balance_loss_clip": 1.05774713, + "balance_loss_mlp": 1.02374983, + "epoch": 0.12422378271719575, + "flos": 37228052937600.0, + "grad_norm": 2.172899737909933, + "language_loss": 0.64352226, + "learning_rate": 3.907600251960687e-06, + "loss": 0.6653294, + "num_input_tokens_seen": 121501525, + "router_z_loss_clip": 0.82910156, + "router_z_loss_mlp": 0.16369629, + "step": 4281, + "time_per_iteration": 2.6929829120635986 + }, + { + "auxiliary_loss_clip": 0.01035757, + "auxiliary_loss_mlp": 0.01001917, + "balance_loss_clip": 1.01457703, + "balance_loss_mlp": 1.00068283, + "epoch": 0.1242528001857118, + "flos": 61936284602880.0, + "grad_norm": 0.6734826319783331, + "language_loss": 0.50745827, + "learning_rate": 3.907543771956647e-06, + "loss": 0.52783501, + "num_input_tokens_seen": 121565085, + "router_z_loss_clip": 0.21191406, + "router_z_loss_mlp": 0.0123291, + "step": 4282, + "time_per_iteration": 3.1035711765289307 + }, + { + "auxiliary_loss_clip": 0.0114395, + "auxiliary_loss_mlp": 0.01047455, + "balance_loss_clip": 1.05900383, + "balance_loss_mlp": 1.03170705, + "epoch": 0.12428181765422784, + "flos": 14388544293120.0, + "grad_norm": 2.6540717359444783, + "language_loss": 0.93256658, + "learning_rate": 3.90748727510441e-06, + "loss": 0.95448065, + "num_input_tokens_seen": 121578245, + "router_z_loss_clip": 0.84912109, + "router_z_loss_mlp": 0.1574707, + "step": 4283, + "time_per_iteration": 2.5762009620666504 + }, + { + "auxiliary_loss_clip": 0.01035307, + "auxiliary_loss_mlp": 0.01007675, + "balance_loss_clip": 1.0143379, + "balance_loss_mlp": 1.00645959, + "epoch": 0.12431083512274389, + "flos": 74795321917440.0, + "grad_norm": 0.6425609734997807, + "language_loss": 0.45109582, + "learning_rate": 3.907430761404474e-06, + "loss": 0.47152567, + "num_input_tokens_seen": 121637845, + "router_z_loss_clip": 0.20996094, + "router_z_loss_mlp": 0.012146, + "step": 4284, + "time_per_iteration": 3.3001155853271484 + }, + { + "auxiliary_loss_clip": 0.01035374, + "auxiliary_loss_mlp": 0.01006239, + "balance_loss_clip": 1.01443672, + "balance_loss_mlp": 1.00513685, + "epoch": 0.12433985259125994, + "flos": 48972065287680.0, + "grad_norm": 0.7034463424725832, + "language_loss": 0.49962905, + "learning_rate": 3.907374230857336e-06, + "loss": 0.52004516, + "num_input_tokens_seen": 121689750, + "router_z_loss_clip": 0.20996094, + "router_z_loss_mlp": 0.01104736, + "step": 4285, + "time_per_iteration": 2.872499942779541 + }, + { + "auxiliary_loss_clip": 0.0114216, + "auxiliary_loss_mlp": 0.01042247, + "balance_loss_clip": 1.05662918, + "balance_loss_mlp": 1.02397192, + "epoch": 0.12436887005977598, + "flos": 23326386520320.0, + "grad_norm": 2.1382308289264054, + "language_loss": 0.82919604, + "learning_rate": 3.907317683463498e-06, + "loss": 0.85104012, + "num_input_tokens_seen": 121706815, + "router_z_loss_clip": 0.85546875, + "router_z_loss_mlp": 0.18261719, + "step": 4286, + "time_per_iteration": 2.56522798538208 + }, + { + "auxiliary_loss_clip": 0.01145903, + "auxiliary_loss_mlp": 0.01056545, + "balance_loss_clip": 1.05897427, + "balance_loss_mlp": 1.03819239, + "epoch": 0.12439788752829203, + "flos": 39594529699200.0, + "grad_norm": 2.1024584801916513, + "language_loss": 0.79581678, + "learning_rate": 3.907261119223458e-06, + "loss": 0.81784129, + "num_input_tokens_seen": 121722690, + "router_z_loss_clip": 0.86865234, + "router_z_loss_mlp": 0.18353271, + "step": 4287, + "time_per_iteration": 2.6539230346679688 + }, + { + "auxiliary_loss_clip": 0.01034727, + "auxiliary_loss_mlp": 0.0100141, + "balance_loss_clip": 1.01374221, + "balance_loss_mlp": 1.00030088, + "epoch": 0.12442690499680809, + "flos": 60268806506880.0, + "grad_norm": 0.7372688578571336, + "language_loss": 0.46562886, + "learning_rate": 3.907204538137716e-06, + "loss": 0.48599023, + "num_input_tokens_seen": 121773065, + "router_z_loss_clip": 0.20996094, + "router_z_loss_mlp": 0.0111084, + "step": 4288, + "time_per_iteration": 2.909061908721924 + }, + { + "auxiliary_loss_clip": 0.01155251, + "auxiliary_loss_mlp": 0.01051899, + "balance_loss_clip": 1.05955112, + "balance_loss_mlp": 1.03059661, + "epoch": 0.12445592246532412, + "flos": 27116182679040.0, + "grad_norm": 3.4580628933358737, + "language_loss": 1.04247403, + "learning_rate": 3.907147940206773e-06, + "loss": 1.06454551, + "num_input_tokens_seen": 121786215, + "router_z_loss_clip": 0.95556641, + "router_z_loss_mlp": 0.2130127, + "step": 4289, + "time_per_iteration": 2.5672664642333984 + }, + { + "auxiliary_loss_clip": 0.01139647, + "auxiliary_loss_mlp": 0.01035273, + "balance_loss_clip": 1.05736017, + "balance_loss_mlp": 1.01854801, + "epoch": 0.12448493993384017, + "flos": 55363012738560.0, + "grad_norm": 2.030885705313889, + "language_loss": 0.52769589, + "learning_rate": 3.907091325431125e-06, + "loss": 0.54944515, + "num_input_tokens_seen": 121809115, + "router_z_loss_clip": 0.82324219, + "router_z_loss_mlp": 0.16723633, + "step": 4290, + "time_per_iteration": 2.7796881198883057 + }, + { + "auxiliary_loss_clip": 0.01034336, + "auxiliary_loss_mlp": 0.01001945, + "balance_loss_clip": 1.01335382, + "balance_loss_mlp": 1.00066316, + "epoch": 0.12451395740235621, + "flos": 63212298324480.0, + "grad_norm": 0.6438553011104515, + "language_loss": 0.49623036, + "learning_rate": 3.907034693811277e-06, + "loss": 0.51659316, + "num_input_tokens_seen": 121872220, + "router_z_loss_clip": 0.20996094, + "router_z_loss_mlp": 0.01281738, + "step": 4291, + "time_per_iteration": 3.0486021041870117 + }, + { + "auxiliary_loss_clip": 0.01138806, + "auxiliary_loss_mlp": 0.01048776, + "balance_loss_clip": 1.05786538, + "balance_loss_mlp": 1.03214645, + "epoch": 0.12454297487087226, + "flos": 11539463374080.0, + "grad_norm": 2.4586506209361896, + "language_loss": 0.73578572, + "learning_rate": 3.906978045347726e-06, + "loss": 0.75766146, + "num_input_tokens_seen": 121884990, + "router_z_loss_clip": 0.81005859, + "router_z_loss_mlp": 0.1663208, + "step": 4292, + "time_per_iteration": 2.562375783920288 + }, + { + "auxiliary_loss_clip": 0.01033657, + "auxiliary_loss_mlp": 0.01002119, + "balance_loss_clip": 1.01276278, + "balance_loss_mlp": 1.00081396, + "epoch": 0.12457199233938832, + "flos": 66495682586880.0, + "grad_norm": 0.630326753443429, + "language_loss": 0.4826045, + "learning_rate": 3.906921380040973e-06, + "loss": 0.50296229, + "num_input_tokens_seen": 121951440, + "router_z_loss_clip": 0.20898438, + "router_z_loss_mlp": 0.01306152, + "step": 4293, + "time_per_iteration": 3.1760497093200684 + }, + { + "auxiliary_loss_clip": 0.01135565, + "auxiliary_loss_mlp": 0.01043931, + "balance_loss_clip": 1.056023, + "balance_loss_mlp": 1.02666974, + "epoch": 0.12460100980790435, + "flos": 31904188773120.0, + "grad_norm": 1.9301991106002383, + "language_loss": 0.86865675, + "learning_rate": 3.90686469789152e-06, + "loss": 0.89045167, + "num_input_tokens_seen": 121976390, + "router_z_loss_clip": 0.79492188, + "router_z_loss_mlp": 0.17248535, + "step": 4294, + "time_per_iteration": 2.638963460922241 + }, + { + "auxiliary_loss_clip": 0.01140926, + "auxiliary_loss_mlp": 0.01034457, + "balance_loss_clip": 1.05577958, + "balance_loss_mlp": 1.01803041, + "epoch": 0.1246300272764204, + "flos": 34124042217600.0, + "grad_norm": 2.4112989525212174, + "language_loss": 0.92922431, + "learning_rate": 3.906807998899866e-06, + "loss": 0.95097816, + "num_input_tokens_seen": 121993740, + "router_z_loss_clip": 0.85107422, + "router_z_loss_mlp": 0.16424561, + "step": 4295, + "time_per_iteration": 2.6435985565185547 + }, + { + "auxiliary_loss_clip": 0.0103319, + "auxiliary_loss_mlp": 0.01001101, + "balance_loss_clip": 1.01232886, + "balance_loss_mlp": 0.99987268, + "epoch": 0.12465904474493646, + "flos": 55978524195840.0, + "grad_norm": 0.6570887762013347, + "language_loss": 0.48360002, + "learning_rate": 3.906751283066511e-06, + "loss": 0.50394297, + "num_input_tokens_seen": 122055320, + "router_z_loss_clip": 0.20898438, + "router_z_loss_mlp": 0.01226807, + "step": 4296, + "time_per_iteration": 3.011446237564087 + }, + { + "auxiliary_loss_clip": 0.01130623, + "auxiliary_loss_mlp": 0.0104444, + "balance_loss_clip": 1.051682, + "balance_loss_mlp": 1.02903163, + "epoch": 0.1246880622134525, + "flos": 13221914463360.0, + "grad_norm": 2.4611469504453805, + "language_loss": 0.70791662, + "learning_rate": 3.906694550391958e-06, + "loss": 0.72966719, + "num_input_tokens_seen": 122068445, + "router_z_loss_clip": 0.78955078, + "router_z_loss_mlp": 0.15411377, + "step": 4297, + "time_per_iteration": 2.500239372253418 + }, + { + "auxiliary_loss_clip": 0.01033683, + "auxiliary_loss_mlp": 0.01001743, + "balance_loss_clip": 1.01279235, + "balance_loss_mlp": 1.00056314, + "epoch": 0.12471707968196855, + "flos": 60650538295680.0, + "grad_norm": 0.6687483468722327, + "language_loss": 0.4832195, + "learning_rate": 3.906637800876706e-06, + "loss": 0.50357378, + "num_input_tokens_seen": 122127275, + "router_z_loss_clip": 0.20898438, + "router_z_loss_mlp": 0.01177979, + "step": 4298, + "time_per_iteration": 2.9980831146240234 + }, + { + "auxiliary_loss_clip": 0.01138607, + "auxiliary_loss_mlp": 0.01047679, + "balance_loss_clip": 1.05860496, + "balance_loss_mlp": 1.03081048, + "epoch": 0.1247460971504846, + "flos": 11359227905280.0, + "grad_norm": 2.70361900869582, + "language_loss": 0.9076488, + "learning_rate": 3.906581034521259e-06, + "loss": 0.92951173, + "num_input_tokens_seen": 122138040, + "router_z_loss_clip": 0.79980469, + "router_z_loss_mlp": 0.16876221, + "step": 4299, + "time_per_iteration": 2.49257755279541 + }, + { + "auxiliary_loss_clip": 0.01033176, + "auxiliary_loss_mlp": 0.01003716, + "balance_loss_clip": 1.01247334, + "balance_loss_mlp": 1.00250649, + "epoch": 0.12477511461900063, + "flos": 62485071569280.0, + "grad_norm": 0.6862407441171949, + "language_loss": 0.53265482, + "learning_rate": 3.906524251326116e-06, + "loss": 0.55302376, + "num_input_tokens_seen": 122201460, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.01208496, + "step": 4300, + "time_per_iteration": 3.054224967956543 + }, + { + "auxiliary_loss_clip": 0.01032208, + "auxiliary_loss_mlp": 0.01005919, + "balance_loss_clip": 1.01146102, + "balance_loss_mlp": 1.00475109, + "epoch": 0.12480413208751669, + "flos": 74795321917440.0, + "grad_norm": 0.5943668536559549, + "language_loss": 0.3998872, + "learning_rate": 3.906467451291779e-06, + "loss": 0.42026848, + "num_input_tokens_seen": 122272725, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.01165771, + "step": 4301, + "time_per_iteration": 3.3166611194610596 + }, + { + "auxiliary_loss_clip": 0.01032451, + "auxiliary_loss_mlp": 0.01006144, + "balance_loss_clip": 1.01177764, + "balance_loss_mlp": 1.00495207, + "epoch": 0.12483314955603274, + "flos": 74347407319680.0, + "grad_norm": 0.7187295718252186, + "language_loss": 0.54350185, + "learning_rate": 3.90641063441875e-06, + "loss": 0.56388772, + "num_input_tokens_seen": 122341065, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.01190186, + "step": 4302, + "time_per_iteration": 3.218709945678711 + }, + { + "auxiliary_loss_clip": 0.01134689, + "auxiliary_loss_mlp": 0.01043924, + "balance_loss_clip": 1.05391896, + "balance_loss_mlp": 1.02742505, + "epoch": 0.12486216702454878, + "flos": 25549867238400.0, + "grad_norm": 2.516802476555308, + "language_loss": 0.97919232, + "learning_rate": 3.906353800707532e-06, + "loss": 1.00097835, + "num_input_tokens_seen": 122355840, + "router_z_loss_clip": 0.80761719, + "router_z_loss_mlp": 0.16491699, + "step": 4303, + "time_per_iteration": 2.5558037757873535 + }, + { + "auxiliary_loss_clip": 0.01144048, + "auxiliary_loss_mlp": 0.01043817, + "balance_loss_clip": 1.05833793, + "balance_loss_mlp": 1.02562547, + "epoch": 0.12489118449306483, + "flos": 22595676145920.0, + "grad_norm": 26.439916911467115, + "language_loss": 0.81652641, + "learning_rate": 3.906296950158625e-06, + "loss": 0.83840501, + "num_input_tokens_seen": 122374215, + "router_z_loss_clip": 0.85742188, + "router_z_loss_mlp": 0.18188477, + "step": 4304, + "time_per_iteration": 2.6086275577545166 + }, + { + "auxiliary_loss_clip": 0.01128946, + "auxiliary_loss_mlp": 0.0104767, + "balance_loss_clip": 1.05329752, + "balance_loss_mlp": 1.03195226, + "epoch": 0.12492020196158087, + "flos": 12633122724480.0, + "grad_norm": 2.0052907478034863, + "language_loss": 0.7415663, + "learning_rate": 3.9062400827725325e-06, + "loss": 0.76333249, + "num_input_tokens_seen": 122390905, + "router_z_loss_clip": 0.75683594, + "router_z_loss_mlp": 0.15716553, + "step": 4305, + "time_per_iteration": 2.535386562347412 + }, + { + "auxiliary_loss_clip": 0.01122488, + "auxiliary_loss_mlp": 0.01033092, + "balance_loss_clip": 1.05033338, + "balance_loss_mlp": 1.02039051, + "epoch": 0.12494921943009692, + "flos": 32483607062400.0, + "grad_norm": 2.3921492313215165, + "language_loss": 0.80288261, + "learning_rate": 3.906183198549755e-06, + "loss": 0.82443845, + "num_input_tokens_seen": 122406360, + "router_z_loss_clip": 0.72216797, + "router_z_loss_mlp": 0.1270752, + "step": 4306, + "time_per_iteration": 2.6185619831085205 + }, + { + "auxiliary_loss_clip": 0.01144209, + "auxiliary_loss_mlp": 0.01040579, + "balance_loss_clip": 1.05855381, + "balance_loss_mlp": 1.02370548, + "epoch": 0.12497823689861297, + "flos": 41681694303360.0, + "grad_norm": 1.9539495967344391, + "language_loss": 0.87526906, + "learning_rate": 3.906126297490797e-06, + "loss": 0.89711696, + "num_input_tokens_seen": 122426770, + "router_z_loss_clip": 0.85742188, + "router_z_loss_mlp": 0.16870117, + "step": 4307, + "time_per_iteration": 2.70687198638916 + }, + { + "auxiliary_loss_clip": 0.01143832, + "auxiliary_loss_mlp": 0.01042358, + "balance_loss_clip": 1.05926716, + "balance_loss_mlp": 1.02545381, + "epoch": 0.125007254367129, + "flos": 28944754295040.0, + "grad_norm": 2.2912242077300897, + "language_loss": 0.66033947, + "learning_rate": 3.90606937959616e-06, + "loss": 0.68220139, + "num_input_tokens_seen": 122440570, + "router_z_loss_clip": 0.84521484, + "router_z_loss_mlp": 0.16906738, + "step": 4308, + "time_per_iteration": 2.526309013366699 + }, + { + "auxiliary_loss_clip": 0.0113397, + "auxiliary_loss_mlp": 0.01046044, + "balance_loss_clip": 1.05357862, + "balance_loss_mlp": 1.02909875, + "epoch": 0.12503627183564506, + "flos": 11322383529600.0, + "grad_norm": 2.1588277897328334, + "language_loss": 0.75499105, + "learning_rate": 3.906012444866346e-06, + "loss": 0.77679116, + "num_input_tokens_seen": 122454440, + "router_z_loss_clip": 0.8046875, + "router_z_loss_mlp": 0.16943359, + "step": 4309, + "time_per_iteration": 2.529794931411743 + }, + { + "auxiliary_loss_clip": 0.01031522, + "auxiliary_loss_mlp": 0.01008414, + "balance_loss_clip": 1.0109551, + "balance_loss_mlp": 1.0071919, + "epoch": 0.1250652893041611, + "flos": 67365869472000.0, + "grad_norm": 0.6951174648150213, + "language_loss": 0.50595278, + "learning_rate": 3.905955493301861e-06, + "loss": 0.52635211, + "num_input_tokens_seen": 122518250, + "router_z_loss_clip": 0.20605469, + "router_z_loss_mlp": 0.01220703, + "step": 4310, + "time_per_iteration": 3.073805332183838 + }, + { + "auxiliary_loss_clip": 0.0114475, + "auxiliary_loss_mlp": 0.01042054, + "balance_loss_clip": 1.06007302, + "balance_loss_mlp": 1.02509093, + "epoch": 0.12509430677267716, + "flos": 17963199941760.0, + "grad_norm": 3.627921974530551, + "language_loss": 0.78812504, + "learning_rate": 3.905898524903204e-06, + "loss": 0.80999309, + "num_input_tokens_seen": 122532440, + "router_z_loss_clip": 0.84570312, + "router_z_loss_mlp": 0.16949463, + "step": 4311, + "time_per_iteration": 2.5016262531280518 + }, + { + "auxiliary_loss_clip": 0.01143437, + "auxiliary_loss_mlp": 0.01036563, + "balance_loss_clip": 1.0595336, + "balance_loss_mlp": 1.02126241, + "epoch": 0.12512332424119318, + "flos": 10442392231680.0, + "grad_norm": 3.0526816465181215, + "language_loss": 0.74484938, + "learning_rate": 3.9058415396708805e-06, + "loss": 0.76664937, + "num_input_tokens_seen": 122542705, + "router_z_loss_clip": 0.83837891, + "router_z_loss_mlp": 0.15307617, + "step": 4312, + "time_per_iteration": 2.4978134632110596 + }, + { + "auxiliary_loss_clip": 0.01031505, + "auxiliary_loss_mlp": 0.01004269, + "balance_loss_clip": 1.01063848, + "balance_loss_mlp": 1.00309515, + "epoch": 0.12515234170970924, + "flos": 59225854181760.0, + "grad_norm": 0.6790802282531619, + "language_loss": 0.48659068, + "learning_rate": 3.905784537605394e-06, + "loss": 0.50694841, + "num_input_tokens_seen": 122600665, + "router_z_loss_clip": 0.20898438, + "router_z_loss_mlp": 0.01171875, + "step": 4313, + "time_per_iteration": 2.928013801574707 + }, + { + "auxiliary_loss_clip": 0.01147745, + "auxiliary_loss_mlp": 0.0105719, + "balance_loss_clip": 1.06070197, + "balance_loss_mlp": 1.04020274, + "epoch": 0.1251813591782253, + "flos": 20263098844800.0, + "grad_norm": 2.4702019863185445, + "language_loss": 0.83416247, + "learning_rate": 3.905727518707247e-06, + "loss": 0.85621178, + "num_input_tokens_seen": 122614070, + "router_z_loss_clip": 0.87060547, + "router_z_loss_mlp": 0.1697998, + "step": 4314, + "time_per_iteration": 2.5151047706604004 + }, + { + "auxiliary_loss_clip": 0.01150598, + "auxiliary_loss_mlp": 0.01061142, + "balance_loss_clip": 1.05976415, + "balance_loss_mlp": 1.04328489, + "epoch": 0.12521037664674134, + "flos": 18544270256640.0, + "grad_norm": 15.552803825644204, + "language_loss": 0.73104608, + "learning_rate": 3.905670482976942e-06, + "loss": 0.75316346, + "num_input_tokens_seen": 122626425, + "router_z_loss_clip": 0.90966797, + "router_z_loss_mlp": 0.17871094, + "step": 4315, + "time_per_iteration": 2.5233960151672363 + }, + { + "auxiliary_loss_clip": 0.01139712, + "auxiliary_loss_mlp": 0.01048806, + "balance_loss_clip": 1.054793, + "balance_loss_mlp": 1.03155637, + "epoch": 0.1252393941152574, + "flos": 26170224053760.0, + "grad_norm": 2.619707406742613, + "language_loss": 0.90008909, + "learning_rate": 3.905613430414986e-06, + "loss": 0.92197424, + "num_input_tokens_seen": 122644825, + "router_z_loss_clip": 0.84912109, + "router_z_loss_mlp": 0.17260742, + "step": 4316, + "time_per_iteration": 2.613960027694702 + }, + { + "auxiliary_loss_clip": 0.01032517, + "auxiliary_loss_mlp": 0.0101186, + "balance_loss_clip": 1.01173699, + "balance_loss_mlp": 1.01070392, + "epoch": 0.12526841158377344, + "flos": 74779340365440.0, + "grad_norm": 0.6254089539174513, + "language_loss": 0.45202816, + "learning_rate": 3.9055563610218805e-06, + "loss": 0.47247192, + "num_input_tokens_seen": 122714850, + "router_z_loss_clip": 0.20800781, + "router_z_loss_mlp": 0.01153564, + "step": 4317, + "time_per_iteration": 3.2748894691467285 + }, + { + "auxiliary_loss_clip": 0.01032764, + "auxiliary_loss_mlp": 0.01007461, + "balance_loss_clip": 1.0120157, + "balance_loss_mlp": 1.00620925, + "epoch": 0.12529742905228947, + "flos": 60776550184320.0, + "grad_norm": 0.6498261554039854, + "language_loss": 0.48810375, + "learning_rate": 3.905499274798129e-06, + "loss": 0.508506, + "num_input_tokens_seen": 122776475, + "router_z_loss_clip": 0.20800781, + "router_z_loss_mlp": 0.01251221, + "step": 4318, + "time_per_iteration": 3.0715222358703613 + }, + { + "auxiliary_loss_clip": 0.01144788, + "auxiliary_loss_mlp": 0.01056758, + "balance_loss_clip": 1.05912709, + "balance_loss_mlp": 1.03617024, + "epoch": 0.12532644652080552, + "flos": 16906708189440.0, + "grad_norm": 2.7944863738775565, + "language_loss": 0.87353075, + "learning_rate": 3.905442171744238e-06, + "loss": 0.8955462, + "num_input_tokens_seen": 122789080, + "router_z_loss_clip": 0.85693359, + "router_z_loss_mlp": 0.20605469, + "step": 4319, + "time_per_iteration": 2.4781742095947266 + }, + { + "auxiliary_loss_clip": 0.01032479, + "auxiliary_loss_mlp": 0.01003543, + "balance_loss_clip": 1.01180124, + "balance_loss_mlp": 1.00234509, + "epoch": 0.12535546398932157, + "flos": 57142209110400.0, + "grad_norm": 0.6964997966093369, + "language_loss": 0.48670727, + "learning_rate": 3.905385051860711e-06, + "loss": 0.5070675, + "num_input_tokens_seen": 122845285, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.01196289, + "step": 4320, + "time_per_iteration": 2.9250524044036865 + }, + { + "auxiliary_loss_clip": 0.01140284, + "auxiliary_loss_mlp": 0.01045768, + "balance_loss_clip": 1.05628753, + "balance_loss_mlp": 1.0284586, + "epoch": 0.12538448145783762, + "flos": 38467150456320.0, + "grad_norm": 2.5166735004971326, + "language_loss": 0.8995558, + "learning_rate": 3.9053279151480515e-06, + "loss": 0.9214164, + "num_input_tokens_seen": 122861700, + "router_z_loss_clip": 0.84179688, + "router_z_loss_mlp": 0.1730957, + "step": 4321, + "time_per_iteration": 2.573535680770874 + }, + { + "auxiliary_loss_clip": 0.01144952, + "auxiliary_loss_mlp": 0.01062803, + "balance_loss_clip": 1.05554461, + "balance_loss_mlp": 1.0441227, + "epoch": 0.12541349892635367, + "flos": 20442113251200.0, + "grad_norm": 9.210898330853352, + "language_loss": 0.84308147, + "learning_rate": 3.905270761606765e-06, + "loss": 0.86515909, + "num_input_tokens_seen": 122874620, + "router_z_loss_clip": 0.89257812, + "router_z_loss_mlp": 0.18676758, + "step": 4322, + "time_per_iteration": 2.490726947784424 + }, + { + "auxiliary_loss_clip": 0.01143287, + "auxiliary_loss_mlp": 0.01043892, + "balance_loss_clip": 1.05530453, + "balance_loss_mlp": 1.02448463, + "epoch": 0.12544251639486972, + "flos": 12496447474560.0, + "grad_norm": 1.8414671063320354, + "language_loss": 0.62617671, + "learning_rate": 3.905213591237356e-06, + "loss": 0.64804852, + "num_input_tokens_seen": 122892190, + "router_z_loss_clip": 0.87939453, + "router_z_loss_mlp": 0.19421387, + "step": 4323, + "time_per_iteration": 2.5338494777679443 + }, + { + "auxiliary_loss_clip": 0.01032137, + "auxiliary_loss_mlp": 0.01000762, + "balance_loss_clip": 1.01159537, + "balance_loss_mlp": 0.99969465, + "epoch": 0.12547153386338575, + "flos": 65053762922880.0, + "grad_norm": 0.688769232350824, + "language_loss": 0.47696054, + "learning_rate": 3.90515640404033e-06, + "loss": 0.49728954, + "num_input_tokens_seen": 122958200, + "router_z_loss_clip": 0.20507812, + "router_z_loss_mlp": 0.01068115, + "step": 4324, + "time_per_iteration": 3.246797561645508 + }, + { + "auxiliary_loss_clip": 0.01151799, + "auxiliary_loss_mlp": 0.01050874, + "balance_loss_clip": 1.06077635, + "balance_loss_mlp": 1.0327307, + "epoch": 0.1255005513319018, + "flos": 34897845934080.0, + "grad_norm": 2.1286103835099768, + "language_loss": 0.86035722, + "learning_rate": 3.905099200016192e-06, + "loss": 0.88238394, + "num_input_tokens_seen": 122976955, + "router_z_loss_clip": 0.91015625, + "router_z_loss_mlp": 0.18127441, + "step": 4325, + "time_per_iteration": 2.656728744506836 + }, + { + "auxiliary_loss_clip": 0.01031153, + "auxiliary_loss_mlp": 0.00998815, + "balance_loss_clip": 1.01059449, + "balance_loss_mlp": 0.99776632, + "epoch": 0.12552956880041785, + "flos": 68063937361920.0, + "grad_norm": 0.7240514993600432, + "language_loss": 0.50423551, + "learning_rate": 3.905041979165446e-06, + "loss": 0.52453518, + "num_input_tokens_seen": 123038310, + "router_z_loss_clip": 0.20507812, + "router_z_loss_mlp": 0.01049805, + "step": 4326, + "time_per_iteration": 3.0823488235473633 + }, + { + "auxiliary_loss_clip": 0.01136487, + "auxiliary_loss_mlp": 0.01045929, + "balance_loss_clip": 1.05608249, + "balance_loss_mlp": 1.03176641, + "epoch": 0.1255585862689339, + "flos": 27956097377280.0, + "grad_norm": 2.7245021247682195, + "language_loss": 0.79028046, + "learning_rate": 3.904984741488598e-06, + "loss": 0.81210458, + "num_input_tokens_seen": 123053145, + "router_z_loss_clip": 0.80517578, + "router_z_loss_mlp": 0.1416626, + "step": 4327, + "time_per_iteration": 2.516397476196289 + }, + { + "auxiliary_loss_clip": 0.01137863, + "auxiliary_loss_mlp": 0.01046468, + "balance_loss_clip": 1.06055665, + "balance_loss_mlp": 1.03130484, + "epoch": 0.12558760373744995, + "flos": 22960313061120.0, + "grad_norm": 2.2490922633427934, + "language_loss": 0.78046191, + "learning_rate": 3.904927486986155e-06, + "loss": 0.80230522, + "num_input_tokens_seen": 123068395, + "router_z_loss_clip": 0.7734375, + "router_z_loss_mlp": 0.15155029, + "step": 4328, + "time_per_iteration": 2.602114677429199 + }, + { + "auxiliary_loss_clip": 0.0114346, + "auxiliary_loss_mlp": 0.01051601, + "balance_loss_clip": 1.05730724, + "balance_loss_mlp": 1.03531063, + "epoch": 0.12561662120596598, + "flos": 17640543047040.0, + "grad_norm": 2.1398957025549623, + "language_loss": 0.7191534, + "learning_rate": 3.904870215658621e-06, + "loss": 0.74110401, + "num_input_tokens_seen": 123085315, + "router_z_loss_clip": 0.86132812, + "router_z_loss_mlp": 0.16290283, + "step": 4329, + "time_per_iteration": 2.5542094707489014 + }, + { + "auxiliary_loss_clip": 0.01146198, + "auxiliary_loss_mlp": 0.0104992, + "balance_loss_clip": 1.0582037, + "balance_loss_mlp": 1.03275418, + "epoch": 0.12564563867448203, + "flos": 16755056968320.0, + "grad_norm": 4.410453955411034, + "language_loss": 0.92074358, + "learning_rate": 3.904812927506503e-06, + "loss": 0.94270474, + "num_input_tokens_seen": 123096210, + "router_z_loss_clip": 0.87988281, + "router_z_loss_mlp": 0.17163086, + "step": 4330, + "time_per_iteration": 2.5475265979766846 + }, + { + "auxiliary_loss_clip": 0.01141451, + "auxiliary_loss_mlp": 0.01050972, + "balance_loss_clip": 1.05638421, + "balance_loss_mlp": 1.03222084, + "epoch": 0.12567465614299808, + "flos": 37115077685760.0, + "grad_norm": 2.175848982800666, + "language_loss": 0.7828967, + "learning_rate": 3.904755622530306e-06, + "loss": 0.80482095, + "num_input_tokens_seen": 123112735, + "router_z_loss_clip": 0.8515625, + "router_z_loss_mlp": 0.1875, + "step": 4331, + "time_per_iteration": 2.672977924346924 + }, + { + "auxiliary_loss_clip": 0.01141503, + "auxiliary_loss_mlp": 0.01040856, + "balance_loss_clip": 1.05791569, + "balance_loss_mlp": 1.02471483, + "epoch": 0.12570367361151413, + "flos": 26212563210240.0, + "grad_norm": 1.7639531612923671, + "language_loss": 0.67427826, + "learning_rate": 3.904698300730537e-06, + "loss": 0.69610184, + "num_input_tokens_seen": 123133345, + "router_z_loss_clip": 0.83544922, + "router_z_loss_mlp": 0.16131592, + "step": 4332, + "time_per_iteration": 2.561616897583008 + }, + { + "auxiliary_loss_clip": 0.01141992, + "auxiliary_loss_mlp": 0.01046666, + "balance_loss_clip": 1.05764198, + "balance_loss_mlp": 1.03195584, + "epoch": 0.12573269108003018, + "flos": 12378803454720.0, + "grad_norm": 2.662943177573125, + "language_loss": 0.78501391, + "learning_rate": 3.904640962107701e-06, + "loss": 0.8069005, + "num_input_tokens_seen": 123146445, + "router_z_loss_clip": 0.84326172, + "router_z_loss_mlp": 0.1472168, + "step": 4333, + "time_per_iteration": 2.4718363285064697 + }, + { + "auxiliary_loss_clip": 0.01138665, + "auxiliary_loss_mlp": 0.01041727, + "balance_loss_clip": 1.05820405, + "balance_loss_mlp": 1.02683783, + "epoch": 0.12576170854854624, + "flos": 31898837646720.0, + "grad_norm": 2.2199157894184567, + "language_loss": 0.81754613, + "learning_rate": 3.904583606662306e-06, + "loss": 0.8393501, + "num_input_tokens_seen": 123163500, + "router_z_loss_clip": 0.80517578, + "router_z_loss_mlp": 0.14898682, + "step": 4334, + "time_per_iteration": 2.6320950984954834 + }, + { + "auxiliary_loss_clip": 0.01135175, + "auxiliary_loss_mlp": 0.01043472, + "balance_loss_clip": 1.05537486, + "balance_loss_mlp": 1.02759886, + "epoch": 0.12579072601706226, + "flos": 30257037774720.0, + "grad_norm": 2.434320414120503, + "language_loss": 0.89717448, + "learning_rate": 3.904526234394858e-06, + "loss": 0.91896093, + "num_input_tokens_seen": 123179145, + "router_z_loss_clip": 0.79833984, + "router_z_loss_mlp": 0.15856934, + "step": 4335, + "time_per_iteration": 2.6448233127593994 + }, + { + "auxiliary_loss_clip": 0.01033465, + "auxiliary_loss_mlp": 0.01000968, + "balance_loss_clip": 1.01274061, + "balance_loss_mlp": 1.00003815, + "epoch": 0.1258197434855783, + "flos": 63987071708160.0, + "grad_norm": 0.6608499771795565, + "language_loss": 0.50003493, + "learning_rate": 3.904468845305863e-06, + "loss": 0.52037925, + "num_input_tokens_seen": 123244260, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.00927734, + "step": 4336, + "time_per_iteration": 3.153576612472534 + }, + { + "auxiliary_loss_clip": 0.01139905, + "auxiliary_loss_mlp": 0.01038291, + "balance_loss_clip": 1.06049466, + "balance_loss_mlp": 1.02222216, + "epoch": 0.12584876095409436, + "flos": 33796429246080.0, + "grad_norm": 2.212789314253854, + "language_loss": 0.74011856, + "learning_rate": 3.904411439395829e-06, + "loss": 0.76190054, + "num_input_tokens_seen": 123261995, + "router_z_loss_clip": 0.79394531, + "router_z_loss_mlp": 0.16052246, + "step": 4337, + "time_per_iteration": 2.703765630722046 + }, + { + "auxiliary_loss_clip": 0.01035416, + "auxiliary_loss_mlp": 0.00999499, + "balance_loss_clip": 1.01486707, + "balance_loss_mlp": 0.99857539, + "epoch": 0.12587777842261041, + "flos": 74776934154240.0, + "grad_norm": 0.6224912951953953, + "language_loss": 0.44734573, + "learning_rate": 3.9043540166652625e-06, + "loss": 0.46769488, + "num_input_tokens_seen": 123329190, + "router_z_loss_clip": 0.20507812, + "router_z_loss_mlp": 0.00921631, + "step": 4338, + "time_per_iteration": 3.1552236080169678 + }, + { + "auxiliary_loss_clip": 0.01147369, + "auxiliary_loss_mlp": 0.01046526, + "balance_loss_clip": 1.06041133, + "balance_loss_mlp": 1.029145, + "epoch": 0.12590679589112647, + "flos": 23617226943360.0, + "grad_norm": 2.2381343097248236, + "language_loss": 0.93014628, + "learning_rate": 3.90429657711467e-06, + "loss": 0.9520852, + "num_input_tokens_seen": 123343300, + "router_z_loss_clip": 0.87011719, + "router_z_loss_mlp": 0.17358398, + "step": 4339, + "time_per_iteration": 4.827743053436279 + }, + { + "auxiliary_loss_clip": 0.01034891, + "auxiliary_loss_mlp": 0.01002668, + "balance_loss_clip": 1.0144968, + "balance_loss_mlp": 1.00172639, + "epoch": 0.12593581335964252, + "flos": 57702021552000.0, + "grad_norm": 0.7140300666670203, + "language_loss": 0.49942911, + "learning_rate": 3.90423912074456e-06, + "loss": 0.51980472, + "num_input_tokens_seen": 123407145, + "router_z_loss_clip": 0.203125, + "router_z_loss_mlp": 0.00939941, + "step": 4340, + "time_per_iteration": 7.889547824859619 + }, + { + "auxiliary_loss_clip": 0.01141462, + "auxiliary_loss_mlp": 0.01046286, + "balance_loss_clip": 1.05748677, + "balance_loss_mlp": 1.02914345, + "epoch": 0.12596483082815854, + "flos": 19528833024000.0, + "grad_norm": 2.3701869662765658, + "language_loss": 1.00436401, + "learning_rate": 3.90418164755544e-06, + "loss": 1.02624154, + "num_input_tokens_seen": 123419395, + "router_z_loss_clip": 0.83886719, + "router_z_loss_mlp": 0.17132568, + "step": 4341, + "time_per_iteration": 2.4892630577087402 + }, + { + "auxiliary_loss_clip": 0.01142254, + "auxiliary_loss_mlp": 0.01036842, + "balance_loss_clip": 1.05921268, + "balance_loss_mlp": 1.02177334, + "epoch": 0.1259938482966746, + "flos": 26754382938240.0, + "grad_norm": 2.1809371040853636, + "language_loss": 0.73974842, + "learning_rate": 3.904124157547817e-06, + "loss": 0.76153934, + "num_input_tokens_seen": 123433890, + "router_z_loss_clip": 0.83056641, + "router_z_loss_mlp": 0.15081787, + "step": 4342, + "time_per_iteration": 4.991887092590332 + }, + { + "auxiliary_loss_clip": 0.01153648, + "auxiliary_loss_mlp": 0.01055614, + "balance_loss_clip": 1.06617963, + "balance_loss_mlp": 1.03731573, + "epoch": 0.12602286576519064, + "flos": 15843285112320.0, + "grad_norm": 2.362749765377428, + "language_loss": 0.72242963, + "learning_rate": 3.9040666507221985e-06, + "loss": 0.74452221, + "num_input_tokens_seen": 123446425, + "router_z_loss_clip": 0.875, + "router_z_loss_mlp": 0.18292236, + "step": 4343, + "time_per_iteration": 2.5058646202087402 + }, + { + "auxiliary_loss_clip": 0.01034695, + "auxiliary_loss_mlp": 0.01005043, + "balance_loss_clip": 1.01407623, + "balance_loss_mlp": 1.00406575, + "epoch": 0.1260518832337067, + "flos": 55222713192960.0, + "grad_norm": 0.7373399973249728, + "language_loss": 0.52355802, + "learning_rate": 3.904009127079093e-06, + "loss": 0.54395533, + "num_input_tokens_seen": 123509040, + "router_z_loss_clip": 0.20605469, + "router_z_loss_mlp": 0.00976562, + "step": 4344, + "time_per_iteration": 3.0444881916046143 + }, + { + "auxiliary_loss_clip": 0.01150433, + "auxiliary_loss_mlp": 0.01046879, + "balance_loss_clip": 1.06110716, + "balance_loss_mlp": 1.02765095, + "epoch": 0.12608090070222275, + "flos": 34197192264960.0, + "grad_norm": 2.799100808237645, + "language_loss": 1.03017664, + "learning_rate": 3.903951586619009e-06, + "loss": 1.05214977, + "num_input_tokens_seen": 123525075, + "router_z_loss_clip": 0.89550781, + "router_z_loss_mlp": 0.19244385, + "step": 4345, + "time_per_iteration": 2.651691198348999 + }, + { + "auxiliary_loss_clip": 0.011492, + "auxiliary_loss_mlp": 0.0103774, + "balance_loss_clip": 1.06232166, + "balance_loss_mlp": 1.02288103, + "epoch": 0.12610991817073877, + "flos": 46966595189760.0, + "grad_norm": 2.1774321815926276, + "language_loss": 0.80825734, + "learning_rate": 3.903894029342453e-06, + "loss": 0.8301267, + "num_input_tokens_seen": 123545165, + "router_z_loss_clip": 0.86816406, + "router_z_loss_mlp": 0.14837646, + "step": 4346, + "time_per_iteration": 2.7631828784942627 + }, + { + "auxiliary_loss_clip": 0.01034318, + "auxiliary_loss_mlp": 0.01002362, + "balance_loss_clip": 1.01384568, + "balance_loss_mlp": 1.00135422, + "epoch": 0.12613893563925482, + "flos": 64148380709760.0, + "grad_norm": 0.7320144220671957, + "language_loss": 0.50357521, + "learning_rate": 3.903836455249935e-06, + "loss": 0.52394199, + "num_input_tokens_seen": 123601105, + "router_z_loss_clip": 0.20507812, + "router_z_loss_mlp": 0.0100708, + "step": 4347, + "time_per_iteration": 2.981874465942383 + }, + { + "auxiliary_loss_clip": 0.01034416, + "auxiliary_loss_mlp": 0.01001462, + "balance_loss_clip": 1.0138706, + "balance_loss_mlp": 1.00046051, + "epoch": 0.12616795310777087, + "flos": 74772301299840.0, + "grad_norm": 0.7195702372068867, + "language_loss": 0.48630565, + "learning_rate": 3.9037788643419635e-06, + "loss": 0.5066644, + "num_input_tokens_seen": 123654925, + "router_z_loss_clip": 0.20507812, + "router_z_loss_mlp": 0.01000977, + "step": 4348, + "time_per_iteration": 3.0674426555633545 + }, + { + "auxiliary_loss_clip": 0.01033513, + "auxiliary_loss_mlp": 0.00999514, + "balance_loss_clip": 1.01285768, + "balance_loss_mlp": 0.99850672, + "epoch": 0.12619697057628693, + "flos": 62405708469120.0, + "grad_norm": 0.7082320234962397, + "language_loss": 0.53031021, + "learning_rate": 3.903721256619046e-06, + "loss": 0.55064052, + "num_input_tokens_seen": 123716735, + "router_z_loss_clip": 0.20605469, + "router_z_loss_mlp": 0.0100708, + "step": 4349, + "time_per_iteration": 3.1099228858947754 + }, + { + "auxiliary_loss_clip": 0.01134274, + "auxiliary_loss_mlp": 0.01044578, + "balance_loss_clip": 1.05621266, + "balance_loss_mlp": 1.0290333, + "epoch": 0.12622598804480298, + "flos": 11869195248000.0, + "grad_norm": 4.597151619567151, + "language_loss": 0.84736645, + "learning_rate": 3.903663632081693e-06, + "loss": 0.86915493, + "num_input_tokens_seen": 123729050, + "router_z_loss_clip": 0.78125, + "router_z_loss_mlp": 0.15557861, + "step": 4350, + "time_per_iteration": 2.4971818923950195 + }, + { + "auxiliary_loss_clip": 0.01148092, + "auxiliary_loss_mlp": 0.01049461, + "balance_loss_clip": 1.06577933, + "balance_loss_mlp": 1.03381443, + "epoch": 0.12625500551331903, + "flos": 19274441927040.0, + "grad_norm": 2.481125754231026, + "language_loss": 0.57678342, + "learning_rate": 3.903605990730411e-06, + "loss": 0.59875894, + "num_input_tokens_seen": 123743560, + "router_z_loss_clip": 0.82275391, + "router_z_loss_mlp": 0.15649414, + "step": 4351, + "time_per_iteration": 2.5181891918182373 + }, + { + "auxiliary_loss_clip": 0.01033951, + "auxiliary_loss_mlp": 0.01004129, + "balance_loss_clip": 1.0133996, + "balance_loss_mlp": 1.00309777, + "epoch": 0.12628402298183505, + "flos": 70005089180160.0, + "grad_norm": 0.6292387337958237, + "language_loss": 0.51049149, + "learning_rate": 3.903548332565712e-06, + "loss": 0.53087229, + "num_input_tokens_seen": 123807385, + "router_z_loss_clip": 0.20507812, + "router_z_loss_mlp": 0.01031494, + "step": 4352, + "time_per_iteration": 3.168180465698242 + }, + { + "auxiliary_loss_clip": 0.01152492, + "auxiliary_loss_mlp": 0.01039447, + "balance_loss_clip": 1.06608748, + "balance_loss_mlp": 1.02201819, + "epoch": 0.1263130404503511, + "flos": 24927858397440.0, + "grad_norm": 3.0211860313002363, + "language_loss": 0.96187061, + "learning_rate": 3.903490657588103e-06, + "loss": 0.98378992, + "num_input_tokens_seen": 123821490, + "router_z_loss_clip": 0.86474609, + "router_z_loss_mlp": 0.17431641, + "step": 4353, + "time_per_iteration": 2.563551425933838 + }, + { + "auxiliary_loss_clip": 0.01138276, + "auxiliary_loss_mlp": 0.01035124, + "balance_loss_clip": 1.05858397, + "balance_loss_mlp": 1.02015114, + "epoch": 0.12634205791886716, + "flos": 28870957802880.0, + "grad_norm": 2.5746772567705425, + "language_loss": 0.82793045, + "learning_rate": 3.9034329657980946e-06, + "loss": 0.84966445, + "num_input_tokens_seen": 123836975, + "router_z_loss_clip": 0.79736328, + "router_z_loss_mlp": 0.14971924, + "step": 4354, + "time_per_iteration": 2.591529607772827 + }, + { + "auxiliary_loss_clip": 0.01145836, + "auxiliary_loss_mlp": 0.01040573, + "balance_loss_clip": 1.06046009, + "balance_loss_mlp": 1.02396083, + "epoch": 0.1263710753873832, + "flos": 35188219480320.0, + "grad_norm": 2.9835540770547295, + "language_loss": 0.85621417, + "learning_rate": 3.903375257196195e-06, + "loss": 0.87807822, + "num_input_tokens_seen": 123855600, + "router_z_loss_clip": 0.85400391, + "router_z_loss_mlp": 0.1661377, + "step": 4355, + "time_per_iteration": 2.6395680904388428 + }, + { + "auxiliary_loss_clip": 0.01155665, + "auxiliary_loss_mlp": 0.01043068, + "balance_loss_clip": 1.06619477, + "balance_loss_mlp": 1.02661157, + "epoch": 0.12640009285589926, + "flos": 37667168703360.0, + "grad_norm": 1.8062433862024359, + "language_loss": 0.9501757, + "learning_rate": 3.9033175317829165e-06, + "loss": 0.97216302, + "num_input_tokens_seen": 123879730, + "router_z_loss_clip": 0.89453125, + "router_z_loss_mlp": 0.16448975, + "step": 4356, + "time_per_iteration": 2.7358815670013428 + }, + { + "auxiliary_loss_clip": 0.01145241, + "auxiliary_loss_mlp": 0.01046114, + "balance_loss_clip": 1.0638833, + "balance_loss_mlp": 1.03019381, + "epoch": 0.1264291103244153, + "flos": 25076169653760.0, + "grad_norm": 2.3612582671032603, + "language_loss": 0.79021156, + "learning_rate": 3.9032597895587666e-06, + "loss": 0.81212509, + "num_input_tokens_seen": 123894475, + "router_z_loss_clip": 0.81396484, + "router_z_loss_mlp": 0.15917969, + "step": 4357, + "time_per_iteration": 2.5619614124298096 + }, + { + "auxiliary_loss_clip": 0.01144622, + "auxiliary_loss_mlp": 0.01042096, + "balance_loss_clip": 1.06245399, + "balance_loss_mlp": 1.02624083, + "epoch": 0.12645812779293134, + "flos": 11757728367360.0, + "grad_norm": 2.6062616453549365, + "language_loss": 0.91137743, + "learning_rate": 3.903202030524256e-06, + "loss": 0.93324459, + "num_input_tokens_seen": 123905720, + "router_z_loss_clip": 0.82128906, + "router_z_loss_mlp": 0.15869141, + "step": 4358, + "time_per_iteration": 2.4787895679473877 + }, + { + "auxiliary_loss_clip": 0.01037783, + "auxiliary_loss_mlp": 0.01007984, + "balance_loss_clip": 1.01689804, + "balance_loss_mlp": 1.00702465, + "epoch": 0.1264871452614474, + "flos": 63723091680000.0, + "grad_norm": 0.7226999308246012, + "language_loss": 0.48227406, + "learning_rate": 3.903144254679895e-06, + "loss": 0.50273174, + "num_input_tokens_seen": 123968350, + "router_z_loss_clip": 0.20898438, + "router_z_loss_mlp": 0.00958252, + "step": 4359, + "time_per_iteration": 3.0855841636657715 + }, + { + "auxiliary_loss_clip": 0.01146818, + "auxiliary_loss_mlp": 0.01041051, + "balance_loss_clip": 1.06040764, + "balance_loss_mlp": 1.02279377, + "epoch": 0.12651616272996344, + "flos": 23871115249920.0, + "grad_norm": 2.487554691931761, + "language_loss": 0.87124556, + "learning_rate": 3.903086462026194e-06, + "loss": 0.89312428, + "num_input_tokens_seen": 123984265, + "router_z_loss_clip": 0.86474609, + "router_z_loss_mlp": 0.18261719, + "step": 4360, + "time_per_iteration": 2.666989326477051 + }, + { + "auxiliary_loss_clip": 0.01037387, + "auxiliary_loss_mlp": 0.010001, + "balance_loss_clip": 1.01671469, + "balance_loss_mlp": 0.99910432, + "epoch": 0.1265451801984795, + "flos": 74795321917440.0, + "grad_norm": 0.6161817047013356, + "language_loss": 0.44245833, + "learning_rate": 3.903028652563663e-06, + "loss": 0.46283323, + "num_input_tokens_seen": 124050510, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.00994873, + "step": 4361, + "time_per_iteration": 3.342769145965576 + }, + { + "auxiliary_loss_clip": 0.01151914, + "auxiliary_loss_mlp": 0.01062783, + "balance_loss_clip": 1.06020677, + "balance_loss_mlp": 1.04411435, + "epoch": 0.12657419766699554, + "flos": 27849945709440.0, + "grad_norm": 2.265090752684267, + "language_loss": 0.82879591, + "learning_rate": 3.902970826292814e-06, + "loss": 0.85094291, + "num_input_tokens_seen": 124067260, + "router_z_loss_clip": 0.91650391, + "router_z_loss_mlp": 0.18676758, + "step": 4362, + "time_per_iteration": 2.5721607208251953 + }, + { + "auxiliary_loss_clip": 0.0115285, + "auxiliary_loss_mlp": 0.01044602, + "balance_loss_clip": 1.06255102, + "balance_loss_mlp": 1.02617264, + "epoch": 0.12660321513551157, + "flos": 31750670044800.0, + "grad_norm": 2.662406423543734, + "language_loss": 1.00804007, + "learning_rate": 3.902912983214155e-06, + "loss": 1.03001451, + "num_input_tokens_seen": 124086600, + "router_z_loss_clip": 0.90234375, + "router_z_loss_mlp": 0.18432617, + "step": 4363, + "time_per_iteration": 2.691321611404419 + }, + { + "auxiliary_loss_clip": 0.01037631, + "auxiliary_loss_mlp": 0.01004187, + "balance_loss_clip": 1.01683664, + "balance_loss_mlp": 1.00313842, + "epoch": 0.12663223260402762, + "flos": 65801924328960.0, + "grad_norm": 0.7056274804589996, + "language_loss": 0.53167391, + "learning_rate": 3.9028551233281985e-06, + "loss": 0.55209208, + "num_input_tokens_seen": 124150145, + "router_z_loss_clip": 0.20800781, + "router_z_loss_mlp": 0.01049805, + "step": 4364, + "time_per_iteration": 3.1714768409729004 + }, + { + "auxiliary_loss_clip": 0.01142971, + "auxiliary_loss_mlp": 0.01046982, + "balance_loss_clip": 1.05957472, + "balance_loss_mlp": 1.03115082, + "epoch": 0.12666125007254367, + "flos": 22266016099200.0, + "grad_norm": 5.368227797649842, + "language_loss": 0.90961266, + "learning_rate": 3.9027972466354565e-06, + "loss": 0.93151218, + "num_input_tokens_seen": 124165865, + "router_z_loss_clip": 0.83398438, + "router_z_loss_mlp": 0.15838623, + "step": 4365, + "time_per_iteration": 2.5192501544952393 + }, + { + "auxiliary_loss_clip": 0.01133087, + "auxiliary_loss_mlp": 0.01043921, + "balance_loss_clip": 1.05684745, + "balance_loss_mlp": 1.02873945, + "epoch": 0.12669026754105972, + "flos": 18800636601600.0, + "grad_norm": 1.925229881972271, + "language_loss": 0.63317943, + "learning_rate": 3.902739353136439e-06, + "loss": 0.65494955, + "num_input_tokens_seen": 124181365, + "router_z_loss_clip": 0.76220703, + "router_z_loss_mlp": 0.15179443, + "step": 4366, + "time_per_iteration": 2.5419936180114746 + }, + { + "auxiliary_loss_clip": 0.0114052, + "auxiliary_loss_mlp": 0.0103414, + "balance_loss_clip": 1.05852234, + "balance_loss_mlp": 1.0188694, + "epoch": 0.12671928500957577, + "flos": 16135813474560.0, + "grad_norm": 2.7590975886150417, + "language_loss": 0.99719411, + "learning_rate": 3.902681442831658e-06, + "loss": 1.01894069, + "num_input_tokens_seen": 124193845, + "router_z_loss_clip": 0.8203125, + "router_z_loss_mlp": 0.15283203, + "step": 4367, + "time_per_iteration": 2.5147626399993896 + }, + { + "auxiliary_loss_clip": 0.01145431, + "auxiliary_loss_mlp": 0.01043837, + "balance_loss_clip": 1.06212211, + "balance_loss_mlp": 1.02738631, + "epoch": 0.12674830247809182, + "flos": 27083899330560.0, + "grad_norm": 2.594655241584102, + "language_loss": 0.90293664, + "learning_rate": 3.902623515721623e-06, + "loss": 0.92482936, + "num_input_tokens_seen": 124210385, + "router_z_loss_clip": 0.83251953, + "router_z_loss_mlp": 0.16442871, + "step": 4368, + "time_per_iteration": 2.571049451828003 + }, + { + "auxiliary_loss_clip": 0.0114934, + "auxiliary_loss_mlp": 0.01049042, + "balance_loss_clip": 1.05924451, + "balance_loss_mlp": 1.03083849, + "epoch": 0.12677731994660785, + "flos": 16318419240960.0, + "grad_norm": 2.8614482941154566, + "language_loss": 0.90291023, + "learning_rate": 3.902565571806849e-06, + "loss": 0.92489403, + "num_input_tokens_seen": 124222565, + "router_z_loss_clip": 0.90039062, + "router_z_loss_mlp": 0.1819458, + "step": 4369, + "time_per_iteration": 2.6600239276885986 + }, + { + "auxiliary_loss_clip": 0.01036966, + "auxiliary_loss_mlp": 0.01005287, + "balance_loss_clip": 1.01579857, + "balance_loss_mlp": 1.00422621, + "epoch": 0.1268063374151239, + "flos": 63867991144320.0, + "grad_norm": 0.6496849636627916, + "language_loss": 0.4874669, + "learning_rate": 3.902507611087845e-06, + "loss": 0.50788945, + "num_input_tokens_seen": 124283845, + "router_z_loss_clip": 0.2109375, + "router_z_loss_mlp": 0.01062012, + "step": 4370, + "time_per_iteration": 3.1039016246795654 + }, + { + "auxiliary_loss_clip": 0.01140208, + "auxiliary_loss_mlp": 0.01052891, + "balance_loss_clip": 1.05788863, + "balance_loss_mlp": 1.03512335, + "epoch": 0.12683535488363995, + "flos": 14534377511040.0, + "grad_norm": 2.4580989128297874, + "language_loss": 0.88510323, + "learning_rate": 3.902449633565124e-06, + "loss": 0.90703428, + "num_input_tokens_seen": 124296535, + "router_z_loss_clip": 0.82177734, + "router_z_loss_mlp": 0.17773438, + "step": 4371, + "time_per_iteration": 2.515202760696411 + }, + { + "auxiliary_loss_clip": 0.01138328, + "auxiliary_loss_mlp": 0.01043241, + "balance_loss_clip": 1.05820191, + "balance_loss_mlp": 1.02757704, + "epoch": 0.126864372352156, + "flos": 24966103403520.0, + "grad_norm": 4.185133966978806, + "language_loss": 0.84597039, + "learning_rate": 3.902391639239199e-06, + "loss": 0.86778605, + "num_input_tokens_seen": 124311450, + "router_z_loss_clip": 0.80029297, + "router_z_loss_mlp": 0.15649414, + "step": 4372, + "time_per_iteration": 2.578489303588867 + }, + { + "auxiliary_loss_clip": 0.01037382, + "auxiliary_loss_mlp": 0.01004378, + "balance_loss_clip": 1.01610935, + "balance_loss_mlp": 1.0033412, + "epoch": 0.12689338982067205, + "flos": 68205101811840.0, + "grad_norm": 0.6901726316530599, + "language_loss": 0.52926236, + "learning_rate": 3.90233362811058e-06, + "loss": 0.54967999, + "num_input_tokens_seen": 124372815, + "router_z_loss_clip": 0.21289062, + "router_z_loss_mlp": 0.01037598, + "step": 4373, + "time_per_iteration": 3.1521637439727783 + }, + { + "auxiliary_loss_clip": 0.01137909, + "auxiliary_loss_mlp": 0.01042442, + "balance_loss_clip": 1.05341315, + "balance_loss_mlp": 1.02583051, + "epoch": 0.1269224072891881, + "flos": 30950041847040.0, + "grad_norm": 1.9786585099485539, + "language_loss": 0.89995933, + "learning_rate": 3.9022756001797805e-06, + "loss": 0.92176288, + "num_input_tokens_seen": 124391845, + "router_z_loss_clip": 0.84472656, + "router_z_loss_mlp": 0.1661377, + "step": 4374, + "time_per_iteration": 2.6807775497436523 + }, + { + "auxiliary_loss_clip": 0.01142774, + "auxiliary_loss_mlp": 0.01046308, + "balance_loss_clip": 1.06007028, + "balance_loss_mlp": 1.02818823, + "epoch": 0.12695142475770413, + "flos": 22157494133760.0, + "grad_norm": 3.380280469059275, + "language_loss": 0.83609426, + "learning_rate": 3.902217555447314e-06, + "loss": 0.85798502, + "num_input_tokens_seen": 124405435, + "router_z_loss_clip": 0.82666016, + "router_z_loss_mlp": 0.18115234, + "step": 4375, + "time_per_iteration": 2.504122257232666 + }, + { + "auxiliary_loss_clip": 0.01142676, + "auxiliary_loss_mlp": 0.01043117, + "balance_loss_clip": 1.06020927, + "balance_loss_mlp": 1.02661252, + "epoch": 0.12698044222622018, + "flos": 52366769798400.0, + "grad_norm": 1.9371598542013575, + "language_loss": 0.80288494, + "learning_rate": 3.902159493913692e-06, + "loss": 0.82474285, + "num_input_tokens_seen": 124426975, + "router_z_loss_clip": 0.82568359, + "router_z_loss_mlp": 0.16503906, + "step": 4376, + "time_per_iteration": 2.7459120750427246 + }, + { + "auxiliary_loss_clip": 0.01154975, + "auxiliary_loss_mlp": 0.01047137, + "balance_loss_clip": 1.06309462, + "balance_loss_mlp": 1.02772951, + "epoch": 0.12700945969473623, + "flos": 36933764808960.0, + "grad_norm": 1.9624536832118937, + "language_loss": 0.84830731, + "learning_rate": 3.902101415579427e-06, + "loss": 0.87032843, + "num_input_tokens_seen": 124451650, + "router_z_loss_clip": 0.91796875, + "router_z_loss_mlp": 0.19421387, + "step": 4377, + "time_per_iteration": 2.760545492172241 + }, + { + "auxiliary_loss_clip": 0.01144073, + "auxiliary_loss_mlp": 0.01049484, + "balance_loss_clip": 1.06176078, + "balance_loss_mlp": 1.03314638, + "epoch": 0.12703847716325228, + "flos": 15516462240000.0, + "grad_norm": 2.1611562726347056, + "language_loss": 0.73676342, + "learning_rate": 3.902043320445033e-06, + "loss": 0.758699, + "num_input_tokens_seen": 124466675, + "router_z_loss_clip": 0.82324219, + "router_z_loss_mlp": 0.16339111, + "step": 4378, + "time_per_iteration": 2.570286512374878 + }, + { + "auxiliary_loss_clip": 0.01135413, + "auxiliary_loss_mlp": 0.01041575, + "balance_loss_clip": 1.05421507, + "balance_loss_mlp": 1.02437937, + "epoch": 0.12706749463176834, + "flos": 24902111324160.0, + "grad_norm": 2.7190272109201783, + "language_loss": 0.77309364, + "learning_rate": 3.901985208511023e-06, + "loss": 0.79486346, + "num_input_tokens_seen": 124486990, + "router_z_loss_clip": 0.8125, + "router_z_loss_mlp": 0.17211914, + "step": 4379, + "time_per_iteration": 2.587698459625244 + }, + { + "auxiliary_loss_clip": 0.01135848, + "auxiliary_loss_mlp": 0.01043944, + "balance_loss_clip": 1.0558238, + "balance_loss_mlp": 1.02733219, + "epoch": 0.12709651210028436, + "flos": 31205187129600.0, + "grad_norm": 3.403629751194209, + "language_loss": 0.9168762, + "learning_rate": 3.90192707977791e-06, + "loss": 0.93867415, + "num_input_tokens_seen": 124501870, + "router_z_loss_clip": 0.79980469, + "router_z_loss_mlp": 0.16601562, + "step": 4380, + "time_per_iteration": 2.595531702041626 + }, + { + "auxiliary_loss_clip": 0.01034639, + "auxiliary_loss_mlp": 0.01001418, + "balance_loss_clip": 1.01340795, + "balance_loss_mlp": 1.00031495, + "epoch": 0.1271255295688004, + "flos": 60077692195200.0, + "grad_norm": 0.7127943951057079, + "language_loss": 0.5472492, + "learning_rate": 3.901868934246208e-06, + "loss": 0.56760979, + "num_input_tokens_seen": 124562555, + "router_z_loss_clip": 0.21289062, + "router_z_loss_mlp": 0.01104736, + "step": 4381, + "time_per_iteration": 3.110912561416626 + }, + { + "auxiliary_loss_clip": 0.01136533, + "auxiliary_loss_mlp": 0.01046705, + "balance_loss_clip": 1.05463839, + "balance_loss_mlp": 1.02934825, + "epoch": 0.12715454703731646, + "flos": 26352578424960.0, + "grad_norm": 2.5875259780205133, + "language_loss": 0.89911211, + "learning_rate": 3.9018107719164285e-06, + "loss": 0.92094457, + "num_input_tokens_seen": 124577830, + "router_z_loss_clip": 0.81884766, + "router_z_loss_mlp": 0.17346191, + "step": 4382, + "time_per_iteration": 2.559520721435547 + }, + { + "auxiliary_loss_clip": 0.01033461, + "auxiliary_loss_mlp": 0.01002631, + "balance_loss_clip": 1.01231134, + "balance_loss_mlp": 1.00167775, + "epoch": 0.1271835645058325, + "flos": 57779696712960.0, + "grad_norm": 0.6782465510076608, + "language_loss": 0.52227187, + "learning_rate": 3.901752592789088e-06, + "loss": 0.54263282, + "num_input_tokens_seen": 124639895, + "router_z_loss_clip": 0.2109375, + "router_z_loss_mlp": 0.00952148, + "step": 4383, + "time_per_iteration": 3.2063064575195312 + }, + { + "auxiliary_loss_clip": 0.01032668, + "auxiliary_loss_mlp": 0.0100324, + "balance_loss_clip": 1.01139522, + "balance_loss_mlp": 1.00217879, + "epoch": 0.12721258197434857, + "flos": 57506055926400.0, + "grad_norm": 0.7292042085542454, + "language_loss": 0.46726626, + "learning_rate": 3.901694396864698e-06, + "loss": 0.48762533, + "num_input_tokens_seen": 124689070, + "router_z_loss_clip": 0.21289062, + "router_z_loss_mlp": 0.01062012, + "step": 4384, + "time_per_iteration": 2.8495426177978516 + }, + { + "auxiliary_loss_clip": 0.01141882, + "auxiliary_loss_mlp": 0.01059142, + "balance_loss_clip": 1.05676961, + "balance_loss_mlp": 1.04003286, + "epoch": 0.12724159944286462, + "flos": 23108839799040.0, + "grad_norm": 1.9854964331057674, + "language_loss": 0.6764589, + "learning_rate": 3.901636184143774e-06, + "loss": 0.69846916, + "num_input_tokens_seen": 124703560, + "router_z_loss_clip": 0.8515625, + "router_z_loss_mlp": 0.19116211, + "step": 4385, + "time_per_iteration": 2.561124086380005 + }, + { + "auxiliary_loss_clip": 0.01142379, + "auxiliary_loss_mlp": 0.01037329, + "balance_loss_clip": 1.05824184, + "balance_loss_mlp": 1.02121139, + "epoch": 0.12727061691138064, + "flos": 32903512030080.0, + "grad_norm": 2.327328991936759, + "language_loss": 0.91163713, + "learning_rate": 3.901577954626829e-06, + "loss": 0.93343413, + "num_input_tokens_seen": 124731940, + "router_z_loss_clip": 0.84179688, + "router_z_loss_mlp": 0.16125488, + "step": 4386, + "time_per_iteration": 2.8910791873931885 + }, + { + "auxiliary_loss_clip": 0.01132154, + "auxiliary_loss_mlp": 0.01049016, + "balance_loss_clip": 1.05409908, + "balance_loss_mlp": 1.03316092, + "epoch": 0.1272996343798967, + "flos": 19784121960960.0, + "grad_norm": 2.361868442112632, + "language_loss": 0.92882729, + "learning_rate": 3.901519708314379e-06, + "loss": 0.95063889, + "num_input_tokens_seen": 124745735, + "router_z_loss_clip": 0.78125, + "router_z_loss_mlp": 0.15863037, + "step": 4387, + "time_per_iteration": 2.627113103866577 + }, + { + "auxiliary_loss_clip": 0.01137574, + "auxiliary_loss_mlp": 0.01046895, + "balance_loss_clip": 1.05501723, + "balance_loss_mlp": 1.030146, + "epoch": 0.12732865184841274, + "flos": 29089330536960.0, + "grad_norm": 3.7548728748177993, + "language_loss": 0.92148709, + "learning_rate": 3.901461445206937e-06, + "loss": 0.94333184, + "num_input_tokens_seen": 124761225, + "router_z_loss_clip": 0.82568359, + "router_z_loss_mlp": 0.16760254, + "step": 4388, + "time_per_iteration": 2.6042673587799072 + }, + { + "auxiliary_loss_clip": 0.01129711, + "auxiliary_loss_mlp": 0.01042417, + "balance_loss_clip": 1.0510149, + "balance_loss_mlp": 1.02725315, + "epoch": 0.1273576693169288, + "flos": 18726983763840.0, + "grad_norm": 3.1044104318141117, + "language_loss": 1.09469104, + "learning_rate": 3.901403165305018e-06, + "loss": 1.11641228, + "num_input_tokens_seen": 124773850, + "router_z_loss_clip": 0.78710938, + "router_z_loss_mlp": 0.15167236, + "step": 4389, + "time_per_iteration": 2.5003390312194824 + }, + { + "auxiliary_loss_clip": 0.01142826, + "auxiliary_loss_mlp": 0.01048439, + "balance_loss_clip": 1.05803752, + "balance_loss_mlp": 1.03171349, + "epoch": 0.12738668678544485, + "flos": 26976095637120.0, + "grad_norm": 2.1184519759877927, + "language_loss": 0.89718592, + "learning_rate": 3.901344868609138e-06, + "loss": 0.91909856, + "num_input_tokens_seen": 124792030, + "router_z_loss_clip": 0.84765625, + "router_z_loss_mlp": 0.16723633, + "step": 4390, + "time_per_iteration": 2.6908628940582275 + }, + { + "auxiliary_loss_clip": 0.01132044, + "auxiliary_loss_mlp": 0.01042899, + "balance_loss_clip": 1.05525661, + "balance_loss_mlp": 1.02846825, + "epoch": 0.12741570425396087, + "flos": 22156704034560.0, + "grad_norm": 5.128764624195539, + "language_loss": 0.75925565, + "learning_rate": 3.90128655511981e-06, + "loss": 0.78100508, + "num_input_tokens_seen": 124804840, + "router_z_loss_clip": 0.76855469, + "router_z_loss_mlp": 0.14428711, + "step": 4391, + "time_per_iteration": 2.544619083404541 + }, + { + "auxiliary_loss_clip": 0.01140051, + "auxiliary_loss_mlp": 0.01044794, + "balance_loss_clip": 1.05616593, + "balance_loss_mlp": 1.02838516, + "epoch": 0.12744472172247692, + "flos": 11318899910400.0, + "grad_norm": 3.5994197757931765, + "language_loss": 0.9324894, + "learning_rate": 3.901228224837549e-06, + "loss": 0.95433789, + "num_input_tokens_seen": 124814995, + "router_z_loss_clip": 0.83935547, + "router_z_loss_mlp": 0.16394043, + "step": 4392, + "time_per_iteration": 2.5448203086853027 + }, + { + "auxiliary_loss_clip": 0.01041532, + "auxiliary_loss_mlp": 0.01007012, + "balance_loss_clip": 1.02017856, + "balance_loss_mlp": 1.00590324, + "epoch": 0.12747373919099297, + "flos": 64203110058240.0, + "grad_norm": 0.6880064637598698, + "language_loss": 0.53429329, + "learning_rate": 3.901169877762872e-06, + "loss": 0.55477875, + "num_input_tokens_seen": 124876650, + "router_z_loss_clip": 0.21289062, + "router_z_loss_mlp": 0.0111084, + "step": 4393, + "time_per_iteration": 3.1704766750335693 + }, + { + "auxiliary_loss_clip": 0.01140533, + "auxiliary_loss_mlp": 0.01035213, + "balance_loss_clip": 1.05972266, + "balance_loss_mlp": 1.01928091, + "epoch": 0.12750275665950903, + "flos": 25039540759680.0, + "grad_norm": 2.3616680466910687, + "language_loss": 0.78641784, + "learning_rate": 3.9011115138962925e-06, + "loss": 0.80817533, + "num_input_tokens_seen": 124890835, + "router_z_loss_clip": 0.80712891, + "router_z_loss_mlp": 0.15930176, + "step": 4394, + "time_per_iteration": 2.4813997745513916 + }, + { + "auxiliary_loss_clip": 0.010404, + "auxiliary_loss_mlp": 0.01003887, + "balance_loss_clip": 1.01916075, + "balance_loss_mlp": 1.00288606, + "epoch": 0.12753177412802508, + "flos": 61318154430720.0, + "grad_norm": 0.689711259593126, + "language_loss": 0.51772368, + "learning_rate": 3.901053133238327e-06, + "loss": 0.53816658, + "num_input_tokens_seen": 124951770, + "router_z_loss_clip": 0.21289062, + "router_z_loss_mlp": 0.01000977, + "step": 4395, + "time_per_iteration": 3.048856258392334 + }, + { + "auxiliary_loss_clip": 0.01131644, + "auxiliary_loss_mlp": 0.01049381, + "balance_loss_clip": 1.05503213, + "balance_loss_mlp": 1.03464127, + "epoch": 0.12756079159654113, + "flos": 25550621424000.0, + "grad_norm": 2.1324744993819107, + "language_loss": 0.67590308, + "learning_rate": 3.900994735789491e-06, + "loss": 0.69771338, + "num_input_tokens_seen": 124967740, + "router_z_loss_clip": 0.76757812, + "router_z_loss_mlp": 0.1473999, + "step": 4396, + "time_per_iteration": 2.547719955444336 + }, + { + "auxiliary_loss_clip": 0.01140673, + "auxiliary_loss_mlp": 0.01039828, + "balance_loss_clip": 1.05586314, + "balance_loss_mlp": 1.02267337, + "epoch": 0.12758980906505715, + "flos": 9970741722240.0, + "grad_norm": 3.174912027617702, + "language_loss": 1.02628374, + "learning_rate": 3.9009363215503005e-06, + "loss": 1.04808879, + "num_input_tokens_seen": 124977200, + "router_z_loss_clip": 0.84814453, + "router_z_loss_mlp": 0.17163086, + "step": 4397, + "time_per_iteration": 2.4756433963775635 + }, + { + "auxiliary_loss_clip": 0.01139214, + "auxiliary_loss_mlp": 0.01045904, + "balance_loss_clip": 1.05453515, + "balance_loss_mlp": 1.03004324, + "epoch": 0.1276188265335732, + "flos": 28650358425600.0, + "grad_norm": 1.8667820463902813, + "language_loss": 0.81302083, + "learning_rate": 3.900877890521271e-06, + "loss": 0.83487201, + "num_input_tokens_seen": 124991355, + "router_z_loss_clip": 0.84716797, + "router_z_loss_mlp": 0.15856934, + "step": 4398, + "time_per_iteration": 2.601442575454712 + }, + { + "auxiliary_loss_clip": 0.01034874, + "auxiliary_loss_mlp": 0.01004419, + "balance_loss_clip": 1.0135591, + "balance_loss_mlp": 1.00341153, + "epoch": 0.12764784400208926, + "flos": 60459887882880.0, + "grad_norm": 0.7152851799691429, + "language_loss": 0.48832172, + "learning_rate": 3.90081944270292e-06, + "loss": 0.50871468, + "num_input_tokens_seen": 125045630, + "router_z_loss_clip": 0.21289062, + "router_z_loss_mlp": 0.0100708, + "step": 4399, + "time_per_iteration": 2.9423201084136963 + }, + { + "auxiliary_loss_clip": 0.01033404, + "auxiliary_loss_mlp": 0.01005531, + "balance_loss_clip": 1.01224017, + "balance_loss_mlp": 1.00459504, + "epoch": 0.1276768614706053, + "flos": 49884555415680.0, + "grad_norm": 0.6942380764358284, + "language_loss": 0.49734974, + "learning_rate": 3.900760978095761e-06, + "loss": 0.51773906, + "num_input_tokens_seen": 125105310, + "router_z_loss_clip": 0.21191406, + "router_z_loss_mlp": 0.00933838, + "step": 4400, + "time_per_iteration": 3.0596959590911865 + }, + { + "auxiliary_loss_clip": 0.01030358, + "auxiliary_loss_mlp": 0.01001328, + "balance_loss_clip": 1.00923836, + "balance_loss_mlp": 1.00036812, + "epoch": 0.12770587893912136, + "flos": 64448558668800.0, + "grad_norm": 0.653471091516153, + "language_loss": 0.47794315, + "learning_rate": 3.900702496700312e-06, + "loss": 0.49826002, + "num_input_tokens_seen": 125161330, + "router_z_loss_clip": 0.2109375, + "router_z_loss_mlp": 0.00958252, + "step": 4401, + "time_per_iteration": 3.0047645568847656 + }, + { + "auxiliary_loss_clip": 0.01139071, + "auxiliary_loss_mlp": 0.01042488, + "balance_loss_clip": 1.05472386, + "balance_loss_mlp": 1.02486897, + "epoch": 0.1277348964076374, + "flos": 23872336312320.0, + "grad_norm": 2.0465355229843594, + "language_loss": 0.76897979, + "learning_rate": 3.90064399851709e-06, + "loss": 0.79079533, + "num_input_tokens_seen": 125178490, + "router_z_loss_clip": 0.84228516, + "router_z_loss_mlp": 0.17626953, + "step": 4402, + "time_per_iteration": 2.5490880012512207 + }, + { + "auxiliary_loss_clip": 0.01130177, + "auxiliary_loss_mlp": 0.0103671, + "balance_loss_clip": 1.04843283, + "balance_loss_mlp": 1.02101588, + "epoch": 0.12776391387615343, + "flos": 20442867436800.0, + "grad_norm": 3.494445995630353, + "language_loss": 0.84772819, + "learning_rate": 3.90058548354661e-06, + "loss": 0.8693971, + "num_input_tokens_seen": 125194175, + "router_z_loss_clip": 0.81787109, + "router_z_loss_mlp": 0.15679932, + "step": 4403, + "time_per_iteration": 2.534198045730591 + }, + { + "auxiliary_loss_clip": 0.01127676, + "auxiliary_loss_mlp": 0.01041839, + "balance_loss_clip": 1.05226588, + "balance_loss_mlp": 1.02778983, + "epoch": 0.12779293134466949, + "flos": 33394337424000.0, + "grad_norm": 2.22350073002956, + "language_loss": 0.80870426, + "learning_rate": 3.900526951789391e-06, + "loss": 0.83039939, + "num_input_tokens_seen": 125212290, + "router_z_loss_clip": 0.75341797, + "router_z_loss_mlp": 0.14044189, + "step": 4404, + "time_per_iteration": 2.6632373332977295 + }, + { + "auxiliary_loss_clip": 0.01146087, + "auxiliary_loss_mlp": 0.01042424, + "balance_loss_clip": 1.05582619, + "balance_loss_mlp": 1.02499557, + "epoch": 0.12782194881318554, + "flos": 13215952805760.0, + "grad_norm": 2.822684846065298, + "language_loss": 0.7922163, + "learning_rate": 3.900468403245949e-06, + "loss": 0.81410146, + "num_input_tokens_seen": 125226525, + "router_z_loss_clip": 0.90234375, + "router_z_loss_mlp": 0.17437744, + "step": 4405, + "time_per_iteration": 2.536038398742676 + }, + { + "auxiliary_loss_clip": 0.01130499, + "auxiliary_loss_mlp": 0.01033851, + "balance_loss_clip": 1.05202031, + "balance_loss_mlp": 1.01925957, + "epoch": 0.1278509662817016, + "flos": 35510912288640.0, + "grad_norm": 1.725697216550484, + "language_loss": 0.69381994, + "learning_rate": 3.9004098379168e-06, + "loss": 0.71546346, + "num_input_tokens_seen": 125246285, + "router_z_loss_clip": 0.78466797, + "router_z_loss_mlp": 0.14587402, + "step": 4406, + "time_per_iteration": 2.681389570236206 + }, + { + "auxiliary_loss_clip": 0.01137782, + "auxiliary_loss_mlp": 0.0104044, + "balance_loss_clip": 1.05480242, + "balance_loss_mlp": 1.02414393, + "epoch": 0.12787998375021764, + "flos": 11175401076480.0, + "grad_norm": 3.8095691051330833, + "language_loss": 0.65822411, + "learning_rate": 3.900351255802463e-06, + "loss": 0.68000633, + "num_input_tokens_seen": 125257820, + "router_z_loss_clip": 0.82958984, + "router_z_loss_mlp": 0.16290283, + "step": 4407, + "time_per_iteration": 2.5100085735321045 + }, + { + "auxiliary_loss_clip": 0.0113246, + "auxiliary_loss_mlp": 0.01035121, + "balance_loss_clip": 1.05173969, + "balance_loss_mlp": 1.02091694, + "epoch": 0.12790900121873366, + "flos": 15625343341440.0, + "grad_norm": 2.482538507202008, + "language_loss": 0.91371346, + "learning_rate": 3.900292656903454e-06, + "loss": 0.93538928, + "num_input_tokens_seen": 125269835, + "router_z_loss_clip": 0.80664062, + "router_z_loss_mlp": 0.14196777, + "step": 4408, + "time_per_iteration": 2.499190330505371 + }, + { + "auxiliary_loss_clip": 0.01132346, + "auxiliary_loss_mlp": 0.01041622, + "balance_loss_clip": 1.05272162, + "balance_loss_mlp": 1.02480745, + "epoch": 0.12793801868724972, + "flos": 16682948415360.0, + "grad_norm": 2.483532393940729, + "language_loss": 0.74911749, + "learning_rate": 3.900234041220292e-06, + "loss": 0.77085721, + "num_input_tokens_seen": 125281530, + "router_z_loss_clip": 0.796875, + "router_z_loss_mlp": 0.16802979, + "step": 4409, + "time_per_iteration": 2.5095722675323486 + }, + { + "auxiliary_loss_clip": 0.0112586, + "auxiliary_loss_mlp": 0.01036853, + "balance_loss_clip": 1.04763687, + "balance_loss_mlp": 1.02250648, + "epoch": 0.12796703615576577, + "flos": 14641103796480.0, + "grad_norm": 3.4321483137201674, + "language_loss": 0.95034033, + "learning_rate": 3.900175408753494e-06, + "loss": 0.97196746, + "num_input_tokens_seen": 125296315, + "router_z_loss_clip": 0.78173828, + "router_z_loss_mlp": 0.14331055, + "step": 4410, + "time_per_iteration": 4.985158443450928 + }, + { + "auxiliary_loss_clip": 0.01132221, + "auxiliary_loss_mlp": 0.01036123, + "balance_loss_clip": 1.05118322, + "balance_loss_mlp": 1.01957655, + "epoch": 0.12799605362428182, + "flos": 74731934413440.0, + "grad_norm": 2.149308332028526, + "language_loss": 0.99677348, + "learning_rate": 3.900116759503578e-06, + "loss": 1.01845694, + "num_input_tokens_seen": 125321410, + "router_z_loss_clip": 0.81005859, + "router_z_loss_mlp": 0.16564941, + "step": 4411, + "time_per_iteration": 5.183081388473511 + }, + { + "auxiliary_loss_clip": 0.01148545, + "auxiliary_loss_mlp": 0.01050893, + "balance_loss_clip": 1.05877888, + "balance_loss_mlp": 1.03416777, + "epoch": 0.12802507109279787, + "flos": 29272187698560.0, + "grad_norm": 2.034982279016075, + "language_loss": 0.95238298, + "learning_rate": 3.900058093471062e-06, + "loss": 0.97437733, + "num_input_tokens_seen": 125339625, + "router_z_loss_clip": 0.89599609, + "router_z_loss_mlp": 0.16723633, + "step": 4412, + "time_per_iteration": 5.112962484359741 + }, + { + "auxiliary_loss_clip": 0.01141797, + "auxiliary_loss_mlp": 0.01045777, + "balance_loss_clip": 1.05378306, + "balance_loss_mlp": 1.02809823, + "epoch": 0.12805408856131392, + "flos": 38759355596160.0, + "grad_norm": 2.226896329067771, + "language_loss": 1.02807641, + "learning_rate": 3.899999410656463e-06, + "loss": 1.04995227, + "num_input_tokens_seen": 125359605, + "router_z_loss_clip": 0.87988281, + "router_z_loss_mlp": 0.17675781, + "step": 4413, + "time_per_iteration": 5.127564907073975 + }, + { + "auxiliary_loss_clip": 0.01146811, + "auxiliary_loss_mlp": 0.01042746, + "balance_loss_clip": 1.05362415, + "balance_loss_mlp": 1.02474523, + "epoch": 0.12808310602982995, + "flos": 14346743840640.0, + "grad_norm": 3.0126827669052267, + "language_loss": 1.05793309, + "learning_rate": 3.899940711060301e-06, + "loss": 1.07982862, + "num_input_tokens_seen": 125371675, + "router_z_loss_clip": 0.93212891, + "router_z_loss_mlp": 0.18005371, + "step": 4414, + "time_per_iteration": 2.504490852355957 + }, + { + "auxiliary_loss_clip": 0.01032811, + "auxiliary_loss_mlp": 0.01011267, + "balance_loss_clip": 1.01181841, + "balance_loss_mlp": 1.01024234, + "epoch": 0.128112123498346, + "flos": 62658232058880.0, + "grad_norm": 0.661085192768792, + "language_loss": 0.41068596, + "learning_rate": 3.899881994683094e-06, + "loss": 0.43112671, + "num_input_tokens_seen": 125433970, + "router_z_loss_clip": 0.20996094, + "router_z_loss_mlp": 0.01025391, + "step": 4415, + "time_per_iteration": 3.111588716506958 + }, + { + "auxiliary_loss_clip": 0.01134428, + "auxiliary_loss_mlp": 0.01038354, + "balance_loss_clip": 1.05455387, + "balance_loss_mlp": 1.02262402, + "epoch": 0.12814114096686205, + "flos": 24876723386880.0, + "grad_norm": 2.71605858197745, + "language_loss": 0.80362338, + "learning_rate": 3.89982326152536e-06, + "loss": 0.82535124, + "num_input_tokens_seen": 125452455, + "router_z_loss_clip": 0.79882812, + "router_z_loss_mlp": 0.15722656, + "step": 4416, + "time_per_iteration": 2.6110010147094727 + }, + { + "auxiliary_loss_clip": 0.01031718, + "auxiliary_loss_mlp": 0.01007885, + "balance_loss_clip": 1.01067436, + "balance_loss_mlp": 1.0068543, + "epoch": 0.1281701584353781, + "flos": 65686291470720.0, + "grad_norm": 0.7389590327733863, + "language_loss": 0.49119234, + "learning_rate": 3.899764511587618e-06, + "loss": 0.51158839, + "num_input_tokens_seen": 125511045, + "router_z_loss_clip": 0.2109375, + "router_z_loss_mlp": 0.01031494, + "step": 4417, + "time_per_iteration": 3.0890309810638428 + }, + { + "auxiliary_loss_clip": 0.01140415, + "auxiliary_loss_mlp": 0.01037921, + "balance_loss_clip": 1.05583, + "balance_loss_mlp": 1.02189887, + "epoch": 0.12819917590389415, + "flos": 17959895890560.0, + "grad_norm": 2.2231838881799635, + "language_loss": 0.79646772, + "learning_rate": 3.899705744870388e-06, + "loss": 0.81825113, + "num_input_tokens_seen": 125524145, + "router_z_loss_clip": 0.84619141, + "router_z_loss_mlp": 0.16015625, + "step": 4418, + "time_per_iteration": 2.512006998062134 + }, + { + "auxiliary_loss_clip": 0.01148279, + "auxiliary_loss_mlp": 0.01057756, + "balance_loss_clip": 1.05631351, + "balance_loss_mlp": 1.03812194, + "epoch": 0.1282281933724102, + "flos": 28873364014080.0, + "grad_norm": 2.1996594458433147, + "language_loss": 0.97059047, + "learning_rate": 3.899646961374188e-06, + "loss": 0.99265081, + "num_input_tokens_seen": 125543915, + "router_z_loss_clip": 0.91894531, + "router_z_loss_mlp": 0.19647217, + "step": 4419, + "time_per_iteration": 2.6176071166992188 + }, + { + "auxiliary_loss_clip": 0.01129861, + "auxiliary_loss_mlp": 0.01041595, + "balance_loss_clip": 1.05271304, + "balance_loss_mlp": 1.02649689, + "epoch": 0.12825721084092623, + "flos": 37224820713600.0, + "grad_norm": 2.402286034494927, + "language_loss": 0.66681665, + "learning_rate": 3.899588161099537e-06, + "loss": 0.68853122, + "num_input_tokens_seen": 125560745, + "router_z_loss_clip": 0.77197266, + "router_z_loss_mlp": 0.15087891, + "step": 4420, + "time_per_iteration": 2.723245143890381 + }, + { + "auxiliary_loss_clip": 0.01127153, + "auxiliary_loss_mlp": 0.01037161, + "balance_loss_clip": 1.05117726, + "balance_loss_mlp": 1.02330327, + "epoch": 0.12828622830944228, + "flos": 17487922158720.0, + "grad_norm": 2.722738756137547, + "language_loss": 0.63179743, + "learning_rate": 3.899529344046955e-06, + "loss": 0.65344059, + "num_input_tokens_seen": 125575595, + "router_z_loss_clip": 0.76025391, + "router_z_loss_mlp": 0.13842773, + "step": 4421, + "time_per_iteration": 2.4717400074005127 + }, + { + "auxiliary_loss_clip": 0.01126433, + "auxiliary_loss_mlp": 0.01043135, + "balance_loss_clip": 1.05261552, + "balance_loss_mlp": 1.02844834, + "epoch": 0.12831524577795833, + "flos": 15918877284480.0, + "grad_norm": 2.454806530152521, + "language_loss": 0.72762966, + "learning_rate": 3.89947051021696e-06, + "loss": 0.74932528, + "num_input_tokens_seen": 125589115, + "router_z_loss_clip": 0.73779297, + "router_z_loss_mlp": 0.14697266, + "step": 4422, + "time_per_iteration": 2.477449417114258 + }, + { + "auxiliary_loss_clip": 0.01028564, + "auxiliary_loss_mlp": 0.0100429, + "balance_loss_clip": 1.00742507, + "balance_loss_mlp": 1.00320566, + "epoch": 0.12834426324647438, + "flos": 74769643693440.0, + "grad_norm": 0.6560663568376798, + "language_loss": 0.47406912, + "learning_rate": 3.899411659610075e-06, + "loss": 0.49439764, + "num_input_tokens_seen": 125650970, + "router_z_loss_clip": 0.2109375, + "router_z_loss_mlp": 0.01086426, + "step": 4423, + "time_per_iteration": 3.150247812271118 + }, + { + "auxiliary_loss_clip": 0.0113534, + "auxiliary_loss_mlp": 0.01049455, + "balance_loss_clip": 1.05582285, + "balance_loss_mlp": 1.03365338, + "epoch": 0.12837328071499043, + "flos": 14056944912000.0, + "grad_norm": 2.45570523330223, + "language_loss": 0.8902241, + "learning_rate": 3.899352792226815e-06, + "loss": 0.912072, + "num_input_tokens_seen": 125664460, + "router_z_loss_clip": 0.79541016, + "router_z_loss_mlp": 0.15789795, + "step": 4424, + "time_per_iteration": 2.51495361328125 + }, + { + "auxiliary_loss_clip": 0.01132626, + "auxiliary_loss_mlp": 0.01038732, + "balance_loss_clip": 1.05626285, + "balance_loss_mlp": 1.02402794, + "epoch": 0.12840229818350646, + "flos": 23031703342080.0, + "grad_norm": 3.9098028849278994, + "language_loss": 0.78754389, + "learning_rate": 3.899293908067705e-06, + "loss": 0.80925751, + "num_input_tokens_seen": 125677580, + "router_z_loss_clip": 0.76318359, + "router_z_loss_mlp": 0.14703369, + "step": 4425, + "time_per_iteration": 2.518244504928589 + }, + { + "auxiliary_loss_clip": 0.01029937, + "auxiliary_loss_mlp": 0.01002567, + "balance_loss_clip": 1.0089407, + "balance_loss_mlp": 1.00148773, + "epoch": 0.1284313156520225, + "flos": 68960158629120.0, + "grad_norm": 0.6911658742715305, + "language_loss": 0.49709645, + "learning_rate": 3.899235007133261e-06, + "loss": 0.51742148, + "num_input_tokens_seen": 125738505, + "router_z_loss_clip": 0.20996094, + "router_z_loss_mlp": 0.01080322, + "step": 4426, + "time_per_iteration": 3.154958724975586 + }, + { + "auxiliary_loss_clip": 0.01031264, + "auxiliary_loss_mlp": 0.01002278, + "balance_loss_clip": 1.01020622, + "balance_loss_mlp": 1.00124133, + "epoch": 0.12846033312053856, + "flos": 74775102560640.0, + "grad_norm": 0.6320621324936273, + "language_loss": 0.45719588, + "learning_rate": 3.899176089424005e-06, + "loss": 0.47753131, + "num_input_tokens_seen": 125802835, + "router_z_loss_clip": 0.2109375, + "router_z_loss_mlp": 0.01037598, + "step": 4427, + "time_per_iteration": 3.1772148609161377 + }, + { + "auxiliary_loss_clip": 0.01140729, + "auxiliary_loss_mlp": 0.01041649, + "balance_loss_clip": 1.05613518, + "balance_loss_mlp": 1.02376795, + "epoch": 0.1284893505890546, + "flos": 25477291808640.0, + "grad_norm": 2.507104189537442, + "language_loss": 0.79701108, + "learning_rate": 3.899117154940458e-06, + "loss": 0.8188349, + "num_input_tokens_seen": 125815590, + "router_z_loss_clip": 0.84716797, + "router_z_loss_mlp": 0.17871094, + "step": 4428, + "time_per_iteration": 2.550044536590576 + }, + { + "auxiliary_loss_clip": 0.0103001, + "auxiliary_loss_mlp": 0.01000858, + "balance_loss_clip": 1.00898361, + "balance_loss_mlp": 0.99985069, + "epoch": 0.12851836805757066, + "flos": 74762748282240.0, + "grad_norm": 0.6319043506359242, + "language_loss": 0.47315812, + "learning_rate": 3.8990582036831395e-06, + "loss": 0.49346679, + "num_input_tokens_seen": 125877510, + "router_z_loss_clip": 0.2109375, + "router_z_loss_mlp": 0.0100708, + "step": 4429, + "time_per_iteration": 3.315164089202881 + }, + { + "auxiliary_loss_clip": 0.01131498, + "auxiliary_loss_mlp": 0.010382, + "balance_loss_clip": 1.05372262, + "balance_loss_mlp": 1.02375162, + "epoch": 0.12854738552608672, + "flos": 29927269987200.0, + "grad_norm": 6.7683376598332945, + "language_loss": 0.84992778, + "learning_rate": 3.8989992356525704e-06, + "loss": 0.87162483, + "num_input_tokens_seen": 125890850, + "router_z_loss_clip": 0.77685547, + "router_z_loss_mlp": 0.14447021, + "step": 4430, + "time_per_iteration": 2.598945379257202 + }, + { + "auxiliary_loss_clip": 0.01137121, + "auxiliary_loss_mlp": 0.01047685, + "balance_loss_clip": 1.05484211, + "balance_loss_mlp": 1.03107905, + "epoch": 0.12857640299460274, + "flos": 25767952663680.0, + "grad_norm": 5.500922591588807, + "language_loss": 0.95602006, + "learning_rate": 3.898940250849272e-06, + "loss": 0.97786814, + "num_input_tokens_seen": 125905155, + "router_z_loss_clip": 0.82226562, + "router_z_loss_mlp": 0.16625977, + "step": 4431, + "time_per_iteration": 2.6179416179656982 + }, + { + "auxiliary_loss_clip": 0.01029334, + "auxiliary_loss_mlp": 0.01004863, + "balance_loss_clip": 1.00845718, + "balance_loss_mlp": 1.00383139, + "epoch": 0.1286054204631188, + "flos": 65247750322560.0, + "grad_norm": 0.690140267704901, + "language_loss": 0.49466237, + "learning_rate": 3.898881249273764e-06, + "loss": 0.5150044, + "num_input_tokens_seen": 125968210, + "router_z_loss_clip": 0.20898438, + "router_z_loss_mlp": 0.01031494, + "step": 4432, + "time_per_iteration": 3.089841604232788 + }, + { + "auxiliary_loss_clip": 0.01148041, + "auxiliary_loss_mlp": 0.0105515, + "balance_loss_clip": 1.06045198, + "balance_loss_mlp": 1.03854394, + "epoch": 0.12863443793163484, + "flos": 28066235454720.0, + "grad_norm": 2.599949908267823, + "language_loss": 0.92369044, + "learning_rate": 3.898822230926569e-06, + "loss": 0.94572234, + "num_input_tokens_seen": 125981235, + "router_z_loss_clip": 0.87597656, + "router_z_loss_mlp": 0.16601562, + "step": 4433, + "time_per_iteration": 2.6536295413970947 + }, + { + "auxiliary_loss_clip": 0.01155686, + "auxiliary_loss_mlp": 0.01057164, + "balance_loss_clip": 1.05932629, + "balance_loss_mlp": 1.03640974, + "epoch": 0.1286634554001509, + "flos": 17664314872320.0, + "grad_norm": 39.51045582431965, + "language_loss": 1.04547238, + "learning_rate": 3.898763195808208e-06, + "loss": 1.06760097, + "num_input_tokens_seen": 125995830, + "router_z_loss_clip": 0.96337891, + "router_z_loss_mlp": 0.2076416, + "step": 4434, + "time_per_iteration": 2.6202597618103027 + }, + { + "auxiliary_loss_clip": 0.01146121, + "auxiliary_loss_mlp": 0.0104541, + "balance_loss_clip": 1.05640507, + "balance_loss_mlp": 1.0279603, + "epoch": 0.12869247286866695, + "flos": 24635796912000.0, + "grad_norm": 2.3443246522739565, + "language_loss": 0.96148747, + "learning_rate": 3.898704143919201e-06, + "loss": 0.98340279, + "num_input_tokens_seen": 126012255, + "router_z_loss_clip": 0.89746094, + "router_z_loss_mlp": 0.17446899, + "step": 4435, + "time_per_iteration": 2.6209068298339844 + }, + { + "auxiliary_loss_clip": 0.0114371, + "auxiliary_loss_mlp": 0.01054326, + "balance_loss_clip": 1.05506897, + "balance_loss_mlp": 1.03572989, + "epoch": 0.128721490337183, + "flos": 16394837425920.0, + "grad_norm": 3.113889578642605, + "language_loss": 0.69448465, + "learning_rate": 3.898645075260071e-06, + "loss": 0.716465, + "num_input_tokens_seen": 126031665, + "router_z_loss_clip": 0.88525391, + "router_z_loss_mlp": 0.18585205, + "step": 4436, + "time_per_iteration": 2.6708810329437256 + }, + { + "auxiliary_loss_clip": 0.01136635, + "auxiliary_loss_mlp": 0.01043946, + "balance_loss_clip": 1.05381691, + "balance_loss_mlp": 1.02812064, + "epoch": 0.12875050780569902, + "flos": 32373145762560.0, + "grad_norm": 1.8532523000773715, + "language_loss": 0.81550741, + "learning_rate": 3.89858598983134e-06, + "loss": 0.83731323, + "num_input_tokens_seen": 126054270, + "router_z_loss_clip": 0.82763672, + "router_z_loss_mlp": 0.15814209, + "step": 4437, + "time_per_iteration": 2.7233054637908936 + }, + { + "auxiliary_loss_clip": 0.01139008, + "auxiliary_loss_mlp": 0.01044986, + "balance_loss_clip": 1.0560596, + "balance_loss_mlp": 1.0289166, + "epoch": 0.12877952527421507, + "flos": 14639344030080.0, + "grad_norm": 3.0230112904126414, + "language_loss": 0.8335619, + "learning_rate": 3.898526887633529e-06, + "loss": 0.85540187, + "num_input_tokens_seen": 126064720, + "router_z_loss_clip": 0.82910156, + "router_z_loss_mlp": 0.1607666, + "step": 4438, + "time_per_iteration": 2.5104281902313232 + }, + { + "auxiliary_loss_clip": 0.01029365, + "auxiliary_loss_mlp": 0.01015941, + "balance_loss_clip": 1.00872993, + "balance_loss_mlp": 1.01468301, + "epoch": 0.12880854274273112, + "flos": 74776072227840.0, + "grad_norm": 0.72482133859417, + "language_loss": 0.46891928, + "learning_rate": 3.89846776866716e-06, + "loss": 0.48937231, + "num_input_tokens_seen": 126112780, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.01257324, + "step": 4439, + "time_per_iteration": 3.1739699840545654 + }, + { + "auxiliary_loss_clip": 0.01028955, + "auxiliary_loss_mlp": 0.01006947, + "balance_loss_clip": 1.00821161, + "balance_loss_mlp": 1.00587392, + "epoch": 0.12883756021124718, + "flos": 72581175757440.0, + "grad_norm": 0.6765740969824976, + "language_loss": 0.46522558, + "learning_rate": 3.898408632932756e-06, + "loss": 0.48558462, + "num_input_tokens_seen": 126179160, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.01074219, + "step": 4440, + "time_per_iteration": 3.2179667949676514 + }, + { + "auxiliary_loss_clip": 0.01138849, + "auxiliary_loss_mlp": 0.01044871, + "balance_loss_clip": 1.05504501, + "balance_loss_mlp": 1.02813363, + "epoch": 0.12886657767976323, + "flos": 62476126104960.0, + "grad_norm": 2.277973486131476, + "language_loss": 0.87599277, + "learning_rate": 3.898349480430839e-06, + "loss": 0.89783001, + "num_input_tokens_seen": 126203005, + "router_z_loss_clip": 0.83789062, + "router_z_loss_mlp": 0.16729736, + "step": 4441, + "time_per_iteration": 2.840393543243408 + }, + { + "auxiliary_loss_clip": 0.01029461, + "auxiliary_loss_mlp": 0.01000982, + "balance_loss_clip": 1.00863159, + "balance_loss_mlp": 0.99989146, + "epoch": 0.12889559514827925, + "flos": 74777221463040.0, + "grad_norm": 0.756165040673209, + "language_loss": 0.55621004, + "learning_rate": 3.89829031116193e-06, + "loss": 0.57651448, + "num_input_tokens_seen": 126264685, + "router_z_loss_clip": 0.20898438, + "router_z_loss_mlp": 0.01092529, + "step": 4442, + "time_per_iteration": 3.133270025253296 + }, + { + "auxiliary_loss_clip": 0.01146348, + "auxiliary_loss_mlp": 0.01049368, + "balance_loss_clip": 1.05679178, + "balance_loss_mlp": 1.03106976, + "epoch": 0.1289246126167953, + "flos": 23798791215360.0, + "grad_norm": 2.4031538297282005, + "language_loss": 0.84461331, + "learning_rate": 3.898231125126553e-06, + "loss": 0.86657047, + "num_input_tokens_seen": 126279200, + "router_z_loss_clip": 0.89550781, + "router_z_loss_mlp": 0.18292236, + "step": 4443, + "time_per_iteration": 2.5612518787384033 + }, + { + "auxiliary_loss_clip": 0.01132058, + "auxiliary_loss_mlp": 0.01044523, + "balance_loss_clip": 1.05101275, + "balance_loss_mlp": 1.02829289, + "epoch": 0.12895363008531135, + "flos": 25841569587840.0, + "grad_norm": 2.2835768228832136, + "language_loss": 0.70863914, + "learning_rate": 3.898171922325232e-06, + "loss": 0.73040491, + "num_input_tokens_seen": 126294840, + "router_z_loss_clip": 0.81054688, + "router_z_loss_mlp": 0.16229248, + "step": 4444, + "time_per_iteration": 2.5618157386779785 + }, + { + "auxiliary_loss_clip": 0.01129788, + "auxiliary_loss_mlp": 0.01037868, + "balance_loss_clip": 1.05227137, + "balance_loss_mlp": 1.02328908, + "epoch": 0.1289826475538274, + "flos": 18072045129600.0, + "grad_norm": 2.0532317798850364, + "language_loss": 0.74004269, + "learning_rate": 3.898112702758487e-06, + "loss": 0.76171929, + "num_input_tokens_seen": 126310265, + "router_z_loss_clip": 0.77539062, + "router_z_loss_mlp": 0.14569092, + "step": 4445, + "time_per_iteration": 2.5162670612335205 + }, + { + "auxiliary_loss_clip": 0.01136121, + "auxiliary_loss_mlp": 0.01041458, + "balance_loss_clip": 1.05467629, + "balance_loss_mlp": 1.02632475, + "epoch": 0.12901166502234346, + "flos": 14933883553920.0, + "grad_norm": 2.74287940003276, + "language_loss": 0.79510349, + "learning_rate": 3.898053466426843e-06, + "loss": 0.81687933, + "num_input_tokens_seen": 126325110, + "router_z_loss_clip": 0.81445312, + "router_z_loss_mlp": 0.15130615, + "step": 4446, + "time_per_iteration": 2.4877607822418213 + }, + { + "auxiliary_loss_clip": 0.01130252, + "auxiliary_loss_mlp": 0.0104726, + "balance_loss_clip": 1.05124855, + "balance_loss_mlp": 1.03092194, + "epoch": 0.1290406824908595, + "flos": 26061845742720.0, + "grad_norm": 3.211752708941245, + "language_loss": 1.0834806, + "learning_rate": 3.897994213330823e-06, + "loss": 1.10525584, + "num_input_tokens_seen": 126338895, + "router_z_loss_clip": 0.78955078, + "router_z_loss_mlp": 0.16320801, + "step": 4447, + "time_per_iteration": 2.6188483238220215 + }, + { + "auxiliary_loss_clip": 0.01033484, + "auxiliary_loss_mlp": 0.01011229, + "balance_loss_clip": 1.01271892, + "balance_loss_mlp": 1.01030529, + "epoch": 0.12906969995937553, + "flos": 69705198529920.0, + "grad_norm": 0.6630078203962716, + "language_loss": 0.48188886, + "learning_rate": 3.89793494347095e-06, + "loss": 0.50233603, + "num_input_tokens_seen": 126402480, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.00921631, + "step": 4448, + "time_per_iteration": 3.1707332134246826 + }, + { + "auxiliary_loss_clip": 0.01133001, + "auxiliary_loss_mlp": 0.01040323, + "balance_loss_clip": 1.05172884, + "balance_loss_mlp": 1.02499914, + "epoch": 0.12909871742789158, + "flos": 16867780824960.0, + "grad_norm": 2.0768566383474965, + "language_loss": 0.78476948, + "learning_rate": 3.897875656847747e-06, + "loss": 0.8065027, + "num_input_tokens_seen": 126416585, + "router_z_loss_clip": 0.8125, + "router_z_loss_mlp": 0.15325928, + "step": 4449, + "time_per_iteration": 2.5153191089630127 + }, + { + "auxiliary_loss_clip": 0.0103394, + "auxiliary_loss_mlp": 0.01011905, + "balance_loss_clip": 1.01313353, + "balance_loss_mlp": 1.01097476, + "epoch": 0.12912773489640764, + "flos": 74779124883840.0, + "grad_norm": 0.672563293407183, + "language_loss": 0.48286545, + "learning_rate": 3.897816353461739e-06, + "loss": 0.50332391, + "num_input_tokens_seen": 126484415, + "router_z_loss_clip": 0.20800781, + "router_z_loss_mlp": 0.00927734, + "step": 4450, + "time_per_iteration": 3.1869421005249023 + }, + { + "auxiliary_loss_clip": 0.01032733, + "auxiliary_loss_mlp": 0.01003553, + "balance_loss_clip": 1.01176, + "balance_loss_mlp": 1.00261104, + "epoch": 0.1291567523649237, + "flos": 65442099836160.0, + "grad_norm": 0.6842308823860025, + "language_loss": 0.51839334, + "learning_rate": 3.8977570333134484e-06, + "loss": 0.53875625, + "num_input_tokens_seen": 126551925, + "router_z_loss_clip": 0.20898438, + "router_z_loss_mlp": 0.00939941, + "step": 4451, + "time_per_iteration": 3.2279953956604004 + }, + { + "auxiliary_loss_clip": 0.01152188, + "auxiliary_loss_mlp": 0.01050422, + "balance_loss_clip": 1.05886507, + "balance_loss_mlp": 1.03089547, + "epoch": 0.12918576983343974, + "flos": 35878134983040.0, + "grad_norm": 2.195590532434233, + "language_loss": 0.92002767, + "learning_rate": 3.8976976964034e-06, + "loss": 0.94205379, + "num_input_tokens_seen": 126572960, + "router_z_loss_clip": 0.93310547, + "router_z_loss_mlp": 0.19543457, + "step": 4452, + "time_per_iteration": 2.6901345252990723 + }, + { + "auxiliary_loss_clip": 0.01143954, + "auxiliary_loss_mlp": 0.01050611, + "balance_loss_clip": 1.05545068, + "balance_loss_mlp": 1.03230643, + "epoch": 0.12921478730195576, + "flos": 10992256606080.0, + "grad_norm": 2.8272234516707693, + "language_loss": 0.81288362, + "learning_rate": 3.897638342732118e-06, + "loss": 0.83482933, + "num_input_tokens_seen": 126584815, + "router_z_loss_clip": 0.88476562, + "router_z_loss_mlp": 0.1829834, + "step": 4453, + "time_per_iteration": 2.5509185791015625 + }, + { + "auxiliary_loss_clip": 0.01142439, + "auxiliary_loss_mlp": 0.01050734, + "balance_loss_clip": 1.0563519, + "balance_loss_mlp": 1.03361535, + "epoch": 0.12924380477047182, + "flos": 31971772212480.0, + "grad_norm": 1.7030012193978175, + "language_loss": 0.77460206, + "learning_rate": 3.897578972300126e-06, + "loss": 0.7965337, + "num_input_tokens_seen": 126607485, + "router_z_loss_clip": 0.86132812, + "router_z_loss_mlp": 0.17102051, + "step": 4454, + "time_per_iteration": 2.7263994216918945 + }, + { + "auxiliary_loss_clip": 0.01031914, + "auxiliary_loss_mlp": 0.0099965, + "balance_loss_clip": 1.01095641, + "balance_loss_mlp": 0.99860716, + "epoch": 0.12927282223898787, + "flos": 71415874730880.0, + "grad_norm": 0.6811754859755006, + "language_loss": 0.50718886, + "learning_rate": 3.897519585107948e-06, + "loss": 0.5275045, + "num_input_tokens_seen": 126671435, + "router_z_loss_clip": 0.20898438, + "router_z_loss_mlp": 0.01043701, + "step": 4455, + "time_per_iteration": 3.0860743522644043 + }, + { + "auxiliary_loss_clip": 0.01144144, + "auxiliary_loss_mlp": 0.01042034, + "balance_loss_clip": 1.0564965, + "balance_loss_mlp": 1.02420068, + "epoch": 0.12930183970750392, + "flos": 41382593752320.0, + "grad_norm": 4.3461509052920855, + "language_loss": 0.90005034, + "learning_rate": 3.89746018115611e-06, + "loss": 0.92191207, + "num_input_tokens_seen": 126686620, + "router_z_loss_clip": 0.87695312, + "router_z_loss_mlp": 0.17834473, + "step": 4456, + "time_per_iteration": 2.7386045455932617 + }, + { + "auxiliary_loss_clip": 0.01136694, + "auxiliary_loss_mlp": 0.01042685, + "balance_loss_clip": 1.05288589, + "balance_loss_mlp": 1.02684176, + "epoch": 0.12933085717601997, + "flos": 20406597678720.0, + "grad_norm": 2.202710917528006, + "language_loss": 0.81913447, + "learning_rate": 3.897400760445136e-06, + "loss": 0.84092832, + "num_input_tokens_seen": 126702535, + "router_z_loss_clip": 0.83789062, + "router_z_loss_mlp": 0.1585083, + "step": 4457, + "time_per_iteration": 2.5117549896240234 + }, + { + "auxiliary_loss_clip": 0.01030964, + "auxiliary_loss_mlp": 0.01010547, + "balance_loss_clip": 1.01024461, + "balance_loss_mlp": 1.00937927, + "epoch": 0.12935987464453602, + "flos": 65397067159680.0, + "grad_norm": 0.6691394656409412, + "language_loss": 0.48725772, + "learning_rate": 3.8973413229755496e-06, + "loss": 0.50767279, + "num_input_tokens_seen": 126762320, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.01165771, + "step": 4458, + "time_per_iteration": 3.0625040531158447 + }, + { + "auxiliary_loss_clip": 0.01132405, + "auxiliary_loss_mlp": 0.01041546, + "balance_loss_clip": 1.05526853, + "balance_loss_mlp": 1.02430785, + "epoch": 0.12938889211305205, + "flos": 28325510801280.0, + "grad_norm": 2.4145931293077583, + "language_loss": 0.74212158, + "learning_rate": 3.897281868747877e-06, + "loss": 0.76386106, + "num_input_tokens_seen": 126777830, + "router_z_loss_clip": 0.77050781, + "router_z_loss_mlp": 0.17260742, + "step": 4459, + "time_per_iteration": 2.560302972793579 + }, + { + "auxiliary_loss_clip": 0.01137115, + "auxiliary_loss_mlp": 0.01038746, + "balance_loss_clip": 1.05412054, + "balance_loss_mlp": 1.02295065, + "epoch": 0.1294179095815681, + "flos": 16099759198080.0, + "grad_norm": 2.3346470769003744, + "language_loss": 0.78765363, + "learning_rate": 3.897222397762644e-06, + "loss": 0.80941224, + "num_input_tokens_seen": 126790320, + "router_z_loss_clip": 0.83056641, + "router_z_loss_mlp": 0.15789795, + "step": 4460, + "time_per_iteration": 2.476433277130127 + }, + { + "auxiliary_loss_clip": 0.01031253, + "auxiliary_loss_mlp": 0.01016163, + "balance_loss_clip": 1.01053333, + "balance_loss_mlp": 1.01498854, + "epoch": 0.12944692705008415, + "flos": 63181379692800.0, + "grad_norm": 0.5897231027477271, + "language_loss": 0.44154838, + "learning_rate": 3.8971629100203754e-06, + "loss": 0.46202251, + "num_input_tokens_seen": 126859375, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.01171875, + "step": 4461, + "time_per_iteration": 3.3351211547851562 + }, + { + "auxiliary_loss_clip": 0.01133526, + "auxiliary_loss_mlp": 0.01037608, + "balance_loss_clip": 1.05273843, + "balance_loss_mlp": 1.02137184, + "epoch": 0.1294759445186002, + "flos": 29672196531840.0, + "grad_norm": 2.7782339799874674, + "language_loss": 0.9038685, + "learning_rate": 3.897103405521595e-06, + "loss": 0.92557985, + "num_input_tokens_seen": 126873155, + "router_z_loss_clip": 0.80859375, + "router_z_loss_mlp": 0.16247559, + "step": 4462, + "time_per_iteration": 2.600863218307495 + }, + { + "auxiliary_loss_clip": 0.01032458, + "auxiliary_loss_mlp": 0.0100989, + "balance_loss_clip": 1.01171827, + "balance_loss_mlp": 1.00876367, + "epoch": 0.12950496198711625, + "flos": 64893959314560.0, + "grad_norm": 0.698580077134859, + "language_loss": 0.54994226, + "learning_rate": 3.89704388426683e-06, + "loss": 0.57036579, + "num_input_tokens_seen": 126937975, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.0112915, + "step": 4463, + "time_per_iteration": 3.1273553371429443 + }, + { + "auxiliary_loss_clip": 0.01033238, + "auxiliary_loss_mlp": 0.01008013, + "balance_loss_clip": 1.0124836, + "balance_loss_mlp": 1.00693977, + "epoch": 0.1295339794556323, + "flos": 74784116874240.0, + "grad_norm": 0.6265535150174186, + "language_loss": 0.48223972, + "learning_rate": 3.896984346256606e-06, + "loss": 0.50265223, + "num_input_tokens_seen": 127006760, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.01074219, + "step": 4464, + "time_per_iteration": 3.2430524826049805 + }, + { + "auxiliary_loss_clip": 0.01031834, + "auxiliary_loss_mlp": 0.01005903, + "balance_loss_clip": 1.01122451, + "balance_loss_mlp": 1.00489604, + "epoch": 0.12956299692414833, + "flos": 69238719578880.0, + "grad_norm": 0.6749932070448089, + "language_loss": 0.4752965, + "learning_rate": 3.896924791491449e-06, + "loss": 0.49567387, + "num_input_tokens_seen": 127069445, + "router_z_loss_clip": 0.20605469, + "router_z_loss_mlp": 0.0100708, + "step": 4465, + "time_per_iteration": 3.1592466831207275 + }, + { + "auxiliary_loss_clip": 0.01031529, + "auxiliary_loss_mlp": 0.01001164, + "balance_loss_clip": 1.01098061, + "balance_loss_mlp": 1.0001626, + "epoch": 0.12959201439266438, + "flos": 64112649655680.0, + "grad_norm": 0.6639664327482503, + "language_loss": 0.48919559, + "learning_rate": 3.896865219971884e-06, + "loss": 0.50952256, + "num_input_tokens_seen": 127124670, + "router_z_loss_clip": 0.20507812, + "router_z_loss_mlp": 0.01000977, + "step": 4466, + "time_per_iteration": 2.934863805770874 + }, + { + "auxiliary_loss_clip": 0.01147456, + "auxiliary_loss_mlp": 0.01051862, + "balance_loss_clip": 1.05744934, + "balance_loss_mlp": 1.03337264, + "epoch": 0.12962103186118043, + "flos": 20224315134720.0, + "grad_norm": 2.719685352680539, + "language_loss": 0.78947419, + "learning_rate": 3.896805631698438e-06, + "loss": 0.81146741, + "num_input_tokens_seen": 127137495, + "router_z_loss_clip": 0.89990234, + "router_z_loss_mlp": 0.18481445, + "step": 4467, + "time_per_iteration": 2.5540103912353516 + }, + { + "auxiliary_loss_clip": 0.01147276, + "auxiliary_loss_mlp": 0.01048839, + "balance_loss_clip": 1.05935001, + "balance_loss_mlp": 1.03139913, + "epoch": 0.12965004932969648, + "flos": 39960854553600.0, + "grad_norm": 2.9882744863994675, + "language_loss": 0.70889461, + "learning_rate": 3.896746026671637e-06, + "loss": 0.7308557, + "num_input_tokens_seen": 127153460, + "router_z_loss_clip": 0.88037109, + "router_z_loss_mlp": 0.17443848, + "step": 4468, + "time_per_iteration": 2.6032660007476807 + }, + { + "auxiliary_loss_clip": 0.01133247, + "auxiliary_loss_mlp": 0.01040533, + "balance_loss_clip": 1.05369091, + "balance_loss_mlp": 1.02395725, + "epoch": 0.12967906679821253, + "flos": 30658231756800.0, + "grad_norm": 4.184337239312273, + "language_loss": 0.61303461, + "learning_rate": 3.896686404892008e-06, + "loss": 0.63477242, + "num_input_tokens_seen": 127169395, + "router_z_loss_clip": 0.79394531, + "router_z_loss_mlp": 0.16595459, + "step": 4469, + "time_per_iteration": 2.573253631591797 + }, + { + "auxiliary_loss_clip": 0.01145289, + "auxiliary_loss_mlp": 0.01045365, + "balance_loss_clip": 1.0581336, + "balance_loss_mlp": 1.02827013, + "epoch": 0.12970808426672856, + "flos": 17048950047360.0, + "grad_norm": 3.2736195382171878, + "language_loss": 0.97510636, + "learning_rate": 3.896626766360077e-06, + "loss": 0.99701285, + "num_input_tokens_seen": 127181220, + "router_z_loss_clip": 0.87109375, + "router_z_loss_mlp": 0.17114258, + "step": 4470, + "time_per_iteration": 2.477449417114258 + }, + { + "auxiliary_loss_clip": 0.01030365, + "auxiliary_loss_mlp": 0.01002639, + "balance_loss_clip": 1.0094856, + "balance_loss_mlp": 1.00152993, + "epoch": 0.1297371017352446, + "flos": 61894735545600.0, + "grad_norm": 0.6407805130230312, + "language_loss": 0.51061952, + "learning_rate": 3.896567111076371e-06, + "loss": 0.53094953, + "num_input_tokens_seen": 127240905, + "router_z_loss_clip": 0.20898438, + "router_z_loss_mlp": 0.0111084, + "step": 4471, + "time_per_iteration": 2.9871888160705566 + }, + { + "auxiliary_loss_clip": 0.01141416, + "auxiliary_loss_mlp": 0.01039278, + "balance_loss_clip": 1.05381215, + "balance_loss_mlp": 1.021873, + "epoch": 0.12976611920376066, + "flos": 30849600441600.0, + "grad_norm": 1.980894092797197, + "language_loss": 0.8290695, + "learning_rate": 3.896507439041417e-06, + "loss": 0.85087651, + "num_input_tokens_seen": 127263770, + "router_z_loss_clip": 0.87646484, + "router_z_loss_mlp": 0.1739502, + "step": 4472, + "time_per_iteration": 2.578770875930786 + }, + { + "auxiliary_loss_clip": 0.01140943, + "auxiliary_loss_mlp": 0.01048642, + "balance_loss_clip": 1.05923343, + "balance_loss_mlp": 1.03177953, + "epoch": 0.1297951366722767, + "flos": 11286041944320.0, + "grad_norm": 2.5287316298417224, + "language_loss": 0.74522108, + "learning_rate": 3.896447750255741e-06, + "loss": 0.76711696, + "num_input_tokens_seen": 127275295, + "router_z_loss_clip": 0.81738281, + "router_z_loss_mlp": 0.16864014, + "step": 4473, + "time_per_iteration": 2.5603187084198 + }, + { + "auxiliary_loss_clip": 0.01032378, + "auxiliary_loss_mlp": 0.01002778, + "balance_loss_clip": 1.01137733, + "balance_loss_mlp": 1.00171745, + "epoch": 0.12982415414079276, + "flos": 62986886524800.0, + "grad_norm": 0.6683678905671295, + "language_loss": 0.4988983, + "learning_rate": 3.896388044719872e-06, + "loss": 0.51924986, + "num_input_tokens_seen": 127341495, + "router_z_loss_clip": 0.20996094, + "router_z_loss_mlp": 0.01062012, + "step": 4474, + "time_per_iteration": 3.128649950027466 + }, + { + "auxiliary_loss_clip": 0.01130379, + "auxiliary_loss_mlp": 0.01038323, + "balance_loss_clip": 1.05278111, + "balance_loss_mlp": 1.02364266, + "epoch": 0.12985317160930882, + "flos": 24018277271040.0, + "grad_norm": 3.1937341537823287, + "language_loss": 0.79167187, + "learning_rate": 3.896328322434335e-06, + "loss": 0.8133589, + "num_input_tokens_seen": 127355630, + "router_z_loss_clip": 0.77685547, + "router_z_loss_mlp": 0.14678955, + "step": 4475, + "time_per_iteration": 2.6414475440979004 + }, + { + "auxiliary_loss_clip": 0.01141781, + "auxiliary_loss_mlp": 0.010483, + "balance_loss_clip": 1.05822492, + "balance_loss_mlp": 1.03108573, + "epoch": 0.12988218907782484, + "flos": 22193404755840.0, + "grad_norm": 2.3976210031512366, + "language_loss": 0.72184741, + "learning_rate": 3.896268583399661e-06, + "loss": 0.74374819, + "num_input_tokens_seen": 127368980, + "router_z_loss_clip": 0.83349609, + "router_z_loss_mlp": 0.17193604, + "step": 4476, + "time_per_iteration": 2.5724921226501465 + }, + { + "auxiliary_loss_clip": 0.01032737, + "auxiliary_loss_mlp": 0.01005917, + "balance_loss_clip": 1.01186597, + "balance_loss_mlp": 1.00499296, + "epoch": 0.1299112065463409, + "flos": 60399271681920.0, + "grad_norm": 0.6613300448685934, + "language_loss": 0.52097499, + "learning_rate": 3.896208827616374e-06, + "loss": 0.54136157, + "num_input_tokens_seen": 127430725, + "router_z_loss_clip": 0.20898438, + "router_z_loss_mlp": 0.00921631, + "step": 4477, + "time_per_iteration": 3.0847065448760986 + }, + { + "auxiliary_loss_clip": 0.01134537, + "auxiliary_loss_mlp": 0.01050177, + "balance_loss_clip": 1.05286264, + "balance_loss_mlp": 1.03367305, + "epoch": 0.12994022401485694, + "flos": 18145410658560.0, + "grad_norm": 2.7068546740726167, + "language_loss": 0.79684991, + "learning_rate": 3.896149055085004e-06, + "loss": 0.81869704, + "num_input_tokens_seen": 127445575, + "router_z_loss_clip": 0.81689453, + "router_z_loss_mlp": 0.16503906, + "step": 4478, + "time_per_iteration": 2.527026414871216 + }, + { + "auxiliary_loss_clip": 0.01135775, + "auxiliary_loss_mlp": 0.01046318, + "balance_loss_clip": 1.05054903, + "balance_loss_mlp": 1.03070796, + "epoch": 0.129969241483373, + "flos": 27014592038400.0, + "grad_norm": 2.380305380662995, + "language_loss": 0.75519973, + "learning_rate": 3.896089265806077e-06, + "loss": 0.77702069, + "num_input_tokens_seen": 127465060, + "router_z_loss_clip": 0.85253906, + "router_z_loss_mlp": 0.15612793, + "step": 4479, + "time_per_iteration": 2.611811876296997 + }, + { + "auxiliary_loss_clip": 0.0103085, + "auxiliary_loss_mlp": 0.01004241, + "balance_loss_clip": 1.00972104, + "balance_loss_mlp": 1.00320995, + "epoch": 0.12999825895188905, + "flos": 74776179968640.0, + "grad_norm": 0.6253836546292401, + "language_loss": 0.48459816, + "learning_rate": 3.896029459780124e-06, + "loss": 0.50494903, + "num_input_tokens_seen": 127532040, + "router_z_loss_clip": 0.2109375, + "router_z_loss_mlp": 0.01031494, + "step": 4480, + "time_per_iteration": 3.2400505542755127 + }, + { + "auxiliary_loss_clip": 0.01150035, + "auxiliary_loss_mlp": 0.01048854, + "balance_loss_clip": 1.06039643, + "balance_loss_mlp": 1.03100848, + "epoch": 0.1300272764204051, + "flos": 26169541695360.0, + "grad_norm": 2.333551422675974, + "language_loss": 0.89589357, + "learning_rate": 3.895969637007671e-06, + "loss": 0.91788244, + "num_input_tokens_seen": 127547225, + "router_z_loss_clip": 0.89550781, + "router_z_loss_mlp": 0.17858887, + "step": 4481, + "time_per_iteration": 4.9962074756622314 + }, + { + "auxiliary_loss_clip": 0.0102909, + "auxiliary_loss_mlp": 0.01002969, + "balance_loss_clip": 1.00788081, + "balance_loss_mlp": 1.00197983, + "epoch": 0.13005629388892112, + "flos": 62187730784640.0, + "grad_norm": 0.707026013642145, + "language_loss": 0.4867104, + "learning_rate": 3.895909797489246e-06, + "loss": 0.50703096, + "num_input_tokens_seen": 127608645, + "router_z_loss_clip": 0.21289062, + "router_z_loss_mlp": 0.0098877, + "step": 4482, + "time_per_iteration": 7.594880819320679 + }, + { + "auxiliary_loss_clip": 0.01136934, + "auxiliary_loss_mlp": 0.01045351, + "balance_loss_clip": 1.05313361, + "balance_loss_mlp": 1.02760124, + "epoch": 0.13008531135743717, + "flos": 31941966902400.0, + "grad_norm": 2.178559864611209, + "language_loss": 0.8163234, + "learning_rate": 3.89584994122538e-06, + "loss": 0.83814633, + "num_input_tokens_seen": 127625460, + "router_z_loss_clip": 0.83935547, + "router_z_loss_mlp": 0.17749023, + "step": 4483, + "time_per_iteration": 2.5763795375823975 + }, + { + "auxiliary_loss_clip": 0.01133911, + "auxiliary_loss_mlp": 0.01046648, + "balance_loss_clip": 1.05545509, + "balance_loss_mlp": 1.03247356, + "epoch": 0.13011432882595322, + "flos": 24600532734720.0, + "grad_norm": 2.399378530242654, + "language_loss": 0.85998261, + "learning_rate": 3.895790068216599e-06, + "loss": 0.88178819, + "num_input_tokens_seen": 127640695, + "router_z_loss_clip": 0.78417969, + "router_z_loss_mlp": 0.14154053, + "step": 4484, + "time_per_iteration": 5.013936281204224 + }, + { + "auxiliary_loss_clip": 0.01143019, + "auxiliary_loss_mlp": 0.01043086, + "balance_loss_clip": 1.05746818, + "balance_loss_mlp": 1.02739763, + "epoch": 0.13014334629446928, + "flos": 12743260801920.0, + "grad_norm": 2.269548292337133, + "language_loss": 0.87744403, + "learning_rate": 3.8957301784634336e-06, + "loss": 0.89930511, + "num_input_tokens_seen": 127653135, + "router_z_loss_clip": 0.85546875, + "router_z_loss_mlp": 0.15679932, + "step": 4485, + "time_per_iteration": 2.5043187141418457 + }, + { + "auxiliary_loss_clip": 0.01130325, + "auxiliary_loss_mlp": 0.0104231, + "balance_loss_clip": 1.05241966, + "balance_loss_mlp": 1.02754009, + "epoch": 0.13017236376298533, + "flos": 20735000749440.0, + "grad_norm": 1.8583568110192104, + "language_loss": 0.71077836, + "learning_rate": 3.895670271966412e-06, + "loss": 0.73250473, + "num_input_tokens_seen": 127668445, + "router_z_loss_clip": 0.77929688, + "router_z_loss_mlp": 0.14764404, + "step": 4486, + "time_per_iteration": 2.4796340465545654 + }, + { + "auxiliary_loss_clip": 0.010288, + "auxiliary_loss_mlp": 0.01002525, + "balance_loss_clip": 1.00757992, + "balance_loss_mlp": 1.00157762, + "epoch": 0.13020138123150135, + "flos": 53691230966400.0, + "grad_norm": 0.7540905607506507, + "language_loss": 0.48648322, + "learning_rate": 3.895610348726063e-06, + "loss": 0.50679648, + "num_input_tokens_seen": 127733515, + "router_z_loss_clip": 0.21289062, + "router_z_loss_mlp": 0.00946045, + "step": 4487, + "time_per_iteration": 3.2135419845581055 + }, + { + "auxiliary_loss_clip": 0.01137485, + "auxiliary_loss_mlp": 0.01048344, + "balance_loss_clip": 1.05240548, + "balance_loss_mlp": 1.03134501, + "epoch": 0.1302303987000174, + "flos": 30403661091840.0, + "grad_norm": 2.344404893913871, + "language_loss": 1.09597993, + "learning_rate": 3.8955504087429175e-06, + "loss": 1.11783826, + "num_input_tokens_seen": 127755760, + "router_z_loss_clip": 0.85058594, + "router_z_loss_mlp": 0.16986084, + "step": 4488, + "time_per_iteration": 2.8955113887786865 + }, + { + "auxiliary_loss_clip": 0.01145024, + "auxiliary_loss_mlp": 0.01053976, + "balance_loss_clip": 1.05682254, + "balance_loss_mlp": 1.03539169, + "epoch": 0.13025941616853345, + "flos": 29054245927680.0, + "grad_norm": 2.8095549243136446, + "language_loss": 0.82565498, + "learning_rate": 3.895490452017503e-06, + "loss": 0.84764504, + "num_input_tokens_seen": 127769350, + "router_z_loss_clip": 0.88330078, + "router_z_loss_mlp": 0.18579102, + "step": 4489, + "time_per_iteration": 2.6257669925689697 + }, + { + "auxiliary_loss_clip": 0.01028368, + "auxiliary_loss_mlp": 0.01002921, + "balance_loss_clip": 1.00720453, + "balance_loss_mlp": 1.00185442, + "epoch": 0.1302884336370495, + "flos": 69962031751680.0, + "grad_norm": 0.6752242317771127, + "language_loss": 0.53472865, + "learning_rate": 3.895430478550349e-06, + "loss": 0.55504155, + "num_input_tokens_seen": 127829640, + "router_z_loss_clip": 0.21191406, + "router_z_loss_mlp": 0.01068115, + "step": 4490, + "time_per_iteration": 3.0691094398498535 + }, + { + "auxiliary_loss_clip": 0.01027812, + "auxiliary_loss_mlp": 0.01001081, + "balance_loss_clip": 1.0065254, + "balance_loss_mlp": 1.00006819, + "epoch": 0.13031745110556556, + "flos": 59087347338240.0, + "grad_norm": 0.665747227337069, + "language_loss": 0.48326522, + "learning_rate": 3.8953704883419875e-06, + "loss": 0.50355411, + "num_input_tokens_seen": 127889880, + "router_z_loss_clip": 0.21289062, + "router_z_loss_mlp": 0.01013184, + "step": 4491, + "time_per_iteration": 3.0244245529174805 + }, + { + "auxiliary_loss_clip": 0.01137316, + "auxiliary_loss_mlp": 0.01041671, + "balance_loss_clip": 1.05045235, + "balance_loss_mlp": 1.02264476, + "epoch": 0.1303464685740816, + "flos": 31973531978880.0, + "grad_norm": 2.46575322159423, + "language_loss": 0.88537884, + "learning_rate": 3.895310481392946e-06, + "loss": 0.90716869, + "num_input_tokens_seen": 127911220, + "router_z_loss_clip": 0.86914062, + "router_z_loss_mlp": 0.19030762, + "step": 4492, + "time_per_iteration": 2.642413377761841 + }, + { + "auxiliary_loss_clip": 0.01137311, + "auxiliary_loss_mlp": 0.01042053, + "balance_loss_clip": 1.05138326, + "balance_loss_mlp": 1.02579331, + "epoch": 0.13037548604259763, + "flos": 32117425862400.0, + "grad_norm": 2.742121195106911, + "language_loss": 1.11067581, + "learning_rate": 3.895250457703756e-06, + "loss": 1.13246942, + "num_input_tokens_seen": 127927425, + "router_z_loss_clip": 0.85986328, + "router_z_loss_mlp": 0.16253662, + "step": 4493, + "time_per_iteration": 2.6120004653930664 + }, + { + "auxiliary_loss_clip": 0.01028151, + "auxiliary_loss_mlp": 0.01000984, + "balance_loss_clip": 1.00667202, + "balance_loss_mlp": 0.99985182, + "epoch": 0.13040450351111368, + "flos": 65543941872000.0, + "grad_norm": 0.7093768481332337, + "language_loss": 0.52568704, + "learning_rate": 3.895190417274947e-06, + "loss": 0.54597837, + "num_input_tokens_seen": 127986005, + "router_z_loss_clip": 0.21484375, + "router_z_loss_mlp": 0.01135254, + "step": 4494, + "time_per_iteration": 3.033872604370117 + }, + { + "auxiliary_loss_clip": 0.01138761, + "auxiliary_loss_mlp": 0.01045681, + "balance_loss_clip": 1.05489147, + "balance_loss_mlp": 1.02927756, + "epoch": 0.13043352097962974, + "flos": 19311465870720.0, + "grad_norm": 5.440853441079469, + "language_loss": 0.99222165, + "learning_rate": 3.895130360107048e-06, + "loss": 1.01406622, + "num_input_tokens_seen": 127999415, + "router_z_loss_clip": 0.83886719, + "router_z_loss_mlp": 0.16394043, + "step": 4495, + "time_per_iteration": 2.477447271347046 + }, + { + "auxiliary_loss_clip": 0.01028283, + "auxiliary_loss_mlp": 0.0100133, + "balance_loss_clip": 1.00696075, + "balance_loss_mlp": 1.0002389, + "epoch": 0.1304625384481458, + "flos": 63792183490560.0, + "grad_norm": 1.8252790037629079, + "language_loss": 0.4570474, + "learning_rate": 3.895070286200592e-06, + "loss": 0.47734353, + "num_input_tokens_seen": 128061475, + "router_z_loss_clip": 0.21289062, + "router_z_loss_mlp": 0.01092529, + "step": 4496, + "time_per_iteration": 3.0449929237365723 + }, + { + "auxiliary_loss_clip": 0.01137335, + "auxiliary_loss_mlp": 0.01041493, + "balance_loss_clip": 1.05177617, + "balance_loss_mlp": 1.02488756, + "epoch": 0.13049155591666184, + "flos": 33579313488000.0, + "grad_norm": 2.442748219305704, + "language_loss": 0.98169482, + "learning_rate": 3.895010195556108e-06, + "loss": 1.00348306, + "num_input_tokens_seen": 128079455, + "router_z_loss_clip": 0.85595703, + "router_z_loss_mlp": 0.16589355, + "step": 4497, + "time_per_iteration": 2.650247097015381 + }, + { + "auxiliary_loss_clip": 0.0114047, + "auxiliary_loss_mlp": 0.01039382, + "balance_loss_clip": 1.05308008, + "balance_loss_mlp": 1.02149487, + "epoch": 0.1305205733851779, + "flos": 11614157706240.0, + "grad_norm": 2.2552760980797233, + "language_loss": 0.7414459, + "learning_rate": 3.894950088174127e-06, + "loss": 0.76324439, + "num_input_tokens_seen": 128091135, + "router_z_loss_clip": 0.87402344, + "router_z_loss_mlp": 0.17871094, + "step": 4498, + "time_per_iteration": 2.471907138824463 + }, + { + "auxiliary_loss_clip": 0.0113583, + "auxiliary_loss_mlp": 0.01046188, + "balance_loss_clip": 1.05269122, + "balance_loss_mlp": 1.02915871, + "epoch": 0.13054959085369391, + "flos": 14172865079040.0, + "grad_norm": 3.1671632442702933, + "language_loss": 0.7853232, + "learning_rate": 3.89488996405518e-06, + "loss": 0.80714333, + "num_input_tokens_seen": 128103725, + "router_z_loss_clip": 0.83105469, + "router_z_loss_mlp": 0.17016602, + "step": 4499, + "time_per_iteration": 2.477105140686035 + }, + { + "auxiliary_loss_clip": 0.01133055, + "auxiliary_loss_mlp": 0.01044417, + "balance_loss_clip": 1.05077338, + "balance_loss_mlp": 1.02801406, + "epoch": 0.13057860832220997, + "flos": 26098259155200.0, + "grad_norm": 3.5295794523319763, + "language_loss": 0.92353976, + "learning_rate": 3.894829823199799e-06, + "loss": 0.94531447, + "num_input_tokens_seen": 128121515, + "router_z_loss_clip": 0.82324219, + "router_z_loss_mlp": 0.16412354, + "step": 4500, + "time_per_iteration": 2.5896317958831787 + }, + { + "auxiliary_loss_clip": 0.01132668, + "auxiliary_loss_mlp": 0.01039457, + "balance_loss_clip": 1.05082488, + "balance_loss_mlp": 1.02215981, + "epoch": 0.13060762579072602, + "flos": 28797233137920.0, + "grad_norm": 2.2037821133728834, + "language_loss": 0.74628925, + "learning_rate": 3.8947696656085135e-06, + "loss": 0.7680105, + "num_input_tokens_seen": 128136800, + "router_z_loss_clip": 0.81835938, + "router_z_loss_mlp": 0.17297363, + "step": 4501, + "time_per_iteration": 2.5847411155700684 + }, + { + "auxiliary_loss_clip": 0.01138879, + "auxiliary_loss_mlp": 0.01044207, + "balance_loss_clip": 1.05389655, + "balance_loss_mlp": 1.02795839, + "epoch": 0.13063664325924207, + "flos": 17306393800320.0, + "grad_norm": 2.481049873750824, + "language_loss": 0.73412901, + "learning_rate": 3.894709491281855e-06, + "loss": 0.75595981, + "num_input_tokens_seen": 128150485, + "router_z_loss_clip": 0.84960938, + "router_z_loss_mlp": 0.16247559, + "step": 4502, + "time_per_iteration": 2.4690349102020264 + }, + { + "auxiliary_loss_clip": 0.01031348, + "auxiliary_loss_mlp": 0.01002875, + "balance_loss_clip": 1.00966549, + "balance_loss_mlp": 1.00174284, + "epoch": 0.13066566072775812, + "flos": 51171486871680.0, + "grad_norm": 0.6950985970531175, + "language_loss": 0.55328745, + "learning_rate": 3.894649300220356e-06, + "loss": 0.57362968, + "num_input_tokens_seen": 128210900, + "router_z_loss_clip": 0.21679688, + "router_z_loss_mlp": 0.01135254, + "step": 4503, + "time_per_iteration": 3.108834981918335 + }, + { + "auxiliary_loss_clip": 0.01141694, + "auxiliary_loss_mlp": 0.01044733, + "balance_loss_clip": 1.05371857, + "balance_loss_mlp": 1.02691758, + "epoch": 0.13069467819627414, + "flos": 28909633772160.0, + "grad_norm": 2.384487621381416, + "language_loss": 0.79665768, + "learning_rate": 3.894589092424549e-06, + "loss": 0.81852198, + "num_input_tokens_seen": 128227930, + "router_z_loss_clip": 0.88085938, + "router_z_loss_mlp": 0.17816162, + "step": 4504, + "time_per_iteration": 2.558032274246216 + }, + { + "auxiliary_loss_clip": 0.01136227, + "auxiliary_loss_mlp": 0.01048283, + "balance_loss_clip": 1.05090487, + "balance_loss_mlp": 1.03098619, + "epoch": 0.1307236956647902, + "flos": 25548969398400.0, + "grad_norm": 2.967442689784206, + "language_loss": 0.83578706, + "learning_rate": 3.894528867894963e-06, + "loss": 0.85763216, + "num_input_tokens_seen": 128242245, + "router_z_loss_clip": 0.85302734, + "router_z_loss_mlp": 0.17303467, + "step": 4505, + "time_per_iteration": 2.6622982025146484 + }, + { + "auxiliary_loss_clip": 0.0113224, + "auxiliary_loss_mlp": 0.01043996, + "balance_loss_clip": 1.05216742, + "balance_loss_mlp": 1.02746797, + "epoch": 0.13075271313330625, + "flos": 28471200364800.0, + "grad_norm": 2.3450232038619974, + "language_loss": 0.7472325, + "learning_rate": 3.8944686266321314e-06, + "loss": 0.76899481, + "num_input_tokens_seen": 128260510, + "router_z_loss_clip": 0.80078125, + "router_z_loss_mlp": 0.16522217, + "step": 4506, + "time_per_iteration": 2.552700996398926 + }, + { + "auxiliary_loss_clip": 0.01134575, + "auxiliary_loss_mlp": 0.01041877, + "balance_loss_clip": 1.05404973, + "balance_loss_mlp": 1.02489603, + "epoch": 0.1307817306018223, + "flos": 26937311927040.0, + "grad_norm": 2.605742996822593, + "language_loss": 0.79228508, + "learning_rate": 3.894408368636586e-06, + "loss": 0.81404954, + "num_input_tokens_seen": 128274985, + "router_z_loss_clip": 0.80517578, + "router_z_loss_mlp": 0.1697998, + "step": 4507, + "time_per_iteration": 2.5462450981140137 + }, + { + "auxiliary_loss_clip": 0.01140888, + "auxiliary_loss_mlp": 0.01045397, + "balance_loss_clip": 1.05572307, + "balance_loss_mlp": 1.02855265, + "epoch": 0.13081074807033835, + "flos": 23214560503680.0, + "grad_norm": 2.282189898599711, + "language_loss": 0.86980343, + "learning_rate": 3.89434809390886e-06, + "loss": 0.89166623, + "num_input_tokens_seen": 128289605, + "router_z_loss_clip": 0.8515625, + "router_z_loss_mlp": 0.16833496, + "step": 4508, + "time_per_iteration": 2.5638716220855713 + }, + { + "auxiliary_loss_clip": 0.01148009, + "auxiliary_loss_mlp": 0.01047481, + "balance_loss_clip": 1.05561757, + "balance_loss_mlp": 1.02971864, + "epoch": 0.1308397655388544, + "flos": 21499897893120.0, + "grad_norm": 2.4984835411827793, + "language_loss": 0.86992729, + "learning_rate": 3.894287802449485e-06, + "loss": 0.89188224, + "num_input_tokens_seen": 128303010, + "router_z_loss_clip": 0.92529297, + "router_z_loss_mlp": 0.17773438, + "step": 4509, + "time_per_iteration": 2.4943323135375977 + }, + { + "auxiliary_loss_clip": 0.0103505, + "auxiliary_loss_mlp": 0.01005132, + "balance_loss_clip": 1.01371026, + "balance_loss_mlp": 1.00409532, + "epoch": 0.13086878300737043, + "flos": 66604707342720.0, + "grad_norm": 0.7720470632012831, + "language_loss": 0.52000761, + "learning_rate": 3.894227494258995e-06, + "loss": 0.54040945, + "num_input_tokens_seen": 128359000, + "router_z_loss_clip": 0.21289062, + "router_z_loss_mlp": 0.01037598, + "step": 4510, + "time_per_iteration": 3.1275908946990967 + }, + { + "auxiliary_loss_clip": 0.01034637, + "auxiliary_loss_mlp": 0.01005108, + "balance_loss_clip": 1.01331353, + "balance_loss_mlp": 1.00411248, + "epoch": 0.13089780047588648, + "flos": 71972347207680.0, + "grad_norm": 0.6408469680746139, + "language_loss": 0.48901454, + "learning_rate": 3.894167169337919e-06, + "loss": 0.50941193, + "num_input_tokens_seen": 128419425, + "router_z_loss_clip": 0.21289062, + "router_z_loss_mlp": 0.00994873, + "step": 4511, + "time_per_iteration": 3.1837031841278076 + }, + { + "auxiliary_loss_clip": 0.01033861, + "auxiliary_loss_mlp": 0.01001731, + "balance_loss_clip": 1.01256716, + "balance_loss_mlp": 1.00068176, + "epoch": 0.13092681794440253, + "flos": 55221025253760.0, + "grad_norm": 0.6262313161691978, + "language_loss": 0.45935392, + "learning_rate": 3.894106827686793e-06, + "loss": 0.47970986, + "num_input_tokens_seen": 128479480, + "router_z_loss_clip": 0.21289062, + "router_z_loss_mlp": 0.01049805, + "step": 4512, + "time_per_iteration": 3.0488641262054443 + }, + { + "auxiliary_loss_clip": 0.01032743, + "auxiliary_loss_mlp": 0.01001072, + "balance_loss_clip": 1.01152945, + "balance_loss_mlp": 1.00000525, + "epoch": 0.13095583541291858, + "flos": 66058111105920.0, + "grad_norm": 0.6768321568175527, + "language_loss": 0.52443993, + "learning_rate": 3.8940464693061484e-06, + "loss": 0.54477811, + "num_input_tokens_seen": 128545160, + "router_z_loss_clip": 0.21289062, + "router_z_loss_mlp": 0.01068115, + "step": 4513, + "time_per_iteration": 3.1750147342681885 + }, + { + "auxiliary_loss_clip": 0.01031946, + "auxiliary_loss_mlp": 0.01002522, + "balance_loss_clip": 1.01085722, + "balance_loss_mlp": 1.00141907, + "epoch": 0.13098485288143463, + "flos": 74778227043840.0, + "grad_norm": 0.6860494015667156, + "language_loss": 0.54213965, + "learning_rate": 3.893986094196519e-06, + "loss": 0.56248432, + "num_input_tokens_seen": 128612160, + "router_z_loss_clip": 0.2109375, + "router_z_loss_mlp": 0.01104736, + "step": 4514, + "time_per_iteration": 3.3063619136810303 + }, + { + "auxiliary_loss_clip": 0.01031378, + "auxiliary_loss_mlp": 0.01001398, + "balance_loss_clip": 1.01021266, + "balance_loss_mlp": 1.00029552, + "epoch": 0.13101387034995068, + "flos": 59473385781120.0, + "grad_norm": 0.692203925289083, + "language_loss": 0.49636996, + "learning_rate": 3.893925702358439e-06, + "loss": 0.5166977, + "num_input_tokens_seen": 128661235, + "router_z_loss_clip": 0.2109375, + "router_z_loss_mlp": 0.01104736, + "step": 4515, + "time_per_iteration": 2.8199994564056396 + }, + { + "auxiliary_loss_clip": 0.01142263, + "auxiliary_loss_mlp": 0.01037813, + "balance_loss_clip": 1.05540276, + "balance_loss_mlp": 1.02037275, + "epoch": 0.1310428878184667, + "flos": 19237705292160.0, + "grad_norm": 2.854875260420256, + "language_loss": 0.81778085, + "learning_rate": 3.893865293792441e-06, + "loss": 0.83958155, + "num_input_tokens_seen": 128674475, + "router_z_loss_clip": 0.86914062, + "router_z_loss_mlp": 0.17443848, + "step": 4516, + "time_per_iteration": 2.553464651107788 + }, + { + "auxiliary_loss_clip": 0.01030533, + "auxiliary_loss_mlp": 0.01002942, + "balance_loss_clip": 1.00936699, + "balance_loss_mlp": 1.00191677, + "epoch": 0.13107190528698276, + "flos": 67147568565120.0, + "grad_norm": 0.6724436178781108, + "language_loss": 0.50745255, + "learning_rate": 3.893804868499058e-06, + "loss": 0.52778733, + "num_input_tokens_seen": 128735720, + "router_z_loss_clip": 0.2109375, + "router_z_loss_mlp": 0.01025391, + "step": 4517, + "time_per_iteration": 3.0472629070281982 + }, + { + "auxiliary_loss_clip": 0.01143297, + "auxiliary_loss_mlp": 0.01049114, + "balance_loss_clip": 1.0576818, + "balance_loss_mlp": 1.03311062, + "epoch": 0.1311009227554988, + "flos": 30258546145920.0, + "grad_norm": 2.349453243469908, + "language_loss": 0.88196409, + "learning_rate": 3.893744426478823e-06, + "loss": 0.90388823, + "num_input_tokens_seen": 128752380, + "router_z_loss_clip": 0.85644531, + "router_z_loss_mlp": 0.16015625, + "step": 4518, + "time_per_iteration": 2.581727981567383 + }, + { + "auxiliary_loss_clip": 0.01137587, + "auxiliary_loss_mlp": 0.01044103, + "balance_loss_clip": 1.05357313, + "balance_loss_mlp": 1.02697873, + "epoch": 0.13112994022401486, + "flos": 24380292493440.0, + "grad_norm": 3.531614875738186, + "language_loss": 0.77835661, + "learning_rate": 3.8936839677322715e-06, + "loss": 0.80017352, + "num_input_tokens_seen": 128764735, + "router_z_loss_clip": 0.84033203, + "router_z_loss_mlp": 0.17126465, + "step": 4519, + "time_per_iteration": 2.5790882110595703 + }, + { + "auxiliary_loss_clip": 0.01030102, + "auxiliary_loss_mlp": 0.01003926, + "balance_loss_clip": 1.00879812, + "balance_loss_mlp": 1.00293016, + "epoch": 0.13115895769253091, + "flos": 69262670972160.0, + "grad_norm": 0.6047255111545066, + "language_loss": 0.4916701, + "learning_rate": 3.893623492259937e-06, + "loss": 0.5120104, + "num_input_tokens_seen": 128827115, + "router_z_loss_clip": 0.21289062, + "router_z_loss_mlp": 0.00994873, + "step": 4520, + "time_per_iteration": 3.0796947479248047 + }, + { + "auxiliary_loss_clip": 0.01030074, + "auxiliary_loss_mlp": 0.01001362, + "balance_loss_clip": 1.00874877, + "balance_loss_mlp": 1.00032508, + "epoch": 0.13118797516104694, + "flos": 56165619162240.0, + "grad_norm": 0.7493186580183607, + "language_loss": 0.44956917, + "learning_rate": 3.893563000062354e-06, + "loss": 0.4698835, + "num_input_tokens_seen": 128877715, + "router_z_loss_clip": 0.21289062, + "router_z_loss_mlp": 0.01037598, + "step": 4521, + "time_per_iteration": 2.9308667182922363 + }, + { + "auxiliary_loss_clip": 0.01029768, + "auxiliary_loss_mlp": 0.01002899, + "balance_loss_clip": 1.00871432, + "balance_loss_mlp": 1.00187421, + "epoch": 0.131216992629563, + "flos": 51167680030080.0, + "grad_norm": 0.6748765182500283, + "language_loss": 0.49780148, + "learning_rate": 3.893502491140055e-06, + "loss": 0.51812816, + "num_input_tokens_seen": 128934735, + "router_z_loss_clip": 0.2109375, + "router_z_loss_mlp": 0.01025391, + "step": 4522, + "time_per_iteration": 2.907515525817871 + }, + { + "auxiliary_loss_clip": 0.01124447, + "auxiliary_loss_mlp": 0.01044897, + "balance_loss_clip": 1.04860568, + "balance_loss_mlp": 1.02941203, + "epoch": 0.13124601009807904, + "flos": 12814615169280.0, + "grad_norm": 3.6237237751415785, + "language_loss": 1.10679829, + "learning_rate": 3.8934419654935775e-06, + "loss": 1.12849188, + "num_input_tokens_seen": 128944255, + "router_z_loss_clip": 0.75878906, + "router_z_loss_mlp": 0.15490723, + "step": 4523, + "time_per_iteration": 2.486482620239258 + }, + { + "auxiliary_loss_clip": 0.01133295, + "auxiliary_loss_mlp": 0.01044461, + "balance_loss_clip": 1.05286956, + "balance_loss_mlp": 1.02887428, + "epoch": 0.1312750275665951, + "flos": 38285119307520.0, + "grad_norm": 2.9110477490065043, + "language_loss": 0.65723777, + "learning_rate": 3.893381423123453e-06, + "loss": 0.67901534, + "num_input_tokens_seen": 128961260, + "router_z_loss_clip": 0.8046875, + "router_z_loss_mlp": 0.15576172, + "step": 4524, + "time_per_iteration": 2.6785104274749756 + }, + { + "auxiliary_loss_clip": 0.0102966, + "auxiliary_loss_mlp": 0.01004581, + "balance_loss_clip": 1.00857186, + "balance_loss_mlp": 1.00345445, + "epoch": 0.13130404503511114, + "flos": 61332480979200.0, + "grad_norm": 0.7347533112570274, + "language_loss": 0.48394737, + "learning_rate": 3.893320864030219e-06, + "loss": 0.50428975, + "num_input_tokens_seen": 129013985, + "router_z_loss_clip": 0.2109375, + "router_z_loss_mlp": 0.0112915, + "step": 4525, + "time_per_iteration": 2.8782992362976074 + }, + { + "auxiliary_loss_clip": 0.01132611, + "auxiliary_loss_mlp": 0.01045057, + "balance_loss_clip": 1.05247068, + "balance_loss_mlp": 1.02715206, + "epoch": 0.1313330625036272, + "flos": 16392036165120.0, + "grad_norm": 2.5621429714172304, + "language_loss": 0.92013705, + "learning_rate": 3.893260288214407e-06, + "loss": 0.94191366, + "num_input_tokens_seen": 129026725, + "router_z_loss_clip": 0.80224609, + "router_z_loss_mlp": 0.17889404, + "step": 4526, + "time_per_iteration": 2.505777359008789 + }, + { + "auxiliary_loss_clip": 0.01138859, + "auxiliary_loss_mlp": 0.01057227, + "balance_loss_clip": 1.05984926, + "balance_loss_mlp": 1.04394698, + "epoch": 0.13136207997214322, + "flos": 32119006060800.0, + "grad_norm": 1.8947937035709628, + "language_loss": 0.65692329, + "learning_rate": 3.893199695676555e-06, + "loss": 0.67888415, + "num_input_tokens_seen": 129044085, + "router_z_loss_clip": 0.78955078, + "router_z_loss_mlp": 0.13293457, + "step": 4527, + "time_per_iteration": 2.6168315410614014 + }, + { + "auxiliary_loss_clip": 0.01146863, + "auxiliary_loss_mlp": 0.0104543, + "balance_loss_clip": 1.06055725, + "balance_loss_mlp": 1.02745271, + "epoch": 0.13139109744065927, + "flos": 24929905472640.0, + "grad_norm": 2.3255766007813503, + "language_loss": 1.02741718, + "learning_rate": 3.893139086417198e-06, + "loss": 1.04934001, + "num_input_tokens_seen": 129059590, + "router_z_loss_clip": 0.86328125, + "router_z_loss_mlp": 0.17974854, + "step": 4528, + "time_per_iteration": 2.5531420707702637 + }, + { + "auxiliary_loss_clip": 0.01147903, + "auxiliary_loss_mlp": 0.01051515, + "balance_loss_clip": 1.06027555, + "balance_loss_mlp": 1.03420627, + "epoch": 0.13142011490917532, + "flos": 50871054539520.0, + "grad_norm": 3.854061449884108, + "language_loss": 0.80709034, + "learning_rate": 3.89307846043687e-06, + "loss": 0.82908452, + "num_input_tokens_seen": 129078745, + "router_z_loss_clip": 0.87597656, + "router_z_loss_mlp": 0.1730957, + "step": 4529, + "time_per_iteration": 2.7652952671051025 + }, + { + "auxiliary_loss_clip": 0.01144696, + "auxiliary_loss_mlp": 0.01057321, + "balance_loss_clip": 1.0590564, + "balance_loss_mlp": 1.04138255, + "epoch": 0.13144913237769137, + "flos": 31201883078400.0, + "grad_norm": 2.4387065541309965, + "language_loss": 0.86688447, + "learning_rate": 3.893017817736107e-06, + "loss": 0.88890463, + "num_input_tokens_seen": 129094070, + "router_z_loss_clip": 0.85546875, + "router_z_loss_mlp": 0.15930176, + "step": 4530, + "time_per_iteration": 2.596179962158203 + }, + { + "auxiliary_loss_clip": 0.01136501, + "auxiliary_loss_mlp": 0.01047513, + "balance_loss_clip": 1.05701268, + "balance_loss_mlp": 1.03233802, + "epoch": 0.13147814984620743, + "flos": 15733039294080.0, + "grad_norm": 11.694995217455315, + "language_loss": 0.86721206, + "learning_rate": 3.892957158315444e-06, + "loss": 0.88905215, + "num_input_tokens_seen": 129104315, + "router_z_loss_clip": 0.79541016, + "router_z_loss_mlp": 0.1519165, + "step": 4531, + "time_per_iteration": 2.4554450511932373 + }, + { + "auxiliary_loss_clip": 0.01034429, + "auxiliary_loss_mlp": 0.01004575, + "balance_loss_clip": 1.01336479, + "balance_loss_mlp": 1.00340068, + "epoch": 0.13150716731472345, + "flos": 53546008279680.0, + "grad_norm": 0.6590406327587942, + "language_loss": 0.48056683, + "learning_rate": 3.892896482175418e-06, + "loss": 0.50095689, + "num_input_tokens_seen": 129162430, + "router_z_loss_clip": 0.2109375, + "router_z_loss_mlp": 0.01171875, + "step": 4532, + "time_per_iteration": 3.0146727561950684 + }, + { + "auxiliary_loss_clip": 0.01139631, + "auxiliary_loss_mlp": 0.01049431, + "balance_loss_clip": 1.05672169, + "balance_loss_mlp": 1.03347492, + "epoch": 0.1315361847832395, + "flos": 30584327523840.0, + "grad_norm": 3.44699911362516, + "language_loss": 0.8725034, + "learning_rate": 3.8928357893165645e-06, + "loss": 0.89439404, + "num_input_tokens_seen": 129177395, + "router_z_loss_clip": 0.82910156, + "router_z_loss_mlp": 0.15936279, + "step": 4533, + "time_per_iteration": 2.617119073867798 + }, + { + "auxiliary_loss_clip": 0.01141138, + "auxiliary_loss_mlp": 0.01044437, + "balance_loss_clip": 1.05850744, + "balance_loss_mlp": 1.02773607, + "epoch": 0.13156520225175555, + "flos": 11866896777600.0, + "grad_norm": 2.333975782652328, + "language_loss": 0.8510741, + "learning_rate": 3.892775079739418e-06, + "loss": 0.87292987, + "num_input_tokens_seen": 129189225, + "router_z_loss_clip": 0.82470703, + "router_z_loss_mlp": 0.16674805, + "step": 4534, + "time_per_iteration": 2.4992165565490723 + }, + { + "auxiliary_loss_clip": 0.01146913, + "auxiliary_loss_mlp": 0.01050742, + "balance_loss_clip": 1.05656731, + "balance_loss_mlp": 1.03208613, + "epoch": 0.1315942197202716, + "flos": 30110881334400.0, + "grad_norm": 2.6558731661639357, + "language_loss": 0.84744954, + "learning_rate": 3.892714353444518e-06, + "loss": 0.86942607, + "num_input_tokens_seen": 129205630, + "router_z_loss_clip": 0.90332031, + "router_z_loss_mlp": 0.18652344, + "step": 4535, + "time_per_iteration": 2.605651617050171 + }, + { + "auxiliary_loss_clip": 0.01134618, + "auxiliary_loss_mlp": 0.01042716, + "balance_loss_clip": 1.05624759, + "balance_loss_mlp": 1.0279938, + "epoch": 0.13162323718878766, + "flos": 28946226752640.0, + "grad_norm": 2.5681509572529757, + "language_loss": 0.90117109, + "learning_rate": 3.892653610432398e-06, + "loss": 0.92294449, + "num_input_tokens_seen": 129224715, + "router_z_loss_clip": 0.78369141, + "router_z_loss_mlp": 0.1472168, + "step": 4536, + "time_per_iteration": 2.6440069675445557 + }, + { + "auxiliary_loss_clip": 0.01136623, + "auxiliary_loss_mlp": 0.01042325, + "balance_loss_clip": 1.05447793, + "balance_loss_mlp": 1.02496839, + "epoch": 0.1316522546573037, + "flos": 11393522415360.0, + "grad_norm": 2.3628336136934656, + "language_loss": 0.75364304, + "learning_rate": 3.892592850703595e-06, + "loss": 0.77543253, + "num_input_tokens_seen": 129235630, + "router_z_loss_clip": 0.82177734, + "router_z_loss_mlp": 0.17358398, + "step": 4537, + "time_per_iteration": 2.466214418411255 + }, + { + "auxiliary_loss_clip": 0.01036981, + "auxiliary_loss_mlp": 0.01002045, + "balance_loss_clip": 1.01612663, + "balance_loss_mlp": 1.00089478, + "epoch": 0.13168127212581973, + "flos": 68892180140160.0, + "grad_norm": 0.6580680072462948, + "language_loss": 0.48275065, + "learning_rate": 3.892532074258647e-06, + "loss": 0.50314093, + "num_input_tokens_seen": 129298350, + "router_z_loss_clip": 0.20898438, + "router_z_loss_mlp": 0.01147461, + "step": 4538, + "time_per_iteration": 3.160919427871704 + }, + { + "auxiliary_loss_clip": 0.01037084, + "auxiliary_loss_mlp": 0.01001799, + "balance_loss_clip": 1.01609683, + "balance_loss_mlp": 1.00054729, + "epoch": 0.13171028959433578, + "flos": 62626307846400.0, + "grad_norm": 0.638312004309839, + "language_loss": 0.4932749, + "learning_rate": 3.892471281098089e-06, + "loss": 0.51366377, + "num_input_tokens_seen": 129361005, + "router_z_loss_clip": 0.20898438, + "router_z_loss_mlp": 0.01251221, + "step": 4539, + "time_per_iteration": 3.104442596435547 + }, + { + "auxiliary_loss_clip": 0.01140834, + "auxiliary_loss_mlp": 0.01046966, + "balance_loss_clip": 1.05816019, + "balance_loss_mlp": 1.03032482, + "epoch": 0.13173930706285183, + "flos": 16755811153920.0, + "grad_norm": 2.2728897086714874, + "language_loss": 0.6050148, + "learning_rate": 3.892410471222459e-06, + "loss": 0.62689275, + "num_input_tokens_seen": 129376220, + "router_z_loss_clip": 0.82714844, + "router_z_loss_mlp": 0.16650391, + "step": 4540, + "time_per_iteration": 2.474043607711792 + }, + { + "auxiliary_loss_clip": 0.01035157, + "auxiliary_loss_mlp": 0.01000987, + "balance_loss_clip": 1.01434326, + "balance_loss_mlp": 0.99974746, + "epoch": 0.1317683245313679, + "flos": 63097599219840.0, + "grad_norm": 0.6473424612189189, + "language_loss": 0.45589599, + "learning_rate": 3.892349644632295e-06, + "loss": 0.47625744, + "num_input_tokens_seen": 129437580, + "router_z_loss_clip": 0.20898438, + "router_z_loss_mlp": 0.01239014, + "step": 4541, + "time_per_iteration": 3.1548469066619873 + }, + { + "auxiliary_loss_clip": 0.01036513, + "auxiliary_loss_mlp": 0.01003111, + "balance_loss_clip": 1.0155642, + "balance_loss_mlp": 1.00190151, + "epoch": 0.13179734199988394, + "flos": 70249532209920.0, + "grad_norm": 0.6654109917788913, + "language_loss": 0.48622742, + "learning_rate": 3.8922888013281324e-06, + "loss": 0.50662363, + "num_input_tokens_seen": 129496185, + "router_z_loss_clip": 0.20898438, + "router_z_loss_mlp": 0.01208496, + "step": 4542, + "time_per_iteration": 3.065640449523926 + }, + { + "auxiliary_loss_clip": 0.0114519, + "auxiliary_loss_mlp": 0.01047868, + "balance_loss_clip": 1.05962348, + "balance_loss_mlp": 1.0305233, + "epoch": 0.1318263594684, + "flos": 34198916117760.0, + "grad_norm": 2.893066960396275, + "language_loss": 1.06571579, + "learning_rate": 3.892227941310509e-06, + "loss": 1.08764648, + "num_input_tokens_seen": 129513435, + "router_z_loss_clip": 0.85644531, + "router_z_loss_mlp": 0.17358398, + "step": 4543, + "time_per_iteration": 2.6381423473358154 + }, + { + "auxiliary_loss_clip": 0.01137995, + "auxiliary_loss_mlp": 0.01039405, + "balance_loss_clip": 1.05935407, + "balance_loss_mlp": 1.02464092, + "epoch": 0.131855376936916, + "flos": 24126619668480.0, + "grad_norm": 5.262931330133199, + "language_loss": 0.82312691, + "learning_rate": 3.892167064579963e-06, + "loss": 0.84490091, + "num_input_tokens_seen": 129525900, + "router_z_loss_clip": 0.78759766, + "router_z_loss_mlp": 0.14764404, + "step": 4544, + "time_per_iteration": 2.5378143787384033 + }, + { + "auxiliary_loss_clip": 0.01149723, + "auxiliary_loss_mlp": 0.01046675, + "balance_loss_clip": 1.06076217, + "balance_loss_mlp": 1.02797067, + "epoch": 0.13188439440543206, + "flos": 25551016473600.0, + "grad_norm": 4.018141860939232, + "language_loss": 0.89809692, + "learning_rate": 3.892106171137032e-06, + "loss": 0.92006087, + "num_input_tokens_seen": 129539915, + "router_z_loss_clip": 0.89013672, + "router_z_loss_mlp": 0.18701172, + "step": 4545, + "time_per_iteration": 2.5160470008850098 + }, + { + "auxiliary_loss_clip": 0.01144234, + "auxiliary_loss_mlp": 0.01049908, + "balance_loss_clip": 1.06236434, + "balance_loss_mlp": 1.03518581, + "epoch": 0.13191341187394812, + "flos": 27884455701120.0, + "grad_norm": 2.2804326513009983, + "language_loss": 0.90917873, + "learning_rate": 3.892045260982254e-06, + "loss": 0.93112016, + "num_input_tokens_seen": 129556010, + "router_z_loss_clip": 0.81933594, + "router_z_loss_mlp": 0.1472168, + "step": 4546, + "time_per_iteration": 2.5702905654907227 + }, + { + "auxiliary_loss_clip": 0.01134945, + "auxiliary_loss_mlp": 0.0104074, + "balance_loss_clip": 1.05645299, + "balance_loss_mlp": 1.02463531, + "epoch": 0.13194242934246417, + "flos": 27302128410240.0, + "grad_norm": 1.5916226564375968, + "language_loss": 0.68850756, + "learning_rate": 3.891984334116166e-06, + "loss": 0.71026438, + "num_input_tokens_seen": 129575295, + "router_z_loss_clip": 0.78515625, + "router_z_loss_mlp": 0.16119385, + "step": 4547, + "time_per_iteration": 2.5637190341949463 + }, + { + "auxiliary_loss_clip": 0.01033654, + "auxiliary_loss_mlp": 0.01006192, + "balance_loss_clip": 1.01268983, + "balance_loss_mlp": 1.00502324, + "epoch": 0.13197144681098022, + "flos": 64821312057600.0, + "grad_norm": 0.6136985533299579, + "language_loss": 0.46787572, + "learning_rate": 3.891923390539307e-06, + "loss": 0.48827419, + "num_input_tokens_seen": 129643220, + "router_z_loss_clip": 0.20898438, + "router_z_loss_mlp": 0.01165771, + "step": 4548, + "time_per_iteration": 3.1511313915252686 + }, + { + "auxiliary_loss_clip": 0.0113054, + "auxiliary_loss_mlp": 0.01032136, + "balance_loss_clip": 1.05552912, + "balance_loss_mlp": 1.01802111, + "epoch": 0.13200046427949624, + "flos": 23799186264960.0, + "grad_norm": 1.66924847963182, + "language_loss": 0.63351631, + "learning_rate": 3.8918624302522145e-06, + "loss": 0.65514302, + "num_input_tokens_seen": 129658650, + "router_z_loss_clip": 0.75048828, + "router_z_loss_mlp": 0.14105225, + "step": 4549, + "time_per_iteration": 2.5424768924713135 + }, + { + "auxiliary_loss_clip": 0.01136731, + "auxiliary_loss_mlp": 0.01040735, + "balance_loss_clip": 1.05403328, + "balance_loss_mlp": 1.02361631, + "epoch": 0.1320294817480123, + "flos": 43721922810240.0, + "grad_norm": 2.302032036135855, + "language_loss": 0.79332292, + "learning_rate": 3.891801453255428e-06, + "loss": 0.81509763, + "num_input_tokens_seen": 129677810, + "router_z_loss_clip": 0.82763672, + "router_z_loss_mlp": 0.17108154, + "step": 4550, + "time_per_iteration": 2.6889164447784424 + }, + { + "auxiliary_loss_clip": 0.01030945, + "auxiliary_loss_mlp": 0.01005707, + "balance_loss_clip": 1.00989866, + "balance_loss_mlp": 1.00449753, + "epoch": 0.13205849921652835, + "flos": 65616409560960.0, + "grad_norm": 0.6823323271591795, + "language_loss": 0.48315069, + "learning_rate": 3.891740459549485e-06, + "loss": 0.50351727, + "num_input_tokens_seen": 129735500, + "router_z_loss_clip": 0.2109375, + "router_z_loss_mlp": 0.01208496, + "step": 4551, + "time_per_iteration": 3.0183262825012207 + }, + { + "auxiliary_loss_clip": 0.01137575, + "auxiliary_loss_mlp": 0.01044539, + "balance_loss_clip": 1.05666494, + "balance_loss_mlp": 1.02908897, + "epoch": 0.1320875166850444, + "flos": 16611450393600.0, + "grad_norm": 3.2079653960847736, + "language_loss": 0.79282022, + "learning_rate": 3.891679449134925e-06, + "loss": 0.81464124, + "num_input_tokens_seen": 129747200, + "router_z_loss_clip": 0.80908203, + "router_z_loss_mlp": 0.15441895, + "step": 4552, + "time_per_iteration": 4.9754860401153564 + }, + { + "auxiliary_loss_clip": 0.01132955, + "auxiliary_loss_mlp": 0.01043679, + "balance_loss_clip": 1.05733156, + "balance_loss_mlp": 1.02867651, + "epoch": 0.13211653415356045, + "flos": 15915537319680.0, + "grad_norm": 2.7539477313437444, + "language_loss": 0.72828853, + "learning_rate": 3.891618422012287e-06, + "loss": 0.75005484, + "num_input_tokens_seen": 129758665, + "router_z_loss_clip": 0.75585938, + "router_z_loss_mlp": 0.15020752, + "step": 4553, + "time_per_iteration": 7.198118209838867 + }, + { + "auxiliary_loss_clip": 0.01142007, + "auxiliary_loss_mlp": 0.01042315, + "balance_loss_clip": 1.05623078, + "balance_loss_mlp": 1.0258764, + "epoch": 0.1321455516220765, + "flos": 22307565156480.0, + "grad_norm": 2.6646561760538385, + "language_loss": 1.05188966, + "learning_rate": 3.89155737818211e-06, + "loss": 1.07373297, + "num_input_tokens_seen": 129778180, + "router_z_loss_clip": 0.859375, + "router_z_loss_mlp": 0.16455078, + "step": 4554, + "time_per_iteration": 2.5325276851654053 + }, + { + "auxiliary_loss_clip": 0.01130974, + "auxiliary_loss_mlp": 0.01046026, + "balance_loss_clip": 1.0541507, + "balance_loss_mlp": 1.03190029, + "epoch": 0.13217456909059253, + "flos": 35401420656000.0, + "grad_norm": 1.6923780725054234, + "language_loss": 0.62142408, + "learning_rate": 3.891496317644932e-06, + "loss": 0.64319408, + "num_input_tokens_seen": 129800635, + "router_z_loss_clip": 0.76953125, + "router_z_loss_mlp": 0.14123535, + "step": 4555, + "time_per_iteration": 5.121221303939819 + }, + { + "auxiliary_loss_clip": 0.01146102, + "auxiliary_loss_mlp": 0.01047397, + "balance_loss_clip": 1.05812311, + "balance_loss_mlp": 1.02952743, + "epoch": 0.13220358655910858, + "flos": 12705662240640.0, + "grad_norm": 2.395275956827882, + "language_loss": 0.90735745, + "learning_rate": 3.891435240401293e-06, + "loss": 0.92929244, + "num_input_tokens_seen": 129813140, + "router_z_loss_clip": 0.88085938, + "router_z_loss_mlp": 0.17858887, + "step": 4556, + "time_per_iteration": 2.56447696685791 + }, + { + "auxiliary_loss_clip": 0.0114281, + "auxiliary_loss_mlp": 0.01057338, + "balance_loss_clip": 1.05621231, + "balance_loss_mlp": 1.03856206, + "epoch": 0.13223260402762463, + "flos": 35445914628480.0, + "grad_norm": 1.818409440864665, + "language_loss": 0.88038522, + "learning_rate": 3.891374146451733e-06, + "loss": 0.90238667, + "num_input_tokens_seen": 129836005, + "router_z_loss_clip": 0.86572266, + "router_z_loss_mlp": 0.18786621, + "step": 4557, + "time_per_iteration": 2.660386800765991 + }, + { + "auxiliary_loss_clip": 0.01141775, + "auxiliary_loss_mlp": 0.01046794, + "balance_loss_clip": 1.06049728, + "balance_loss_mlp": 1.03161907, + "epoch": 0.13226162149614068, + "flos": 34569083727360.0, + "grad_norm": 2.0232403409742763, + "language_loss": 0.80305398, + "learning_rate": 3.8913130357967915e-06, + "loss": 0.82493967, + "num_input_tokens_seen": 129856165, + "router_z_loss_clip": 0.81201172, + "router_z_loss_mlp": 0.15179443, + "step": 4558, + "time_per_iteration": 2.6726605892181396 + }, + { + "auxiliary_loss_clip": 0.0103299, + "auxiliary_loss_mlp": 0.01001332, + "balance_loss_clip": 1.01167476, + "balance_loss_mlp": 1.00014579, + "epoch": 0.13229063896465673, + "flos": 59156439148800.0, + "grad_norm": 0.6960535173206731, + "language_loss": 0.4891471, + "learning_rate": 3.891251908437008e-06, + "loss": 0.50949031, + "num_input_tokens_seen": 129910560, + "router_z_loss_clip": 0.21289062, + "router_z_loss_mlp": 0.01184082, + "step": 4559, + "time_per_iteration": 2.935359001159668 + }, + { + "auxiliary_loss_clip": 0.01133138, + "auxiliary_loss_mlp": 0.01033091, + "balance_loss_clip": 1.0543865, + "balance_loss_mlp": 1.01798093, + "epoch": 0.13231965643317278, + "flos": 21067893020160.0, + "grad_norm": 1.6187505227275072, + "language_loss": 0.62124705, + "learning_rate": 3.8911907643729216e-06, + "loss": 0.64290935, + "num_input_tokens_seen": 129929280, + "router_z_loss_clip": 0.78759766, + "router_z_loss_mlp": 0.15106201, + "step": 4560, + "time_per_iteration": 2.565417528152466 + }, + { + "auxiliary_loss_clip": 0.01140952, + "auxiliary_loss_mlp": 0.0103784, + "balance_loss_clip": 1.05841553, + "balance_loss_mlp": 1.02103734, + "epoch": 0.1323486739016888, + "flos": 32227779421440.0, + "grad_norm": 2.6127581863761913, + "language_loss": 0.92712951, + "learning_rate": 3.8911296036050736e-06, + "loss": 0.94891745, + "num_input_tokens_seen": 129944165, + "router_z_loss_clip": 0.82568359, + "router_z_loss_mlp": 0.16809082, + "step": 4561, + "time_per_iteration": 2.6147403717041016 + }, + { + "auxiliary_loss_clip": 0.01141367, + "auxiliary_loss_mlp": 0.01038012, + "balance_loss_clip": 1.05581808, + "balance_loss_mlp": 1.02059007, + "epoch": 0.13237769137020486, + "flos": 17193921338880.0, + "grad_norm": 2.9740516958435417, + "language_loss": 0.85573876, + "learning_rate": 3.8910684261340035e-06, + "loss": 0.87753248, + "num_input_tokens_seen": 129956605, + "router_z_loss_clip": 0.85595703, + "router_z_loss_mlp": 0.17431641, + "step": 4562, + "time_per_iteration": 2.49383807182312 + }, + { + "auxiliary_loss_clip": 0.01033529, + "auxiliary_loss_mlp": 0.01007581, + "balance_loss_clip": 1.01219916, + "balance_loss_mlp": 1.0063889, + "epoch": 0.1324067088387209, + "flos": 63606022277760.0, + "grad_norm": 0.7593178892466828, + "language_loss": 0.52063918, + "learning_rate": 3.891007231960252e-06, + "loss": 0.54105031, + "num_input_tokens_seen": 130006270, + "router_z_loss_clip": 0.21289062, + "router_z_loss_mlp": 0.01190186, + "step": 4563, + "time_per_iteration": 2.9368972778320312 + }, + { + "auxiliary_loss_clip": 0.01141189, + "auxiliary_loss_mlp": 0.01043267, + "balance_loss_clip": 1.06002617, + "balance_loss_mlp": 1.0277344, + "epoch": 0.13243572630723696, + "flos": 38977476935040.0, + "grad_norm": 7.351188681631272, + "language_loss": 0.90323979, + "learning_rate": 3.890946021084359e-06, + "loss": 0.92508435, + "num_input_tokens_seen": 130022560, + "router_z_loss_clip": 0.81152344, + "router_z_loss_mlp": 0.15539551, + "step": 4564, + "time_per_iteration": 2.7691266536712646 + }, + { + "auxiliary_loss_clip": 0.01141804, + "auxiliary_loss_mlp": 0.01042091, + "balance_loss_clip": 1.05743122, + "balance_loss_mlp": 1.02416158, + "epoch": 0.132464743775753, + "flos": 31570937366400.0, + "grad_norm": 2.2110827744353876, + "language_loss": 0.86689836, + "learning_rate": 3.890884793506865e-06, + "loss": 0.88873726, + "num_input_tokens_seen": 130042530, + "router_z_loss_clip": 0.84472656, + "router_z_loss_mlp": 0.17932129, + "step": 4565, + "time_per_iteration": 2.8241026401519775 + }, + { + "auxiliary_loss_clip": 0.01032844, + "auxiliary_loss_mlp": 0.00998871, + "balance_loss_clip": 1.0116775, + "balance_loss_mlp": 0.99782246, + "epoch": 0.13249376124426904, + "flos": 74769176816640.0, + "grad_norm": 0.6461315319417223, + "language_loss": 0.44889709, + "learning_rate": 3.8908235492283125e-06, + "loss": 0.46921426, + "num_input_tokens_seen": 130107465, + "router_z_loss_clip": 0.2109375, + "router_z_loss_mlp": 0.01049805, + "step": 4566, + "time_per_iteration": 3.2967276573181152 + }, + { + "auxiliary_loss_clip": 0.01032642, + "auxiliary_loss_mlp": 0.00999075, + "balance_loss_clip": 1.01149416, + "balance_loss_mlp": 0.99791259, + "epoch": 0.1325227787127851, + "flos": 71024090112000.0, + "grad_norm": 0.6732535071719272, + "language_loss": 0.54730725, + "learning_rate": 3.890762288249241e-06, + "loss": 0.56762445, + "num_input_tokens_seen": 130171750, + "router_z_loss_clip": 0.2109375, + "router_z_loss_mlp": 0.01159668, + "step": 4567, + "time_per_iteration": 3.1363182067871094 + }, + { + "auxiliary_loss_clip": 0.01032073, + "auxiliary_loss_mlp": 0.00998378, + "balance_loss_clip": 1.01100945, + "balance_loss_mlp": 0.99726373, + "epoch": 0.13255179618130114, + "flos": 74771690768640.0, + "grad_norm": 0.6997433532615775, + "language_loss": 0.54359865, + "learning_rate": 3.890701010570192e-06, + "loss": 0.56390321, + "num_input_tokens_seen": 130229705, + "router_z_loss_clip": 0.2109375, + "router_z_loss_mlp": 0.01116943, + "step": 4568, + "time_per_iteration": 3.0589969158172607 + }, + { + "auxiliary_loss_clip": 0.01144952, + "auxiliary_loss_mlp": 0.01043456, + "balance_loss_clip": 1.05608439, + "balance_loss_mlp": 1.02530015, + "epoch": 0.1325808136498172, + "flos": 74732580858240.0, + "grad_norm": 2.212711157896897, + "language_loss": 0.83510256, + "learning_rate": 3.890639716191706e-06, + "loss": 0.85698664, + "num_input_tokens_seen": 130251850, + "router_z_loss_clip": 0.88867188, + "router_z_loss_mlp": 0.18157959, + "step": 4569, + "time_per_iteration": 2.9290411472320557 + }, + { + "auxiliary_loss_clip": 0.01141421, + "auxiliary_loss_mlp": 0.01033556, + "balance_loss_clip": 1.05839992, + "balance_loss_mlp": 1.01739752, + "epoch": 0.13260983111833324, + "flos": 32887135428480.0, + "grad_norm": 2.347931380671544, + "language_loss": 0.74068904, + "learning_rate": 3.890578405114325e-06, + "loss": 0.76243883, + "num_input_tokens_seen": 130269885, + "router_z_loss_clip": 0.83056641, + "router_z_loss_mlp": 0.16162109, + "step": 4570, + "time_per_iteration": 2.6318178176879883 + }, + { + "auxiliary_loss_clip": 0.01030966, + "auxiliary_loss_mlp": 0.01005158, + "balance_loss_clip": 1.01002717, + "balance_loss_mlp": 1.00415063, + "epoch": 0.1326388485868493, + "flos": 69341998158720.0, + "grad_norm": 0.7042842920462666, + "language_loss": 0.46737915, + "learning_rate": 3.890517077338591e-06, + "loss": 0.48774043, + "num_input_tokens_seen": 130329670, + "router_z_loss_clip": 0.20898438, + "router_z_loss_mlp": 0.0100708, + "step": 4571, + "time_per_iteration": 3.05651593208313 + }, + { + "auxiliary_loss_clip": 0.01134154, + "auxiliary_loss_mlp": 0.01040607, + "balance_loss_clip": 1.05650997, + "balance_loss_mlp": 1.02406049, + "epoch": 0.13266786605536532, + "flos": 39814123495680.0, + "grad_norm": 2.4474374065115367, + "language_loss": 0.57563365, + "learning_rate": 3.890455732865045e-06, + "loss": 0.59738129, + "num_input_tokens_seen": 130350170, + "router_z_loss_clip": 0.77490234, + "router_z_loss_mlp": 0.16558838, + "step": 4572, + "time_per_iteration": 2.6920716762542725 + }, + { + "auxiliary_loss_clip": 0.01031058, + "auxiliary_loss_mlp": 0.01005804, + "balance_loss_clip": 1.01006842, + "balance_loss_mlp": 1.00481415, + "epoch": 0.13269688352388137, + "flos": 74774096979840.0, + "grad_norm": 0.6312574566670467, + "language_loss": 0.44731939, + "learning_rate": 3.890394371694228e-06, + "loss": 0.46768802, + "num_input_tokens_seen": 130416000, + "router_z_loss_clip": 0.20996094, + "router_z_loss_mlp": 0.0098877, + "step": 4573, + "time_per_iteration": 3.263016700744629 + }, + { + "auxiliary_loss_clip": 0.01134148, + "auxiliary_loss_mlp": 0.01046007, + "balance_loss_clip": 1.05685544, + "balance_loss_mlp": 1.03105783, + "epoch": 0.13272590099239742, + "flos": 22851252391680.0, + "grad_norm": 4.5124141822556405, + "language_loss": 0.67336488, + "learning_rate": 3.890332993826685e-06, + "loss": 0.69516641, + "num_input_tokens_seen": 130430320, + "router_z_loss_clip": 0.77294922, + "router_z_loss_mlp": 0.14959717, + "step": 4574, + "time_per_iteration": 2.5996243953704834 + }, + { + "auxiliary_loss_clip": 0.0103125, + "auxiliary_loss_mlp": 0.01001474, + "balance_loss_clip": 1.0105238, + "balance_loss_mlp": 1.00049019, + "epoch": 0.13275491846091347, + "flos": 69696794747520.0, + "grad_norm": 0.6719299141183388, + "language_loss": 0.49929661, + "learning_rate": 3.890271599262955e-06, + "loss": 0.51962382, + "num_input_tokens_seen": 130486335, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.00982666, + "step": 4575, + "time_per_iteration": 3.0010266304016113 + }, + { + "auxiliary_loss_clip": 0.01139678, + "auxiliary_loss_mlp": 0.01049501, + "balance_loss_clip": 1.05909204, + "balance_loss_mlp": 1.03165555, + "epoch": 0.13278393592942953, + "flos": 35982311402880.0, + "grad_norm": 2.241839893670826, + "language_loss": 0.77236938, + "learning_rate": 3.890210188003581e-06, + "loss": 0.79426116, + "num_input_tokens_seen": 130503095, + "router_z_loss_clip": 0.80566406, + "router_z_loss_mlp": 0.1784668, + "step": 4576, + "time_per_iteration": 2.6283767223358154 + }, + { + "auxiliary_loss_clip": 0.01147136, + "auxiliary_loss_mlp": 0.01044171, + "balance_loss_clip": 1.05928636, + "balance_loss_mlp": 1.02680171, + "epoch": 0.13281295339794558, + "flos": 36167862084480.0, + "grad_norm": 2.2961773786800808, + "language_loss": 0.87266779, + "learning_rate": 3.890148760049106e-06, + "loss": 0.8945809, + "num_input_tokens_seen": 130529340, + "router_z_loss_clip": 0.87841797, + "router_z_loss_mlp": 0.17352295, + "step": 4577, + "time_per_iteration": 2.778831958770752 + }, + { + "auxiliary_loss_clip": 0.01147301, + "auxiliary_loss_mlp": 0.0105352, + "balance_loss_clip": 1.05877173, + "balance_loss_mlp": 1.03515029, + "epoch": 0.1328419708664616, + "flos": 16977595680000.0, + "grad_norm": 2.5290269091551014, + "language_loss": 0.81523287, + "learning_rate": 3.890087315400072e-06, + "loss": 0.83724117, + "num_input_tokens_seen": 130546925, + "router_z_loss_clip": 0.88525391, + "router_z_loss_mlp": 0.18359375, + "step": 4578, + "time_per_iteration": 2.604288101196289 + }, + { + "auxiliary_loss_clip": 0.01141977, + "auxiliary_loss_mlp": 0.01049775, + "balance_loss_clip": 1.0578835, + "balance_loss_mlp": 1.03352022, + "epoch": 0.13287098833497765, + "flos": 28802548350720.0, + "grad_norm": 2.5559464473228517, + "language_loss": 0.89304775, + "learning_rate": 3.890025854057022e-06, + "loss": 0.91496527, + "num_input_tokens_seen": 130562385, + "router_z_loss_clip": 0.84033203, + "router_z_loss_mlp": 0.1628418, + "step": 4579, + "time_per_iteration": 2.587254047393799 + }, + { + "auxiliary_loss_clip": 0.01031515, + "auxiliary_loss_mlp": 0.0100117, + "balance_loss_clip": 1.01057959, + "balance_loss_mlp": 1.00008476, + "epoch": 0.1329000058034937, + "flos": 58176688803840.0, + "grad_norm": 0.6674467343818734, + "language_loss": 0.48088717, + "learning_rate": 3.8899643760204994e-06, + "loss": 0.50121403, + "num_input_tokens_seen": 130624070, + "router_z_loss_clip": 0.20898438, + "router_z_loss_mlp": 0.01086426, + "step": 4580, + "time_per_iteration": 3.073911666870117 + }, + { + "auxiliary_loss_clip": 0.01135997, + "auxiliary_loss_mlp": 0.01041025, + "balance_loss_clip": 1.05410194, + "balance_loss_mlp": 1.02299452, + "epoch": 0.13292902327200976, + "flos": 17122423317120.0, + "grad_norm": 2.642591517056057, + "language_loss": 0.69417131, + "learning_rate": 3.889902881291046e-06, + "loss": 0.71594155, + "num_input_tokens_seen": 130636005, + "router_z_loss_clip": 0.81933594, + "router_z_loss_mlp": 0.18023682, + "step": 4581, + "time_per_iteration": 2.4621636867523193 + }, + { + "auxiliary_loss_clip": 0.01145642, + "auxiliary_loss_mlp": 0.01038897, + "balance_loss_clip": 1.05700207, + "balance_loss_mlp": 1.02248228, + "epoch": 0.1329580407405258, + "flos": 32378173666560.0, + "grad_norm": 2.013842411195742, + "language_loss": 0.78476477, + "learning_rate": 3.889841369869207e-06, + "loss": 0.80661017, + "num_input_tokens_seen": 130653015, + "router_z_loss_clip": 0.88720703, + "router_z_loss_mlp": 0.16394043, + "step": 4582, + "time_per_iteration": 2.4933695793151855 + }, + { + "auxiliary_loss_clip": 0.0114076, + "auxiliary_loss_mlp": 0.0105103, + "balance_loss_clip": 1.05456471, + "balance_loss_mlp": 1.03397155, + "epoch": 0.13298705820904183, + "flos": 30584686659840.0, + "grad_norm": 1.937740101674764, + "language_loss": 0.88842815, + "learning_rate": 3.8897798417555225e-06, + "loss": 0.91034609, + "num_input_tokens_seen": 130676655, + "router_z_loss_clip": 0.86230469, + "router_z_loss_mlp": 0.17059326, + "step": 4583, + "time_per_iteration": 2.8013501167297363 + }, + { + "auxiliary_loss_clip": 0.0114063, + "auxiliary_loss_mlp": 0.01041708, + "balance_loss_clip": 1.05314481, + "balance_loss_mlp": 1.02408886, + "epoch": 0.13301607567755788, + "flos": 16317880536960.0, + "grad_norm": 2.979730532224237, + "language_loss": 1.04116118, + "learning_rate": 3.889718296950539e-06, + "loss": 1.06298459, + "num_input_tokens_seen": 130690240, + "router_z_loss_clip": 0.87548828, + "router_z_loss_mlp": 0.17602539, + "step": 4584, + "time_per_iteration": 2.4828028678894043 + }, + { + "auxiliary_loss_clip": 0.01128916, + "auxiliary_loss_mlp": 0.01035107, + "balance_loss_clip": 1.05182016, + "balance_loss_mlp": 1.02124906, + "epoch": 0.13304509314607393, + "flos": 15662870075520.0, + "grad_norm": 2.074456204375843, + "language_loss": 0.64049381, + "learning_rate": 3.889656735454798e-06, + "loss": 0.66213405, + "num_input_tokens_seen": 130702495, + "router_z_loss_clip": 0.77050781, + "router_z_loss_mlp": 0.13867188, + "step": 4585, + "time_per_iteration": 2.4693210124969482 + }, + { + "auxiliary_loss_clip": 0.01147145, + "auxiliary_loss_mlp": 0.010457, + "balance_loss_clip": 1.05732274, + "balance_loss_mlp": 1.02717447, + "epoch": 0.13307411061458999, + "flos": 32810322193920.0, + "grad_norm": 2.2260466834387804, + "language_loss": 0.96153313, + "learning_rate": 3.889595157268844e-06, + "loss": 0.98346162, + "num_input_tokens_seen": 130719900, + "router_z_loss_clip": 0.89746094, + "router_z_loss_mlp": 0.18518066, + "step": 4586, + "time_per_iteration": 2.581008195877075 + }, + { + "auxiliary_loss_clip": 0.01136968, + "auxiliary_loss_mlp": 0.01038303, + "balance_loss_clip": 1.05700707, + "balance_loss_mlp": 1.0226326, + "epoch": 0.13310312808310604, + "flos": 35108533157760.0, + "grad_norm": 2.1212317385745525, + "language_loss": 0.73915231, + "learning_rate": 3.889533562393222e-06, + "loss": 0.76090503, + "num_input_tokens_seen": 130735815, + "router_z_loss_clip": 0.80078125, + "router_z_loss_mlp": 0.15661621, + "step": 4587, + "time_per_iteration": 2.6573917865753174 + }, + { + "auxiliary_loss_clip": 0.01137008, + "auxiliary_loss_mlp": 0.01043166, + "balance_loss_clip": 1.05396605, + "balance_loss_mlp": 1.02595866, + "epoch": 0.1331321455516221, + "flos": 28831958611200.0, + "grad_norm": 5.701074545343175, + "language_loss": 0.97204936, + "learning_rate": 3.8894719508284735e-06, + "loss": 0.99385118, + "num_input_tokens_seen": 130751590, + "router_z_loss_clip": 0.83105469, + "router_z_loss_mlp": 0.17224121, + "step": 4588, + "time_per_iteration": 2.584394693374634 + }, + { + "auxiliary_loss_clip": 0.01035069, + "auxiliary_loss_mlp": 0.01015377, + "balance_loss_clip": 1.01382399, + "balance_loss_mlp": 1.01422024, + "epoch": 0.1331611630201381, + "flos": 60470302826880.0, + "grad_norm": 0.6394440443697214, + "language_loss": 0.46462831, + "learning_rate": 3.889410322575145e-06, + "loss": 0.48513281, + "num_input_tokens_seen": 130817530, + "router_z_loss_clip": 0.21289062, + "router_z_loss_mlp": 0.01153564, + "step": 4589, + "time_per_iteration": 3.114431858062744 + }, + { + "auxiliary_loss_clip": 0.0103377, + "auxiliary_loss_mlp": 0.01011497, + "balance_loss_clip": 1.0126158, + "balance_loss_mlp": 1.01031101, + "epoch": 0.13319018048865416, + "flos": 74769823261440.0, + "grad_norm": 0.7134663511392042, + "language_loss": 0.45336902, + "learning_rate": 3.88934867763378e-06, + "loss": 0.4738217, + "num_input_tokens_seen": 130875410, + "router_z_loss_clip": 0.21191406, + "router_z_loss_mlp": 0.01184082, + "step": 4590, + "time_per_iteration": 3.054816246032715 + }, + { + "auxiliary_loss_clip": 0.01134611, + "auxiliary_loss_mlp": 0.01043054, + "balance_loss_clip": 1.05433345, + "balance_loss_mlp": 1.02782524, + "epoch": 0.13321919795717022, + "flos": 20880223436160.0, + "grad_norm": 2.430427665347871, + "language_loss": 0.76742578, + "learning_rate": 3.889287016004923e-06, + "loss": 0.78920245, + "num_input_tokens_seen": 130888580, + "router_z_loss_clip": 0.80322266, + "router_z_loss_mlp": 0.15216064, + "step": 4591, + "time_per_iteration": 2.518481731414795 + }, + { + "auxiliary_loss_clip": 0.01134411, + "auxiliary_loss_mlp": 0.0103707, + "balance_loss_clip": 1.05519652, + "balance_loss_mlp": 1.02252054, + "epoch": 0.13324821542568627, + "flos": 30886588471680.0, + "grad_norm": 2.064405895615558, + "language_loss": 0.7337836, + "learning_rate": 3.889225337689118e-06, + "loss": 0.75549835, + "num_input_tokens_seen": 130908065, + "router_z_loss_clip": 0.79150391, + "router_z_loss_mlp": 0.14544678, + "step": 4592, + "time_per_iteration": 2.6352622509002686 + }, + { + "auxiliary_loss_clip": 0.01132107, + "auxiliary_loss_mlp": 0.01042237, + "balance_loss_clip": 1.05297029, + "balance_loss_mlp": 1.02756858, + "epoch": 0.13327723289420232, + "flos": 16755452017920.0, + "grad_norm": 2.7751393963290694, + "language_loss": 0.85273284, + "learning_rate": 3.889163642686911e-06, + "loss": 0.87447619, + "num_input_tokens_seen": 130921805, + "router_z_loss_clip": 0.79248047, + "router_z_loss_mlp": 0.14678955, + "step": 4593, + "time_per_iteration": 2.4704034328460693 + }, + { + "auxiliary_loss_clip": 0.01135851, + "auxiliary_loss_mlp": 0.01037201, + "balance_loss_clip": 1.05697131, + "balance_loss_mlp": 1.02307475, + "epoch": 0.13330625036271834, + "flos": 40145327827200.0, + "grad_norm": 1.9778834567702668, + "language_loss": 0.90364343, + "learning_rate": 3.8891019309988456e-06, + "loss": 0.92537403, + "num_input_tokens_seen": 130938955, + "router_z_loss_clip": 0.79003906, + "router_z_loss_mlp": 0.14141846, + "step": 4594, + "time_per_iteration": 2.7009568214416504 + }, + { + "auxiliary_loss_clip": 0.01031576, + "auxiliary_loss_mlp": 0.00998899, + "balance_loss_clip": 1.01017618, + "balance_loss_mlp": 0.99781388, + "epoch": 0.1333352678312344, + "flos": 65619785439360.0, + "grad_norm": 0.6396811452159638, + "language_loss": 0.5218336, + "learning_rate": 3.889040202625468e-06, + "loss": 0.54213834, + "num_input_tokens_seen": 130999005, + "router_z_loss_clip": 0.21484375, + "router_z_loss_mlp": 0.01086426, + "step": 4595, + "time_per_iteration": 3.0335681438446045 + }, + { + "auxiliary_loss_clip": 0.01030731, + "auxiliary_loss_mlp": 0.00999057, + "balance_loss_clip": 1.0094527, + "balance_loss_mlp": 0.99787062, + "epoch": 0.13336428529975045, + "flos": 61317723467520.0, + "grad_norm": 0.6366396503987465, + "language_loss": 0.46837381, + "learning_rate": 3.888978457567322e-06, + "loss": 0.48867169, + "num_input_tokens_seen": 131058630, + "router_z_loss_clip": 0.21289062, + "router_z_loss_mlp": 0.01184082, + "step": 4596, + "time_per_iteration": 3.0820183753967285 + }, + { + "auxiliary_loss_clip": 0.01138229, + "auxiliary_loss_mlp": 0.01040763, + "balance_loss_clip": 1.05598426, + "balance_loss_mlp": 1.02270234, + "epoch": 0.1333933027682665, + "flos": 33795423665280.0, + "grad_norm": 2.4054350754526057, + "language_loss": 0.926723, + "learning_rate": 3.8889166958249544e-06, + "loss": 0.94851291, + "num_input_tokens_seen": 131076075, + "router_z_loss_clip": 0.82324219, + "router_z_loss_mlp": 0.18066406, + "step": 4597, + "time_per_iteration": 2.610854387283325 + }, + { + "auxiliary_loss_clip": 0.01130594, + "auxiliary_loss_mlp": 0.01036209, + "balance_loss_clip": 1.05403495, + "balance_loss_mlp": 1.02168322, + "epoch": 0.13342232023678255, + "flos": 25479302970240.0, + "grad_norm": 2.5194885689485598, + "language_loss": 0.75069475, + "learning_rate": 3.888854917398911e-06, + "loss": 0.77236283, + "num_input_tokens_seen": 131095435, + "router_z_loss_clip": 0.76660156, + "router_z_loss_mlp": 0.14532471, + "step": 4598, + "time_per_iteration": 2.6467230319976807 + }, + { + "auxiliary_loss_clip": 0.011346, + "auxiliary_loss_mlp": 0.01037949, + "balance_loss_clip": 1.05221069, + "balance_loss_mlp": 1.02129531, + "epoch": 0.1334513377052986, + "flos": 16066075219200.0, + "grad_norm": 3.4865749112098827, + "language_loss": 0.93017668, + "learning_rate": 3.888793122289736e-06, + "loss": 0.95190215, + "num_input_tokens_seen": 131107250, + "router_z_loss_clip": 0.82324219, + "router_z_loss_mlp": 0.16650391, + "step": 4599, + "time_per_iteration": 2.5471270084381104 + }, + { + "auxiliary_loss_clip": 0.01141189, + "auxiliary_loss_mlp": 0.01047944, + "balance_loss_clip": 1.05590439, + "balance_loss_mlp": 1.03081357, + "epoch": 0.13348035517381462, + "flos": 58027261248000.0, + "grad_norm": 2.1914073155406495, + "language_loss": 0.83140242, + "learning_rate": 3.888731310497976e-06, + "loss": 0.85329378, + "num_input_tokens_seen": 131127980, + "router_z_loss_clip": 0.85351562, + "router_z_loss_mlp": 0.17126465, + "step": 4600, + "time_per_iteration": 2.7908823490142822 + }, + { + "auxiliary_loss_clip": 0.01138686, + "auxiliary_loss_mlp": 0.01043879, + "balance_loss_clip": 1.05426884, + "balance_loss_mlp": 1.02720153, + "epoch": 0.13350937264233068, + "flos": 12341851338240.0, + "grad_norm": 2.7470498998123616, + "language_loss": 0.99517751, + "learning_rate": 3.888669482024176e-06, + "loss": 1.0170033, + "num_input_tokens_seen": 131139465, + "router_z_loss_clip": 0.84521484, + "router_z_loss_mlp": 0.16674805, + "step": 4601, + "time_per_iteration": 2.4810683727264404 + }, + { + "auxiliary_loss_clip": 0.01141664, + "auxiliary_loss_mlp": 0.01039072, + "balance_loss_clip": 1.05557203, + "balance_loss_mlp": 1.02276409, + "epoch": 0.13353839011084673, + "flos": 14058237801600.0, + "grad_norm": 3.000351377899566, + "language_loss": 0.8332423, + "learning_rate": 3.888607636868884e-06, + "loss": 0.85504961, + "num_input_tokens_seen": 131153165, + "router_z_loss_clip": 0.86230469, + "router_z_loss_mlp": 0.16290283, + "step": 4602, + "time_per_iteration": 2.4994399547576904 + }, + { + "auxiliary_loss_clip": 0.01142632, + "auxiliary_loss_mlp": 0.01044925, + "balance_loss_clip": 1.05789292, + "balance_loss_mlp": 1.02729964, + "epoch": 0.13356740757936278, + "flos": 26901473132160.0, + "grad_norm": 1.7773741029190713, + "language_loss": 0.76842976, + "learning_rate": 3.8885457750326445e-06, + "loss": 0.79030526, + "num_input_tokens_seen": 131170420, + "router_z_loss_clip": 0.84814453, + "router_z_loss_mlp": 0.1762085, + "step": 4603, + "time_per_iteration": 2.618025541305542 + }, + { + "auxiliary_loss_clip": 0.01137783, + "auxiliary_loss_mlp": 0.0104279, + "balance_loss_clip": 1.05702829, + "balance_loss_mlp": 1.02710843, + "epoch": 0.13359642504787883, + "flos": 16650270017280.0, + "grad_norm": 5.933238480403077, + "language_loss": 0.61877298, + "learning_rate": 3.888483896516004e-06, + "loss": 0.64057875, + "num_input_tokens_seen": 131185085, + "router_z_loss_clip": 0.80712891, + "router_z_loss_mlp": 0.15692139, + "step": 4604, + "time_per_iteration": 2.476064443588257 + }, + { + "auxiliary_loss_clip": 0.01141495, + "auxiliary_loss_mlp": 0.01050777, + "balance_loss_clip": 1.05433619, + "balance_loss_mlp": 1.03358674, + "epoch": 0.13362544251639488, + "flos": 26504121905280.0, + "grad_norm": 2.0356836121454966, + "language_loss": 0.6922673, + "learning_rate": 3.8884220013195106e-06, + "loss": 0.71419001, + "num_input_tokens_seen": 131206015, + "router_z_loss_clip": 0.87255859, + "router_z_loss_mlp": 0.17199707, + "step": 4605, + "time_per_iteration": 2.5723490715026855 + }, + { + "auxiliary_loss_clip": 0.01031688, + "auxiliary_loss_mlp": 0.01007514, + "balance_loss_clip": 1.01105142, + "balance_loss_mlp": 1.00645924, + "epoch": 0.1336544599849109, + "flos": 74776467277440.0, + "grad_norm": 0.6967906557950294, + "language_loss": 0.48290482, + "learning_rate": 3.88836008944371e-06, + "loss": 0.50329685, + "num_input_tokens_seen": 131269770, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.01055908, + "step": 4606, + "time_per_iteration": 3.1777613162994385 + }, + { + "auxiliary_loss_clip": 0.01139873, + "auxiliary_loss_mlp": 0.01040188, + "balance_loss_clip": 1.0526185, + "balance_loss_mlp": 1.02372551, + "epoch": 0.13368347745342696, + "flos": 23361435216000.0, + "grad_norm": 2.8803517155276204, + "language_loss": 0.9648878, + "learning_rate": 3.888298160889148e-06, + "loss": 0.98668838, + "num_input_tokens_seen": 131286120, + "router_z_loss_clip": 0.87207031, + "router_z_loss_mlp": 0.16461182, + "step": 4607, + "time_per_iteration": 2.5212998390197754 + }, + { + "auxiliary_loss_clip": 0.01151493, + "auxiliary_loss_mlp": 0.01042846, + "balance_loss_clip": 1.06020534, + "balance_loss_mlp": 1.02454782, + "epoch": 0.133712494921943, + "flos": 29563530912000.0, + "grad_norm": 1.9188168224224877, + "language_loss": 0.87644619, + "learning_rate": 3.888236215656373e-06, + "loss": 0.89838958, + "num_input_tokens_seen": 131305715, + "router_z_loss_clip": 0.91308594, + "router_z_loss_mlp": 0.18286133, + "step": 4608, + "time_per_iteration": 2.578601598739624 + }, + { + "auxiliary_loss_clip": 0.01144747, + "auxiliary_loss_mlp": 0.0105158, + "balance_loss_clip": 1.05659747, + "balance_loss_mlp": 1.03425264, + "epoch": 0.13374151239045906, + "flos": 14676906677760.0, + "grad_norm": 2.6900860009315317, + "language_loss": 0.97416544, + "learning_rate": 3.888174253745931e-06, + "loss": 0.9961288, + "num_input_tokens_seen": 131318220, + "router_z_loss_clip": 0.88134766, + "router_z_loss_mlp": 0.17327881, + "step": 4609, + "time_per_iteration": 2.465113639831543 + }, + { + "auxiliary_loss_clip": 0.01142396, + "auxiliary_loss_mlp": 0.01036313, + "balance_loss_clip": 1.0583334, + "balance_loss_mlp": 1.02049446, + "epoch": 0.1337705298589751, + "flos": 37889958810240.0, + "grad_norm": 2.049213925249036, + "language_loss": 0.99824697, + "learning_rate": 3.888112275158371e-06, + "loss": 1.02003407, + "num_input_tokens_seen": 131342095, + "router_z_loss_clip": 0.84130859, + "router_z_loss_mlp": 0.15820312, + "step": 4610, + "time_per_iteration": 2.6813952922821045 + }, + { + "auxiliary_loss_clip": 0.01131959, + "auxiliary_loss_mlp": 0.01039631, + "balance_loss_clip": 1.05427313, + "balance_loss_mlp": 1.02545655, + "epoch": 0.13379954732749114, + "flos": 31497356355840.0, + "grad_norm": 2.541485487361971, + "language_loss": 0.91071117, + "learning_rate": 3.888050279894239e-06, + "loss": 0.93242705, + "num_input_tokens_seen": 131356475, + "router_z_loss_clip": 0.77685547, + "router_z_loss_mlp": 0.14172363, + "step": 4611, + "time_per_iteration": 2.5976879596710205 + }, + { + "auxiliary_loss_clip": 0.01136375, + "auxiliary_loss_mlp": 0.01040317, + "balance_loss_clip": 1.05817711, + "balance_loss_mlp": 1.02441406, + "epoch": 0.1338285647960072, + "flos": 27998939324160.0, + "grad_norm": 1.9571498114931167, + "language_loss": 0.8550576, + "learning_rate": 3.8879882679540824e-06, + "loss": 0.8768245, + "num_input_tokens_seen": 131379440, + "router_z_loss_clip": 0.78173828, + "router_z_loss_mlp": 0.15887451, + "step": 4612, + "time_per_iteration": 2.709146738052368 + }, + { + "auxiliary_loss_clip": 0.01143799, + "auxiliary_loss_mlp": 0.0103911, + "balance_loss_clip": 1.05740619, + "balance_loss_mlp": 1.02162218, + "epoch": 0.13385758226452324, + "flos": 29745705715200.0, + "grad_norm": 2.4511964245554227, + "language_loss": 0.69081408, + "learning_rate": 3.88792623933845e-06, + "loss": 0.71264315, + "num_input_tokens_seen": 131395510, + "router_z_loss_clip": 0.86279297, + "router_z_loss_mlp": 0.17480469, + "step": 4613, + "time_per_iteration": 2.6429762840270996 + }, + { + "auxiliary_loss_clip": 0.0103164, + "auxiliary_loss_mlp": 0.01005185, + "balance_loss_clip": 1.01111937, + "balance_loss_mlp": 1.00415409, + "epoch": 0.1338865997330393, + "flos": 69513255227520.0, + "grad_norm": 0.7126733217492607, + "language_loss": 0.49702096, + "learning_rate": 3.887864194047889e-06, + "loss": 0.51738918, + "num_input_tokens_seen": 131451495, + "router_z_loss_clip": 0.20507812, + "router_z_loss_mlp": 0.01031494, + "step": 4614, + "time_per_iteration": 3.0409188270568848 + }, + { + "auxiliary_loss_clip": 0.01031992, + "auxiliary_loss_mlp": 0.01005731, + "balance_loss_clip": 1.01132953, + "balance_loss_mlp": 1.00472975, + "epoch": 0.13391561720155534, + "flos": 70251794766720.0, + "grad_norm": 0.7129473344992144, + "language_loss": 0.52505559, + "learning_rate": 3.887802132082947e-06, + "loss": 0.54543287, + "num_input_tokens_seen": 131516760, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.01000977, + "step": 4615, + "time_per_iteration": 3.1188578605651855 + }, + { + "auxiliary_loss_clip": 0.01146121, + "auxiliary_loss_mlp": 0.01046142, + "balance_loss_clip": 1.05821669, + "balance_loss_mlp": 1.02780724, + "epoch": 0.1339446346700714, + "flos": 21500903473920.0, + "grad_norm": 2.557807965877623, + "language_loss": 1.05446267, + "learning_rate": 3.8877400534441735e-06, + "loss": 1.07638526, + "num_input_tokens_seen": 131537805, + "router_z_loss_clip": 0.87890625, + "router_z_loss_mlp": 0.18347168, + "step": 4616, + "time_per_iteration": 2.6840872764587402 + }, + { + "auxiliary_loss_clip": 0.01030691, + "auxiliary_loss_mlp": 0.01000881, + "balance_loss_clip": 1.01017618, + "balance_loss_mlp": 0.99982566, + "epoch": 0.13397365213858742, + "flos": 62333779484160.0, + "grad_norm": 0.7604504146153284, + "language_loss": 0.51641476, + "learning_rate": 3.887677958132115e-06, + "loss": 0.53673053, + "num_input_tokens_seen": 131598015, + "router_z_loss_clip": 0.20507812, + "router_z_loss_mlp": 0.01055908, + "step": 4617, + "time_per_iteration": 3.006387233734131 + }, + { + "auxiliary_loss_clip": 0.01030038, + "auxiliary_loss_mlp": 0.01001531, + "balance_loss_clip": 1.00954425, + "balance_loss_mlp": 1.00047588, + "epoch": 0.13400266960710347, + "flos": 67331970011520.0, + "grad_norm": 0.6179931272674869, + "language_loss": 0.46198761, + "learning_rate": 3.887615846147322e-06, + "loss": 0.48230332, + "num_input_tokens_seen": 131662050, + "router_z_loss_clip": 0.20507812, + "router_z_loss_mlp": 0.01055908, + "step": 4618, + "time_per_iteration": 3.113569974899292 + }, + { + "auxiliary_loss_clip": 0.01142269, + "auxiliary_loss_mlp": 0.01048975, + "balance_loss_clip": 1.0609045, + "balance_loss_mlp": 1.03315604, + "epoch": 0.13403168707561952, + "flos": 26608837029120.0, + "grad_norm": 2.129198137435166, + "language_loss": 0.895123, + "learning_rate": 3.887553717490341e-06, + "loss": 0.91703552, + "num_input_tokens_seen": 131678830, + "router_z_loss_clip": 0.81396484, + "router_z_loss_mlp": 0.15820312, + "step": 4619, + "time_per_iteration": 2.6130075454711914 + }, + { + "auxiliary_loss_clip": 0.01135234, + "auxiliary_loss_mlp": 0.01040577, + "balance_loss_clip": 1.05922961, + "balance_loss_mlp": 1.02671289, + "epoch": 0.13406070454413557, + "flos": 31425966074880.0, + "grad_norm": 1.7201123312692819, + "language_loss": 0.82712466, + "learning_rate": 3.887491572161722e-06, + "loss": 0.84888268, + "num_input_tokens_seen": 131699490, + "router_z_loss_clip": 0.75927734, + "router_z_loss_mlp": 0.13861084, + "step": 4620, + "time_per_iteration": 2.645801544189453 + }, + { + "auxiliary_loss_clip": 0.01137101, + "auxiliary_loss_mlp": 0.01049129, + "balance_loss_clip": 1.05652642, + "balance_loss_mlp": 1.03405523, + "epoch": 0.13408972201265162, + "flos": 17231017109760.0, + "grad_norm": 2.610929095324164, + "language_loss": 0.77904689, + "learning_rate": 3.8874294101620145e-06, + "loss": 0.80090928, + "num_input_tokens_seen": 131711440, + "router_z_loss_clip": 0.80566406, + "router_z_loss_mlp": 0.15081787, + "step": 4621, + "time_per_iteration": 2.5089149475097656 + }, + { + "auxiliary_loss_clip": 0.01131996, + "auxiliary_loss_mlp": 0.01038124, + "balance_loss_clip": 1.05599988, + "balance_loss_mlp": 1.02352715, + "epoch": 0.13411873948116768, + "flos": 26243517755520.0, + "grad_norm": 2.8935173193364006, + "language_loss": 0.76819789, + "learning_rate": 3.887367231491765e-06, + "loss": 0.78989911, + "num_input_tokens_seen": 131726205, + "router_z_loss_clip": 0.75976562, + "router_z_loss_mlp": 0.14599609, + "step": 4622, + "time_per_iteration": 2.5939102172851562 + }, + { + "auxiliary_loss_clip": 0.01141159, + "auxiliary_loss_mlp": 0.01055747, + "balance_loss_clip": 1.05616415, + "balance_loss_mlp": 1.03883123, + "epoch": 0.1341477569496837, + "flos": 11649278229120.0, + "grad_norm": 2.6707311890361085, + "language_loss": 0.81671095, + "learning_rate": 3.887305036151526e-06, + "loss": 0.83868003, + "num_input_tokens_seen": 131737755, + "router_z_loss_clip": 0.85107422, + "router_z_loss_mlp": 0.16925049, + "step": 4623, + "time_per_iteration": 4.997305154800415 + }, + { + "auxiliary_loss_clip": 0.01134843, + "auxiliary_loss_mlp": 0.010491, + "balance_loss_clip": 1.05420458, + "balance_loss_mlp": 1.03316164, + "epoch": 0.13417677441819975, + "flos": 44669928510720.0, + "grad_norm": 3.2269546649700196, + "language_loss": 0.91316831, + "learning_rate": 3.887242824141845e-06, + "loss": 0.93500769, + "num_input_tokens_seen": 131754615, + "router_z_loss_clip": 0.80761719, + "router_z_loss_mlp": 0.15930176, + "step": 4624, + "time_per_iteration": 7.475244760513306 + }, + { + "auxiliary_loss_clip": 0.01031091, + "auxiliary_loss_mlp": 0.01016528, + "balance_loss_clip": 1.0100317, + "balance_loss_mlp": 1.01547897, + "epoch": 0.1342057918867158, + "flos": 74783111293440.0, + "grad_norm": 0.6175830914628654, + "language_loss": 0.46761945, + "learning_rate": 3.887180595463271e-06, + "loss": 0.48809564, + "num_input_tokens_seen": 131819725, + "router_z_loss_clip": 0.2109375, + "router_z_loss_mlp": 0.01049805, + "step": 4625, + "time_per_iteration": 3.2043488025665283 + }, + { + "auxiliary_loss_clip": 0.0113613, + "auxiliary_loss_mlp": 0.01036718, + "balance_loss_clip": 1.05535197, + "balance_loss_mlp": 1.01998067, + "epoch": 0.13423480935523185, + "flos": 31829243045760.0, + "grad_norm": 1.9253651927196482, + "language_loss": 0.84273052, + "learning_rate": 3.887118350116355e-06, + "loss": 0.86445898, + "num_input_tokens_seen": 131839875, + "router_z_loss_clip": 0.80761719, + "router_z_loss_mlp": 0.16729736, + "step": 4626, + "time_per_iteration": 5.071018934249878 + }, + { + "auxiliary_loss_clip": 0.01030765, + "auxiliary_loss_mlp": 0.01004853, + "balance_loss_clip": 1.00958252, + "balance_loss_mlp": 1.00379205, + "epoch": 0.1342638268237479, + "flos": 72117282585600.0, + "grad_norm": 0.6099974817660898, + "language_loss": 0.44209248, + "learning_rate": 3.887056088101645e-06, + "loss": 0.46244866, + "num_input_tokens_seen": 131905310, + "router_z_loss_clip": 0.21191406, + "router_z_loss_mlp": 0.01062012, + "step": 4627, + "time_per_iteration": 3.162574529647827 + }, + { + "auxiliary_loss_clip": 0.01031165, + "auxiliary_loss_mlp": 0.01001336, + "balance_loss_clip": 1.00994229, + "balance_loss_mlp": 1.00018013, + "epoch": 0.13429284429226393, + "flos": 69304543251840.0, + "grad_norm": 0.6745890985823578, + "language_loss": 0.46852148, + "learning_rate": 3.886993809419693e-06, + "loss": 0.48884648, + "num_input_tokens_seen": 131962840, + "router_z_loss_clip": 0.21289062, + "router_z_loss_mlp": 0.01153564, + "step": 4628, + "time_per_iteration": 3.0598132610321045 + }, + { + "auxiliary_loss_clip": 0.0114923, + "auxiliary_loss_mlp": 0.01044961, + "balance_loss_clip": 1.0597676, + "balance_loss_mlp": 1.02629232, + "epoch": 0.13432186176077998, + "flos": 42416283346560.0, + "grad_norm": 2.2936876107752697, + "language_loss": 0.82957971, + "learning_rate": 3.886931514071047e-06, + "loss": 0.85152161, + "num_input_tokens_seen": 131983575, + "router_z_loss_clip": 0.89453125, + "router_z_loss_mlp": 0.18676758, + "step": 4629, + "time_per_iteration": 2.6600570678710938 + }, + { + "auxiliary_loss_clip": 0.01141626, + "auxiliary_loss_mlp": 0.01050329, + "balance_loss_clip": 1.05837047, + "balance_loss_mlp": 1.0322088, + "epoch": 0.13435087922929603, + "flos": 29854910039040.0, + "grad_norm": 2.871648700728485, + "language_loss": 0.60137039, + "learning_rate": 3.88686920205626e-06, + "loss": 0.62328994, + "num_input_tokens_seen": 131999200, + "router_z_loss_clip": 0.83154297, + "router_z_loss_mlp": 0.18103027, + "step": 4630, + "time_per_iteration": 2.5607664585113525 + }, + { + "auxiliary_loss_clip": 0.01144098, + "auxiliary_loss_mlp": 0.01047617, + "balance_loss_clip": 1.05594039, + "balance_loss_mlp": 1.02841234, + "epoch": 0.13437989669781208, + "flos": 24601753797120.0, + "grad_norm": 2.2832996721769185, + "language_loss": 0.75077367, + "learning_rate": 3.88680687337588e-06, + "loss": 0.77269077, + "num_input_tokens_seen": 132012310, + "router_z_loss_clip": 0.87939453, + "router_z_loss_mlp": 0.19213867, + "step": 4631, + "time_per_iteration": 2.493544816970825 + }, + { + "auxiliary_loss_clip": 0.01147233, + "auxiliary_loss_mlp": 0.01055863, + "balance_loss_clip": 1.05812287, + "balance_loss_mlp": 1.03544247, + "epoch": 0.13440891416632814, + "flos": 28980593089920.0, + "grad_norm": 2.0646024708956388, + "language_loss": 0.92800796, + "learning_rate": 3.886744528030458e-06, + "loss": 0.95003891, + "num_input_tokens_seen": 132027875, + "router_z_loss_clip": 0.89111328, + "router_z_loss_mlp": 0.20422363, + "step": 4632, + "time_per_iteration": 2.569823980331421 + }, + { + "auxiliary_loss_clip": 0.01140938, + "auxiliary_loss_mlp": 0.01044137, + "balance_loss_clip": 1.05843115, + "balance_loss_mlp": 1.02749586, + "epoch": 0.1344379316348442, + "flos": 27518705464320.0, + "grad_norm": 2.5423417004114364, + "language_loss": 1.08610559, + "learning_rate": 3.886682166020544e-06, + "loss": 1.10795641, + "num_input_tokens_seen": 132043125, + "router_z_loss_clip": 0.82421875, + "router_z_loss_mlp": 0.16625977, + "step": 4633, + "time_per_iteration": 2.588501214981079 + }, + { + "auxiliary_loss_clip": 0.011481, + "auxiliary_loss_mlp": 0.01053448, + "balance_loss_clip": 1.05961609, + "balance_loss_mlp": 1.03476775, + "epoch": 0.1344669491033602, + "flos": 12085233598080.0, + "grad_norm": 3.1250177296236132, + "language_loss": 0.72041547, + "learning_rate": 3.8866197873466915e-06, + "loss": 0.74243093, + "num_input_tokens_seen": 132053885, + "router_z_loss_clip": 0.88476562, + "router_z_loss_mlp": 0.18695068, + "step": 4634, + "time_per_iteration": 2.4832680225372314 + }, + { + "auxiliary_loss_clip": 0.01138086, + "auxiliary_loss_mlp": 0.01047005, + "balance_loss_clip": 1.05814695, + "balance_loss_mlp": 1.03180623, + "epoch": 0.13449596657187626, + "flos": 28504848430080.0, + "grad_norm": 8.278510930599106, + "language_loss": 0.84870982, + "learning_rate": 3.8865573920094484e-06, + "loss": 0.87056065, + "num_input_tokens_seen": 132070890, + "router_z_loss_clip": 0.79980469, + "router_z_loss_mlp": 0.15197754, + "step": 4635, + "time_per_iteration": 2.6751081943511963 + }, + { + "auxiliary_loss_clip": 0.01141791, + "auxiliary_loss_mlp": 0.01051075, + "balance_loss_clip": 1.05668569, + "balance_loss_mlp": 1.03315818, + "epoch": 0.13452498404039231, + "flos": 15444210032640.0, + "grad_norm": 2.7462378403714434, + "language_loss": 0.8324737, + "learning_rate": 3.8864949800093665e-06, + "loss": 0.8544023, + "num_input_tokens_seen": 132084110, + "router_z_loss_clip": 0.8515625, + "router_z_loss_mlp": 0.17907715, + "step": 4636, + "time_per_iteration": 2.5658156871795654 + }, + { + "auxiliary_loss_clip": 0.01034526, + "auxiliary_loss_mlp": 0.01033787, + "balance_loss_clip": 1.012887, + "balance_loss_mlp": 1.03251195, + "epoch": 0.13455400150890837, + "flos": 65173846089600.0, + "grad_norm": 0.7733732346187047, + "language_loss": 0.41358572, + "learning_rate": 3.886432551346998e-06, + "loss": 0.43426883, + "num_input_tokens_seen": 132139935, + "router_z_loss_clip": 0.21679688, + "router_z_loss_mlp": 0.01275635, + "step": 4637, + "time_per_iteration": 3.0194220542907715 + }, + { + "auxiliary_loss_clip": 0.01144686, + "auxiliary_loss_mlp": 0.01061484, + "balance_loss_clip": 1.05636609, + "balance_loss_mlp": 1.04269624, + "epoch": 0.13458301897742442, + "flos": 17199021070080.0, + "grad_norm": 3.5223287938716576, + "language_loss": 0.79545677, + "learning_rate": 3.886370106022895e-06, + "loss": 0.81751841, + "num_input_tokens_seen": 132158385, + "router_z_loss_clip": 0.88330078, + "router_z_loss_mlp": 0.18811035, + "step": 4638, + "time_per_iteration": 2.5615901947021484 + }, + { + "auxiliary_loss_clip": 0.01135557, + "auxiliary_loss_mlp": 0.01046942, + "balance_loss_clip": 1.05314553, + "balance_loss_mlp": 1.03073597, + "epoch": 0.13461203644594047, + "flos": 14423880297600.0, + "grad_norm": 3.4029007008967183, + "language_loss": 0.8725453, + "learning_rate": 3.886307644037606e-06, + "loss": 0.89437026, + "num_input_tokens_seen": 132171260, + "router_z_loss_clip": 0.82470703, + "router_z_loss_mlp": 0.16210938, + "step": 4639, + "time_per_iteration": 2.512737989425659 + }, + { + "auxiliary_loss_clip": 0.01143219, + "auxiliary_loss_mlp": 0.01048298, + "balance_loss_clip": 1.06054044, + "balance_loss_mlp": 1.02964211, + "epoch": 0.1346410539144565, + "flos": 25260427445760.0, + "grad_norm": 2.4340647984924417, + "language_loss": 0.97348607, + "learning_rate": 3.886245165391686e-06, + "loss": 0.99540126, + "num_input_tokens_seen": 132185145, + "router_z_loss_clip": 0.82666016, + "router_z_loss_mlp": 0.18664551, + "step": 4640, + "time_per_iteration": 2.577934741973877 + }, + { + "auxiliary_loss_clip": 0.01137836, + "auxiliary_loss_mlp": 0.01043871, + "balance_loss_clip": 1.05726731, + "balance_loss_mlp": 1.02780175, + "epoch": 0.13467007138297254, + "flos": 31096341941760.0, + "grad_norm": 2.4461467851971883, + "language_loss": 0.80927408, + "learning_rate": 3.886182670085685e-06, + "loss": 0.83109117, + "num_input_tokens_seen": 132203275, + "router_z_loss_clip": 0.80517578, + "router_z_loss_mlp": 0.16064453, + "step": 4641, + "time_per_iteration": 2.72802996635437 + }, + { + "auxiliary_loss_clip": 0.01140509, + "auxiliary_loss_mlp": 0.01048682, + "balance_loss_clip": 1.06116486, + "balance_loss_mlp": 1.03235579, + "epoch": 0.1346990888514886, + "flos": 44087098429440.0, + "grad_norm": 1.915128688540128, + "language_loss": 0.71028364, + "learning_rate": 3.8861201581201554e-06, + "loss": 0.73217559, + "num_input_tokens_seen": 132222920, + "router_z_loss_clip": 0.79345703, + "router_z_loss_mlp": 0.16339111, + "step": 4642, + "time_per_iteration": 2.8010621070861816 + }, + { + "auxiliary_loss_clip": 0.01138961, + "auxiliary_loss_mlp": 0.0104354, + "balance_loss_clip": 1.05551934, + "balance_loss_mlp": 1.02746499, + "epoch": 0.13472810632000465, + "flos": 11104693153920.0, + "grad_norm": 2.8453175324397377, + "language_loss": 0.73422956, + "learning_rate": 3.886057629495649e-06, + "loss": 0.75605458, + "num_input_tokens_seen": 132234410, + "router_z_loss_clip": 0.83447266, + "router_z_loss_mlp": 0.1605835, + "step": 4643, + "time_per_iteration": 2.5187346935272217 + }, + { + "auxiliary_loss_clip": 0.01146333, + "auxiliary_loss_mlp": 0.01057227, + "balance_loss_clip": 1.06023312, + "balance_loss_mlp": 1.03748608, + "epoch": 0.1347571237885207, + "flos": 27665831571840.0, + "grad_norm": 2.5260922556447394, + "language_loss": 0.82367432, + "learning_rate": 3.885995084212719e-06, + "loss": 0.84570998, + "num_input_tokens_seen": 132249020, + "router_z_loss_clip": 0.86035156, + "router_z_loss_mlp": 0.19750977, + "step": 4644, + "time_per_iteration": 2.6120893955230713 + }, + { + "auxiliary_loss_clip": 0.01045384, + "auxiliary_loss_mlp": 0.01016091, + "balance_loss_clip": 1.0234735, + "balance_loss_mlp": 1.01522684, + "epoch": 0.13478614125703672, + "flos": 74769464125440.0, + "grad_norm": 0.7071174044767357, + "language_loss": 0.5337652, + "learning_rate": 3.8859325222719174e-06, + "loss": 0.55437994, + "num_input_tokens_seen": 132308525, + "router_z_loss_clip": 0.21875, + "router_z_loss_mlp": 0.00866699, + "step": 4645, + "time_per_iteration": 3.0978200435638428 + }, + { + "auxiliary_loss_clip": 0.01043855, + "auxiliary_loss_mlp": 0.01010775, + "balance_loss_clip": 1.02181494, + "balance_loss_mlp": 1.00980902, + "epoch": 0.13481515872555278, + "flos": 60223274017920.0, + "grad_norm": 0.6801700594103569, + "language_loss": 0.49986246, + "learning_rate": 3.8858699436737955e-06, + "loss": 0.52040875, + "num_input_tokens_seen": 132369715, + "router_z_loss_clip": 0.22070312, + "router_z_loss_mlp": 0.00964355, + "step": 4646, + "time_per_iteration": 3.082294464111328 + }, + { + "auxiliary_loss_clip": 0.0113852, + "auxiliary_loss_mlp": 0.01056153, + "balance_loss_clip": 1.05623603, + "balance_loss_mlp": 1.04105532, + "epoch": 0.13484417619406883, + "flos": 19054991784960.0, + "grad_norm": 2.0001364509450106, + "language_loss": 0.67368245, + "learning_rate": 3.885807348418908e-06, + "loss": 0.69562918, + "num_input_tokens_seen": 132384790, + "router_z_loss_clip": 0.82275391, + "router_z_loss_mlp": 0.15081787, + "step": 4647, + "time_per_iteration": 2.5407845973968506 + }, + { + "auxiliary_loss_clip": 0.01143475, + "auxiliary_loss_mlp": 0.01052435, + "balance_loss_clip": 1.0607698, + "balance_loss_mlp": 1.03626418, + "epoch": 0.13487319366258488, + "flos": 22121691252480.0, + "grad_norm": 2.405817065355854, + "language_loss": 0.72502369, + "learning_rate": 3.885744736507807e-06, + "loss": 0.74698275, + "num_input_tokens_seen": 132399530, + "router_z_loss_clip": 0.82666016, + "router_z_loss_mlp": 0.16168213, + "step": 4648, + "time_per_iteration": 2.5589003562927246 + }, + { + "auxiliary_loss_clip": 0.0114438, + "auxiliary_loss_mlp": 0.01047893, + "balance_loss_clip": 1.05830216, + "balance_loss_mlp": 1.02833092, + "epoch": 0.13490221113110093, + "flos": 72614677190400.0, + "grad_norm": 2.534664497134903, + "language_loss": 0.92831242, + "learning_rate": 3.885682107941045e-06, + "loss": 0.95023519, + "num_input_tokens_seen": 132418300, + "router_z_loss_clip": 0.86132812, + "router_z_loss_mlp": 0.19555664, + "step": 4649, + "time_per_iteration": 2.915313482284546 + }, + { + "auxiliary_loss_clip": 0.01148039, + "auxiliary_loss_mlp": 0.01047931, + "balance_loss_clip": 1.05833197, + "balance_loss_mlp": 1.02939951, + "epoch": 0.13493122859961698, + "flos": 25111685226240.0, + "grad_norm": 1.9989246587439593, + "language_loss": 0.7207576, + "learning_rate": 3.885619462719175e-06, + "loss": 0.74271727, + "num_input_tokens_seen": 132433290, + "router_z_loss_clip": 0.8984375, + "router_z_loss_mlp": 0.18518066, + "step": 4650, + "time_per_iteration": 2.5684521198272705 + }, + { + "auxiliary_loss_clip": 0.01144984, + "auxiliary_loss_mlp": 0.01048114, + "balance_loss_clip": 1.06088018, + "balance_loss_mlp": 1.0295409, + "epoch": 0.134960246068133, + "flos": 28615776606720.0, + "grad_norm": 3.407078710111746, + "language_loss": 0.64409459, + "learning_rate": 3.885556800842753e-06, + "loss": 0.66602558, + "num_input_tokens_seen": 132448625, + "router_z_loss_clip": 0.84130859, + "router_z_loss_mlp": 0.18579102, + "step": 4651, + "time_per_iteration": 2.5097711086273193 + }, + { + "auxiliary_loss_clip": 0.01144029, + "auxiliary_loss_mlp": 0.0104935, + "balance_loss_clip": 1.05930007, + "balance_loss_mlp": 1.03053927, + "epoch": 0.13498926353664906, + "flos": 13144993488000.0, + "grad_norm": 2.33917267740273, + "language_loss": 0.84587133, + "learning_rate": 3.885494122312327e-06, + "loss": 0.86780512, + "num_input_tokens_seen": 132460660, + "router_z_loss_clip": 0.84765625, + "router_z_loss_mlp": 0.18811035, + "step": 4652, + "time_per_iteration": 2.4693658351898193 + }, + { + "auxiliary_loss_clip": 0.0114396, + "auxiliary_loss_mlp": 0.01047724, + "balance_loss_clip": 1.06326377, + "balance_loss_mlp": 1.03116632, + "epoch": 0.1350182810051651, + "flos": 13327204204800.0, + "grad_norm": 2.44104080214384, + "language_loss": 0.74435425, + "learning_rate": 3.885431427128457e-06, + "loss": 0.76627111, + "num_input_tokens_seen": 132472395, + "router_z_loss_clip": 0.80664062, + "router_z_loss_mlp": 0.16564941, + "step": 4653, + "time_per_iteration": 2.5079457759857178 + }, + { + "auxiliary_loss_clip": 0.01141922, + "auxiliary_loss_mlp": 0.01043362, + "balance_loss_clip": 1.05791211, + "balance_loss_mlp": 1.02680969, + "epoch": 0.13504729847368116, + "flos": 12707960711040.0, + "grad_norm": 2.174023198119254, + "language_loss": 0.72676659, + "learning_rate": 3.885368715291692e-06, + "loss": 0.74861944, + "num_input_tokens_seen": 132487380, + "router_z_loss_clip": 0.83984375, + "router_z_loss_mlp": 0.16546631, + "step": 4654, + "time_per_iteration": 2.469074010848999 + }, + { + "auxiliary_loss_clip": 0.01042897, + "auxiliary_loss_mlp": 0.01004348, + "balance_loss_clip": 1.02148557, + "balance_loss_mlp": 1.00347769, + "epoch": 0.1350763159421972, + "flos": 72475562793600.0, + "grad_norm": 0.6817850936940406, + "language_loss": 0.50449461, + "learning_rate": 3.8853059868025885e-06, + "loss": 0.52496701, + "num_input_tokens_seen": 132550370, + "router_z_loss_clip": 0.21484375, + "router_z_loss_mlp": 0.00872803, + "step": 4655, + "time_per_iteration": 3.2029788494110107 + }, + { + "auxiliary_loss_clip": 0.0104177, + "auxiliary_loss_mlp": 0.01004638, + "balance_loss_clip": 1.02029061, + "balance_loss_mlp": 1.00379741, + "epoch": 0.13510533341071326, + "flos": 66453989875200.0, + "grad_norm": 0.7486661603851791, + "language_loss": 0.48701212, + "learning_rate": 3.885243241661699e-06, + "loss": 0.50747621, + "num_input_tokens_seen": 132611460, + "router_z_loss_clip": 0.21484375, + "router_z_loss_mlp": 0.00842285, + "step": 4656, + "time_per_iteration": 3.0602338314056396 + }, + { + "auxiliary_loss_clip": 0.01040121, + "auxiliary_loss_mlp": 0.01001256, + "balance_loss_clip": 1.01886749, + "balance_loss_mlp": 1.00040364, + "epoch": 0.1351343508792293, + "flos": 64739470919040.0, + "grad_norm": 0.6961877164244508, + "language_loss": 0.499284, + "learning_rate": 3.885180479869578e-06, + "loss": 0.51969779, + "num_input_tokens_seen": 132671710, + "router_z_loss_clip": 0.21289062, + "router_z_loss_mlp": 0.00854492, + "step": 4657, + "time_per_iteration": 3.0446038246154785 + }, + { + "auxiliary_loss_clip": 0.0104019, + "auxiliary_loss_mlp": 0.0100344, + "balance_loss_clip": 1.01897597, + "balance_loss_mlp": 1.00245059, + "epoch": 0.13516336834774534, + "flos": 74294832787200.0, + "grad_norm": 0.6983283727321068, + "language_loss": 0.53939319, + "learning_rate": 3.885117701426781e-06, + "loss": 0.55982947, + "num_input_tokens_seen": 132734250, + "router_z_loss_clip": 0.21289062, + "router_z_loss_mlp": 0.0098877, + "step": 4658, + "time_per_iteration": 3.094665765762329 + }, + { + "auxiliary_loss_clip": 0.01039482, + "auxiliary_loss_mlp": 0.01005092, + "balance_loss_clip": 1.01813018, + "balance_loss_mlp": 1.00413227, + "epoch": 0.1351923858162614, + "flos": 57480667989120.0, + "grad_norm": 0.6213146253313934, + "language_loss": 0.46558797, + "learning_rate": 3.885054906333861e-06, + "loss": 0.48603368, + "num_input_tokens_seen": 132795370, + "router_z_loss_clip": 0.21289062, + "router_z_loss_mlp": 0.00958252, + "step": 4659, + "time_per_iteration": 3.1183764934539795 + }, + { + "auxiliary_loss_clip": 0.01143155, + "auxiliary_loss_mlp": 0.01052717, + "balance_loss_clip": 1.05984688, + "balance_loss_mlp": 1.03556275, + "epoch": 0.13522140328477744, + "flos": 25299678032640.0, + "grad_norm": 3.124077641695618, + "language_loss": 1.06880045, + "learning_rate": 3.884992094591373e-06, + "loss": 1.09075916, + "num_input_tokens_seen": 132815895, + "router_z_loss_clip": 0.83300781, + "router_z_loss_mlp": 0.17156982, + "step": 4660, + "time_per_iteration": 2.68601393699646 + }, + { + "auxiliary_loss_clip": 0.01039007, + "auxiliary_loss_mlp": 0.01008373, + "balance_loss_clip": 1.01784933, + "balance_loss_mlp": 1.00737154, + "epoch": 0.1352504207532935, + "flos": 62690264012160.0, + "grad_norm": 0.765212680849512, + "language_loss": 0.53989142, + "learning_rate": 3.8849292661998734e-06, + "loss": 0.5603652, + "num_input_tokens_seen": 132871560, + "router_z_loss_clip": 0.21191406, + "router_z_loss_mlp": 0.01000977, + "step": 4661, + "time_per_iteration": 2.96913480758667 + }, + { + "auxiliary_loss_clip": 0.0114321, + "auxiliary_loss_mlp": 0.01050484, + "balance_loss_clip": 1.05884182, + "balance_loss_mlp": 1.03188729, + "epoch": 0.13527943822180952, + "flos": 10953975686400.0, + "grad_norm": 3.302006215737982, + "language_loss": 1.01093245, + "learning_rate": 3.884866421159915e-06, + "loss": 1.03286934, + "num_input_tokens_seen": 132881595, + "router_z_loss_clip": 0.84277344, + "router_z_loss_mlp": 0.18603516, + "step": 4662, + "time_per_iteration": 2.5086493492126465 + }, + { + "auxiliary_loss_clip": 0.01037091, + "auxiliary_loss_mlp": 0.01008052, + "balance_loss_clip": 1.01589537, + "balance_loss_mlp": 1.00714612, + "epoch": 0.13530845569032557, + "flos": 71777674471680.0, + "grad_norm": 0.7211350559444647, + "language_loss": 0.47862682, + "learning_rate": 3.8848035594720535e-06, + "loss": 0.49907824, + "num_input_tokens_seen": 132937290, + "router_z_loss_clip": 0.21289062, + "router_z_loss_mlp": 0.0090332, + "step": 4663, + "time_per_iteration": 3.0389328002929688 + }, + { + "auxiliary_loss_clip": 0.0113691, + "auxiliary_loss_mlp": 0.01043189, + "balance_loss_clip": 1.05789924, + "balance_loss_mlp": 1.02775717, + "epoch": 0.13533747315884162, + "flos": 25002947779200.0, + "grad_norm": 1.8918889869289186, + "language_loss": 0.72274661, + "learning_rate": 3.884740681136844e-06, + "loss": 0.74454761, + "num_input_tokens_seen": 132953845, + "router_z_loss_clip": 0.79003906, + "router_z_loss_mlp": 0.15435791, + "step": 4664, + "time_per_iteration": 2.560314893722534 + }, + { + "auxiliary_loss_clip": 0.01035079, + "auxiliary_loss_mlp": 0.01002225, + "balance_loss_clip": 1.01401448, + "balance_loss_mlp": 1.001212, + "epoch": 0.13536649062735767, + "flos": 72348620129280.0, + "grad_norm": 0.6830261576651799, + "language_loss": 0.51076293, + "learning_rate": 3.884677786154843e-06, + "loss": 0.53113592, + "num_input_tokens_seen": 133022550, + "router_z_loss_clip": 0.2109375, + "router_z_loss_mlp": 0.01013184, + "step": 4665, + "time_per_iteration": 3.254441976547241 + }, + { + "auxiliary_loss_clip": 0.01034359, + "auxiliary_loss_mlp": 0.01004654, + "balance_loss_clip": 1.0133636, + "balance_loss_mlp": 1.00364041, + "epoch": 0.13539550809587372, + "flos": 66422029749120.0, + "grad_norm": 0.6875982421235548, + "language_loss": 0.48172265, + "learning_rate": 3.884614874526604e-06, + "loss": 0.50211281, + "num_input_tokens_seen": 133085495, + "router_z_loss_clip": 0.20996094, + "router_z_loss_mlp": 0.01013184, + "step": 4666, + "time_per_iteration": 3.09926176071167 + }, + { + "auxiliary_loss_clip": 0.01033978, + "auxiliary_loss_mlp": 0.01002229, + "balance_loss_clip": 1.01283848, + "balance_loss_mlp": 1.00123394, + "epoch": 0.13542452556438977, + "flos": 64231694305920.0, + "grad_norm": 0.7556069222652556, + "language_loss": 0.54010093, + "learning_rate": 3.884551946252684e-06, + "loss": 0.56046295, + "num_input_tokens_seen": 133144345, + "router_z_loss_clip": 0.21191406, + "router_z_loss_mlp": 0.00994873, + "step": 4667, + "time_per_iteration": 3.029157876968384 + }, + { + "auxiliary_loss_clip": 0.01140246, + "auxiliary_loss_mlp": 0.01055926, + "balance_loss_clip": 1.05599284, + "balance_loss_mlp": 1.0371387, + "epoch": 0.1354535430329058, + "flos": 23724707414400.0, + "grad_norm": 2.504809079665183, + "language_loss": 1.13559318, + "learning_rate": 3.88448900133364e-06, + "loss": 1.15755486, + "num_input_tokens_seen": 133159600, + "router_z_loss_clip": 0.84277344, + "router_z_loss_mlp": 0.18786621, + "step": 4668, + "time_per_iteration": 2.54457950592041 + }, + { + "auxiliary_loss_clip": 0.01034347, + "auxiliary_loss_mlp": 0.0100069, + "balance_loss_clip": 1.0134418, + "balance_loss_mlp": 0.99978429, + "epoch": 0.13548256050142185, + "flos": 64743241847040.0, + "grad_norm": 0.6279182739076575, + "language_loss": 0.45843518, + "learning_rate": 3.8844260397700255e-06, + "loss": 0.47878557, + "num_input_tokens_seen": 133224710, + "router_z_loss_clip": 0.20898438, + "router_z_loss_mlp": 0.0090332, + "step": 4669, + "time_per_iteration": 3.263540267944336 + }, + { + "auxiliary_loss_clip": 0.011484, + "auxiliary_loss_mlp": 0.010491, + "balance_loss_clip": 1.06023431, + "balance_loss_mlp": 1.03183877, + "epoch": 0.1355115779699379, + "flos": 16100118334080.0, + "grad_norm": 4.359675686936328, + "language_loss": 0.91156483, + "learning_rate": 3.884363061562397e-06, + "loss": 0.93353981, + "num_input_tokens_seen": 133237425, + "router_z_loss_clip": 0.88183594, + "router_z_loss_mlp": 0.17266846, + "step": 4670, + "time_per_iteration": 2.479613780975342 + }, + { + "auxiliary_loss_clip": 0.01139901, + "auxiliary_loss_mlp": 0.01047642, + "balance_loss_clip": 1.05571985, + "balance_loss_mlp": 1.02892637, + "epoch": 0.13554059543845395, + "flos": 16501563711360.0, + "grad_norm": 2.2960720419669296, + "language_loss": 0.75572002, + "learning_rate": 3.884300066711313e-06, + "loss": 0.77759552, + "num_input_tokens_seen": 133250970, + "router_z_loss_clip": 0.84130859, + "router_z_loss_mlp": 0.18707275, + "step": 4671, + "time_per_iteration": 2.510000467300415 + }, + { + "auxiliary_loss_clip": 0.01032908, + "auxiliary_loss_mlp": 0.0100403, + "balance_loss_clip": 1.01226449, + "balance_loss_mlp": 1.00307012, + "epoch": 0.13556961290697, + "flos": 65867029729920.0, + "grad_norm": 0.6810945432978632, + "language_loss": 0.47997424, + "learning_rate": 3.884237055217327e-06, + "loss": 0.50034356, + "num_input_tokens_seen": 133307290, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.00958252, + "step": 4672, + "time_per_iteration": 2.9715840816497803 + }, + { + "auxiliary_loss_clip": 0.01031882, + "auxiliary_loss_mlp": 0.01009642, + "balance_loss_clip": 1.01124048, + "balance_loss_mlp": 1.00872409, + "epoch": 0.13559863037548603, + "flos": 62779359697920.0, + "grad_norm": 0.6217052675522351, + "language_loss": 0.49146008, + "learning_rate": 3.8841740270809974e-06, + "loss": 0.51187533, + "num_input_tokens_seen": 133374450, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.00915527, + "step": 4673, + "time_per_iteration": 3.1554737091064453 + }, + { + "auxiliary_loss_clip": 0.01138043, + "auxiliary_loss_mlp": 0.01050508, + "balance_loss_clip": 1.05804873, + "balance_loss_mlp": 1.03520131, + "epoch": 0.13562764784400208, + "flos": 40699142697600.0, + "grad_norm": 1.8226822344401556, + "language_loss": 1.0446018, + "learning_rate": 3.88411098230288e-06, + "loss": 1.06648731, + "num_input_tokens_seen": 133403210, + "router_z_loss_clip": 0.79882812, + "router_z_loss_mlp": 0.1529541, + "step": 4674, + "time_per_iteration": 2.789560317993164 + }, + { + "auxiliary_loss_clip": 0.01146156, + "auxiliary_loss_mlp": 0.01052541, + "balance_loss_clip": 1.06005836, + "balance_loss_mlp": 1.0346595, + "epoch": 0.13565666531251813, + "flos": 15116561147520.0, + "grad_norm": 2.7622495087273027, + "language_loss": 0.81948638, + "learning_rate": 3.884047920883532e-06, + "loss": 0.84147334, + "num_input_tokens_seen": 133417000, + "router_z_loss_clip": 0.86181641, + "router_z_loss_mlp": 0.17883301, + "step": 4675, + "time_per_iteration": 2.561453342437744 + }, + { + "auxiliary_loss_clip": 0.01031933, + "auxiliary_loss_mlp": 0.01003583, + "balance_loss_clip": 1.01151633, + "balance_loss_mlp": 1.00259972, + "epoch": 0.13568568278103418, + "flos": 74768171235840.0, + "grad_norm": 0.6669520200771435, + "language_loss": 0.45923376, + "learning_rate": 3.883984842823512e-06, + "loss": 0.47958893, + "num_input_tokens_seen": 133475575, + "router_z_loss_clip": 0.20410156, + "router_z_loss_mlp": 0.00982666, + "step": 4676, + "time_per_iteration": 3.0782864093780518 + }, + { + "auxiliary_loss_clip": 0.01141641, + "auxiliary_loss_mlp": 0.01044637, + "balance_loss_clip": 1.06021094, + "balance_loss_mlp": 1.02744114, + "epoch": 0.13571470024955024, + "flos": 17195896586880.0, + "grad_norm": 2.206978232564331, + "language_loss": 0.77218282, + "learning_rate": 3.883921748123374e-06, + "loss": 0.79404557, + "num_input_tokens_seen": 133490005, + "router_z_loss_clip": 0.81445312, + "router_z_loss_mlp": 0.17211914, + "step": 4677, + "time_per_iteration": 2.5463905334472656 + }, + { + "auxiliary_loss_clip": 0.01138069, + "auxiliary_loss_mlp": 0.01051763, + "balance_loss_clip": 1.05858374, + "balance_loss_mlp": 1.0367074, + "epoch": 0.1357437177180663, + "flos": 29563710480000.0, + "grad_norm": 1.7804703946845797, + "language_loss": 0.8656131, + "learning_rate": 3.883858636783676e-06, + "loss": 0.88751149, + "num_input_tokens_seen": 133513735, + "router_z_loss_clip": 0.79589844, + "router_z_loss_mlp": 0.15057373, + "step": 4678, + "time_per_iteration": 2.8913180828094482 + }, + { + "auxiliary_loss_clip": 0.01138607, + "auxiliary_loss_mlp": 0.01048853, + "balance_loss_clip": 1.05989754, + "balance_loss_mlp": 1.0339514, + "epoch": 0.1357727351865823, + "flos": 10920435361920.0, + "grad_norm": 2.6781662233106824, + "language_loss": 0.85476238, + "learning_rate": 3.883795508804978e-06, + "loss": 0.87663686, + "num_input_tokens_seen": 133523660, + "router_z_loss_clip": 0.78710938, + "router_z_loss_mlp": 0.14898682, + "step": 4679, + "time_per_iteration": 2.4462122917175293 + }, + { + "auxiliary_loss_clip": 0.01142502, + "auxiliary_loss_mlp": 0.01066033, + "balance_loss_clip": 1.05973625, + "balance_loss_mlp": 1.04809833, + "epoch": 0.13580175265509836, + "flos": 16354473517440.0, + "grad_norm": 4.534470193143981, + "language_loss": 0.96825325, + "learning_rate": 3.8837323641878345e-06, + "loss": 0.9903385, + "num_input_tokens_seen": 133534385, + "router_z_loss_clip": 0.82666016, + "router_z_loss_mlp": 0.17932129, + "step": 4680, + "time_per_iteration": 2.4912092685699463 + }, + { + "auxiliary_loss_clip": 0.01030516, + "auxiliary_loss_mlp": 0.01008327, + "balance_loss_clip": 1.00986814, + "balance_loss_mlp": 1.00730789, + "epoch": 0.1358307701236144, + "flos": 70103806732800.0, + "grad_norm": 0.7103223026425516, + "language_loss": 0.51256597, + "learning_rate": 3.883669202932805e-06, + "loss": 0.53295439, + "num_input_tokens_seen": 133595740, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.01019287, + "step": 4681, + "time_per_iteration": 3.051616907119751 + }, + { + "auxiliary_loss_clip": 0.01144466, + "auxiliary_loss_mlp": 0.01054507, + "balance_loss_clip": 1.0547998, + "balance_loss_mlp": 1.0362736, + "epoch": 0.13585978759213047, + "flos": 25111757053440.0, + "grad_norm": 2.772303476895539, + "language_loss": 0.81521153, + "learning_rate": 3.883606025040447e-06, + "loss": 0.8372013, + "num_input_tokens_seen": 133612145, + "router_z_loss_clip": 0.89697266, + "router_z_loss_mlp": 0.18237305, + "step": 4682, + "time_per_iteration": 2.538768768310547 + }, + { + "auxiliary_loss_clip": 0.01129364, + "auxiliary_loss_mlp": 0.01038939, + "balance_loss_clip": 1.05510783, + "balance_loss_mlp": 1.02393639, + "epoch": 0.13588880506064652, + "flos": 26208864109440.0, + "grad_norm": 2.4066700342626284, + "language_loss": 0.90149164, + "learning_rate": 3.883542830511318e-06, + "loss": 0.92317468, + "num_input_tokens_seen": 133632015, + "router_z_loss_clip": 0.74169922, + "router_z_loss_mlp": 0.15002441, + "step": 4683, + "time_per_iteration": 2.5505857467651367 + }, + { + "auxiliary_loss_clip": 0.01029345, + "auxiliary_loss_mlp": 0.01008823, + "balance_loss_clip": 1.008641, + "balance_loss_mlp": 1.00790524, + "epoch": 0.13591782252916257, + "flos": 69302819399040.0, + "grad_norm": 0.699081458956037, + "language_loss": 0.46841547, + "learning_rate": 3.8834796193459766e-06, + "loss": 0.48879719, + "num_input_tokens_seen": 133688230, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.00915527, + "step": 4684, + "time_per_iteration": 3.0364575386047363 + }, + { + "auxiliary_loss_clip": 0.01139375, + "auxiliary_loss_mlp": 0.01040615, + "balance_loss_clip": 1.05690742, + "balance_loss_mlp": 1.02424765, + "epoch": 0.1359468399976786, + "flos": 18763145781120.0, + "grad_norm": 3.0866765507382894, + "language_loss": 0.85621703, + "learning_rate": 3.883416391544981e-06, + "loss": 0.87801695, + "num_input_tokens_seen": 133700095, + "router_z_loss_clip": 0.82568359, + "router_z_loss_mlp": 0.16369629, + "step": 4685, + "time_per_iteration": 2.5052387714385986 + }, + { + "auxiliary_loss_clip": 0.01030557, + "auxiliary_loss_mlp": 0.01003968, + "balance_loss_clip": 1.00958216, + "balance_loss_mlp": 1.00299597, + "epoch": 0.13597585746619464, + "flos": 74782105712640.0, + "grad_norm": 0.6471335803198334, + "language_loss": 0.50772512, + "learning_rate": 3.88335314710889e-06, + "loss": 0.52807045, + "num_input_tokens_seen": 133768045, + "router_z_loss_clip": 0.20996094, + "router_z_loss_mlp": 0.00970459, + "step": 4686, + "time_per_iteration": 3.1713407039642334 + }, + { + "auxiliary_loss_clip": 0.01030564, + "auxiliary_loss_mlp": 0.01001733, + "balance_loss_clip": 1.00959873, + "balance_loss_mlp": 1.00080895, + "epoch": 0.1360048749347107, + "flos": 74768889507840.0, + "grad_norm": 0.6378518143247154, + "language_loss": 0.44737095, + "learning_rate": 3.883289886038262e-06, + "loss": 0.46769392, + "num_input_tokens_seen": 133833890, + "router_z_loss_clip": 0.20996094, + "router_z_loss_mlp": 0.00921631, + "step": 4687, + "time_per_iteration": 3.3171546459198 + }, + { + "auxiliary_loss_clip": 0.01030167, + "auxiliary_loss_mlp": 0.00998832, + "balance_loss_clip": 1.00933719, + "balance_loss_mlp": 0.99787867, + "epoch": 0.13603389240322675, + "flos": 56169928794240.0, + "grad_norm": 0.7279702768365285, + "language_loss": 0.51568627, + "learning_rate": 3.883226608333655e-06, + "loss": 0.53597629, + "num_input_tokens_seen": 133891460, + "router_z_loss_clip": 0.20898438, + "router_z_loss_mlp": 0.00952148, + "step": 4688, + "time_per_iteration": 3.149625301361084 + }, + { + "auxiliary_loss_clip": 0.01029159, + "auxiliary_loss_mlp": 0.01002516, + "balance_loss_clip": 1.00841272, + "balance_loss_mlp": 1.00160432, + "epoch": 0.1360629098717428, + "flos": 59230558863360.0, + "grad_norm": 0.6454006400482845, + "language_loss": 0.52735698, + "learning_rate": 3.883163313995629e-06, + "loss": 0.5476737, + "num_input_tokens_seen": 133953480, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.00909424, + "step": 4689, + "time_per_iteration": 3.132324695587158 + }, + { + "auxiliary_loss_clip": 0.01130799, + "auxiliary_loss_mlp": 0.010453, + "balance_loss_clip": 1.05391073, + "balance_loss_mlp": 1.02890313, + "epoch": 0.13609192734025882, + "flos": 23982510303360.0, + "grad_norm": 3.381168529612469, + "language_loss": 0.77902418, + "learning_rate": 3.883100003024743e-06, + "loss": 0.80078512, + "num_input_tokens_seen": 133969580, + "router_z_loss_clip": 0.77001953, + "router_z_loss_mlp": 0.16387939, + "step": 4690, + "time_per_iteration": 2.5555853843688965 + }, + { + "auxiliary_loss_clip": 0.01028024, + "auxiliary_loss_mlp": 0.01007609, + "balance_loss_clip": 1.00725007, + "balance_loss_mlp": 1.00664377, + "epoch": 0.13612094480877487, + "flos": 74775282128640.0, + "grad_norm": 0.63737037861722, + "language_loss": 0.48436856, + "learning_rate": 3.883036675421555e-06, + "loss": 0.50472492, + "num_input_tokens_seen": 134038295, + "router_z_loss_clip": 0.20800781, + "router_z_loss_mlp": 0.00964355, + "step": 4691, + "time_per_iteration": 3.25032639503479 + }, + { + "auxiliary_loss_clip": 0.01136932, + "auxiliary_loss_mlp": 0.01045332, + "balance_loss_clip": 1.05454516, + "balance_loss_mlp": 1.02874947, + "epoch": 0.13614996227729093, + "flos": 13434648762240.0, + "grad_norm": 2.8280715960538885, + "language_loss": 0.87836361, + "learning_rate": 3.882973331186625e-06, + "loss": 0.90018624, + "num_input_tokens_seen": 134049135, + "router_z_loss_clip": 0.82373047, + "router_z_loss_mlp": 0.16558838, + "step": 4692, + "time_per_iteration": 2.5081818103790283 + }, + { + "auxiliary_loss_clip": 0.01025821, + "auxiliary_loss_mlp": 0.01008662, + "balance_loss_clip": 1.00523305, + "balance_loss_mlp": 1.00770879, + "epoch": 0.13617897974580698, + "flos": 70719099730560.0, + "grad_norm": 3.1236223467234168, + "language_loss": 0.42471251, + "learning_rate": 3.882909970320513e-06, + "loss": 0.44505733, + "num_input_tokens_seen": 134102070, + "router_z_loss_clip": 0.20605469, + "router_z_loss_mlp": 0.00952148, + "step": 4693, + "time_per_iteration": 3.005439043045044 + }, + { + "auxiliary_loss_clip": 0.01143852, + "auxiliary_loss_mlp": 0.01046973, + "balance_loss_clip": 1.05573285, + "balance_loss_mlp": 1.02775621, + "epoch": 0.13620799721432303, + "flos": 18035164840320.0, + "grad_norm": 2.3558634717823237, + "language_loss": 0.7982586, + "learning_rate": 3.8828465928237784e-06, + "loss": 0.82016683, + "num_input_tokens_seen": 134117085, + "router_z_loss_clip": 0.88183594, + "router_z_loss_mlp": 0.19213867, + "step": 4694, + "time_per_iteration": 5.131720542907715 + }, + { + "auxiliary_loss_clip": 0.0113562, + "auxiliary_loss_mlp": 0.01051039, + "balance_loss_clip": 1.05522346, + "balance_loss_mlp": 1.03278208, + "epoch": 0.13623701468283908, + "flos": 16318850204160.0, + "grad_norm": 2.310463851185235, + "language_loss": 0.74140507, + "learning_rate": 3.88278319869698e-06, + "loss": 0.76327163, + "num_input_tokens_seen": 134129670, + "router_z_loss_clip": 0.80419922, + "router_z_loss_mlp": 0.18243408, + "step": 4695, + "time_per_iteration": 4.863981485366821 + }, + { + "auxiliary_loss_clip": 0.01133175, + "auxiliary_loss_mlp": 0.01041255, + "balance_loss_clip": 1.05458248, + "balance_loss_mlp": 1.0260855, + "epoch": 0.1362660321513551, + "flos": 18069710745600.0, + "grad_norm": 2.4756761490735064, + "language_loss": 0.74455297, + "learning_rate": 3.882719787940679e-06, + "loss": 0.76629722, + "num_input_tokens_seen": 134141495, + "router_z_loss_clip": 0.78613281, + "router_z_loss_mlp": 0.15185547, + "step": 4696, + "time_per_iteration": 4.7853851318359375 + }, + { + "auxiliary_loss_clip": 0.01137611, + "auxiliary_loss_mlp": 0.01038144, + "balance_loss_clip": 1.05721021, + "balance_loss_mlp": 1.02308154, + "epoch": 0.13629504961987116, + "flos": 20113853834880.0, + "grad_norm": 2.605421213203596, + "language_loss": 0.85143352, + "learning_rate": 3.882656360555435e-06, + "loss": 0.87319106, + "num_input_tokens_seen": 134154030, + "router_z_loss_clip": 0.8046875, + "router_z_loss_mlp": 0.15057373, + "step": 4697, + "time_per_iteration": 4.838900804519653 + }, + { + "auxiliary_loss_clip": 0.01135667, + "auxiliary_loss_mlp": 0.01049626, + "balance_loss_clip": 1.05260038, + "balance_loss_mlp": 1.02858508, + "epoch": 0.1363240670883872, + "flos": 25077749852160.0, + "grad_norm": 2.247540822763646, + "language_loss": 0.68743157, + "learning_rate": 3.882592916541808e-06, + "loss": 0.70928454, + "num_input_tokens_seen": 134171320, + "router_z_loss_clip": 0.83007812, + "router_z_loss_mlp": 0.21014404, + "step": 4698, + "time_per_iteration": 2.5988121032714844 + }, + { + "auxiliary_loss_clip": 0.01137606, + "auxiliary_loss_mlp": 0.01040017, + "balance_loss_clip": 1.05633819, + "balance_loss_mlp": 1.02438831, + "epoch": 0.13635308455690326, + "flos": 11357073089280.0, + "grad_norm": 2.5645754639825293, + "language_loss": 0.75505179, + "learning_rate": 3.882529455900359e-06, + "loss": 0.77682805, + "num_input_tokens_seen": 134182020, + "router_z_loss_clip": 0.81347656, + "router_z_loss_mlp": 0.15631104, + "step": 4699, + "time_per_iteration": 2.4638044834136963 + }, + { + "auxiliary_loss_clip": 0.01029633, + "auxiliary_loss_mlp": 0.01007762, + "balance_loss_clip": 1.00896668, + "balance_loss_mlp": 1.0068202, + "epoch": 0.1363821020254193, + "flos": 74774815251840.0, + "grad_norm": 0.6298590353273544, + "language_loss": 0.44057715, + "learning_rate": 3.8824659786316474e-06, + "loss": 0.46095109, + "num_input_tokens_seen": 134250615, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.00939941, + "step": 4700, + "time_per_iteration": 3.3383960723876953 + }, + { + "auxiliary_loss_clip": 0.011358, + "auxiliary_loss_mlp": 0.01047884, + "balance_loss_clip": 1.05451107, + "balance_loss_mlp": 1.031183, + "epoch": 0.13641111949393536, + "flos": 14055041491200.0, + "grad_norm": 2.8201299285436052, + "language_loss": 0.75511217, + "learning_rate": 3.882402484736235e-06, + "loss": 0.77694905, + "num_input_tokens_seen": 134262230, + "router_z_loss_clip": 0.81201172, + "router_z_loss_mlp": 0.16705322, + "step": 4701, + "time_per_iteration": 2.5220730304718018 + }, + { + "auxiliary_loss_clip": 0.01133289, + "auxiliary_loss_mlp": 0.01037603, + "balance_loss_clip": 1.05490565, + "balance_loss_mlp": 1.02267766, + "epoch": 0.13644013696245139, + "flos": 20550635216640.0, + "grad_norm": 2.381888132730375, + "language_loss": 0.73794782, + "learning_rate": 3.8823389742146816e-06, + "loss": 0.75965667, + "num_input_tokens_seen": 134274630, + "router_z_loss_clip": 0.78369141, + "router_z_loss_mlp": 0.14929199, + "step": 4702, + "time_per_iteration": 2.5608458518981934 + }, + { + "auxiliary_loss_clip": 0.01029977, + "auxiliary_loss_mlp": 0.01012141, + "balance_loss_clip": 1.00926232, + "balance_loss_mlp": 1.01115739, + "epoch": 0.13646915443096744, + "flos": 66162395266560.0, + "grad_norm": 0.6841096024596433, + "language_loss": 0.50199497, + "learning_rate": 3.88227544706755e-06, + "loss": 0.52241611, + "num_input_tokens_seen": 134337495, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.00982666, + "step": 4703, + "time_per_iteration": 3.0603480339050293 + }, + { + "auxiliary_loss_clip": 0.01148342, + "auxiliary_loss_mlp": 0.01049088, + "balance_loss_clip": 1.05942214, + "balance_loss_mlp": 1.03028238, + "epoch": 0.1364981718994835, + "flos": 25330129787520.0, + "grad_norm": 2.6302613915596864, + "language_loss": 1.05149436, + "learning_rate": 3.882211903295399e-06, + "loss": 1.07346869, + "num_input_tokens_seen": 134350945, + "router_z_loss_clip": 0.88867188, + "router_z_loss_mlp": 0.18798828, + "step": 4704, + "time_per_iteration": 2.5465939044952393 + }, + { + "auxiliary_loss_clip": 0.0114161, + "auxiliary_loss_mlp": 0.01040764, + "balance_loss_clip": 1.05636501, + "balance_loss_mlp": 1.02406228, + "epoch": 0.13652718936799954, + "flos": 30622680270720.0, + "grad_norm": 2.4831818686193676, + "language_loss": 0.93909967, + "learning_rate": 3.882148342898791e-06, + "loss": 0.96092343, + "num_input_tokens_seen": 134370750, + "router_z_loss_clip": 0.85253906, + "router_z_loss_mlp": 0.16693115, + "step": 4705, + "time_per_iteration": 2.595412492752075 + }, + { + "auxiliary_loss_clip": 0.01030669, + "auxiliary_loss_mlp": 0.01000492, + "balance_loss_clip": 1.01013803, + "balance_loss_mlp": 0.9994964, + "epoch": 0.1365562068365156, + "flos": 74634189505920.0, + "grad_norm": 0.6518149476307991, + "language_loss": 0.52629828, + "learning_rate": 3.882084765878287e-06, + "loss": 0.54660988, + "num_input_tokens_seen": 134437385, + "router_z_loss_clip": 0.20507812, + "router_z_loss_mlp": 0.00994873, + "step": 4706, + "time_per_iteration": 3.2041146755218506 + }, + { + "auxiliary_loss_clip": 0.01134194, + "auxiliary_loss_mlp": 0.01043533, + "balance_loss_clip": 1.05546761, + "balance_loss_mlp": 1.02672446, + "epoch": 0.13658522430503162, + "flos": 27192313555200.0, + "grad_norm": 2.2429617729946734, + "language_loss": 0.74900258, + "learning_rate": 3.882021172234449e-06, + "loss": 0.77077985, + "num_input_tokens_seen": 134453295, + "router_z_loss_clip": 0.78710938, + "router_z_loss_mlp": 0.16815186, + "step": 4707, + "time_per_iteration": 2.584233045578003 + }, + { + "auxiliary_loss_clip": 0.01134464, + "auxiliary_loss_mlp": 0.01046247, + "balance_loss_clip": 1.05400443, + "balance_loss_mlp": 1.03028488, + "epoch": 0.13661424177354767, + "flos": 28248841221120.0, + "grad_norm": 2.2524383932916434, + "language_loss": 0.87564111, + "learning_rate": 3.8819575619678384e-06, + "loss": 0.89744818, + "num_input_tokens_seen": 134476635, + "router_z_loss_clip": 0.80566406, + "router_z_loss_mlp": 0.15960693, + "step": 4708, + "time_per_iteration": 2.859844923019409 + }, + { + "auxiliary_loss_clip": 0.01139324, + "auxiliary_loss_mlp": 0.01051499, + "balance_loss_clip": 1.0569787, + "balance_loss_mlp": 1.03463721, + "epoch": 0.13664325924206372, + "flos": 31899843227520.0, + "grad_norm": 2.174078985552856, + "language_loss": 0.90355039, + "learning_rate": 3.881893935079017e-06, + "loss": 0.92545855, + "num_input_tokens_seen": 134493600, + "router_z_loss_clip": 0.82324219, + "router_z_loss_mlp": 0.16876221, + "step": 4709, + "time_per_iteration": 2.6592252254486084 + }, + { + "auxiliary_loss_clip": 0.01031373, + "auxiliary_loss_mlp": 0.01012672, + "balance_loss_clip": 1.01073408, + "balance_loss_mlp": 1.01180768, + "epoch": 0.13667227671057977, + "flos": 59235048063360.0, + "grad_norm": 0.6674759797645744, + "language_loss": 0.48618174, + "learning_rate": 3.881830291568546e-06, + "loss": 0.5066222, + "num_input_tokens_seen": 134556420, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.00866699, + "step": 4710, + "time_per_iteration": 3.1142075061798096 + }, + { + "auxiliary_loss_clip": 0.01139443, + "auxiliary_loss_mlp": 0.01045434, + "balance_loss_clip": 1.05787253, + "balance_loss_mlp": 1.02861321, + "epoch": 0.13670129417909582, + "flos": 23141266801920.0, + "grad_norm": 2.8376425948151147, + "language_loss": 0.87927359, + "learning_rate": 3.88176663143699e-06, + "loss": 0.90112233, + "num_input_tokens_seen": 134568695, + "router_z_loss_clip": 0.81591797, + "router_z_loss_mlp": 0.16821289, + "step": 4711, + "time_per_iteration": 2.5299205780029297 + }, + { + "auxiliary_loss_clip": 0.01135608, + "auxiliary_loss_mlp": 0.0104671, + "balance_loss_clip": 1.05377269, + "balance_loss_mlp": 1.03145742, + "epoch": 0.13673031164761187, + "flos": 28868623418880.0, + "grad_norm": 2.4395920733321415, + "language_loss": 0.80126393, + "learning_rate": 3.881702954684908e-06, + "loss": 0.8230871, + "num_input_tokens_seen": 134582285, + "router_z_loss_clip": 0.81884766, + "router_z_loss_mlp": 0.15240479, + "step": 4712, + "time_per_iteration": 2.5820608139038086 + }, + { + "auxiliary_loss_clip": 0.01032633, + "auxiliary_loss_mlp": 0.010069, + "balance_loss_clip": 1.01216781, + "balance_loss_mlp": 1.006024, + "epoch": 0.1367593291161279, + "flos": 67731907017600.0, + "grad_norm": 0.6800525384213757, + "language_loss": 0.47316682, + "learning_rate": 3.8816392613128654e-06, + "loss": 0.49356219, + "num_input_tokens_seen": 134641820, + "router_z_loss_clip": 0.20507812, + "router_z_loss_mlp": 0.00878906, + "step": 4713, + "time_per_iteration": 3.0254104137420654 + }, + { + "auxiliary_loss_clip": 0.01138282, + "auxiliary_loss_mlp": 0.01042534, + "balance_loss_clip": 1.05616188, + "balance_loss_mlp": 1.02400851, + "epoch": 0.13678834658464395, + "flos": 11719986151680.0, + "grad_norm": 2.8706708487734462, + "language_loss": 0.90128702, + "learning_rate": 3.881575551321423e-06, + "loss": 0.92309523, + "num_input_tokens_seen": 134651690, + "router_z_loss_clip": 0.82080078, + "router_z_loss_mlp": 0.18518066, + "step": 4714, + "time_per_iteration": 2.4819846153259277 + }, + { + "auxiliary_loss_clip": 0.01142577, + "auxiliary_loss_mlp": 0.01046504, + "balance_loss_clip": 1.06005096, + "balance_loss_mlp": 1.02827048, + "epoch": 0.13681736405316, + "flos": 13291006273920.0, + "grad_norm": 2.678035428669311, + "language_loss": 0.84188116, + "learning_rate": 3.881511824711143e-06, + "loss": 0.86377192, + "num_input_tokens_seen": 134663315, + "router_z_loss_clip": 0.82519531, + "router_z_loss_mlp": 0.18237305, + "step": 4715, + "time_per_iteration": 2.472198724746704 + }, + { + "auxiliary_loss_clip": 0.01135631, + "auxiliary_loss_mlp": 0.01043716, + "balance_loss_clip": 1.05552733, + "balance_loss_mlp": 1.02771759, + "epoch": 0.13684638152167605, + "flos": 57293426390400.0, + "grad_norm": 2.02801766628139, + "language_loss": 0.74942774, + "learning_rate": 3.88144808148259e-06, + "loss": 0.77122122, + "num_input_tokens_seen": 134687420, + "router_z_loss_clip": 0.80078125, + "router_z_loss_mlp": 0.16009521, + "step": 4716, + "time_per_iteration": 2.8415191173553467 + }, + { + "auxiliary_loss_clip": 0.01141897, + "auxiliary_loss_mlp": 0.01048826, + "balance_loss_clip": 1.05916762, + "balance_loss_mlp": 1.03215492, + "epoch": 0.1368753989901921, + "flos": 51240252481920.0, + "grad_norm": 1.7352170247695597, + "language_loss": 0.77602357, + "learning_rate": 3.881384321636327e-06, + "loss": 0.79793078, + "num_input_tokens_seen": 134718355, + "router_z_loss_clip": 0.82763672, + "router_z_loss_mlp": 0.16668701, + "step": 4717, + "time_per_iteration": 2.9569268226623535 + }, + { + "auxiliary_loss_clip": 0.01143884, + "auxiliary_loss_mlp": 0.01049016, + "balance_loss_clip": 1.05670714, + "balance_loss_mlp": 1.0310688, + "epoch": 0.13690441645870816, + "flos": 15917871703680.0, + "grad_norm": 2.8155216937117746, + "language_loss": 0.78400332, + "learning_rate": 3.881320545172915e-06, + "loss": 0.80593234, + "num_input_tokens_seen": 134732355, + "router_z_loss_clip": 0.87207031, + "router_z_loss_mlp": 0.17950439, + "step": 4718, + "time_per_iteration": 2.5433125495910645 + }, + { + "auxiliary_loss_clip": 0.01134787, + "auxiliary_loss_mlp": 0.01040563, + "balance_loss_clip": 1.05615723, + "balance_loss_mlp": 1.024261, + "epoch": 0.13693343392722418, + "flos": 28835262662400.0, + "grad_norm": 5.084628167448223, + "language_loss": 0.72872818, + "learning_rate": 3.88125675209292e-06, + "loss": 0.75048167, + "num_input_tokens_seen": 134749280, + "router_z_loss_clip": 0.78613281, + "router_z_loss_mlp": 0.16290283, + "step": 4719, + "time_per_iteration": 2.5617454051971436 + }, + { + "auxiliary_loss_clip": 0.0103749, + "auxiliary_loss_mlp": 0.01006322, + "balance_loss_clip": 1.01691878, + "balance_loss_mlp": 1.00529051, + "epoch": 0.13696245139574023, + "flos": 74779986810240.0, + "grad_norm": 0.7267791102528102, + "language_loss": 0.45942783, + "learning_rate": 3.881192942396903e-06, + "loss": 0.47986591, + "num_input_tokens_seen": 134817295, + "router_z_loss_clip": 0.20605469, + "router_z_loss_mlp": 0.01031494, + "step": 4720, + "time_per_iteration": 3.205029010772705 + }, + { + "auxiliary_loss_clip": 0.01148328, + "auxiliary_loss_mlp": 0.01040883, + "balance_loss_clip": 1.06147635, + "balance_loss_mlp": 1.02339506, + "epoch": 0.13699146886425628, + "flos": 29234876446080.0, + "grad_norm": 2.4167829965353844, + "language_loss": 0.79394257, + "learning_rate": 3.8811291160854285e-06, + "loss": 0.81583464, + "num_input_tokens_seen": 134831285, + "router_z_loss_clip": 0.86914062, + "router_z_loss_mlp": 0.17480469, + "step": 4721, + "time_per_iteration": 2.506464958190918 + }, + { + "auxiliary_loss_clip": 0.011375, + "auxiliary_loss_mlp": 0.01043949, + "balance_loss_clip": 1.05816889, + "balance_loss_mlp": 1.02790356, + "epoch": 0.13702048633277233, + "flos": 12014382021120.0, + "grad_norm": 2.4638604495884096, + "language_loss": 0.75382137, + "learning_rate": 3.8810652731590615e-06, + "loss": 0.77563584, + "num_input_tokens_seen": 134843360, + "router_z_loss_clip": 0.79345703, + "router_z_loss_mlp": 0.16040039, + "step": 4722, + "time_per_iteration": 2.481217622756958 + }, + { + "auxiliary_loss_clip": 0.01139656, + "auxiliary_loss_mlp": 0.01039348, + "balance_loss_clip": 1.05769467, + "balance_loss_mlp": 1.02344549, + "epoch": 0.13704950380128839, + "flos": 18177622179840.0, + "grad_norm": 3.25825200188013, + "language_loss": 0.79119295, + "learning_rate": 3.881001413618364e-06, + "loss": 0.81298304, + "num_input_tokens_seen": 134854710, + "router_z_loss_clip": 0.81982422, + "router_z_loss_mlp": 0.15917969, + "step": 4723, + "time_per_iteration": 2.5058627128601074 + }, + { + "auxiliary_loss_clip": 0.01037307, + "auxiliary_loss_mlp": 0.01002663, + "balance_loss_clip": 1.0167079, + "balance_loss_mlp": 1.00170326, + "epoch": 0.1370785212698044, + "flos": 64034399877120.0, + "grad_norm": 0.7314815823769246, + "language_loss": 0.49901366, + "learning_rate": 3.880937537463901e-06, + "loss": 0.51941335, + "num_input_tokens_seen": 134902590, + "router_z_loss_clip": 0.20605469, + "router_z_loss_mlp": 0.00958252, + "step": 4724, + "time_per_iteration": 2.8975706100463867 + }, + { + "auxiliary_loss_clip": 0.01146117, + "auxiliary_loss_mlp": 0.01038947, + "balance_loss_clip": 1.0598526, + "balance_loss_mlp": 1.02126217, + "epoch": 0.13710753873832046, + "flos": 14970404707200.0, + "grad_norm": 2.875023369357012, + "language_loss": 0.83118606, + "learning_rate": 3.880873644696237e-06, + "loss": 0.85303676, + "num_input_tokens_seen": 134915285, + "router_z_loss_clip": 0.86279297, + "router_z_loss_mlp": 0.17681885, + "step": 4725, + "time_per_iteration": 2.4729137420654297 + }, + { + "auxiliary_loss_clip": 0.01036296, + "auxiliary_loss_mlp": 0.01007498, + "balance_loss_clip": 1.01556492, + "balance_loss_mlp": 1.00654399, + "epoch": 0.1371365562068365, + "flos": 67400774513280.0, + "grad_norm": 0.673889676237605, + "language_loss": 0.44364259, + "learning_rate": 3.880809735315935e-06, + "loss": 0.46408051, + "num_input_tokens_seen": 134972375, + "router_z_loss_clip": 0.20800781, + "router_z_loss_mlp": 0.00952148, + "step": 4726, + "time_per_iteration": 3.0072219371795654 + }, + { + "auxiliary_loss_clip": 0.01147595, + "auxiliary_loss_mlp": 0.01042501, + "balance_loss_clip": 1.05788779, + "balance_loss_mlp": 1.02340984, + "epoch": 0.13716557367535256, + "flos": 38321209497600.0, + "grad_norm": 2.3587090253195995, + "language_loss": 0.94671589, + "learning_rate": 3.880745809323561e-06, + "loss": 0.96861684, + "num_input_tokens_seen": 134992315, + "router_z_loss_clip": 0.89599609, + "router_z_loss_mlp": 0.190979, + "step": 4727, + "time_per_iteration": 2.701022148132324 + }, + { + "auxiliary_loss_clip": 0.01140162, + "auxiliary_loss_mlp": 0.01041969, + "balance_loss_clip": 1.0582279, + "balance_loss_mlp": 1.02461195, + "epoch": 0.13719459114386862, + "flos": 21171746217600.0, + "grad_norm": 3.155760783144763, + "language_loss": 0.73026425, + "learning_rate": 3.880681866719679e-06, + "loss": 0.75208557, + "num_input_tokens_seen": 135005075, + "router_z_loss_clip": 0.82080078, + "router_z_loss_mlp": 0.17358398, + "step": 4728, + "time_per_iteration": 2.5405099391937256 + }, + { + "auxiliary_loss_clip": 0.01137281, + "auxiliary_loss_mlp": 0.0105041, + "balance_loss_clip": 1.05712414, + "balance_loss_mlp": 1.03575945, + "epoch": 0.13722360861238467, + "flos": 9423534954240.0, + "grad_norm": 2.9500570005715754, + "language_loss": 0.82850963, + "learning_rate": 3.880617907504854e-06, + "loss": 0.85038656, + "num_input_tokens_seen": 135015710, + "router_z_loss_clip": 0.80126953, + "router_z_loss_mlp": 0.14642334, + "step": 4729, + "time_per_iteration": 2.52919602394104 + }, + { + "auxiliary_loss_clip": 0.0112682, + "auxiliary_loss_mlp": 0.01042318, + "balance_loss_clip": 1.05152035, + "balance_loss_mlp": 1.02783442, + "epoch": 0.1372526260809007, + "flos": 22632376867200.0, + "grad_norm": 2.2271609288972973, + "language_loss": 0.79836583, + "learning_rate": 3.88055393167965e-06, + "loss": 0.82005727, + "num_input_tokens_seen": 135031490, + "router_z_loss_clip": 0.75341797, + "router_z_loss_mlp": 0.14471436, + "step": 4730, + "time_per_iteration": 2.5403215885162354 + }, + { + "auxiliary_loss_clip": 0.01134227, + "auxiliary_loss_mlp": 0.01041664, + "balance_loss_clip": 1.05297112, + "balance_loss_mlp": 1.02582061, + "epoch": 0.13728164354941674, + "flos": 13910465249280.0, + "grad_norm": 2.746763258231609, + "language_loss": 0.92525041, + "learning_rate": 3.880489939244633e-06, + "loss": 0.94700933, + "num_input_tokens_seen": 135043795, + "router_z_loss_clip": 0.81201172, + "router_z_loss_mlp": 0.15844727, + "step": 4731, + "time_per_iteration": 2.5390713214874268 + }, + { + "auxiliary_loss_clip": 0.01154703, + "auxiliary_loss_mlp": 0.01055715, + "balance_loss_clip": 1.06104267, + "balance_loss_mlp": 1.0351572, + "epoch": 0.1373106610179328, + "flos": 11537631780480.0, + "grad_norm": 2.2589079312767533, + "language_loss": 0.74131107, + "learning_rate": 3.880425930200368e-06, + "loss": 0.76341534, + "num_input_tokens_seen": 135055140, + "router_z_loss_clip": 0.93554688, + "router_z_loss_mlp": 0.20587158, + "step": 4732, + "time_per_iteration": 2.494671106338501 + }, + { + "auxiliary_loss_clip": 0.01136562, + "auxiliary_loss_mlp": 0.01044532, + "balance_loss_clip": 1.05457044, + "balance_loss_mlp": 1.02754426, + "epoch": 0.13733967848644885, + "flos": 31901567080320.0, + "grad_norm": 2.9366381293009898, + "language_loss": 1.02570486, + "learning_rate": 3.88036190454742e-06, + "loss": 1.04751587, + "num_input_tokens_seen": 135073165, + "router_z_loss_clip": 0.81982422, + "router_z_loss_mlp": 0.16973877, + "step": 4733, + "time_per_iteration": 2.7500221729278564 + }, + { + "auxiliary_loss_clip": 0.01138557, + "auxiliary_loss_mlp": 0.01043988, + "balance_loss_clip": 1.05633068, + "balance_loss_mlp": 1.0264045, + "epoch": 0.1373686959549649, + "flos": 16623373708800.0, + "grad_norm": 2.2664160835285125, + "language_loss": 0.66582608, + "learning_rate": 3.880297862286355e-06, + "loss": 0.68765152, + "num_input_tokens_seen": 135088555, + "router_z_loss_clip": 0.82275391, + "router_z_loss_mlp": 0.17565918, + "step": 4734, + "time_per_iteration": 2.596923828125 + }, + { + "auxiliary_loss_clip": 0.01135602, + "auxiliary_loss_mlp": 0.01039435, + "balance_loss_clip": 1.0564034, + "balance_loss_mlp": 1.02219737, + "epoch": 0.13739771342348092, + "flos": 13514766048000.0, + "grad_norm": 2.238665081924681, + "language_loss": 0.72158736, + "learning_rate": 3.880233803417738e-06, + "loss": 0.74333769, + "num_input_tokens_seen": 135102305, + "router_z_loss_clip": 0.79248047, + "router_z_loss_mlp": 0.17254639, + "step": 4735, + "time_per_iteration": 2.4918932914733887 + }, + { + "auxiliary_loss_clip": 0.01038816, + "auxiliary_loss_mlp": 0.01018941, + "balance_loss_clip": 1.01782918, + "balance_loss_mlp": 1.01794612, + "epoch": 0.13742673089199697, + "flos": 64746581811840.0, + "grad_norm": 0.5982722713505857, + "language_loss": 0.47643441, + "learning_rate": 3.880169727942135e-06, + "loss": 0.49701196, + "num_input_tokens_seen": 135167735, + "router_z_loss_clip": 0.20996094, + "router_z_loss_mlp": 0.00994873, + "step": 4736, + "time_per_iteration": 3.2671401500701904 + }, + { + "auxiliary_loss_clip": 0.01143125, + "auxiliary_loss_mlp": 0.01044797, + "balance_loss_clip": 1.05750799, + "balance_loss_mlp": 1.02609253, + "epoch": 0.13745574836051302, + "flos": 15990411219840.0, + "grad_norm": 3.672053568115514, + "language_loss": 1.19926453, + "learning_rate": 3.8801056358601125e-06, + "loss": 1.22114372, + "num_input_tokens_seen": 135178365, + "router_z_loss_clip": 0.85644531, + "router_z_loss_mlp": 0.18695068, + "step": 4737, + "time_per_iteration": 2.5092740058898926 + }, + { + "auxiliary_loss_clip": 0.01139671, + "auxiliary_loss_mlp": 0.01045786, + "balance_loss_clip": 1.05760956, + "balance_loss_mlp": 1.02857208, + "epoch": 0.13748476582902908, + "flos": 17128169493120.0, + "grad_norm": 2.0795692164024406, + "language_loss": 0.74735057, + "learning_rate": 3.880041527172237e-06, + "loss": 0.76920515, + "num_input_tokens_seen": 135193470, + "router_z_loss_clip": 0.82177734, + "router_z_loss_mlp": 0.17211914, + "step": 4738, + "time_per_iteration": 2.4690780639648438 + }, + { + "auxiliary_loss_clip": 0.01125719, + "auxiliary_loss_mlp": 0.01037356, + "balance_loss_clip": 1.05278206, + "balance_loss_mlp": 1.02335489, + "epoch": 0.13751378329754513, + "flos": 15589253151360.0, + "grad_norm": 2.958096077642216, + "language_loss": 0.94672728, + "learning_rate": 3.879977401879073e-06, + "loss": 0.96835804, + "num_input_tokens_seen": 135205510, + "router_z_loss_clip": 0.72998047, + "router_z_loss_mlp": 0.14007568, + "step": 4739, + "time_per_iteration": 2.579399347305298 + }, + { + "auxiliary_loss_clip": 0.01145469, + "auxiliary_loss_mlp": 0.01042961, + "balance_loss_clip": 1.05898619, + "balance_loss_mlp": 1.0249244, + "epoch": 0.13754280076606118, + "flos": 32049950163840.0, + "grad_norm": 2.0457319906225546, + "language_loss": 0.82085985, + "learning_rate": 3.8799132599811875e-06, + "loss": 0.84274411, + "num_input_tokens_seen": 135224380, + "router_z_loss_clip": 0.86523438, + "router_z_loss_mlp": 0.18029785, + "step": 4740, + "time_per_iteration": 2.677981376647949 + }, + { + "auxiliary_loss_clip": 0.01133363, + "auxiliary_loss_mlp": 0.01043615, + "balance_loss_clip": 1.05156827, + "balance_loss_mlp": 1.02652025, + "epoch": 0.1375718182345772, + "flos": 16574749672320.0, + "grad_norm": 3.0541641196925395, + "language_loss": 0.9391489, + "learning_rate": 3.879849101479148e-06, + "loss": 0.96091872, + "num_input_tokens_seen": 135238255, + "router_z_loss_clip": 0.81884766, + "router_z_loss_mlp": 0.17077637, + "step": 4741, + "time_per_iteration": 2.6249372959136963 + }, + { + "auxiliary_loss_clip": 0.01146754, + "auxiliary_loss_mlp": 0.01048264, + "balance_loss_clip": 1.05779362, + "balance_loss_mlp": 1.02922654, + "epoch": 0.13760083570309325, + "flos": 16612851024000.0, + "grad_norm": 2.3514433344288057, + "language_loss": 0.86694217, + "learning_rate": 3.879784926373521e-06, + "loss": 0.88889229, + "num_input_tokens_seen": 135253140, + "router_z_loss_clip": 0.88916016, + "router_z_loss_mlp": 0.19030762, + "step": 4742, + "time_per_iteration": 2.5120253562927246 + }, + { + "auxiliary_loss_clip": 0.01139214, + "auxiliary_loss_mlp": 0.01045699, + "balance_loss_clip": 1.05756199, + "balance_loss_mlp": 1.02697158, + "epoch": 0.1376298531716093, + "flos": 19976496226560.0, + "grad_norm": 3.452039779325381, + "language_loss": 0.89933956, + "learning_rate": 3.879720734664872e-06, + "loss": 0.92118871, + "num_input_tokens_seen": 135268845, + "router_z_loss_clip": 0.81542969, + "router_z_loss_mlp": 0.18719482, + "step": 4743, + "time_per_iteration": 2.502593994140625 + }, + { + "auxiliary_loss_clip": 0.01141985, + "auxiliary_loss_mlp": 0.01043509, + "balance_loss_clip": 1.05771708, + "balance_loss_mlp": 1.02632523, + "epoch": 0.13765887064012536, + "flos": 44777444895360.0, + "grad_norm": 2.312629033961349, + "language_loss": 0.76377189, + "learning_rate": 3.879656526353769e-06, + "loss": 0.78562683, + "num_input_tokens_seen": 135285445, + "router_z_loss_clip": 0.84375, + "router_z_loss_mlp": 0.171875, + "step": 4744, + "time_per_iteration": 2.6560473442077637 + }, + { + "auxiliary_loss_clip": 0.01043787, + "auxiliary_loss_mlp": 0.01001767, + "balance_loss_clip": 1.02251458, + "balance_loss_mlp": 1.00041378, + "epoch": 0.1376878881086414, + "flos": 63162596880000.0, + "grad_norm": 0.7610077063029524, + "language_loss": 0.50284392, + "learning_rate": 3.879592301440779e-06, + "loss": 0.52329946, + "num_input_tokens_seen": 135338580, + "router_z_loss_clip": 0.21289062, + "router_z_loss_mlp": 0.0135498, + "step": 4745, + "time_per_iteration": 3.016068458557129 + }, + { + "auxiliary_loss_clip": 0.0113175, + "auxiliary_loss_mlp": 0.01047569, + "balance_loss_clip": 1.05664086, + "balance_loss_mlp": 1.03168464, + "epoch": 0.13771690557715746, + "flos": 15288575379840.0, + "grad_norm": 2.033376506877686, + "language_loss": 0.79230344, + "learning_rate": 3.8795280599264695e-06, + "loss": 0.81409663, + "num_input_tokens_seen": 135355230, + "router_z_loss_clip": 0.75146484, + "router_z_loss_mlp": 0.15875244, + "step": 4746, + "time_per_iteration": 2.6371121406555176 + }, + { + "auxiliary_loss_clip": 0.01043828, + "auxiliary_loss_mlp": 0.01002615, + "balance_loss_clip": 1.02221394, + "balance_loss_mlp": 1.00129783, + "epoch": 0.13774592304567349, + "flos": 52553760001920.0, + "grad_norm": 0.6185614535709613, + "language_loss": 0.46372941, + "learning_rate": 3.879463801811408e-06, + "loss": 0.48419386, + "num_input_tokens_seen": 135414820, + "router_z_loss_clip": 0.21679688, + "router_z_loss_mlp": 0.01318359, + "step": 4747, + "time_per_iteration": 3.101811170578003 + }, + { + "auxiliary_loss_clip": 0.01134256, + "auxiliary_loss_mlp": 0.01037019, + "balance_loss_clip": 1.05255091, + "balance_loss_mlp": 1.02136099, + "epoch": 0.13777494051418954, + "flos": 16756385771520.0, + "grad_norm": 2.37536784316223, + "language_loss": 0.66379333, + "learning_rate": 3.87939952709616e-06, + "loss": 0.68550599, + "num_input_tokens_seen": 135427055, + "router_z_loss_clip": 0.81640625, + "router_z_loss_mlp": 0.15655518, + "step": 4748, + "time_per_iteration": 2.5664212703704834 + }, + { + "auxiliary_loss_clip": 0.01130718, + "auxiliary_loss_mlp": 0.0103616, + "balance_loss_clip": 1.05337989, + "balance_loss_mlp": 1.02175999, + "epoch": 0.1378039579827056, + "flos": 17230478405760.0, + "grad_norm": 2.2589600451080774, + "language_loss": 0.81026787, + "learning_rate": 3.879335235781297e-06, + "loss": 0.83193672, + "num_input_tokens_seen": 135439870, + "router_z_loss_clip": 0.7734375, + "router_z_loss_mlp": 0.14398193, + "step": 4749, + "time_per_iteration": 2.4822285175323486 + }, + { + "auxiliary_loss_clip": 0.01141178, + "auxiliary_loss_mlp": 0.01049877, + "balance_loss_clip": 1.05605793, + "balance_loss_mlp": 1.03190064, + "epoch": 0.13783297545122164, + "flos": 25339180014720.0, + "grad_norm": 1.5275016145899982, + "language_loss": 0.85525751, + "learning_rate": 3.8792709278673824e-06, + "loss": 0.87716812, + "num_input_tokens_seen": 135465265, + "router_z_loss_clip": 0.85058594, + "router_z_loss_mlp": 0.17980957, + "step": 4750, + "time_per_iteration": 2.7514097690582275 + }, + { + "auxiliary_loss_clip": 0.01153003, + "auxiliary_loss_mlp": 0.01051096, + "balance_loss_clip": 1.06250238, + "balance_loss_mlp": 1.03189147, + "epoch": 0.1378619929197377, + "flos": 29489447111040.0, + "grad_norm": 2.323179216964378, + "language_loss": 0.9853549, + "learning_rate": 3.8792066033549885e-06, + "loss": 1.00739598, + "num_input_tokens_seen": 135485220, + "router_z_loss_clip": 0.90527344, + "router_z_loss_mlp": 0.19189453, + "step": 4751, + "time_per_iteration": 2.5663609504699707 + }, + { + "auxiliary_loss_clip": 0.01042391, + "auxiliary_loss_mlp": 0.00998537, + "balance_loss_clip": 1.0209074, + "balance_loss_mlp": 0.9972679, + "epoch": 0.13789101038825372, + "flos": 73571233305600.0, + "grad_norm": 0.6445342090846006, + "language_loss": 0.49147573, + "learning_rate": 3.879142262244681e-06, + "loss": 0.51188505, + "num_input_tokens_seen": 135549555, + "router_z_loss_clip": 0.21484375, + "router_z_loss_mlp": 0.01269531, + "step": 4752, + "time_per_iteration": 3.1911542415618896 + }, + { + "auxiliary_loss_clip": 0.0114089, + "auxiliary_loss_mlp": 0.01044642, + "balance_loss_clip": 1.05641448, + "balance_loss_mlp": 1.02502036, + "epoch": 0.13792002785676977, + "flos": 23474410467840.0, + "grad_norm": 2.0123018149380703, + "language_loss": 0.5984236, + "learning_rate": 3.8790779045370275e-06, + "loss": 0.62027895, + "num_input_tokens_seen": 135564205, + "router_z_loss_clip": 0.84619141, + "router_z_loss_mlp": 0.19622803, + "step": 4753, + "time_per_iteration": 2.561474561691284 + }, + { + "auxiliary_loss_clip": 0.01041404, + "auxiliary_loss_mlp": 0.00998587, + "balance_loss_clip": 1.02016199, + "balance_loss_mlp": 0.99750185, + "epoch": 0.13794904532528582, + "flos": 68285937369600.0, + "grad_norm": 1.0866798640470734, + "language_loss": 0.47107264, + "learning_rate": 3.879013530232599e-06, + "loss": 0.49147254, + "num_input_tokens_seen": 135628705, + "router_z_loss_clip": 0.21289062, + "router_z_loss_mlp": 0.01086426, + "step": 4754, + "time_per_iteration": 3.1131529808044434 + }, + { + "auxiliary_loss_clip": 0.01134386, + "auxiliary_loss_mlp": 0.0103527, + "balance_loss_clip": 1.05306292, + "balance_loss_mlp": 1.01841402, + "epoch": 0.13797806279380187, + "flos": 12232539273600.0, + "grad_norm": 3.493618205769162, + "language_loss": 0.76265138, + "learning_rate": 3.878949139331961e-06, + "loss": 0.78434795, + "num_input_tokens_seen": 135640790, + "router_z_loss_clip": 0.81298828, + "router_z_loss_mlp": 0.16870117, + "step": 4755, + "time_per_iteration": 2.529543161392212 + }, + { + "auxiliary_loss_clip": 0.0113039, + "auxiliary_loss_mlp": 0.01039777, + "balance_loss_clip": 1.05204654, + "balance_loss_mlp": 1.02300382, + "epoch": 0.13800708026231792, + "flos": 16684205391360.0, + "grad_norm": 3.7249925957603205, + "language_loss": 0.81261265, + "learning_rate": 3.878884731835686e-06, + "loss": 0.83431429, + "num_input_tokens_seen": 135653715, + "router_z_loss_clip": 0.78417969, + "router_z_loss_mlp": 0.16760254, + "step": 4756, + "time_per_iteration": 2.4987809658050537 + }, + { + "auxiliary_loss_clip": 0.01133491, + "auxiliary_loss_mlp": 0.01043714, + "balance_loss_clip": 1.05373466, + "balance_loss_mlp": 1.02779937, + "epoch": 0.13803609773083397, + "flos": 23505724149120.0, + "grad_norm": 3.3724728779111905, + "language_loss": 0.96064723, + "learning_rate": 3.87882030774434e-06, + "loss": 0.98241925, + "num_input_tokens_seen": 135666495, + "router_z_loss_clip": 0.79785156, + "router_z_loss_mlp": 0.15911865, + "step": 4757, + "time_per_iteration": 2.588646173477173 + }, + { + "auxiliary_loss_clip": 0.01134066, + "auxiliary_loss_mlp": 0.01039509, + "balance_loss_clip": 1.05487883, + "balance_loss_mlp": 1.02126443, + "epoch": 0.13806511519935, + "flos": 25157938965120.0, + "grad_norm": 1.7583854567667836, + "language_loss": 0.76637268, + "learning_rate": 3.878755867058492e-06, + "loss": 0.78810841, + "num_input_tokens_seen": 135683785, + "router_z_loss_clip": 0.79199219, + "router_z_loss_mlp": 0.18243408, + "step": 4758, + "time_per_iteration": 2.544475793838501 + }, + { + "auxiliary_loss_clip": 0.01135207, + "auxiliary_loss_mlp": 0.01039073, + "balance_loss_clip": 1.05222225, + "balance_loss_mlp": 1.02241337, + "epoch": 0.13809413266786605, + "flos": 24127589335680.0, + "grad_norm": 2.2377214712416342, + "language_loss": 0.86549461, + "learning_rate": 3.878691409778712e-06, + "loss": 0.88723743, + "num_input_tokens_seen": 135699135, + "router_z_loss_clip": 0.83056641, + "router_z_loss_mlp": 0.16644287, + "step": 4759, + "time_per_iteration": 2.5863850116729736 + }, + { + "auxiliary_loss_clip": 0.0113663, + "auxiliary_loss_mlp": 0.01045849, + "balance_loss_clip": 1.05645156, + "balance_loss_mlp": 1.03048897, + "epoch": 0.1381231501363821, + "flos": 29636321823360.0, + "grad_norm": 1.8744660542948353, + "language_loss": 0.74866539, + "learning_rate": 3.87862693590557e-06, + "loss": 0.77049029, + "num_input_tokens_seen": 135716815, + "router_z_loss_clip": 0.80273438, + "router_z_loss_mlp": 0.15362549, + "step": 4760, + "time_per_iteration": 2.596696138381958 + }, + { + "auxiliary_loss_clip": 0.01137135, + "auxiliary_loss_mlp": 0.01052245, + "balance_loss_clip": 1.05620718, + "balance_loss_mlp": 1.03559756, + "epoch": 0.13815216760489815, + "flos": 38609715536640.0, + "grad_norm": 2.37258456600434, + "language_loss": 0.93779689, + "learning_rate": 3.878562445439634e-06, + "loss": 0.95969069, + "num_input_tokens_seen": 135735765, + "router_z_loss_clip": 0.80859375, + "router_z_loss_mlp": 0.16638184, + "step": 4761, + "time_per_iteration": 2.6988770961761475 + }, + { + "auxiliary_loss_clip": 0.01131726, + "auxiliary_loss_mlp": 0.01040258, + "balance_loss_clip": 1.05332315, + "balance_loss_mlp": 1.0257498, + "epoch": 0.1381811850734142, + "flos": 24351313196160.0, + "grad_norm": 2.211366036187055, + "language_loss": 0.88229591, + "learning_rate": 3.878497938381475e-06, + "loss": 0.90401572, + "num_input_tokens_seen": 135754305, + "router_z_loss_clip": 0.78417969, + "router_z_loss_mlp": 0.14508057, + "step": 4762, + "time_per_iteration": 2.6050801277160645 + }, + { + "auxiliary_loss_clip": 0.01037791, + "auxiliary_loss_mlp": 0.01032494, + "balance_loss_clip": 1.01724422, + "balance_loss_mlp": 1.03145659, + "epoch": 0.13821020254193025, + "flos": 64810142928000.0, + "grad_norm": 0.6985362934041578, + "language_loss": 0.53627908, + "learning_rate": 3.8784334147316614e-06, + "loss": 0.55698192, + "num_input_tokens_seen": 135815535, + "router_z_loss_clip": 0.20605469, + "router_z_loss_mlp": 0.01037598, + "step": 4763, + "time_per_iteration": 3.141598701477051 + }, + { + "auxiliary_loss_clip": 0.011252, + "auxiliary_loss_mlp": 0.01043242, + "balance_loss_clip": 1.05181336, + "balance_loss_mlp": 1.02970624, + "epoch": 0.13823922001044628, + "flos": 11647985339520.0, + "grad_norm": 2.70511117463231, + "language_loss": 0.75683713, + "learning_rate": 3.8783688744907645e-06, + "loss": 0.77852154, + "num_input_tokens_seen": 135826500, + "router_z_loss_clip": 0.734375, + "router_z_loss_mlp": 0.13537598, + "step": 4764, + "time_per_iteration": 2.570347785949707 + }, + { + "auxiliary_loss_clip": 0.01147166, + "auxiliary_loss_mlp": 0.01056759, + "balance_loss_clip": 1.05795169, + "balance_loss_mlp": 1.0380013, + "epoch": 0.13826823747896233, + "flos": 20587587333120.0, + "grad_norm": 3.305070073083128, + "language_loss": 0.94755149, + "learning_rate": 3.8783043176593526e-06, + "loss": 0.96959072, + "num_input_tokens_seen": 135840870, + "router_z_loss_clip": 0.89208984, + "router_z_loss_mlp": 0.18756104, + "step": 4765, + "time_per_iteration": 5.044306039810181 + }, + { + "auxiliary_loss_clip": 0.01142757, + "auxiliary_loss_mlp": 0.01057485, + "balance_loss_clip": 1.05815637, + "balance_loss_mlp": 1.03787518, + "epoch": 0.13829725494747838, + "flos": 26061199297920.0, + "grad_norm": 2.1941394665408582, + "language_loss": 0.64814651, + "learning_rate": 3.878239744237997e-06, + "loss": 0.67014897, + "num_input_tokens_seen": 135856830, + "router_z_loss_clip": 0.84570312, + "router_z_loss_mlp": 0.19604492, + "step": 4766, + "time_per_iteration": 5.001543283462524 + }, + { + "auxiliary_loss_clip": 0.01147679, + "auxiliary_loss_mlp": 0.01045816, + "balance_loss_clip": 1.06117141, + "balance_loss_mlp": 1.026963, + "epoch": 0.13832627241599443, + "flos": 20004864992640.0, + "grad_norm": 2.5840043860792443, + "language_loss": 0.76266539, + "learning_rate": 3.878175154227269e-06, + "loss": 0.78460038, + "num_input_tokens_seen": 135869795, + "router_z_loss_clip": 0.86523438, + "router_z_loss_mlp": 0.18878174, + "step": 4767, + "time_per_iteration": 2.5000078678131104 + }, + { + "auxiliary_loss_clip": 0.01139828, + "auxiliary_loss_mlp": 0.01045975, + "balance_loss_clip": 1.05591798, + "balance_loss_mlp": 1.02797461, + "epoch": 0.13835528988451049, + "flos": 13983220247040.0, + "grad_norm": 2.8787423760991486, + "language_loss": 0.86811006, + "learning_rate": 3.878110547627737e-06, + "loss": 0.8899681, + "num_input_tokens_seen": 135880500, + "router_z_loss_clip": 0.83886719, + "router_z_loss_mlp": 0.1796875, + "step": 4768, + "time_per_iteration": 2.4869701862335205 + }, + { + "auxiliary_loss_clip": 0.01041403, + "auxiliary_loss_mlp": 0.01003249, + "balance_loss_clip": 1.02058268, + "balance_loss_mlp": 1.00214624, + "epoch": 0.1383843073530265, + "flos": 69113210480640.0, + "grad_norm": 0.7562193581302254, + "language_loss": 0.45968187, + "learning_rate": 3.878045924439974e-06, + "loss": 0.48012835, + "num_input_tokens_seen": 135937085, + "router_z_loss_clip": 0.20898438, + "router_z_loss_mlp": 0.01104736, + "step": 4769, + "time_per_iteration": 5.358396530151367 + }, + { + "auxiliary_loss_clip": 0.01137154, + "auxiliary_loss_mlp": 0.01033671, + "balance_loss_clip": 1.05459547, + "balance_loss_mlp": 1.01839983, + "epoch": 0.13841332482154256, + "flos": 36678296304000.0, + "grad_norm": 1.9846197095446605, + "language_loss": 0.7233994, + "learning_rate": 3.877981284664548e-06, + "loss": 0.74510765, + "num_input_tokens_seen": 135955760, + "router_z_loss_clip": 0.82470703, + "router_z_loss_mlp": 0.15264893, + "step": 4770, + "time_per_iteration": 2.6122562885284424 + }, + { + "auxiliary_loss_clip": 0.01136106, + "auxiliary_loss_mlp": 0.0103817, + "balance_loss_clip": 1.0569011, + "balance_loss_mlp": 1.02300668, + "epoch": 0.1384423422900586, + "flos": 41464614458880.0, + "grad_norm": 2.696024814080405, + "language_loss": 0.76403934, + "learning_rate": 3.877916628302031e-06, + "loss": 0.7857821, + "num_input_tokens_seen": 135974400, + "router_z_loss_clip": 0.79101562, + "router_z_loss_mlp": 0.15167236, + "step": 4771, + "time_per_iteration": 2.623539447784424 + }, + { + "auxiliary_loss_clip": 0.0115452, + "auxiliary_loss_mlp": 0.01049506, + "balance_loss_clip": 1.06103051, + "balance_loss_mlp": 1.02868056, + "epoch": 0.13847135975857466, + "flos": 32226486531840.0, + "grad_norm": 2.5054937728022346, + "language_loss": 0.85467064, + "learning_rate": 3.877851955352996e-06, + "loss": 0.87671089, + "num_input_tokens_seen": 135989515, + "router_z_loss_clip": 0.93652344, + "router_z_loss_mlp": 0.20825195, + "step": 4772, + "time_per_iteration": 2.5925161838531494 + }, + { + "auxiliary_loss_clip": 0.01041892, + "auxiliary_loss_mlp": 0.01000678, + "balance_loss_clip": 1.02118206, + "balance_loss_mlp": 0.99948579, + "epoch": 0.13850037722709072, + "flos": 74770649274240.0, + "grad_norm": 0.7002381873016341, + "language_loss": 0.50773132, + "learning_rate": 3.877787265818011e-06, + "loss": 0.528157, + "num_input_tokens_seen": 136055365, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.01190186, + "step": 4773, + "time_per_iteration": 3.2232236862182617 + }, + { + "auxiliary_loss_clip": 0.01138743, + "auxiliary_loss_mlp": 0.01046623, + "balance_loss_clip": 1.05604649, + "balance_loss_mlp": 1.02834797, + "epoch": 0.13852939469560677, + "flos": 28250062283520.0, + "grad_norm": 3.422061093820181, + "language_loss": 0.89392686, + "learning_rate": 3.8777225596976506e-06, + "loss": 0.91578048, + "num_input_tokens_seen": 136075035, + "router_z_loss_clip": 0.82666016, + "router_z_loss_mlp": 0.18261719, + "step": 4774, + "time_per_iteration": 2.577626943588257 + }, + { + "auxiliary_loss_clip": 0.01139122, + "auxiliary_loss_mlp": 0.01037967, + "balance_loss_clip": 1.05727816, + "balance_loss_mlp": 1.02241659, + "epoch": 0.1385584121641228, + "flos": 74731431623040.0, + "grad_norm": 3.7336543255297805, + "language_loss": 0.66548997, + "learning_rate": 3.877657836992484e-06, + "loss": 0.68726087, + "num_input_tokens_seen": 136096115, + "router_z_loss_clip": 0.81884766, + "router_z_loss_mlp": 0.15551758, + "step": 4775, + "time_per_iteration": 2.9111697673797607 + }, + { + "auxiliary_loss_clip": 0.01040863, + "auxiliary_loss_mlp": 0.01001649, + "balance_loss_clip": 1.02013183, + "balance_loss_mlp": 1.00053453, + "epoch": 0.13858742963263884, + "flos": 72332710404480.0, + "grad_norm": 0.6492684415306446, + "language_loss": 0.48730826, + "learning_rate": 3.877593097703084e-06, + "loss": 0.50773335, + "num_input_tokens_seen": 136160945, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.01116943, + "step": 4776, + "time_per_iteration": 3.17451548576355 + }, + { + "auxiliary_loss_clip": 0.01133202, + "auxiliary_loss_mlp": 0.01032499, + "balance_loss_clip": 1.05601931, + "balance_loss_mlp": 1.01719213, + "epoch": 0.1386164471011549, + "flos": 16940248513920.0, + "grad_norm": 2.669061627006619, + "language_loss": 0.65362257, + "learning_rate": 3.877528341830021e-06, + "loss": 0.67527956, + "num_input_tokens_seen": 136176325, + "router_z_loss_clip": 0.77099609, + "router_z_loss_mlp": 0.15313721, + "step": 4777, + "time_per_iteration": 2.4839446544647217 + }, + { + "auxiliary_loss_clip": 0.01130641, + "auxiliary_loss_mlp": 0.01039214, + "balance_loss_clip": 1.05439591, + "balance_loss_mlp": 1.02468276, + "epoch": 0.13864546456967095, + "flos": 11430797754240.0, + "grad_norm": 2.626383751625317, + "language_loss": 0.860461, + "learning_rate": 3.8774635693738685e-06, + "loss": 0.88215959, + "num_input_tokens_seen": 136188395, + "router_z_loss_clip": 0.76318359, + "router_z_loss_mlp": 0.14538574, + "step": 4778, + "time_per_iteration": 2.508650541305542 + }, + { + "auxiliary_loss_clip": 0.01038709, + "auxiliary_loss_mlp": 0.01003071, + "balance_loss_clip": 1.01823735, + "balance_loss_mlp": 1.00189102, + "epoch": 0.138674482038187, + "flos": 72293998521600.0, + "grad_norm": 0.6134362438226334, + "language_loss": 0.4816592, + "learning_rate": 3.877398780335199e-06, + "loss": 0.50207698, + "num_input_tokens_seen": 136254080, + "router_z_loss_clip": 0.20507812, + "router_z_loss_mlp": 0.01177979, + "step": 4779, + "time_per_iteration": 3.1919963359832764 + }, + { + "auxiliary_loss_clip": 0.01129305, + "auxiliary_loss_mlp": 0.01035581, + "balance_loss_clip": 1.05281734, + "balance_loss_mlp": 1.02063167, + "epoch": 0.13870349950670305, + "flos": 31023084153600.0, + "grad_norm": 2.3820641393744295, + "language_loss": 0.83279181, + "learning_rate": 3.877333974714582e-06, + "loss": 0.85444063, + "num_input_tokens_seen": 136271430, + "router_z_loss_clip": 0.76464844, + "router_z_loss_mlp": 0.1494751, + "step": 4780, + "time_per_iteration": 2.610779285430908 + }, + { + "auxiliary_loss_clip": 0.01138799, + "auxiliary_loss_mlp": 0.01046341, + "balance_loss_clip": 1.05685163, + "balance_loss_mlp": 1.03008723, + "epoch": 0.13873251697521907, + "flos": 40141269590400.0, + "grad_norm": 2.778252245774382, + "language_loss": 0.69260991, + "learning_rate": 3.877269152512593e-06, + "loss": 0.71446133, + "num_input_tokens_seen": 136287685, + "router_z_loss_clip": 0.81884766, + "router_z_loss_mlp": 0.16265869, + "step": 4781, + "time_per_iteration": 2.6585285663604736 + }, + { + "auxiliary_loss_clip": 0.01038895, + "auxiliary_loss_mlp": 0.01004628, + "balance_loss_clip": 1.01822758, + "balance_loss_mlp": 1.00340581, + "epoch": 0.13876153444373512, + "flos": 71596648903680.0, + "grad_norm": 0.6536762321440938, + "language_loss": 0.47937241, + "learning_rate": 3.877204313729802e-06, + "loss": 0.49980763, + "num_input_tokens_seen": 136355700, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.01220703, + "step": 4782, + "time_per_iteration": 3.238786458969116 + }, + { + "auxiliary_loss_clip": 0.01038369, + "auxiliary_loss_mlp": 0.01006603, + "balance_loss_clip": 1.01761222, + "balance_loss_mlp": 1.00547075, + "epoch": 0.13879055191225118, + "flos": 61562848855680.0, + "grad_norm": 0.7886081470854837, + "language_loss": 0.50094962, + "learning_rate": 3.877139458366783e-06, + "loss": 0.52139932, + "num_input_tokens_seen": 136416835, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.01135254, + "step": 4783, + "time_per_iteration": 3.0247137546539307 + }, + { + "auxiliary_loss_clip": 0.01037254, + "auxiliary_loss_mlp": 0.0100204, + "balance_loss_clip": 1.01660872, + "balance_loss_mlp": 1.00088394, + "epoch": 0.13881956938076723, + "flos": 65879743144320.0, + "grad_norm": 0.6714345073757055, + "language_loss": 0.46745718, + "learning_rate": 3.87707458642411e-06, + "loss": 0.4878501, + "num_input_tokens_seen": 136483870, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.01153564, + "step": 4784, + "time_per_iteration": 3.2695322036743164 + }, + { + "auxiliary_loss_clip": 0.01035936, + "auxiliary_loss_mlp": 0.0100216, + "balance_loss_clip": 1.0153178, + "balance_loss_mlp": 1.00103378, + "epoch": 0.13884858684928328, + "flos": 73599817553280.0, + "grad_norm": 1.7135518956942128, + "language_loss": 0.51383644, + "learning_rate": 3.877009697902354e-06, + "loss": 0.53421742, + "num_input_tokens_seen": 136541115, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.0112915, + "step": 4785, + "time_per_iteration": 3.065211534500122 + }, + { + "auxiliary_loss_clip": 0.01035216, + "auxiliary_loss_mlp": 0.01001093, + "balance_loss_clip": 1.01469147, + "balance_loss_mlp": 1.00000858, + "epoch": 0.1388776043177993, + "flos": 71479938637440.0, + "grad_norm": 0.7466862669673942, + "language_loss": 0.52175772, + "learning_rate": 3.8769447928020885e-06, + "loss": 0.54212087, + "num_input_tokens_seen": 136596710, + "router_z_loss_clip": 0.20507812, + "router_z_loss_mlp": 0.01086426, + "step": 4786, + "time_per_iteration": 2.9729747772216797 + }, + { + "auxiliary_loss_clip": 0.01144707, + "auxiliary_loss_mlp": 0.01049789, + "balance_loss_clip": 1.05986297, + "balance_loss_mlp": 1.03315926, + "epoch": 0.13890662178631535, + "flos": 40290011809920.0, + "grad_norm": 2.242053537810187, + "language_loss": 0.82707727, + "learning_rate": 3.8768798711238875e-06, + "loss": 0.84902221, + "num_input_tokens_seen": 136612115, + "router_z_loss_clip": 0.84863281, + "router_z_loss_mlp": 0.16650391, + "step": 4787, + "time_per_iteration": 2.7385048866271973 + }, + { + "auxiliary_loss_clip": 0.01121775, + "auxiliary_loss_mlp": 0.01047865, + "balance_loss_clip": 1.04935741, + "balance_loss_mlp": 1.03472173, + "epoch": 0.1389356392548314, + "flos": 42516976147200.0, + "grad_norm": 2.1895107500482838, + "language_loss": 0.68735969, + "learning_rate": 3.876814932868323e-06, + "loss": 0.70905614, + "num_input_tokens_seen": 136628895, + "router_z_loss_clip": 0.72412109, + "router_z_loss_mlp": 0.13122559, + "step": 4788, + "time_per_iteration": 2.6996772289276123 + }, + { + "auxiliary_loss_clip": 0.0113641, + "auxiliary_loss_mlp": 0.01051748, + "balance_loss_clip": 1.05492878, + "balance_loss_mlp": 1.03446889, + "epoch": 0.13896465672334746, + "flos": 20513826754560.0, + "grad_norm": 3.708490588591475, + "language_loss": 0.79010546, + "learning_rate": 3.8767499780359704e-06, + "loss": 0.81198704, + "num_input_tokens_seen": 136642490, + "router_z_loss_clip": 0.81445312, + "router_z_loss_mlp": 0.17285156, + "step": 4789, + "time_per_iteration": 2.539327383041382 + }, + { + "auxiliary_loss_clip": 0.01144172, + "auxiliary_loss_mlp": 0.01044077, + "balance_loss_clip": 1.05701685, + "balance_loss_mlp": 1.02682734, + "epoch": 0.1389936741918635, + "flos": 31466545464960.0, + "grad_norm": 2.2780150170209503, + "language_loss": 0.81317937, + "learning_rate": 3.876685006627403e-06, + "loss": 0.83506191, + "num_input_tokens_seen": 136658795, + "router_z_loss_clip": 0.87060547, + "router_z_loss_mlp": 0.17248535, + "step": 4790, + "time_per_iteration": 2.598761558532715 + }, + { + "auxiliary_loss_clip": 0.01034917, + "auxiliary_loss_mlp": 0.01003872, + "balance_loss_clip": 1.01448798, + "balance_loss_mlp": 1.00285268, + "epoch": 0.13902269166037956, + "flos": 74774779338240.0, + "grad_norm": 0.5926150604464158, + "language_loss": 0.44421786, + "learning_rate": 3.8766200186431935e-06, + "loss": 0.46460569, + "num_input_tokens_seen": 136727835, + "router_z_loss_clip": 0.20410156, + "router_z_loss_mlp": 0.01019287, + "step": 4791, + "time_per_iteration": 3.2647151947021484 + }, + { + "auxiliary_loss_clip": 0.01138302, + "auxiliary_loss_mlp": 0.01058582, + "balance_loss_clip": 1.05658853, + "balance_loss_mlp": 1.04055166, + "epoch": 0.13905170912889558, + "flos": 28360236274560.0, + "grad_norm": 2.1254266026136013, + "language_loss": 0.83040774, + "learning_rate": 3.876555014083916e-06, + "loss": 0.85237658, + "num_input_tokens_seen": 136743840, + "router_z_loss_clip": 0.81591797, + "router_z_loss_mlp": 0.18017578, + "step": 4792, + "time_per_iteration": 2.557465076446533 + }, + { + "auxiliary_loss_clip": 0.01135363, + "auxiliary_loss_mlp": 0.01049581, + "balance_loss_clip": 1.05371034, + "balance_loss_mlp": 1.03283763, + "epoch": 0.13908072659741164, + "flos": 26649452332800.0, + "grad_norm": 2.202397506066105, + "language_loss": 0.8511939, + "learning_rate": 3.876489992950147e-06, + "loss": 0.87304324, + "num_input_tokens_seen": 136764625, + "router_z_loss_clip": 0.81738281, + "router_z_loss_mlp": 0.16760254, + "step": 4793, + "time_per_iteration": 2.5646839141845703 + }, + { + "auxiliary_loss_clip": 0.0113697, + "auxiliary_loss_mlp": 0.01045271, + "balance_loss_clip": 1.05646813, + "balance_loss_mlp": 1.03001809, + "epoch": 0.1391097440659277, + "flos": 21135943336320.0, + "grad_norm": 2.5868354678139993, + "language_loss": 0.91635859, + "learning_rate": 3.876424955242458e-06, + "loss": 0.93818098, + "num_input_tokens_seen": 136779125, + "router_z_loss_clip": 0.80419922, + "router_z_loss_mlp": 0.15252686, + "step": 4794, + "time_per_iteration": 2.5585744380950928 + }, + { + "auxiliary_loss_clip": 0.01035576, + "auxiliary_loss_mlp": 0.01009364, + "balance_loss_clip": 1.01503694, + "balance_loss_mlp": 1.00829685, + "epoch": 0.13913876153444374, + "flos": 72801846961920.0, + "grad_norm": 0.8777572844352342, + "language_loss": 0.43607107, + "learning_rate": 3.876359900961424e-06, + "loss": 0.45652044, + "num_input_tokens_seen": 136840325, + "router_z_loss_clip": 0.20507812, + "router_z_loss_mlp": 0.01068115, + "step": 4795, + "time_per_iteration": 3.1121253967285156 + }, + { + "auxiliary_loss_clip": 0.01145583, + "auxiliary_loss_mlp": 0.01044293, + "balance_loss_clip": 1.05772972, + "balance_loss_mlp": 1.02614987, + "epoch": 0.1391677790029598, + "flos": 33215466672000.0, + "grad_norm": 2.05067089045658, + "language_loss": 0.78827864, + "learning_rate": 3.876294830107621e-06, + "loss": 0.81017745, + "num_input_tokens_seen": 136861125, + "router_z_loss_clip": 0.87841797, + "router_z_loss_mlp": 0.18145752, + "step": 4796, + "time_per_iteration": 2.7585527896881104 + }, + { + "auxiliary_loss_clip": 0.01135391, + "auxiliary_loss_mlp": 0.01057369, + "balance_loss_clip": 1.05531526, + "balance_loss_mlp": 1.04065585, + "epoch": 0.13919679647147584, + "flos": 18619000502400.0, + "grad_norm": 3.0213599302573098, + "language_loss": 0.88365078, + "learning_rate": 3.876229742681622e-06, + "loss": 0.90557849, + "num_input_tokens_seen": 136878275, + "router_z_loss_clip": 0.80126953, + "router_z_loss_mlp": 0.16711426, + "step": 4797, + "time_per_iteration": 2.540637493133545 + }, + { + "auxiliary_loss_clip": 0.010375, + "auxiliary_loss_mlp": 0.00998728, + "balance_loss_clip": 1.01711583, + "balance_loss_mlp": 0.99761385, + "epoch": 0.13922581393999187, + "flos": 51751228383360.0, + "grad_norm": 0.6712736387783712, + "language_loss": 0.46641502, + "learning_rate": 3.876164638684004e-06, + "loss": 0.48677737, + "num_input_tokens_seen": 136937245, + "router_z_loss_clip": 0.20410156, + "router_z_loss_mlp": 0.01116943, + "step": 4798, + "time_per_iteration": 3.060976028442383 + }, + { + "auxiliary_loss_clip": 0.01138807, + "auxiliary_loss_mlp": 0.01044668, + "balance_loss_clip": 1.05567443, + "balance_loss_mlp": 1.02711391, + "epoch": 0.13925483140850792, + "flos": 20301451591680.0, + "grad_norm": 3.2219907950007114, + "language_loss": 0.75970042, + "learning_rate": 3.87609951811534e-06, + "loss": 0.78153515, + "num_input_tokens_seen": 136953605, + "router_z_loss_clip": 0.83056641, + "router_z_loss_mlp": 0.17541504, + "step": 4799, + "time_per_iteration": 2.5340938568115234 + }, + { + "auxiliary_loss_clip": 0.01142425, + "auxiliary_loss_mlp": 0.01046537, + "balance_loss_clip": 1.05808163, + "balance_loss_mlp": 1.03042626, + "epoch": 0.13928384887702397, + "flos": 38138675558400.0, + "grad_norm": 1.9567211361850017, + "language_loss": 0.91711247, + "learning_rate": 3.876034380976205e-06, + "loss": 0.93900216, + "num_input_tokens_seen": 136974105, + "router_z_loss_clip": 0.84228516, + "router_z_loss_mlp": 0.16101074, + "step": 4800, + "time_per_iteration": 2.701084613800049 + }, + { + "auxiliary_loss_clip": 0.01120595, + "auxiliary_loss_mlp": 0.01042796, + "balance_loss_clip": 1.0537343, + "balance_loss_mlp": 1.02946281, + "epoch": 0.13931286634554002, + "flos": 37189053745920.0, + "grad_norm": 1.855527529375164, + "language_loss": 0.64324874, + "learning_rate": 3.875969227267176e-06, + "loss": 0.66488266, + "num_input_tokens_seen": 136992400, + "router_z_loss_clip": 0.66943359, + "router_z_loss_mlp": 0.13330078, + "step": 4801, + "time_per_iteration": 2.687310218811035 + }, + { + "auxiliary_loss_clip": 0.01140666, + "auxiliary_loss_mlp": 0.01059656, + "balance_loss_clip": 1.05624366, + "balance_loss_mlp": 1.04247785, + "epoch": 0.13934188381405607, + "flos": 15699786278400.0, + "grad_norm": 2.98857755584889, + "language_loss": 0.99227965, + "learning_rate": 3.875904056988828e-06, + "loss": 1.01428294, + "num_input_tokens_seen": 137004530, + "router_z_loss_clip": 0.84423828, + "router_z_loss_mlp": 0.17181396, + "step": 4802, + "time_per_iteration": 2.4954421520233154 + }, + { + "auxiliary_loss_clip": 0.01034362, + "auxiliary_loss_mlp": 0.01008428, + "balance_loss_clip": 1.01394522, + "balance_loss_mlp": 1.00733685, + "epoch": 0.1393709012825721, + "flos": 62012271824640.0, + "grad_norm": 0.6601570262324175, + "language_loss": 0.45212865, + "learning_rate": 3.875838870141735e-06, + "loss": 0.47255653, + "num_input_tokens_seen": 137072705, + "router_z_loss_clip": 0.20507812, + "router_z_loss_mlp": 0.01092529, + "step": 4803, + "time_per_iteration": 3.2598044872283936 + }, + { + "auxiliary_loss_clip": 0.01152979, + "auxiliary_loss_mlp": 0.01063621, + "balance_loss_clip": 1.06203139, + "balance_loss_mlp": 1.04502439, + "epoch": 0.13939991875108815, + "flos": 16611522220800.0, + "grad_norm": 2.568969442533264, + "language_loss": 0.74421799, + "learning_rate": 3.875773666726475e-06, + "loss": 0.76638401, + "num_input_tokens_seen": 137086220, + "router_z_loss_clip": 0.91015625, + "router_z_loss_mlp": 0.18579102, + "step": 4804, + "time_per_iteration": 2.5123701095581055 + }, + { + "auxiliary_loss_clip": 0.01032498, + "auxiliary_loss_mlp": 0.00999641, + "balance_loss_clip": 1.01215303, + "balance_loss_mlp": 0.99860984, + "epoch": 0.1394289362196042, + "flos": 67000909334400.0, + "grad_norm": 0.6632214775424402, + "language_loss": 0.4751702, + "learning_rate": 3.875708446743623e-06, + "loss": 0.49549156, + "num_input_tokens_seen": 137148555, + "router_z_loss_clip": 0.203125, + "router_z_loss_mlp": 0.01031494, + "step": 4805, + "time_per_iteration": 3.1051366329193115 + }, + { + "auxiliary_loss_clip": 0.01132472, + "auxiliary_loss_mlp": 0.01038677, + "balance_loss_clip": 1.05590177, + "balance_loss_mlp": 1.0236448, + "epoch": 0.13945795368812025, + "flos": 30440433640320.0, + "grad_norm": 2.382361358406432, + "language_loss": 0.90655708, + "learning_rate": 3.875643210193755e-06, + "loss": 0.92826861, + "num_input_tokens_seen": 137166505, + "router_z_loss_clip": 0.765625, + "router_z_loss_mlp": 0.15057373, + "step": 4806, + "time_per_iteration": 2.583423376083374 + }, + { + "auxiliary_loss_clip": 0.0103165, + "auxiliary_loss_mlp": 0.00999133, + "balance_loss_clip": 1.01115322, + "balance_loss_mlp": 0.99806577, + "epoch": 0.1394869711566363, + "flos": 62164641317760.0, + "grad_norm": 0.6452362243077646, + "language_loss": 0.49036235, + "learning_rate": 3.875577957077447e-06, + "loss": 0.51067019, + "num_input_tokens_seen": 137227595, + "router_z_loss_clip": 0.20507812, + "router_z_loss_mlp": 0.01068115, + "step": 4807, + "time_per_iteration": 3.111605405807495 + }, + { + "auxiliary_loss_clip": 0.01143697, + "auxiliary_loss_mlp": 0.01043886, + "balance_loss_clip": 1.05587614, + "balance_loss_mlp": 1.02750647, + "epoch": 0.13951598862515235, + "flos": 27711546606720.0, + "grad_norm": 2.127837136022934, + "language_loss": 0.95291507, + "learning_rate": 3.875512687395275e-06, + "loss": 0.97479105, + "num_input_tokens_seen": 137248075, + "router_z_loss_clip": 0.87792969, + "router_z_loss_mlp": 0.16369629, + "step": 4808, + "time_per_iteration": 2.6192197799682617 + }, + { + "auxiliary_loss_clip": 0.01031432, + "auxiliary_loss_mlp": 0.010017, + "balance_loss_clip": 1.01098132, + "balance_loss_mlp": 1.0006392, + "epoch": 0.13954500609366838, + "flos": 71863789328640.0, + "grad_norm": 0.7128279920946932, + "language_loss": 0.54074287, + "learning_rate": 3.875447401147817e-06, + "loss": 0.56107426, + "num_input_tokens_seen": 137313275, + "router_z_loss_clip": 0.20507812, + "router_z_loss_mlp": 0.01062012, + "step": 4809, + "time_per_iteration": 3.1806044578552246 + }, + { + "auxiliary_loss_clip": 0.01029913, + "auxiliary_loss_mlp": 0.01002055, + "balance_loss_clip": 1.0096488, + "balance_loss_mlp": 1.00104165, + "epoch": 0.13957402356218443, + "flos": 74779448106240.0, + "grad_norm": 0.591544973309317, + "language_loss": 0.4742226, + "learning_rate": 3.875382098335648e-06, + "loss": 0.4945423, + "num_input_tokens_seen": 137389890, + "router_z_loss_clip": 0.203125, + "router_z_loss_mlp": 0.01013184, + "step": 4810, + "time_per_iteration": 3.316650867462158 + }, + { + "auxiliary_loss_clip": 0.01142115, + "auxiliary_loss_mlp": 0.01048681, + "balance_loss_clip": 1.05375707, + "balance_loss_mlp": 1.03012037, + "epoch": 0.13960304103070048, + "flos": 25695664542720.0, + "grad_norm": 3.3188897715365258, + "language_loss": 0.86450738, + "learning_rate": 3.875316778959346e-06, + "loss": 0.88641536, + "num_input_tokens_seen": 137404365, + "router_z_loss_clip": 0.88476562, + "router_z_loss_mlp": 0.18566895, + "step": 4811, + "time_per_iteration": 2.573604106903076 + }, + { + "auxiliary_loss_clip": 0.01133533, + "auxiliary_loss_mlp": 0.01045176, + "balance_loss_clip": 1.05315399, + "balance_loss_mlp": 1.02861786, + "epoch": 0.13963205849921653, + "flos": 27009887356800.0, + "grad_norm": 1.8590141944490344, + "language_loss": 0.78787351, + "learning_rate": 3.875251443019486e-06, + "loss": 0.80966055, + "num_input_tokens_seen": 137423170, + "router_z_loss_clip": 0.80419922, + "router_z_loss_mlp": 0.16564941, + "step": 4812, + "time_per_iteration": 2.6607797145843506 + }, + { + "auxiliary_loss_clip": 0.0112968, + "auxiliary_loss_mlp": 0.01037894, + "balance_loss_clip": 1.05074763, + "balance_loss_mlp": 1.02372015, + "epoch": 0.13966107596773258, + "flos": 15334215609600.0, + "grad_norm": 2.800567347213856, + "language_loss": 0.74662113, + "learning_rate": 3.875186090516648e-06, + "loss": 0.7682969, + "num_input_tokens_seen": 137435045, + "router_z_loss_clip": 0.7890625, + "router_z_loss_mlp": 0.14178467, + "step": 4813, + "time_per_iteration": 2.483286142349243 + }, + { + "auxiliary_loss_clip": 0.0114667, + "auxiliary_loss_mlp": 0.01053717, + "balance_loss_clip": 1.05617714, + "balance_loss_mlp": 1.03638363, + "epoch": 0.1396900934362486, + "flos": 15625666563840.0, + "grad_norm": 4.663290937283138, + "language_loss": 0.88924432, + "learning_rate": 3.875120721451406e-06, + "loss": 0.91124821, + "num_input_tokens_seen": 137448110, + "router_z_loss_clip": 0.90380859, + "router_z_loss_mlp": 0.17321777, + "step": 4814, + "time_per_iteration": 2.5261054039001465 + }, + { + "auxiliary_loss_clip": 0.0103149, + "auxiliary_loss_mlp": 0.01025805, + "balance_loss_clip": 1.01097608, + "balance_loss_mlp": 1.02483296, + "epoch": 0.13971911090476466, + "flos": 61384983684480.0, + "grad_norm": 0.6531302836112987, + "language_loss": 0.48057067, + "learning_rate": 3.87505533582434e-06, + "loss": 0.50114357, + "num_input_tokens_seen": 137506595, + "router_z_loss_clip": 0.20507812, + "router_z_loss_mlp": 0.00970459, + "step": 4815, + "time_per_iteration": 2.992741346359253 + }, + { + "auxiliary_loss_clip": 0.01149962, + "auxiliary_loss_mlp": 0.01039456, + "balance_loss_clip": 1.05738556, + "balance_loss_mlp": 1.02166986, + "epoch": 0.1397481283732807, + "flos": 17488604517120.0, + "grad_norm": 2.827440946642964, + "language_loss": 0.94193059, + "learning_rate": 3.874989933636027e-06, + "loss": 0.96382475, + "num_input_tokens_seen": 137520995, + "router_z_loss_clip": 0.92724609, + "router_z_loss_mlp": 0.17791748, + "step": 4816, + "time_per_iteration": 2.554550886154175 + }, + { + "auxiliary_loss_clip": 0.01130656, + "auxiliary_loss_mlp": 0.01039565, + "balance_loss_clip": 1.05279756, + "balance_loss_mlp": 1.02421117, + "epoch": 0.13977714584179676, + "flos": 24526628501760.0, + "grad_norm": 3.151826877646541, + "language_loss": 0.8705917, + "learning_rate": 3.874924514887043e-06, + "loss": 0.89229393, + "num_input_tokens_seen": 137535430, + "router_z_loss_clip": 0.77832031, + "router_z_loss_mlp": 0.15350342, + "step": 4817, + "time_per_iteration": 2.5914409160614014 + }, + { + "auxiliary_loss_clip": 0.01137265, + "auxiliary_loss_mlp": 0.01042884, + "balance_loss_clip": 1.05646253, + "balance_loss_mlp": 1.02656484, + "epoch": 0.13980616331031281, + "flos": 25694048430720.0, + "grad_norm": 3.0938304344081464, + "language_loss": 0.85210836, + "learning_rate": 3.874859079577968e-06, + "loss": 0.87390983, + "num_input_tokens_seen": 137552295, + "router_z_loss_clip": 0.80810547, + "router_z_loss_mlp": 0.16296387, + "step": 4818, + "time_per_iteration": 2.5796096324920654 + }, + { + "auxiliary_loss_clip": 0.01131953, + "auxiliary_loss_mlp": 0.01041926, + "balance_loss_clip": 1.05340195, + "balance_loss_mlp": 1.02632761, + "epoch": 0.13983518077882887, + "flos": 29454901205760.0, + "grad_norm": 2.0757880628417498, + "language_loss": 0.72224295, + "learning_rate": 3.874793627709379e-06, + "loss": 0.74398178, + "num_input_tokens_seen": 137567470, + "router_z_loss_clip": 0.78466797, + "router_z_loss_mlp": 0.15600586, + "step": 4819, + "time_per_iteration": 2.595346689224243 + }, + { + "auxiliary_loss_clip": 0.0112603, + "auxiliary_loss_mlp": 0.01037557, + "balance_loss_clip": 1.05216956, + "balance_loss_mlp": 1.02381837, + "epoch": 0.1398641982473449, + "flos": 19128034091520.0, + "grad_norm": 1.850163108861516, + "language_loss": 0.77968943, + "learning_rate": 3.874728159281853e-06, + "loss": 0.80132532, + "num_input_tokens_seen": 137581025, + "router_z_loss_clip": 0.73828125, + "router_z_loss_mlp": 0.13739014, + "step": 4820, + "time_per_iteration": 2.5468435287475586 + }, + { + "auxiliary_loss_clip": 0.01132498, + "auxiliary_loss_mlp": 0.01039856, + "balance_loss_clip": 1.0541054, + "balance_loss_mlp": 1.02522278, + "epoch": 0.13989321571586094, + "flos": 21719850825600.0, + "grad_norm": 2.914873591067294, + "language_loss": 0.68335372, + "learning_rate": 3.8746626742959705e-06, + "loss": 0.70507723, + "num_input_tokens_seen": 137598045, + "router_z_loss_clip": 0.78417969, + "router_z_loss_mlp": 0.1461792, + "step": 4821, + "time_per_iteration": 2.539030075073242 + }, + { + "auxiliary_loss_clip": 0.0114259, + "auxiliary_loss_mlp": 0.01045359, + "balance_loss_clip": 1.05952787, + "balance_loss_mlp": 1.02915812, + "epoch": 0.139922233184377, + "flos": 20442005510400.0, + "grad_norm": 2.351943998821958, + "language_loss": 0.80879867, + "learning_rate": 3.874597172752308e-06, + "loss": 0.83067811, + "num_input_tokens_seen": 137611355, + "router_z_loss_clip": 0.83056641, + "router_z_loss_mlp": 0.1619873, + "step": 4822, + "time_per_iteration": 2.5196011066436768 + }, + { + "auxiliary_loss_clip": 0.0103303, + "auxiliary_loss_mlp": 0.00999307, + "balance_loss_clip": 1.01243711, + "balance_loss_mlp": 0.99804956, + "epoch": 0.13995125065289304, + "flos": 50359079013120.0, + "grad_norm": 0.716325414459115, + "language_loss": 0.51807982, + "learning_rate": 3.874531654651444e-06, + "loss": 0.53840321, + "num_input_tokens_seen": 137659970, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.01257324, + "step": 4823, + "time_per_iteration": 2.919110059738159 + }, + { + "auxiliary_loss_clip": 0.01139598, + "auxiliary_loss_mlp": 0.0104268, + "balance_loss_clip": 1.05603886, + "balance_loss_mlp": 1.02619302, + "epoch": 0.1399802681214091, + "flos": 29490201296640.0, + "grad_norm": 3.6549504316965535, + "language_loss": 0.75351548, + "learning_rate": 3.874466119993959e-06, + "loss": 0.77533823, + "num_input_tokens_seen": 137674305, + "router_z_loss_clip": 0.83544922, + "router_z_loss_mlp": 0.16485596, + "step": 4824, + "time_per_iteration": 2.5845086574554443 + }, + { + "auxiliary_loss_clip": 0.01133754, + "auxiliary_loss_mlp": 0.01041491, + "balance_loss_clip": 1.05353069, + "balance_loss_mlp": 1.02543926, + "epoch": 0.14000928558992515, + "flos": 28507075073280.0, + "grad_norm": 1.9229263237333258, + "language_loss": 0.76230758, + "learning_rate": 3.87440056878043e-06, + "loss": 0.78406, + "num_input_tokens_seen": 137692280, + "router_z_loss_clip": 0.80273438, + "router_z_loss_mlp": 0.16064453, + "step": 4825, + "time_per_iteration": 2.6051838397979736 + }, + { + "auxiliary_loss_clip": 0.01148742, + "auxiliary_loss_mlp": 0.01053301, + "balance_loss_clip": 1.06024075, + "balance_loss_mlp": 1.03530598, + "epoch": 0.14003830305844117, + "flos": 19603635096960.0, + "grad_norm": 1.8269469667906544, + "language_loss": 0.8269183, + "learning_rate": 3.874335001011437e-06, + "loss": 0.8489387, + "num_input_tokens_seen": 137709770, + "router_z_loss_clip": 0.88476562, + "router_z_loss_mlp": 0.18005371, + "step": 4826, + "time_per_iteration": 2.551316261291504 + }, + { + "auxiliary_loss_clip": 0.01031351, + "auxiliary_loss_mlp": 0.010014, + "balance_loss_clip": 1.01097417, + "balance_loss_mlp": 1.00019014, + "epoch": 0.14006732052695722, + "flos": 72949511773440.0, + "grad_norm": 0.6442895268398915, + "language_loss": 0.49326968, + "learning_rate": 3.874269416687559e-06, + "loss": 0.51359719, + "num_input_tokens_seen": 137774005, + "router_z_loss_clip": 0.203125, + "router_z_loss_mlp": 0.01208496, + "step": 4827, + "time_per_iteration": 3.1713433265686035 + }, + { + "auxiliary_loss_clip": 0.01134772, + "auxiliary_loss_mlp": 0.0104575, + "balance_loss_clip": 1.05162454, + "balance_loss_mlp": 1.03074765, + "epoch": 0.14009633799547327, + "flos": 42369490903680.0, + "grad_norm": 2.2269204278389756, + "language_loss": 0.75204289, + "learning_rate": 3.874203815809375e-06, + "loss": 0.77384818, + "num_input_tokens_seen": 137792685, + "router_z_loss_clip": 0.83203125, + "router_z_loss_mlp": 0.15002441, + "step": 4828, + "time_per_iteration": 2.7547075748443604 + }, + { + "auxiliary_loss_clip": 0.01135176, + "auxiliary_loss_mlp": 0.01044105, + "balance_loss_clip": 1.05186403, + "balance_loss_mlp": 1.02816629, + "epoch": 0.14012535546398933, + "flos": 34964351965440.0, + "grad_norm": 2.4158751080158645, + "language_loss": 0.80116093, + "learning_rate": 3.874138198377465e-06, + "loss": 0.82295376, + "num_input_tokens_seen": 137808845, + "router_z_loss_clip": 0.83300781, + "router_z_loss_mlp": 0.15942383, + "step": 4829, + "time_per_iteration": 2.6468420028686523 + }, + { + "auxiliary_loss_clip": 0.01136, + "auxiliary_loss_mlp": 0.0104432, + "balance_loss_clip": 1.05273676, + "balance_loss_mlp": 1.02763081, + "epoch": 0.14015437293250538, + "flos": 30406821488640.0, + "grad_norm": 2.685495957455549, + "language_loss": 0.73376399, + "learning_rate": 3.874072564392407e-06, + "loss": 0.75556713, + "num_input_tokens_seen": 137828915, + "router_z_loss_clip": 0.83251953, + "router_z_loss_mlp": 0.16687012, + "step": 4830, + "time_per_iteration": 2.578169107437134 + }, + { + "auxiliary_loss_clip": 0.0113031, + "auxiliary_loss_mlp": 0.01044232, + "balance_loss_clip": 1.05255914, + "balance_loss_mlp": 1.02885985, + "epoch": 0.1401833904010214, + "flos": 17780270952960.0, + "grad_norm": 2.269293009441252, + "language_loss": 0.85696775, + "learning_rate": 3.874006913854782e-06, + "loss": 0.87871313, + "num_input_tokens_seen": 137843705, + "router_z_loss_clip": 0.77832031, + "router_z_loss_mlp": 0.15356445, + "step": 4831, + "time_per_iteration": 2.5409467220306396 + }, + { + "auxiliary_loss_clip": 0.01142943, + "auxiliary_loss_mlp": 0.01038336, + "balance_loss_clip": 1.05732369, + "balance_loss_mlp": 1.02099109, + "epoch": 0.14021240786953745, + "flos": 74732329463040.0, + "grad_norm": 1.9595995950917076, + "language_loss": 0.89328754, + "learning_rate": 3.87394124676517e-06, + "loss": 0.91510028, + "num_input_tokens_seen": 137868145, + "router_z_loss_clip": 0.85546875, + "router_z_loss_mlp": 0.17346191, + "step": 4832, + "time_per_iteration": 2.976545572280884 + }, + { + "auxiliary_loss_clip": 0.0103287, + "auxiliary_loss_mlp": 0.01007561, + "balance_loss_clip": 1.01238048, + "balance_loss_mlp": 1.00638056, + "epoch": 0.1402414253380535, + "flos": 65108201984640.0, + "grad_norm": 0.6413347345027086, + "language_loss": 0.47254458, + "learning_rate": 3.87387556312415e-06, + "loss": 0.49294892, + "num_input_tokens_seen": 137931400, + "router_z_loss_clip": 0.20507812, + "router_z_loss_mlp": 0.01177979, + "step": 4833, + "time_per_iteration": 3.109222650527954 + }, + { + "auxiliary_loss_clip": 0.01124594, + "auxiliary_loss_mlp": 0.01038858, + "balance_loss_clip": 1.05157292, + "balance_loss_mlp": 1.02365887, + "epoch": 0.14027044280656956, + "flos": 17271452845440.0, + "grad_norm": 3.0732103747675144, + "language_loss": 0.8277303, + "learning_rate": 3.873809862932303e-06, + "loss": 0.84936476, + "num_input_tokens_seen": 137946265, + "router_z_loss_clip": 0.72998047, + "router_z_loss_mlp": 0.15197754, + "step": 4834, + "time_per_iteration": 2.496126413345337 + }, + { + "auxiliary_loss_clip": 0.01135641, + "auxiliary_loss_mlp": 0.01046365, + "balance_loss_clip": 1.05516648, + "balance_loss_mlp": 1.03029573, + "epoch": 0.1402994602750856, + "flos": 26790437214720.0, + "grad_norm": 2.243883043735353, + "language_loss": 0.74622148, + "learning_rate": 3.873744146190209e-06, + "loss": 0.76804155, + "num_input_tokens_seen": 137961195, + "router_z_loss_clip": 0.8046875, + "router_z_loss_mlp": 0.16046143, + "step": 4835, + "time_per_iteration": 2.586590051651001 + }, + { + "auxiliary_loss_clip": 0.01148913, + "auxiliary_loss_mlp": 0.01055114, + "balance_loss_clip": 1.05962241, + "balance_loss_mlp": 1.03753066, + "epoch": 0.14032847774360166, + "flos": 25111326090240.0, + "grad_norm": 2.703242320845928, + "language_loss": 1.05823672, + "learning_rate": 3.8736784128984494e-06, + "loss": 1.08027697, + "num_input_tokens_seen": 137977000, + "router_z_loss_clip": 0.89257812, + "router_z_loss_mlp": 0.17578125, + "step": 4836, + "time_per_iteration": 5.042152643203735 + }, + { + "auxiliary_loss_clip": 0.01143429, + "auxiliary_loss_mlp": 0.01046049, + "balance_loss_clip": 1.05544055, + "balance_loss_mlp": 1.02836418, + "epoch": 0.14035749521211768, + "flos": 40910368625280.0, + "grad_norm": 2.89715942223603, + "language_loss": 0.92177171, + "learning_rate": 3.873612663057603e-06, + "loss": 0.94366652, + "num_input_tokens_seen": 137994130, + "router_z_loss_clip": 0.88085938, + "router_z_loss_mlp": 0.17687988, + "step": 4837, + "time_per_iteration": 7.388551712036133 + }, + { + "auxiliary_loss_clip": 0.01036658, + "auxiliary_loss_mlp": 0.0099999, + "balance_loss_clip": 1.01623392, + "balance_loss_mlp": 0.99895936, + "epoch": 0.14038651268063373, + "flos": 68508547908480.0, + "grad_norm": 0.6512233079052084, + "language_loss": 0.51201242, + "learning_rate": 3.8735468966682515e-06, + "loss": 0.53237885, + "num_input_tokens_seen": 138060150, + "router_z_loss_clip": 0.20507812, + "router_z_loss_mlp": 0.01031494, + "step": 4838, + "time_per_iteration": 3.147433042526245 + }, + { + "auxiliary_loss_clip": 0.01147708, + "auxiliary_loss_mlp": 0.01046591, + "balance_loss_clip": 1.05881798, + "balance_loss_mlp": 1.0287329, + "epoch": 0.1404155301491498, + "flos": 9678967545600.0, + "grad_norm": 4.002254946016148, + "language_loss": 0.93191582, + "learning_rate": 3.873481113730976e-06, + "loss": 0.95385885, + "num_input_tokens_seen": 138071850, + "router_z_loss_clip": 0.88867188, + "router_z_loss_mlp": 0.17858887, + "step": 4839, + "time_per_iteration": 2.5056519508361816 + }, + { + "auxiliary_loss_clip": 0.01137057, + "auxiliary_loss_mlp": 0.0104832, + "balance_loss_clip": 1.05646765, + "balance_loss_mlp": 1.0307132, + "epoch": 0.14044454761766584, + "flos": 48425071023360.0, + "grad_norm": 2.5701964556637846, + "language_loss": 0.8663727, + "learning_rate": 3.8734153142463565e-06, + "loss": 0.88822651, + "num_input_tokens_seen": 138090460, + "router_z_loss_clip": 0.80615234, + "router_z_loss_mlp": 0.17614746, + "step": 4840, + "time_per_iteration": 5.199331760406494 + }, + { + "auxiliary_loss_clip": 0.01036143, + "auxiliary_loss_mlp": 0.01001951, + "balance_loss_clip": 1.01572394, + "balance_loss_mlp": 1.00078261, + "epoch": 0.1404735650861819, + "flos": 56235357417600.0, + "grad_norm": 0.6459942220094979, + "language_loss": 0.43637639, + "learning_rate": 3.873349498214975e-06, + "loss": 0.45675731, + "num_input_tokens_seen": 138149905, + "router_z_loss_clip": 0.20410156, + "router_z_loss_mlp": 0.01165771, + "step": 4841, + "time_per_iteration": 3.019399881362915 + }, + { + "auxiliary_loss_clip": 0.01132671, + "auxiliary_loss_mlp": 0.01048944, + "balance_loss_clip": 1.05638552, + "balance_loss_mlp": 1.03302956, + "epoch": 0.14050258255469794, + "flos": 31280096943360.0, + "grad_norm": 1.8702947135054695, + "language_loss": 0.63887823, + "learning_rate": 3.873283665637414e-06, + "loss": 0.66069436, + "num_input_tokens_seen": 138166930, + "router_z_loss_clip": 0.76269531, + "router_z_loss_mlp": 0.15899658, + "step": 4842, + "time_per_iteration": 2.637796640396118 + }, + { + "auxiliary_loss_clip": 0.01134373, + "auxiliary_loss_mlp": 0.01036849, + "balance_loss_clip": 1.05495572, + "balance_loss_mlp": 1.02098179, + "epoch": 0.14053160002321397, + "flos": 17340257347200.0, + "grad_norm": 3.7138305045290005, + "language_loss": 0.6894477, + "learning_rate": 3.873217816514251e-06, + "loss": 0.71115994, + "num_input_tokens_seen": 138179785, + "router_z_loss_clip": 0.79443359, + "router_z_loss_mlp": 0.15856934, + "step": 4843, + "time_per_iteration": 2.5067203044891357 + }, + { + "auxiliary_loss_clip": 0.01144129, + "auxiliary_loss_mlp": 0.01053784, + "balance_loss_clip": 1.05797088, + "balance_loss_mlp": 1.03318477, + "epoch": 0.14056061749173002, + "flos": 16209861361920.0, + "grad_norm": 2.7774940172829305, + "language_loss": 0.77174938, + "learning_rate": 3.873151950846072e-06, + "loss": 0.79372853, + "num_input_tokens_seen": 138194625, + "router_z_loss_clip": 0.86181641, + "router_z_loss_mlp": 0.20605469, + "step": 4844, + "time_per_iteration": 2.682889461517334 + }, + { + "auxiliary_loss_clip": 0.01139004, + "auxiliary_loss_mlp": 0.01042889, + "balance_loss_clip": 1.05921519, + "balance_loss_mlp": 1.02641416, + "epoch": 0.14058963496024607, + "flos": 17922405070080.0, + "grad_norm": 2.7191051354007776, + "language_loss": 0.81595314, + "learning_rate": 3.873086068633457e-06, + "loss": 0.83777201, + "num_input_tokens_seen": 138205845, + "router_z_loss_clip": 0.79785156, + "router_z_loss_mlp": 0.16485596, + "step": 4845, + "time_per_iteration": 2.5941154956817627 + }, + { + "auxiliary_loss_clip": 0.01134339, + "auxiliary_loss_mlp": 0.01046658, + "balance_loss_clip": 1.05672717, + "balance_loss_mlp": 1.02968216, + "epoch": 0.14061865242876212, + "flos": 30914167138560.0, + "grad_norm": 2.3486746568539005, + "language_loss": 0.74634957, + "learning_rate": 3.873020169876988e-06, + "loss": 0.76815957, + "num_input_tokens_seen": 138222690, + "router_z_loss_clip": 0.77636719, + "router_z_loss_mlp": 0.16973877, + "step": 4846, + "time_per_iteration": 2.6411900520324707 + }, + { + "auxiliary_loss_clip": 0.01137361, + "auxiliary_loss_mlp": 0.01038296, + "balance_loss_clip": 1.06032813, + "balance_loss_mlp": 1.02402616, + "epoch": 0.14064766989727817, + "flos": 18580432273920.0, + "grad_norm": 2.8103618018614975, + "language_loss": 0.84413171, + "learning_rate": 3.8729542545772465e-06, + "loss": 0.86588824, + "num_input_tokens_seen": 138234370, + "router_z_loss_clip": 0.76953125, + "router_z_loss_mlp": 0.14282227, + "step": 4847, + "time_per_iteration": 2.5708131790161133 + }, + { + "auxiliary_loss_clip": 0.01141809, + "auxiliary_loss_mlp": 0.01048897, + "balance_loss_clip": 1.05581856, + "balance_loss_mlp": 1.02937078, + "epoch": 0.1406766873657942, + "flos": 16467376942080.0, + "grad_norm": 2.5007260008100958, + "language_loss": 0.86883616, + "learning_rate": 3.872888322734815e-06, + "loss": 0.89074332, + "num_input_tokens_seen": 138251555, + "router_z_loss_clip": 0.859375, + "router_z_loss_mlp": 0.1953125, + "step": 4848, + "time_per_iteration": 2.5163450241088867 + }, + { + "auxiliary_loss_clip": 0.01151345, + "auxiliary_loss_mlp": 0.01041181, + "balance_loss_clip": 1.06375384, + "balance_loss_mlp": 1.02503991, + "epoch": 0.14070570483431025, + "flos": 30293415273600.0, + "grad_norm": 2.70646029323703, + "language_loss": 0.76879764, + "learning_rate": 3.8728223743502766e-06, + "loss": 0.79072285, + "num_input_tokens_seen": 138268145, + "router_z_loss_clip": 0.87646484, + "router_z_loss_mlp": 0.16131592, + "step": 4849, + "time_per_iteration": 2.6141226291656494 + }, + { + "auxiliary_loss_clip": 0.01148045, + "auxiliary_loss_mlp": 0.01041729, + "balance_loss_clip": 1.06324553, + "balance_loss_mlp": 1.02576065, + "epoch": 0.1407347223028263, + "flos": 13656253720320.0, + "grad_norm": 3.9513162173835052, + "language_loss": 0.67228019, + "learning_rate": 3.872756409424212e-06, + "loss": 0.69417799, + "num_input_tokens_seen": 138280320, + "router_z_loss_clip": 0.84814453, + "router_z_loss_mlp": 0.15960693, + "step": 4850, + "time_per_iteration": 2.5127506256103516 + }, + { + "auxiliary_loss_clip": 0.01146361, + "auxiliary_loss_mlp": 0.01037749, + "balance_loss_clip": 1.06236577, + "balance_loss_mlp": 1.02215672, + "epoch": 0.14076373977134235, + "flos": 21537209145600.0, + "grad_norm": 2.7267507379921296, + "language_loss": 0.90288168, + "learning_rate": 3.872690427957206e-06, + "loss": 0.92472279, + "num_input_tokens_seen": 138295305, + "router_z_loss_clip": 0.83935547, + "router_z_loss_mlp": 0.15612793, + "step": 4851, + "time_per_iteration": 2.5952212810516357 + }, + { + "auxiliary_loss_clip": 0.01046594, + "auxiliary_loss_mlp": 0.01000905, + "balance_loss_clip": 1.02620387, + "balance_loss_mlp": 0.99980795, + "epoch": 0.1407927572398584, + "flos": 58611459024000.0, + "grad_norm": 0.6733007860081177, + "language_loss": 0.45657787, + "learning_rate": 3.8726244299498394e-06, + "loss": 0.47705287, + "num_input_tokens_seen": 138357115, + "router_z_loss_clip": 0.20410156, + "router_z_loss_mlp": 0.01098633, + "step": 4852, + "time_per_iteration": 3.1565682888031006 + }, + { + "auxiliary_loss_clip": 0.0113725, + "auxiliary_loss_mlp": 0.0104437, + "balance_loss_clip": 1.05855322, + "balance_loss_mlp": 1.02918863, + "epoch": 0.14082177470837445, + "flos": 22776953109120.0, + "grad_norm": 2.556873875788818, + "language_loss": 0.83251739, + "learning_rate": 3.872558415402697e-06, + "loss": 0.85433352, + "num_input_tokens_seen": 138370255, + "router_z_loss_clip": 0.78710938, + "router_z_loss_mlp": 0.15179443, + "step": 4853, + "time_per_iteration": 2.5768792629241943 + }, + { + "auxiliary_loss_clip": 0.01046906, + "auxiliary_loss_mlp": 0.01002083, + "balance_loss_clip": 1.02648783, + "balance_loss_mlp": 1.00096822, + "epoch": 0.14085079217689048, + "flos": 62838036564480.0, + "grad_norm": 0.6819815055249016, + "language_loss": 0.47247708, + "learning_rate": 3.87249238431636e-06, + "loss": 0.49296695, + "num_input_tokens_seen": 138426110, + "router_z_loss_clip": 0.20507812, + "router_z_loss_mlp": 0.01116943, + "step": 4854, + "time_per_iteration": 2.961224317550659 + }, + { + "auxiliary_loss_clip": 0.01142081, + "auxiliary_loss_mlp": 0.01037941, + "balance_loss_clip": 1.06090021, + "balance_loss_mlp": 1.02151382, + "epoch": 0.14087980964540653, + "flos": 15705496540800.0, + "grad_norm": 15.551520343420757, + "language_loss": 0.6687296, + "learning_rate": 3.872426336691413e-06, + "loss": 0.69052982, + "num_input_tokens_seen": 138441855, + "router_z_loss_clip": 0.8125, + "router_z_loss_mlp": 0.16418457, + "step": 4855, + "time_per_iteration": 2.5609006881713867 + }, + { + "auxiliary_loss_clip": 0.01148726, + "auxiliary_loss_mlp": 0.01045113, + "balance_loss_clip": 1.06200814, + "balance_loss_mlp": 1.02572942, + "epoch": 0.14090882711392258, + "flos": 25075846431360.0, + "grad_norm": 2.578756046789267, + "language_loss": 0.80335188, + "learning_rate": 3.8723602725284396e-06, + "loss": 0.82529026, + "num_input_tokens_seen": 138456145, + "router_z_loss_clip": 0.8671875, + "router_z_loss_mlp": 0.19366455, + "step": 4856, + "time_per_iteration": 2.587242364883423 + }, + { + "auxiliary_loss_clip": 0.01045558, + "auxiliary_loss_mlp": 0.01002516, + "balance_loss_clip": 1.0250361, + "balance_loss_mlp": 1.00139511, + "epoch": 0.14093784458243863, + "flos": 73001978565120.0, + "grad_norm": 0.6341936990030597, + "language_loss": 0.45593882, + "learning_rate": 3.872294191828022e-06, + "loss": 0.47641957, + "num_input_tokens_seen": 138526920, + "router_z_loss_clip": 0.20507812, + "router_z_loss_mlp": 0.01123047, + "step": 4857, + "time_per_iteration": 3.2615764141082764 + }, + { + "auxiliary_loss_clip": 0.01148811, + "auxiliary_loss_mlp": 0.01056404, + "balance_loss_clip": 1.06323862, + "balance_loss_mlp": 1.03866613, + "epoch": 0.14096686205095468, + "flos": 43099411178880.0, + "grad_norm": 1.967466509250964, + "language_loss": 0.73360419, + "learning_rate": 3.872228094590745e-06, + "loss": 0.75565636, + "num_input_tokens_seen": 138547950, + "router_z_loss_clip": 0.85449219, + "router_z_loss_mlp": 0.17718506, + "step": 4858, + "time_per_iteration": 2.8257577419281006 + }, + { + "auxiliary_loss_clip": 0.01141335, + "auxiliary_loss_mlp": 0.01039249, + "balance_loss_clip": 1.06379437, + "balance_loss_mlp": 1.02263665, + "epoch": 0.14099587951947073, + "flos": 34393696594560.0, + "grad_norm": 1.5908966098294015, + "language_loss": 0.77073371, + "learning_rate": 3.872161980817191e-06, + "loss": 0.7925396, + "num_input_tokens_seen": 138568180, + "router_z_loss_clip": 0.77587891, + "router_z_loss_mlp": 0.16619873, + "step": 4859, + "time_per_iteration": 2.777057647705078 + }, + { + "auxiliary_loss_clip": 0.0114311, + "auxiliary_loss_mlp": 0.01043684, + "balance_loss_clip": 1.06293964, + "balance_loss_mlp": 1.02756119, + "epoch": 0.14102489698798676, + "flos": 36687059222400.0, + "grad_norm": 2.555630823612608, + "language_loss": 0.81714511, + "learning_rate": 3.872095850507945e-06, + "loss": 0.8390131, + "num_input_tokens_seen": 138584465, + "router_z_loss_clip": 0.80224609, + "router_z_loss_mlp": 0.16113281, + "step": 4860, + "time_per_iteration": 2.7339999675750732 + }, + { + "auxiliary_loss_clip": 0.01142199, + "auxiliary_loss_mlp": 0.01043891, + "balance_loss_clip": 1.06247258, + "balance_loss_mlp": 1.02743983, + "epoch": 0.1410539144565028, + "flos": 31242713863680.0, + "grad_norm": 2.2861108119954885, + "language_loss": 0.78664827, + "learning_rate": 3.87202970366359e-06, + "loss": 0.80850911, + "num_input_tokens_seen": 138600570, + "router_z_loss_clip": 0.79785156, + "router_z_loss_mlp": 0.16461182, + "step": 4861, + "time_per_iteration": 2.6712629795074463 + }, + { + "auxiliary_loss_clip": 0.01142971, + "auxiliary_loss_mlp": 0.01043944, + "balance_loss_clip": 1.06180716, + "balance_loss_mlp": 1.02801776, + "epoch": 0.14108293192501886, + "flos": 27813963260160.0, + "grad_norm": 1.8369543677019429, + "language_loss": 0.80347389, + "learning_rate": 3.871963540284713e-06, + "loss": 0.82534301, + "num_input_tokens_seen": 138618145, + "router_z_loss_clip": 0.81201172, + "router_z_loss_mlp": 0.15924072, + "step": 4862, + "time_per_iteration": 2.6695103645324707 + }, + { + "auxiliary_loss_clip": 0.01152599, + "auxiliary_loss_mlp": 0.01049101, + "balance_loss_clip": 1.06378055, + "balance_loss_mlp": 1.03166628, + "epoch": 0.1411119493935349, + "flos": 41685285663360.0, + "grad_norm": 2.3796818848747465, + "language_loss": 0.90535736, + "learning_rate": 3.871897360371896e-06, + "loss": 0.92737436, + "num_input_tokens_seen": 138642405, + "router_z_loss_clip": 0.88818359, + "router_z_loss_mlp": 0.17425537, + "step": 4863, + "time_per_iteration": 2.6889212131500244 + }, + { + "auxiliary_loss_clip": 0.01044251, + "auxiliary_loss_mlp": 0.01010766, + "balance_loss_clip": 1.02350461, + "balance_loss_mlp": 1.00964534, + "epoch": 0.14114096686205096, + "flos": 59080380099840.0, + "grad_norm": 0.659051010823423, + "language_loss": 0.50772107, + "learning_rate": 3.871831163925724e-06, + "loss": 0.5282712, + "num_input_tokens_seen": 138702555, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.01123047, + "step": 4864, + "time_per_iteration": 3.0518760681152344 + }, + { + "auxiliary_loss_clip": 0.0113984, + "auxiliary_loss_mlp": 0.0103918, + "balance_loss_clip": 1.05910182, + "balance_loss_mlp": 1.02418983, + "epoch": 0.141169984330567, + "flos": 30183025800960.0, + "grad_norm": 1.9601211144578292, + "language_loss": 0.79384083, + "learning_rate": 3.8717649509467804e-06, + "loss": 0.81563103, + "num_input_tokens_seen": 138719480, + "router_z_loss_clip": 0.80712891, + "router_z_loss_mlp": 0.14990234, + "step": 4865, + "time_per_iteration": 2.624220848083496 + }, + { + "auxiliary_loss_clip": 0.01139086, + "auxiliary_loss_mlp": 0.01042824, + "balance_loss_clip": 1.0591464, + "balance_loss_mlp": 1.0273087, + "epoch": 0.14119900179908304, + "flos": 15443132624640.0, + "grad_norm": 2.3975728639012384, + "language_loss": 0.75471532, + "learning_rate": 3.871698721435652e-06, + "loss": 0.77653438, + "num_input_tokens_seen": 138731635, + "router_z_loss_clip": 0.79882812, + "router_z_loss_mlp": 0.15509033, + "step": 4866, + "time_per_iteration": 2.4848239421844482 + }, + { + "auxiliary_loss_clip": 0.01134667, + "auxiliary_loss_mlp": 0.01040275, + "balance_loss_clip": 1.05702925, + "balance_loss_mlp": 1.02626765, + "epoch": 0.1412280192675991, + "flos": 29123589133440.0, + "grad_norm": 2.6745251658230154, + "language_loss": 0.67010736, + "learning_rate": 3.871632475392924e-06, + "loss": 0.69185686, + "num_input_tokens_seen": 138745750, + "router_z_loss_clip": 0.77587891, + "router_z_loss_mlp": 0.14007568, + "step": 4867, + "time_per_iteration": 2.574270486831665 + }, + { + "auxiliary_loss_clip": 0.01142076, + "auxiliary_loss_mlp": 0.01048309, + "balance_loss_clip": 1.06117272, + "balance_loss_mlp": 1.02992058, + "epoch": 0.14125703673611514, + "flos": 38721828862080.0, + "grad_norm": 2.255822857795721, + "language_loss": 0.84493887, + "learning_rate": 3.87156621281918e-06, + "loss": 0.86684275, + "num_input_tokens_seen": 138761630, + "router_z_loss_clip": 0.80908203, + "router_z_loss_mlp": 0.18395996, + "step": 4868, + "time_per_iteration": 2.6667580604553223 + }, + { + "auxiliary_loss_clip": 0.01141687, + "auxiliary_loss_mlp": 0.01046719, + "balance_loss_clip": 1.06323195, + "balance_loss_mlp": 1.03085816, + "epoch": 0.1412860542046312, + "flos": 16610229331200.0, + "grad_norm": 2.36009107727824, + "language_loss": 0.68669313, + "learning_rate": 3.871499933715006e-06, + "loss": 0.70857722, + "num_input_tokens_seen": 138774850, + "router_z_loss_clip": 0.78417969, + "router_z_loss_mlp": 0.15863037, + "step": 4869, + "time_per_iteration": 2.4975266456604004 + }, + { + "auxiliary_loss_clip": 0.01135625, + "auxiliary_loss_mlp": 0.01040121, + "balance_loss_clip": 1.0557735, + "balance_loss_mlp": 1.02443361, + "epoch": 0.14131507167314725, + "flos": 29976504554880.0, + "grad_norm": 1.5002828486854176, + "language_loss": 0.765948, + "learning_rate": 3.8714336380809875e-06, + "loss": 0.78770548, + "num_input_tokens_seen": 138798380, + "router_z_loss_clip": 0.79882812, + "router_z_loss_mlp": 0.15686035, + "step": 4870, + "time_per_iteration": 2.623304843902588 + }, + { + "auxiliary_loss_clip": 0.01132673, + "auxiliary_loss_mlp": 0.01038434, + "balance_loss_clip": 1.05827689, + "balance_loss_mlp": 1.02479017, + "epoch": 0.14134408914166327, + "flos": 71631910103040.0, + "grad_norm": 1.9274918517725137, + "language_loss": 0.86165571, + "learning_rate": 3.871367325917709e-06, + "loss": 0.88336682, + "num_input_tokens_seen": 138824920, + "router_z_loss_clip": 0.74414062, + "router_z_loss_mlp": 0.13647461, + "step": 4871, + "time_per_iteration": 2.989299774169922 + }, + { + "auxiliary_loss_clip": 0.01137251, + "auxiliary_loss_mlp": 0.01036637, + "balance_loss_clip": 1.0553062, + "balance_loss_mlp": 1.02101421, + "epoch": 0.14137310661017932, + "flos": 45252686764800.0, + "grad_norm": 1.9858941024520078, + "language_loss": 0.76414287, + "learning_rate": 3.871300997225758e-06, + "loss": 0.78588176, + "num_input_tokens_seen": 138843630, + "router_z_loss_clip": 0.81933594, + "router_z_loss_mlp": 0.15606689, + "step": 4872, + "time_per_iteration": 2.650395393371582 + }, + { + "auxiliary_loss_clip": 0.01042976, + "auxiliary_loss_mlp": 0.0099962, + "balance_loss_clip": 1.02258015, + "balance_loss_mlp": 0.99853534, + "epoch": 0.14140212407869537, + "flos": 74778298871040.0, + "grad_norm": 0.7702154410047721, + "language_loss": 0.53368306, + "learning_rate": 3.8712346520057185e-06, + "loss": 0.55410898, + "num_input_tokens_seen": 138909345, + "router_z_loss_clip": 0.20410156, + "router_z_loss_mlp": 0.01086426, + "step": 4873, + "time_per_iteration": 3.193732500076294 + }, + { + "auxiliary_loss_clip": 0.01042825, + "auxiliary_loss_mlp": 0.01001381, + "balance_loss_clip": 1.02233732, + "balance_loss_mlp": 1.00019455, + "epoch": 0.14143114154721143, + "flos": 74778047475840.0, + "grad_norm": 0.6832432965605508, + "language_loss": 0.5012306, + "learning_rate": 3.871168290258178e-06, + "loss": 0.52167273, + "num_input_tokens_seen": 138970240, + "router_z_loss_clip": 0.20507812, + "router_z_loss_mlp": 0.01184082, + "step": 4874, + "time_per_iteration": 3.160438060760498 + }, + { + "auxiliary_loss_clip": 0.01140039, + "auxiliary_loss_mlp": 0.01039913, + "balance_loss_clip": 1.05786407, + "balance_loss_mlp": 1.02135789, + "epoch": 0.14146015901572748, + "flos": 34526529089280.0, + "grad_norm": 2.3831086234315544, + "language_loss": 0.79813272, + "learning_rate": 3.871101911983722e-06, + "loss": 0.81993222, + "num_input_tokens_seen": 138986040, + "router_z_loss_clip": 0.82128906, + "router_z_loss_mlp": 0.18536377, + "step": 4875, + "time_per_iteration": 2.6445155143737793 + }, + { + "auxiliary_loss_clip": 0.01142524, + "auxiliary_loss_mlp": 0.01043544, + "balance_loss_clip": 1.05952275, + "balance_loss_mlp": 1.02540028, + "epoch": 0.14148917648424353, + "flos": 15988866935040.0, + "grad_norm": 2.7812867269276196, + "language_loss": 0.88548434, + "learning_rate": 3.871035517182936e-06, + "loss": 0.907345, + "num_input_tokens_seen": 138998735, + "router_z_loss_clip": 0.82910156, + "router_z_loss_mlp": 0.18145752, + "step": 4876, + "time_per_iteration": 2.563889741897583 + }, + { + "auxiliary_loss_clip": 0.01136845, + "auxiliary_loss_mlp": 0.01044738, + "balance_loss_clip": 1.05701232, + "balance_loss_mlp": 1.028162, + "epoch": 0.14151819395275955, + "flos": 32080617400320.0, + "grad_norm": 2.2353423876804652, + "language_loss": 0.75808829, + "learning_rate": 3.870969105856408e-06, + "loss": 0.77990413, + "num_input_tokens_seen": 139014785, + "router_z_loss_clip": 0.79980469, + "router_z_loss_mlp": 0.16577148, + "step": 4877, + "time_per_iteration": 2.6775615215301514 + }, + { + "auxiliary_loss_clip": 0.01133698, + "auxiliary_loss_mlp": 0.01040358, + "balance_loss_clip": 1.05787349, + "balance_loss_mlp": 1.02579689, + "epoch": 0.1415472114212756, + "flos": 19128321400320.0, + "grad_norm": 2.380773674628119, + "language_loss": 0.77762568, + "learning_rate": 3.8709026780047225e-06, + "loss": 0.79936618, + "num_input_tokens_seen": 139028520, + "router_z_loss_clip": 0.75732422, + "router_z_loss_mlp": 0.14562988, + "step": 4878, + "time_per_iteration": 2.515897750854492 + }, + { + "auxiliary_loss_clip": 0.0104033, + "auxiliary_loss_mlp": 0.01006068, + "balance_loss_clip": 1.02014017, + "balance_loss_mlp": 1.00497723, + "epoch": 0.14157622888979166, + "flos": 62702295068160.0, + "grad_norm": 0.6801887998294847, + "language_loss": 0.54005414, + "learning_rate": 3.870836233628469e-06, + "loss": 0.56051809, + "num_input_tokens_seen": 139093780, + "router_z_loss_clip": 0.20214844, + "router_z_loss_mlp": 0.01092529, + "step": 4879, + "time_per_iteration": 3.1593995094299316 + }, + { + "auxiliary_loss_clip": 0.01134713, + "auxiliary_loss_mlp": 0.0104707, + "balance_loss_clip": 1.05813575, + "balance_loss_mlp": 1.03299713, + "epoch": 0.1416052463583077, + "flos": 18661734708480.0, + "grad_norm": 2.7945101354134922, + "language_loss": 0.89697278, + "learning_rate": 3.870769772728232e-06, + "loss": 0.91879058, + "num_input_tokens_seen": 139107700, + "router_z_loss_clip": 0.76611328, + "router_z_loss_mlp": 0.14074707, + "step": 4880, + "time_per_iteration": 2.5962445735931396 + }, + { + "auxiliary_loss_clip": 0.01040249, + "auxiliary_loss_mlp": 0.01008887, + "balance_loss_clip": 1.02005506, + "balance_loss_mlp": 1.00786233, + "epoch": 0.14163426382682376, + "flos": 64371494039040.0, + "grad_norm": 0.7162921461123597, + "language_loss": 0.52961195, + "learning_rate": 3.8707032953046e-06, + "loss": 0.55010331, + "num_input_tokens_seen": 139169030, + "router_z_loss_clip": 0.20214844, + "router_z_loss_mlp": 0.01025391, + "step": 4881, + "time_per_iteration": 3.0596299171447754 + }, + { + "auxiliary_loss_clip": 0.01123674, + "auxiliary_loss_mlp": 0.01043445, + "balance_loss_clip": 1.05277491, + "balance_loss_mlp": 1.03009343, + "epoch": 0.14166328129533978, + "flos": 32373792207360.0, + "grad_norm": 1.9797551924044114, + "language_loss": 0.88998407, + "learning_rate": 3.870636801358158e-06, + "loss": 0.91165531, + "num_input_tokens_seen": 139188470, + "router_z_loss_clip": 0.70947266, + "router_z_loss_mlp": 0.13354492, + "step": 4882, + "time_per_iteration": 2.6604249477386475 + }, + { + "auxiliary_loss_clip": 0.01040639, + "auxiliary_loss_mlp": 0.01005812, + "balance_loss_clip": 1.02029037, + "balance_loss_mlp": 1.00469732, + "epoch": 0.14169229876385583, + "flos": 66053226856320.0, + "grad_norm": 0.6627048444875465, + "language_loss": 0.45569983, + "learning_rate": 3.870570290889496e-06, + "loss": 0.47616431, + "num_input_tokens_seen": 139246410, + "router_z_loss_clip": 0.203125, + "router_z_loss_mlp": 0.01116943, + "step": 4883, + "time_per_iteration": 3.0200178623199463 + }, + { + "auxiliary_loss_clip": 0.0114198, + "auxiliary_loss_mlp": 0.01034033, + "balance_loss_clip": 1.0571568, + "balance_loss_mlp": 1.0173198, + "epoch": 0.14172131623237189, + "flos": 10594582156800.0, + "grad_norm": 3.0282041003462172, + "language_loss": 0.94374013, + "learning_rate": 3.870503763899201e-06, + "loss": 0.96550024, + "num_input_tokens_seen": 139258725, + "router_z_loss_clip": 0.84814453, + "router_z_loss_mlp": 0.16711426, + "step": 4884, + "time_per_iteration": 2.54642915725708 + }, + { + "auxiliary_loss_clip": 0.01135163, + "auxiliary_loss_mlp": 0.01047378, + "balance_loss_clip": 1.05391014, + "balance_loss_mlp": 1.03046238, + "epoch": 0.14175033370088794, + "flos": 22851216478080.0, + "grad_norm": 2.3006250210383863, + "language_loss": 0.77065599, + "learning_rate": 3.870437220387858e-06, + "loss": 0.79248136, + "num_input_tokens_seen": 139275895, + "router_z_loss_clip": 0.81201172, + "router_z_loss_mlp": 0.16912842, + "step": 4885, + "time_per_iteration": 2.5467286109924316 + }, + { + "auxiliary_loss_clip": 0.01138867, + "auxiliary_loss_mlp": 0.01048972, + "balance_loss_clip": 1.05556417, + "balance_loss_mlp": 1.03143644, + "epoch": 0.141779351169404, + "flos": 18950707624320.0, + "grad_norm": 2.9619796213734935, + "language_loss": 0.8451317, + "learning_rate": 3.870370660356058e-06, + "loss": 0.86701012, + "num_input_tokens_seen": 139295140, + "router_z_loss_clip": 0.83251953, + "router_z_loss_mlp": 0.17541504, + "step": 4886, + "time_per_iteration": 2.7101471424102783 + }, + { + "auxiliary_loss_clip": 0.01145971, + "auxiliary_loss_mlp": 0.01054628, + "balance_loss_clip": 1.05640364, + "balance_loss_mlp": 1.03523231, + "epoch": 0.14180836863792004, + "flos": 33246959921280.0, + "grad_norm": 2.315596481013439, + "language_loss": 0.75386995, + "learning_rate": 3.870304083804387e-06, + "loss": 0.77587593, + "num_input_tokens_seen": 139313020, + "router_z_loss_clip": 0.89550781, + "router_z_loss_mlp": 0.19378662, + "step": 4887, + "time_per_iteration": 2.6045734882354736 + }, + { + "auxiliary_loss_clip": 0.01132263, + "auxiliary_loss_mlp": 0.01041204, + "balance_loss_clip": 1.05274248, + "balance_loss_mlp": 1.0256238, + "epoch": 0.14183738610643606, + "flos": 15193733518080.0, + "grad_norm": 2.350235696698148, + "language_loss": 0.77248675, + "learning_rate": 3.870237490733433e-06, + "loss": 0.79422146, + "num_input_tokens_seen": 139326510, + "router_z_loss_clip": 0.79589844, + "router_z_loss_mlp": 0.15588379, + "step": 4888, + "time_per_iteration": 2.470623254776001 + }, + { + "auxiliary_loss_clip": 0.0113405, + "auxiliary_loss_mlp": 0.01041249, + "balance_loss_clip": 1.05469501, + "balance_loss_mlp": 1.02493548, + "epoch": 0.14186640357495212, + "flos": 24894066677760.0, + "grad_norm": 1.8275331204462368, + "language_loss": 0.72385669, + "learning_rate": 3.870170881143785e-06, + "loss": 0.74560964, + "num_input_tokens_seen": 139342750, + "router_z_loss_clip": 0.79345703, + "router_z_loss_mlp": 0.16320801, + "step": 4889, + "time_per_iteration": 2.5167481899261475 + }, + { + "auxiliary_loss_clip": 0.01035613, + "auxiliary_loss_mlp": 0.01002898, + "balance_loss_clip": 1.01498866, + "balance_loss_mlp": 1.00168216, + "epoch": 0.14189542104346817, + "flos": 62044231950720.0, + "grad_norm": 0.6682543736452219, + "language_loss": 0.49795434, + "learning_rate": 3.870104255036031e-06, + "loss": 0.51833946, + "num_input_tokens_seen": 139402785, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.012146, + "step": 4890, + "time_per_iteration": 3.0888750553131104 + }, + { + "auxiliary_loss_clip": 0.01036184, + "auxiliary_loss_mlp": 0.01001499, + "balance_loss_clip": 1.0153873, + "balance_loss_mlp": 1.00043201, + "epoch": 0.14192443851198422, + "flos": 74772337213440.0, + "grad_norm": 0.6931629463435032, + "language_loss": 0.5029971, + "learning_rate": 3.87003761241076e-06, + "loss": 0.52337396, + "num_input_tokens_seen": 139462290, + "router_z_loss_clip": 0.20800781, + "router_z_loss_mlp": 0.01068115, + "step": 4891, + "time_per_iteration": 3.116624593734741 + }, + { + "auxiliary_loss_clip": 0.01130632, + "auxiliary_loss_mlp": 0.01034727, + "balance_loss_clip": 1.05176294, + "balance_loss_mlp": 1.01933134, + "epoch": 0.14195345598050027, + "flos": 23616365016960.0, + "grad_norm": 2.178533829773247, + "language_loss": 0.83056003, + "learning_rate": 3.8699709532685605e-06, + "loss": 0.85221356, + "num_input_tokens_seen": 139476790, + "router_z_loss_clip": 0.7890625, + "router_z_loss_mlp": 0.15405273, + "step": 4892, + "time_per_iteration": 2.57806134223938 + }, + { + "auxiliary_loss_clip": 0.01128022, + "auxiliary_loss_mlp": 0.01037438, + "balance_loss_clip": 1.05034411, + "balance_loss_mlp": 1.02128482, + "epoch": 0.1419824734490163, + "flos": 22667497390080.0, + "grad_norm": 2.471861600480913, + "language_loss": 1.01733315, + "learning_rate": 3.869904277610019e-06, + "loss": 1.03898776, + "num_input_tokens_seen": 139491180, + "router_z_loss_clip": 0.77636719, + "router_z_loss_mlp": 0.16149902, + "step": 4893, + "time_per_iteration": 2.5370430946350098 + }, + { + "auxiliary_loss_clip": 0.01133976, + "auxiliary_loss_mlp": 0.01038459, + "balance_loss_clip": 1.05135286, + "balance_loss_mlp": 1.02106595, + "epoch": 0.14201149091753235, + "flos": 17011495140480.0, + "grad_norm": 3.8140927066671932, + "language_loss": 0.83182847, + "learning_rate": 3.869837585435727e-06, + "loss": 0.85355282, + "num_input_tokens_seen": 139504000, + "router_z_loss_clip": 0.82519531, + "router_z_loss_mlp": 0.1739502, + "step": 4894, + "time_per_iteration": 2.4825875759124756 + }, + { + "auxiliary_loss_clip": 0.01038486, + "auxiliary_loss_mlp": 0.0101308, + "balance_loss_clip": 1.01791489, + "balance_loss_mlp": 1.01203096, + "epoch": 0.1420405083860484, + "flos": 74014263653760.0, + "grad_norm": 0.8516348968603117, + "language_loss": 0.52189422, + "learning_rate": 3.869770876746274e-06, + "loss": 0.5424099, + "num_input_tokens_seen": 139569210, + "router_z_loss_clip": 0.20605469, + "router_z_loss_mlp": 0.01049805, + "step": 4895, + "time_per_iteration": 3.1489338874816895 + }, + { + "auxiliary_loss_clip": 0.01138923, + "auxiliary_loss_mlp": 0.01047506, + "balance_loss_clip": 1.05330443, + "balance_loss_mlp": 1.03093028, + "epoch": 0.14206952585456445, + "flos": 24603298081920.0, + "grad_norm": 1.8734606908658977, + "language_loss": 0.73122865, + "learning_rate": 3.869704151542247e-06, + "loss": 0.75309294, + "num_input_tokens_seen": 139586180, + "router_z_loss_clip": 0.85498047, + "router_z_loss_mlp": 0.16583252, + "step": 4896, + "time_per_iteration": 2.5291125774383545 + }, + { + "auxiliary_loss_clip": 0.01139379, + "auxiliary_loss_mlp": 0.01048249, + "balance_loss_clip": 1.05443168, + "balance_loss_mlp": 1.03211963, + "epoch": 0.1420985433230805, + "flos": 24965672440320.0, + "grad_norm": 3.609565590279844, + "language_loss": 0.80591857, + "learning_rate": 3.869637409824237e-06, + "loss": 0.82779491, + "num_input_tokens_seen": 139601060, + "router_z_loss_clip": 0.85009766, + "router_z_loss_mlp": 0.16125488, + "step": 4897, + "time_per_iteration": 2.5432560443878174 + }, + { + "auxiliary_loss_clip": 0.01139936, + "auxiliary_loss_mlp": 0.01052998, + "balance_loss_clip": 1.05490613, + "balance_loss_mlp": 1.03642774, + "epoch": 0.14212756079159655, + "flos": 16610983516800.0, + "grad_norm": 2.2435078026102966, + "language_loss": 0.86036205, + "learning_rate": 3.869570651592831e-06, + "loss": 0.88229144, + "num_input_tokens_seen": 139614830, + "router_z_loss_clip": 0.8515625, + "router_z_loss_mlp": 0.16577148, + "step": 4898, + "time_per_iteration": 2.5295605659484863 + }, + { + "auxiliary_loss_clip": 0.01143016, + "auxiliary_loss_mlp": 0.01055635, + "balance_loss_clip": 1.06037104, + "balance_loss_mlp": 1.03892803, + "epoch": 0.14215657826011258, + "flos": 33650739682560.0, + "grad_norm": 2.7828212580039073, + "language_loss": 0.77384019, + "learning_rate": 3.869503876848623e-06, + "loss": 0.79582667, + "num_input_tokens_seen": 139632640, + "router_z_loss_clip": 0.82568359, + "router_z_loss_mlp": 0.16711426, + "step": 4899, + "time_per_iteration": 2.6775197982788086 + }, + { + "auxiliary_loss_clip": 0.0104097, + "auxiliary_loss_mlp": 0.01018045, + "balance_loss_clip": 1.02020228, + "balance_loss_mlp": 1.01701379, + "epoch": 0.14218559572862863, + "flos": 74772157645440.0, + "grad_norm": 0.644893854743401, + "language_loss": 0.51624131, + "learning_rate": 3.869437085592198e-06, + "loss": 0.5368315, + "num_input_tokens_seen": 139693945, + "router_z_loss_clip": 0.20800781, + "router_z_loss_mlp": 0.01031494, + "step": 4900, + "time_per_iteration": 3.150329113006592 + }, + { + "auxiliary_loss_clip": 0.01039526, + "auxiliary_loss_mlp": 0.01005565, + "balance_loss_clip": 1.01884079, + "balance_loss_mlp": 1.00448596, + "epoch": 0.14221461319714468, + "flos": 70803311166720.0, + "grad_norm": 0.6140382091398525, + "language_loss": 0.45669395, + "learning_rate": 3.86937027782415e-06, + "loss": 0.47714484, + "num_input_tokens_seen": 139764910, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.01080322, + "step": 4901, + "time_per_iteration": 3.2044754028320312 + }, + { + "auxiliary_loss_clip": 0.0112862, + "auxiliary_loss_mlp": 0.01032102, + "balance_loss_clip": 1.05286062, + "balance_loss_mlp": 1.01875055, + "epoch": 0.14224363066566073, + "flos": 28109077401600.0, + "grad_norm": 1.8109477890976924, + "language_loss": 0.81801057, + "learning_rate": 3.869303453545066e-06, + "loss": 0.83961779, + "num_input_tokens_seen": 139782930, + "router_z_loss_clip": 0.75732422, + "router_z_loss_mlp": 0.13360596, + "step": 4902, + "time_per_iteration": 2.6155200004577637 + }, + { + "auxiliary_loss_clip": 0.01133689, + "auxiliary_loss_mlp": 0.01037929, + "balance_loss_clip": 1.05609763, + "balance_loss_mlp": 1.02008367, + "epoch": 0.14227264813417678, + "flos": 16361799891840.0, + "grad_norm": 3.3673159874665703, + "language_loss": 0.86309552, + "learning_rate": 3.869236612755538e-06, + "loss": 0.8848117, + "num_input_tokens_seen": 139795940, + "router_z_loss_clip": 0.77539062, + "router_z_loss_mlp": 0.17858887, + "step": 4903, + "time_per_iteration": 2.5342650413513184 + }, + { + "auxiliary_loss_clip": 0.01130294, + "auxiliary_loss_mlp": 0.01040845, + "balance_loss_clip": 1.05288863, + "balance_loss_mlp": 1.02604496, + "epoch": 0.14230166560269283, + "flos": 25696095505920.0, + "grad_norm": 2.3837837484433257, + "language_loss": 0.75092852, + "learning_rate": 3.869169755456156e-06, + "loss": 0.77263987, + "num_input_tokens_seen": 139810465, + "router_z_loss_clip": 0.77392578, + "router_z_loss_mlp": 0.14794922, + "step": 4904, + "time_per_iteration": 2.608191967010498 + }, + { + "auxiliary_loss_clip": 0.01139039, + "auxiliary_loss_mlp": 0.01035645, + "balance_loss_clip": 1.05642796, + "balance_loss_mlp": 1.01869321, + "epoch": 0.14233068307120886, + "flos": 26720124341760.0, + "grad_norm": 1.5361015975884424, + "language_loss": 0.67477906, + "learning_rate": 3.86910288164751e-06, + "loss": 0.69652587, + "num_input_tokens_seen": 139826505, + "router_z_loss_clip": 0.82568359, + "router_z_loss_mlp": 0.16955566, + "step": 4905, + "time_per_iteration": 2.575138568878174 + }, + { + "auxiliary_loss_clip": 0.01036724, + "auxiliary_loss_mlp": 0.01004708, + "balance_loss_clip": 1.01589036, + "balance_loss_mlp": 1.00340319, + "epoch": 0.1423597005397249, + "flos": 71775304174080.0, + "grad_norm": 0.7236562609966034, + "language_loss": 0.4974595, + "learning_rate": 3.869035991330192e-06, + "loss": 0.51787382, + "num_input_tokens_seen": 139883105, + "router_z_loss_clip": 0.20898438, + "router_z_loss_mlp": 0.01306152, + "step": 4906, + "time_per_iteration": 3.072145700454712 + }, + { + "auxiliary_loss_clip": 0.01034631, + "auxiliary_loss_mlp": 0.0100285, + "balance_loss_clip": 1.01377594, + "balance_loss_mlp": 1.00159216, + "epoch": 0.14238871800824096, + "flos": 74776539104640.0, + "grad_norm": 0.6779300966110842, + "language_loss": 0.49506202, + "learning_rate": 3.86896908450479e-06, + "loss": 0.51543683, + "num_input_tokens_seen": 139949495, + "router_z_loss_clip": 0.20898438, + "router_z_loss_mlp": 0.01257324, + "step": 4907, + "time_per_iteration": 5.505699872970581 + }, + { + "auxiliary_loss_clip": 0.01139612, + "auxiliary_loss_mlp": 0.01045953, + "balance_loss_clip": 1.05622864, + "balance_loss_mlp": 1.02962136, + "epoch": 0.142417735476757, + "flos": 18288801751680.0, + "grad_norm": 3.2726140160287653, + "language_loss": 0.96448469, + "learning_rate": 3.868902161171897e-06, + "loss": 0.98634028, + "num_input_tokens_seen": 139960970, + "router_z_loss_clip": 0.83398438, + "router_z_loss_mlp": 0.16339111, + "step": 4908, + "time_per_iteration": 4.872607707977295 + }, + { + "auxiliary_loss_clip": 0.011326, + "auxiliary_loss_mlp": 0.01036325, + "balance_loss_clip": 1.05385971, + "balance_loss_mlp": 1.021173, + "epoch": 0.14244675294527306, + "flos": 36714566062080.0, + "grad_norm": 2.074973827412667, + "language_loss": 0.85750151, + "learning_rate": 3.868835221332105e-06, + "loss": 0.87919074, + "num_input_tokens_seen": 139980940, + "router_z_loss_clip": 0.78710938, + "router_z_loss_mlp": 0.15142822, + "step": 4909, + "time_per_iteration": 5.049127817153931 + }, + { + "auxiliary_loss_clip": 0.01031711, + "auxiliary_loss_mlp": 0.01000868, + "balance_loss_clip": 1.01122534, + "balance_loss_mlp": 0.99957508, + "epoch": 0.1424757704137891, + "flos": 64005779715840.0, + "grad_norm": 0.6690465378663297, + "language_loss": 0.5082444, + "learning_rate": 3.868768264986004e-06, + "loss": 0.52857018, + "num_input_tokens_seen": 140042780, + "router_z_loss_clip": 0.20507812, + "router_z_loss_mlp": 0.01293945, + "step": 4910, + "time_per_iteration": 3.072807550430298 + }, + { + "auxiliary_loss_clip": 0.0103243, + "auxiliary_loss_mlp": 0.01001745, + "balance_loss_clip": 1.0119369, + "balance_loss_mlp": 1.00046945, + "epoch": 0.14250478788230514, + "flos": 70830638438400.0, + "grad_norm": 0.6180427699908123, + "language_loss": 0.46497697, + "learning_rate": 3.868701292134185e-06, + "loss": 0.48531872, + "num_input_tokens_seen": 140105470, + "router_z_loss_clip": 0.20507812, + "router_z_loss_mlp": 0.01275635, + "step": 4911, + "time_per_iteration": 5.584520578384399 + }, + { + "auxiliary_loss_clip": 0.01139028, + "auxiliary_loss_mlp": 0.01047118, + "balance_loss_clip": 1.05799389, + "balance_loss_mlp": 1.02918279, + "epoch": 0.1425338053508212, + "flos": 13145137142400.0, + "grad_norm": 2.442896512727032, + "language_loss": 0.76220739, + "learning_rate": 3.86863430277724e-06, + "loss": 0.78406882, + "num_input_tokens_seen": 140118160, + "router_z_loss_clip": 0.81005859, + "router_z_loss_mlp": 0.17938232, + "step": 4912, + "time_per_iteration": 2.5148701667785645 + }, + { + "auxiliary_loss_clip": 0.01137191, + "auxiliary_loss_mlp": 0.01048414, + "balance_loss_clip": 1.05519795, + "balance_loss_mlp": 1.03136671, + "epoch": 0.14256282281933724, + "flos": 24968186392320.0, + "grad_norm": 1.9948164667714323, + "language_loss": 0.85315764, + "learning_rate": 3.868567296915761e-06, + "loss": 0.87501365, + "num_input_tokens_seen": 140138420, + "router_z_loss_clip": 0.82128906, + "router_z_loss_mlp": 0.17028809, + "step": 4913, + "time_per_iteration": 2.590649127960205 + }, + { + "auxiliary_loss_clip": 0.01031171, + "auxiliary_loss_mlp": 0.01006113, + "balance_loss_clip": 1.01048231, + "balance_loss_mlp": 1.00487947, + "epoch": 0.1425918402878533, + "flos": 61859722763520.0, + "grad_norm": 0.6524074470903133, + "language_loss": 0.46428663, + "learning_rate": 3.868500274550339e-06, + "loss": 0.48465949, + "num_input_tokens_seen": 140198775, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.0123291, + "step": 4914, + "time_per_iteration": 3.157270669937134 + }, + { + "auxiliary_loss_clip": 0.01030901, + "auxiliary_loss_mlp": 0.01008522, + "balance_loss_clip": 1.01026869, + "balance_loss_mlp": 1.00728846, + "epoch": 0.14262085775636935, + "flos": 74780058637440.0, + "grad_norm": 0.6125614842892331, + "language_loss": 0.45914, + "learning_rate": 3.868433235681566e-06, + "loss": 0.47953421, + "num_input_tokens_seen": 140265290, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.0123291, + "step": 4915, + "time_per_iteration": 3.186086416244507 + }, + { + "auxiliary_loss_clip": 0.01131054, + "auxiliary_loss_mlp": 0.01048812, + "balance_loss_clip": 1.05369067, + "balance_loss_mlp": 1.03156829, + "epoch": 0.14264987522488537, + "flos": 14677660863360.0, + "grad_norm": 2.9337191443322173, + "language_loss": 0.88733703, + "learning_rate": 3.868366180310036e-06, + "loss": 0.9091357, + "num_input_tokens_seen": 140276615, + "router_z_loss_clip": 0.77441406, + "router_z_loss_mlp": 0.17248535, + "step": 4916, + "time_per_iteration": 2.5091326236724854 + }, + { + "auxiliary_loss_clip": 0.01029617, + "auxiliary_loss_mlp": 0.01002579, + "balance_loss_clip": 1.00910044, + "balance_loss_mlp": 1.00128555, + "epoch": 0.14267889269340142, + "flos": 65868214878720.0, + "grad_norm": 0.6569768842946321, + "language_loss": 0.46570441, + "learning_rate": 3.868299108436339e-06, + "loss": 0.48602638, + "num_input_tokens_seen": 140333190, + "router_z_loss_clip": 0.20507812, + "router_z_loss_mlp": 0.01293945, + "step": 4917, + "time_per_iteration": 2.99479603767395 + }, + { + "auxiliary_loss_clip": 0.01145221, + "auxiliary_loss_mlp": 0.01049061, + "balance_loss_clip": 1.05881238, + "balance_loss_mlp": 1.03045189, + "epoch": 0.14270791016191747, + "flos": 29380350528000.0, + "grad_norm": 5.1556074926289375, + "language_loss": 0.89805579, + "learning_rate": 3.868232020061068e-06, + "loss": 0.91999853, + "num_input_tokens_seen": 140350165, + "router_z_loss_clip": 0.86376953, + "router_z_loss_mlp": 0.18603516, + "step": 4918, + "time_per_iteration": 2.6703014373779297 + }, + { + "auxiliary_loss_clip": 0.01029758, + "auxiliary_loss_mlp": 0.01002015, + "balance_loss_clip": 1.00940084, + "balance_loss_mlp": 1.00071549, + "epoch": 0.14273692763043352, + "flos": 60802225430400.0, + "grad_norm": 0.6137933779945763, + "language_loss": 0.45983768, + "learning_rate": 3.868164915184817e-06, + "loss": 0.48015541, + "num_input_tokens_seen": 140412370, + "router_z_loss_clip": 0.20410156, + "router_z_loss_mlp": 0.01300049, + "step": 4919, + "time_per_iteration": 3.105689525604248 + }, + { + "auxiliary_loss_clip": 0.01137491, + "auxiliary_loss_mlp": 0.01047608, + "balance_loss_clip": 1.05318117, + "balance_loss_mlp": 1.02942824, + "epoch": 0.14276594509894958, + "flos": 26864808324480.0, + "grad_norm": 2.8746462341109313, + "language_loss": 0.93070972, + "learning_rate": 3.868097793808176e-06, + "loss": 0.95256072, + "num_input_tokens_seen": 140428275, + "router_z_loss_clip": 0.84228516, + "router_z_loss_mlp": 0.18164062, + "step": 4920, + "time_per_iteration": 2.628908157348633 + }, + { + "auxiliary_loss_clip": 0.01133468, + "auxiliary_loss_mlp": 0.01046339, + "balance_loss_clip": 1.05377352, + "balance_loss_mlp": 1.03214133, + "epoch": 0.14279496256746563, + "flos": 22265261913600.0, + "grad_norm": 2.35921017130727, + "language_loss": 0.78797674, + "learning_rate": 3.8680306559317405e-06, + "loss": 0.80977488, + "num_input_tokens_seen": 140442500, + "router_z_loss_clip": 0.79736328, + "router_z_loss_mlp": 0.14208984, + "step": 4921, + "time_per_iteration": 2.62174654006958 + }, + { + "auxiliary_loss_clip": 0.01131828, + "auxiliary_loss_mlp": 0.01038612, + "balance_loss_clip": 1.05474365, + "balance_loss_mlp": 1.0234127, + "epoch": 0.14282398003598165, + "flos": 26373049176960.0, + "grad_norm": 1.76419853279882, + "language_loss": 0.56751406, + "learning_rate": 3.867963501556102e-06, + "loss": 0.5892185, + "num_input_tokens_seen": 140462010, + "router_z_loss_clip": 0.77099609, + "router_z_loss_mlp": 0.15209961, + "step": 4922, + "time_per_iteration": 2.6906585693359375 + }, + { + "auxiliary_loss_clip": 0.01030805, + "auxiliary_loss_mlp": 0.01006404, + "balance_loss_clip": 1.01030362, + "balance_loss_mlp": 1.0050863, + "epoch": 0.1428529975044977, + "flos": 55538582417280.0, + "grad_norm": 0.6835310125529301, + "language_loss": 0.50274026, + "learning_rate": 3.867896330681854e-06, + "loss": 0.52311236, + "num_input_tokens_seen": 140516450, + "router_z_loss_clip": 0.20507812, + "router_z_loss_mlp": 0.01318359, + "step": 4923, + "time_per_iteration": 2.9832491874694824 + }, + { + "auxiliary_loss_clip": 0.01142136, + "auxiliary_loss_mlp": 0.01043589, + "balance_loss_clip": 1.0599184, + "balance_loss_mlp": 1.02730513, + "epoch": 0.14288201497301375, + "flos": 59374234287360.0, + "grad_norm": 1.5309082229630673, + "language_loss": 0.66004938, + "learning_rate": 3.86782914330959e-06, + "loss": 0.68190658, + "num_input_tokens_seen": 140544355, + "router_z_loss_clip": 0.82275391, + "router_z_loss_mlp": 0.16290283, + "step": 4924, + "time_per_iteration": 2.904679536819458 + }, + { + "auxiliary_loss_clip": 0.01133636, + "auxiliary_loss_mlp": 0.01047543, + "balance_loss_clip": 1.05501759, + "balance_loss_mlp": 1.03307676, + "epoch": 0.1429110324415298, + "flos": 13475228152320.0, + "grad_norm": 1.7699828335193464, + "language_loss": 0.6681748, + "learning_rate": 3.867761939439902e-06, + "loss": 0.68998659, + "num_input_tokens_seen": 140562625, + "router_z_loss_clip": 0.78662109, + "router_z_loss_mlp": 0.14471436, + "step": 4925, + "time_per_iteration": 2.5806381702423096 + }, + { + "auxiliary_loss_clip": 0.01127116, + "auxiliary_loss_mlp": 0.01032903, + "balance_loss_clip": 1.05209672, + "balance_loss_mlp": 1.01936054, + "epoch": 0.14294004991004586, + "flos": 29712704094720.0, + "grad_norm": 1.8008988519090345, + "language_loss": 0.70278007, + "learning_rate": 3.8676947190733855e-06, + "loss": 0.7243802, + "num_input_tokens_seen": 140580935, + "router_z_loss_clip": 0.75, + "router_z_loss_mlp": 0.13537598, + "step": 4926, + "time_per_iteration": 2.6123099327087402 + }, + { + "auxiliary_loss_clip": 0.01137364, + "auxiliary_loss_mlp": 0.01041032, + "balance_loss_clip": 1.05658603, + "balance_loss_mlp": 1.02515984, + "epoch": 0.14296906737856188, + "flos": 33871410887040.0, + "grad_norm": 2.094695660120688, + "language_loss": 0.76669168, + "learning_rate": 3.867627482210634e-06, + "loss": 0.78847563, + "num_input_tokens_seen": 140597075, + "router_z_loss_clip": 0.80810547, + "router_z_loss_mlp": 0.15863037, + "step": 4927, + "time_per_iteration": 2.6681995391845703 + }, + { + "auxiliary_loss_clip": 0.01133857, + "auxiliary_loss_mlp": 0.01050281, + "balance_loss_clip": 1.05411291, + "balance_loss_mlp": 1.03456306, + "epoch": 0.14299808484707793, + "flos": 24017738567040.0, + "grad_norm": 2.217812965637683, + "language_loss": 0.86031616, + "learning_rate": 3.86756022885224e-06, + "loss": 0.88215756, + "num_input_tokens_seen": 140611865, + "router_z_loss_clip": 0.79736328, + "router_z_loss_mlp": 0.15722656, + "step": 4928, + "time_per_iteration": 2.5467543601989746 + }, + { + "auxiliary_loss_clip": 0.01137709, + "auxiliary_loss_mlp": 0.01036824, + "balance_loss_clip": 1.05676126, + "balance_loss_mlp": 1.01947868, + "epoch": 0.14302710231559398, + "flos": 25623591903360.0, + "grad_norm": 2.525011934978011, + "language_loss": 0.86509764, + "learning_rate": 3.867492958998799e-06, + "loss": 0.88684297, + "num_input_tokens_seen": 140627065, + "router_z_loss_clip": 0.80957031, + "router_z_loss_mlp": 0.17346191, + "step": 4929, + "time_per_iteration": 2.5634350776672363 + }, + { + "auxiliary_loss_clip": 0.01135899, + "auxiliary_loss_mlp": 0.01041206, + "balance_loss_clip": 1.05697155, + "balance_loss_mlp": 1.02604878, + "epoch": 0.14305611978411004, + "flos": 11392696402560.0, + "grad_norm": 3.2756869823920898, + "language_loss": 0.74827552, + "learning_rate": 3.867425672650904e-06, + "loss": 0.77004659, + "num_input_tokens_seen": 140638110, + "router_z_loss_clip": 0.78857422, + "router_z_loss_mlp": 0.15155029, + "step": 4930, + "time_per_iteration": 2.498816728591919 + }, + { + "auxiliary_loss_clip": 0.01128811, + "auxiliary_loss_mlp": 0.01036558, + "balance_loss_clip": 1.05361533, + "balance_loss_mlp": 1.02205658, + "epoch": 0.1430851372526261, + "flos": 74732975907840.0, + "grad_norm": 1.981469202270812, + "language_loss": 0.71366984, + "learning_rate": 3.86735836980915e-06, + "loss": 0.73532355, + "num_input_tokens_seen": 140660835, + "router_z_loss_clip": 0.75244141, + "router_z_loss_mlp": 0.14501953, + "step": 4931, + "time_per_iteration": 2.947847843170166 + }, + { + "auxiliary_loss_clip": 0.0114236, + "auxiliary_loss_mlp": 0.01039227, + "balance_loss_clip": 1.05718899, + "balance_loss_mlp": 1.02343154, + "epoch": 0.14311415472114214, + "flos": 16573600437120.0, + "grad_norm": 17.791327418945926, + "language_loss": 0.84188706, + "learning_rate": 3.8672910504741315e-06, + "loss": 0.86370295, + "num_input_tokens_seen": 140673115, + "router_z_loss_clip": 0.85253906, + "router_z_loss_mlp": 0.15795898, + "step": 4932, + "time_per_iteration": 2.4890995025634766 + }, + { + "auxiliary_loss_clip": 0.01032193, + "auxiliary_loss_mlp": 0.01005251, + "balance_loss_clip": 1.01181126, + "balance_loss_mlp": 1.00408292, + "epoch": 0.14314317218965816, + "flos": 60756905445120.0, + "grad_norm": 0.7347098301953535, + "language_loss": 0.52495444, + "learning_rate": 3.867223714646442e-06, + "loss": 0.54532886, + "num_input_tokens_seen": 140725625, + "router_z_loss_clip": 0.20410156, + "router_z_loss_mlp": 0.01165771, + "step": 4933, + "time_per_iteration": 2.956071376800537 + }, + { + "auxiliary_loss_clip": 0.01141405, + "auxiliary_loss_mlp": 0.01044096, + "balance_loss_clip": 1.05638278, + "balance_loss_mlp": 1.02684677, + "epoch": 0.14317218965817421, + "flos": 34086695051520.0, + "grad_norm": 2.6959989632956725, + "language_loss": 0.93915802, + "learning_rate": 3.867156362326678e-06, + "loss": 0.96101302, + "num_input_tokens_seen": 140748190, + "router_z_loss_clip": 0.8515625, + "router_z_loss_mlp": 0.17242432, + "step": 4934, + "time_per_iteration": 2.8190910816192627 + }, + { + "auxiliary_loss_clip": 0.01033442, + "auxiliary_loss_mlp": 0.01006728, + "balance_loss_clip": 1.01315784, + "balance_loss_mlp": 1.00564313, + "epoch": 0.14320120712669027, + "flos": 60623606073600.0, + "grad_norm": 0.6459523665224712, + "language_loss": 0.46920559, + "learning_rate": 3.867088993515432e-06, + "loss": 0.4896073, + "num_input_tokens_seen": 140813230, + "router_z_loss_clip": 0.203125, + "router_z_loss_mlp": 0.01086426, + "step": 4935, + "time_per_iteration": 3.1406805515289307 + }, + { + "auxiliary_loss_clip": 0.01128188, + "auxiliary_loss_mlp": 0.01041794, + "balance_loss_clip": 1.05026305, + "balance_loss_mlp": 1.0258317, + "epoch": 0.14323022459520632, + "flos": 37044657072000.0, + "grad_norm": 2.6323431326097606, + "language_loss": 1.0021286, + "learning_rate": 3.867021608213302e-06, + "loss": 1.02382851, + "num_input_tokens_seen": 140834150, + "router_z_loss_clip": 0.77978516, + "router_z_loss_mlp": 0.15985107, + "step": 4936, + "time_per_iteration": 2.690967321395874 + }, + { + "auxiliary_loss_clip": 0.01127537, + "auxiliary_loss_mlp": 0.01035306, + "balance_loss_clip": 1.05065536, + "balance_loss_mlp": 1.01979113, + "epoch": 0.14325924206372237, + "flos": 22996726473600.0, + "grad_norm": 2.5927422818790293, + "language_loss": 0.9887467, + "learning_rate": 3.866954206420881e-06, + "loss": 1.01037502, + "num_input_tokens_seen": 140849965, + "router_z_loss_clip": 0.76855469, + "router_z_loss_mlp": 0.15515137, + "step": 4937, + "time_per_iteration": 2.5362958908081055 + }, + { + "auxiliary_loss_clip": 0.01150152, + "auxiliary_loss_mlp": 0.01054286, + "balance_loss_clip": 1.05987799, + "balance_loss_mlp": 1.03721559, + "epoch": 0.14328825953223842, + "flos": 15664234792320.0, + "grad_norm": 2.563658014802729, + "language_loss": 0.98330295, + "learning_rate": 3.866886788138765e-06, + "loss": 1.00534725, + "num_input_tokens_seen": 140865175, + "router_z_loss_clip": 0.90185547, + "router_z_loss_mlp": 0.17077637, + "step": 4938, + "time_per_iteration": 2.516519546508789 + }, + { + "auxiliary_loss_clip": 0.01139949, + "auxiliary_loss_mlp": 0.01041519, + "balance_loss_clip": 1.05877137, + "balance_loss_mlp": 1.02369761, + "epoch": 0.14331727700075444, + "flos": 15005848452480.0, + "grad_norm": 3.361380018449044, + "language_loss": 0.79069984, + "learning_rate": 3.86681935336755e-06, + "loss": 0.81251454, + "num_input_tokens_seen": 140877400, + "router_z_loss_clip": 0.81103516, + "router_z_loss_mlp": 0.17816162, + "step": 4939, + "time_per_iteration": 2.496870756149292 + }, + { + "auxiliary_loss_clip": 0.01132932, + "auxiliary_loss_mlp": 0.01041067, + "balance_loss_clip": 1.05271626, + "balance_loss_mlp": 1.02517033, + "epoch": 0.1433462944692705, + "flos": 74740122714240.0, + "grad_norm": 1.5777875076197503, + "language_loss": 0.84337407, + "learning_rate": 3.866751902107831e-06, + "loss": 0.86511409, + "num_input_tokens_seen": 140907505, + "router_z_loss_clip": 0.80322266, + "router_z_loss_mlp": 0.15887451, + "step": 4940, + "time_per_iteration": 3.0069897174835205 + }, + { + "auxiliary_loss_clip": 0.01033142, + "auxiliary_loss_mlp": 0.0100299, + "balance_loss_clip": 1.01274788, + "balance_loss_mlp": 1.00188112, + "epoch": 0.14337531193778655, + "flos": 74787959629440.0, + "grad_norm": 0.669437623913046, + "language_loss": 0.53050631, + "learning_rate": 3.866684434360203e-06, + "loss": 0.55086768, + "num_input_tokens_seen": 140978730, + "router_z_loss_clip": 0.20410156, + "router_z_loss_mlp": 0.0111084, + "step": 4941, + "time_per_iteration": 3.2857863903045654 + }, + { + "auxiliary_loss_clip": 0.0113859, + "auxiliary_loss_mlp": 0.01040271, + "balance_loss_clip": 1.05380464, + "balance_loss_mlp": 1.02383161, + "epoch": 0.1434043294063026, + "flos": 31789633322880.0, + "grad_norm": 2.451413261765814, + "language_loss": 0.80445915, + "learning_rate": 3.866616950125263e-06, + "loss": 0.82624775, + "num_input_tokens_seen": 141000495, + "router_z_loss_clip": 0.84814453, + "router_z_loss_mlp": 0.16442871, + "step": 4942, + "time_per_iteration": 2.7218306064605713 + }, + { + "auxiliary_loss_clip": 0.01132402, + "auxiliary_loss_mlp": 0.01034769, + "balance_loss_clip": 1.05385852, + "balance_loss_mlp": 1.01932573, + "epoch": 0.14343334687481865, + "flos": 14024302427520.0, + "grad_norm": 2.2706411421853536, + "language_loss": 0.66879535, + "learning_rate": 3.866549449403607e-06, + "loss": 0.690467, + "num_input_tokens_seen": 141016315, + "router_z_loss_clip": 0.78515625, + "router_z_loss_mlp": 0.15435791, + "step": 4943, + "time_per_iteration": 2.512547492980957 + }, + { + "auxiliary_loss_clip": 0.01129525, + "auxiliary_loss_mlp": 0.01037399, + "balance_loss_clip": 1.05351663, + "balance_loss_mlp": 1.02221179, + "epoch": 0.14346236434333468, + "flos": 15406647384960.0, + "grad_norm": 2.8678825018170544, + "language_loss": 0.7449218, + "learning_rate": 3.866481932195831e-06, + "loss": 0.76659107, + "num_input_tokens_seen": 141029335, + "router_z_loss_clip": 0.76074219, + "router_z_loss_mlp": 0.15161133, + "step": 4944, + "time_per_iteration": 2.5435614585876465 + }, + { + "auxiliary_loss_clip": 0.0113913, + "auxiliary_loss_mlp": 0.01045359, + "balance_loss_clip": 1.0574863, + "balance_loss_mlp": 1.02816868, + "epoch": 0.14349138181185073, + "flos": 13473432472320.0, + "grad_norm": 2.405184186481755, + "language_loss": 0.7112807, + "learning_rate": 3.866414398502531e-06, + "loss": 0.73312551, + "num_input_tokens_seen": 141042830, + "router_z_loss_clip": 0.81591797, + "router_z_loss_mlp": 0.17193604, + "step": 4945, + "time_per_iteration": 2.4929263591766357 + }, + { + "auxiliary_loss_clip": 0.01128759, + "auxiliary_loss_mlp": 0.01033902, + "balance_loss_clip": 1.05390537, + "balance_loss_mlp": 1.01863074, + "epoch": 0.14352039928036678, + "flos": 37408216579200.0, + "grad_norm": 2.105774234892691, + "language_loss": 0.84842753, + "learning_rate": 3.866346848324304e-06, + "loss": 0.87005419, + "num_input_tokens_seen": 141059870, + "router_z_loss_clip": 0.74902344, + "router_z_loss_mlp": 0.15270996, + "step": 4946, + "time_per_iteration": 2.6978468894958496 + }, + { + "auxiliary_loss_clip": 0.01136362, + "auxiliary_loss_mlp": 0.01035324, + "balance_loss_clip": 1.05913401, + "balance_loss_mlp": 1.01959443, + "epoch": 0.14354941674888283, + "flos": 18618605452800.0, + "grad_norm": 2.5485184213173318, + "language_loss": 0.85488474, + "learning_rate": 3.8662792816617465e-06, + "loss": 0.87660164, + "num_input_tokens_seen": 141073325, + "router_z_loss_clip": 0.77099609, + "router_z_loss_mlp": 0.1572876, + "step": 4947, + "time_per_iteration": 2.5526373386383057 + }, + { + "auxiliary_loss_clip": 0.0113648, + "auxiliary_loss_mlp": 0.01040312, + "balance_loss_clip": 1.05617678, + "balance_loss_mlp": 1.02432573, + "epoch": 0.14357843421739888, + "flos": 14167118903040.0, + "grad_norm": 2.294644543940826, + "language_loss": 0.90294981, + "learning_rate": 3.8662116985154545e-06, + "loss": 0.92471778, + "num_input_tokens_seen": 141086265, + "router_z_loss_clip": 0.80322266, + "router_z_loss_mlp": 0.15985107, + "step": 4948, + "time_per_iteration": 2.551799774169922 + }, + { + "auxiliary_loss_clip": 0.01143069, + "auxiliary_loss_mlp": 0.01043024, + "balance_loss_clip": 1.05888653, + "balance_loss_mlp": 1.02692473, + "epoch": 0.14360745168591493, + "flos": 25515788209920.0, + "grad_norm": 2.003643086137624, + "language_loss": 0.87538755, + "learning_rate": 3.866144098886027e-06, + "loss": 0.89724845, + "num_input_tokens_seen": 141102335, + "router_z_loss_clip": 0.84179688, + "router_z_loss_mlp": 0.16119385, + "step": 4949, + "time_per_iteration": 2.5535032749176025 + }, + { + "auxiliary_loss_clip": 0.0113575, + "auxiliary_loss_mlp": 0.01047592, + "balance_loss_clip": 1.05692208, + "balance_loss_mlp": 1.03202891, + "epoch": 0.14363646915443096, + "flos": 12631937575680.0, + "grad_norm": 4.38570366668038, + "language_loss": 0.87900138, + "learning_rate": 3.866076482774058e-06, + "loss": 0.9008348, + "num_input_tokens_seen": 141114270, + "router_z_loss_clip": 0.78808594, + "router_z_loss_mlp": 0.15551758, + "step": 4950, + "time_per_iteration": 2.491421937942505 + }, + { + "auxiliary_loss_clip": 0.01132346, + "auxiliary_loss_mlp": 0.01047535, + "balance_loss_clip": 1.05371666, + "balance_loss_mlp": 1.02967763, + "epoch": 0.143665486622947, + "flos": 15922145422080.0, + "grad_norm": 9.478342402877116, + "language_loss": 0.78963387, + "learning_rate": 3.8660088501801474e-06, + "loss": 0.81143272, + "num_input_tokens_seen": 141128730, + "router_z_loss_clip": 0.78515625, + "router_z_loss_mlp": 0.1786499, + "step": 4951, + "time_per_iteration": 2.503417730331421 + }, + { + "auxiliary_loss_clip": 0.01134005, + "auxiliary_loss_mlp": 0.01040574, + "balance_loss_clip": 1.05315721, + "balance_loss_mlp": 1.02309227, + "epoch": 0.14369450409146306, + "flos": 26386406058240.0, + "grad_norm": 2.609593892444954, + "language_loss": 0.95483232, + "learning_rate": 3.865941201104891e-06, + "loss": 0.97657812, + "num_input_tokens_seen": 141142240, + "router_z_loss_clip": 0.80761719, + "router_z_loss_mlp": 0.17492676, + "step": 4952, + "time_per_iteration": 2.606959819793701 + }, + { + "auxiliary_loss_clip": 0.01142412, + "auxiliary_loss_mlp": 0.01049636, + "balance_loss_clip": 1.05669916, + "balance_loss_mlp": 1.03171301, + "epoch": 0.1437235215599791, + "flos": 41530725440640.0, + "grad_norm": 2.3132214305683014, + "language_loss": 0.93299067, + "learning_rate": 3.8658735355488875e-06, + "loss": 0.95491117, + "num_input_tokens_seen": 141159210, + "router_z_loss_clip": 0.85742188, + "router_z_loss_mlp": 0.17932129, + "step": 4953, + "time_per_iteration": 2.6961007118225098 + }, + { + "auxiliary_loss_clip": 0.01043037, + "auxiliary_loss_mlp": 0.01004853, + "balance_loss_clip": 1.02236176, + "balance_loss_mlp": 1.00357711, + "epoch": 0.14375253902849516, + "flos": 62504282367360.0, + "grad_norm": 0.767603683931171, + "language_loss": 0.48996675, + "learning_rate": 3.865805853512733e-06, + "loss": 0.51044559, + "num_input_tokens_seen": 141211385, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.01275635, + "step": 4954, + "time_per_iteration": 2.930591344833374 + }, + { + "auxiliary_loss_clip": 0.01041595, + "auxiliary_loss_mlp": 0.01001632, + "balance_loss_clip": 1.02089858, + "balance_loss_mlp": 1.00063026, + "epoch": 0.1437815564970112, + "flos": 62991914428800.0, + "grad_norm": 0.699209569937205, + "language_loss": 0.49687487, + "learning_rate": 3.865738154997027e-06, + "loss": 0.51730716, + "num_input_tokens_seen": 141274490, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.01000977, + "step": 4955, + "time_per_iteration": 3.086735248565674 + }, + { + "auxiliary_loss_clip": 0.01137625, + "auxiliary_loss_mlp": 0.01042562, + "balance_loss_clip": 1.05781484, + "balance_loss_mlp": 1.02606297, + "epoch": 0.14381057396552724, + "flos": 42014119697280.0, + "grad_norm": 1.821808455791847, + "language_loss": 0.91846108, + "learning_rate": 3.865670440002366e-06, + "loss": 0.94026291, + "num_input_tokens_seen": 141299290, + "router_z_loss_clip": 0.79785156, + "router_z_loss_mlp": 0.16503906, + "step": 4956, + "time_per_iteration": 2.7302777767181396 + }, + { + "auxiliary_loss_clip": 0.01139069, + "auxiliary_loss_mlp": 0.01052524, + "balance_loss_clip": 1.0552547, + "balance_loss_mlp": 1.03381395, + "epoch": 0.1438395914340433, + "flos": 36091874862720.0, + "grad_norm": 2.5723041186873368, + "language_loss": 0.81727374, + "learning_rate": 3.86560270852935e-06, + "loss": 0.83918971, + "num_input_tokens_seen": 141314325, + "router_z_loss_clip": 0.83789062, + "router_z_loss_mlp": 0.18725586, + "step": 4957, + "time_per_iteration": 2.6504342555999756 + }, + { + "auxiliary_loss_clip": 0.01134004, + "auxiliary_loss_mlp": 0.01042041, + "balance_loss_clip": 1.05589843, + "balance_loss_mlp": 1.02634728, + "epoch": 0.14386860890255934, + "flos": 11503732320000.0, + "grad_norm": 3.33320371693475, + "language_loss": 0.76811755, + "learning_rate": 3.865534960578574e-06, + "loss": 0.78987801, + "num_input_tokens_seen": 141325265, + "router_z_loss_clip": 0.78125, + "router_z_loss_mlp": 0.15692139, + "step": 4958, + "time_per_iteration": 2.4755747318267822 + }, + { + "auxiliary_loss_clip": 0.01037942, + "auxiliary_loss_mlp": 0.01005029, + "balance_loss_clip": 1.01768851, + "balance_loss_mlp": 1.00393188, + "epoch": 0.1438976263710754, + "flos": 58534645789440.0, + "grad_norm": 0.7072399964535455, + "language_loss": 0.51144505, + "learning_rate": 3.865467196150639e-06, + "loss": 0.53187478, + "num_input_tokens_seen": 141385290, + "router_z_loss_clip": 0.203125, + "router_z_loss_mlp": 0.01098633, + "step": 4959, + "time_per_iteration": 3.0763087272644043 + }, + { + "auxiliary_loss_clip": 0.01035081, + "auxiliary_loss_mlp": 0.01001449, + "balance_loss_clip": 1.01481628, + "balance_loss_mlp": 1.00027454, + "epoch": 0.14392664383959144, + "flos": 64845371191680.0, + "grad_norm": 0.7162018960785128, + "language_loss": 0.51490164, + "learning_rate": 3.865399415246144e-06, + "loss": 0.53526694, + "num_input_tokens_seen": 141442665, + "router_z_loss_clip": 0.203125, + "router_z_loss_mlp": 0.01171875, + "step": 4960, + "time_per_iteration": 2.928488254547119 + }, + { + "auxiliary_loss_clip": 0.01129235, + "auxiliary_loss_mlp": 0.01043465, + "balance_loss_clip": 1.05079949, + "balance_loss_mlp": 1.02697802, + "epoch": 0.14395566130810747, + "flos": 35728961800320.0, + "grad_norm": 3.1777632643818263, + "language_loss": 1.02415252, + "learning_rate": 3.865331617865686e-06, + "loss": 1.0458796, + "num_input_tokens_seen": 141458015, + "router_z_loss_clip": 0.78369141, + "router_z_loss_mlp": 0.16479492, + "step": 4961, + "time_per_iteration": 2.702775239944458 + }, + { + "auxiliary_loss_clip": 0.01032773, + "auxiliary_loss_mlp": 0.00999993, + "balance_loss_clip": 1.01241314, + "balance_loss_mlp": 0.9988904, + "epoch": 0.14398467877662352, + "flos": 74775784919040.0, + "grad_norm": 0.6796829197005638, + "language_loss": 0.50307196, + "learning_rate": 3.865263804009863e-06, + "loss": 0.52339965, + "num_input_tokens_seen": 141520365, + "router_z_loss_clip": 0.20410156, + "router_z_loss_mlp": 0.01104736, + "step": 4962, + "time_per_iteration": 3.167388916015625 + }, + { + "auxiliary_loss_clip": 0.01131184, + "auxiliary_loss_mlp": 0.01052399, + "balance_loss_clip": 1.05102134, + "balance_loss_mlp": 1.03504181, + "epoch": 0.14401369624513957, + "flos": 17233064184960.0, + "grad_norm": 3.333293588453396, + "language_loss": 0.83296144, + "learning_rate": 3.865195973679277e-06, + "loss": 0.85479718, + "num_input_tokens_seen": 141534935, + "router_z_loss_clip": 0.80175781, + "router_z_loss_mlp": 0.17333984, + "step": 4963, + "time_per_iteration": 2.5304529666900635 + }, + { + "auxiliary_loss_clip": 0.01032932, + "auxiliary_loss_mlp": 0.01002674, + "balance_loss_clip": 1.01256621, + "balance_loss_mlp": 1.00158966, + "epoch": 0.14404271371365562, + "flos": 60859070703360.0, + "grad_norm": 0.7156553897379482, + "language_loss": 0.45812225, + "learning_rate": 3.8651281268745245e-06, + "loss": 0.47847831, + "num_input_tokens_seen": 141586535, + "router_z_loss_clip": 0.203125, + "router_z_loss_mlp": 0.01086426, + "step": 4964, + "time_per_iteration": 2.9373536109924316 + }, + { + "auxiliary_loss_clip": 0.01133444, + "auxiliary_loss_mlp": 0.01041145, + "balance_loss_clip": 1.05211341, + "balance_loss_mlp": 1.0256958, + "epoch": 0.14407173118217168, + "flos": 40952420472960.0, + "grad_norm": 1.705344311580374, + "language_loss": 0.78642112, + "learning_rate": 3.865060263596206e-06, + "loss": 0.80816698, + "num_input_tokens_seen": 141607730, + "router_z_loss_clip": 0.81103516, + "router_z_loss_mlp": 0.15441895, + "step": 4965, + "time_per_iteration": 2.677769422531128 + }, + { + "auxiliary_loss_clip": 0.01134916, + "auxiliary_loss_mlp": 0.01040646, + "balance_loss_clip": 1.05443621, + "balance_loss_mlp": 1.02496433, + "epoch": 0.14410074865068773, + "flos": 35071078250880.0, + "grad_norm": 2.1598188457718064, + "language_loss": 0.70287704, + "learning_rate": 3.864992383844921e-06, + "loss": 0.72463268, + "num_input_tokens_seen": 141624060, + "router_z_loss_clip": 0.8046875, + "router_z_loss_mlp": 0.15679932, + "step": 4966, + "time_per_iteration": 2.741475820541382 + }, + { + "auxiliary_loss_clip": 0.01135752, + "auxiliary_loss_mlp": 0.01041581, + "balance_loss_clip": 1.05400443, + "balance_loss_mlp": 1.02623844, + "epoch": 0.14412976611920375, + "flos": 29052593902080.0, + "grad_norm": 2.101518074303409, + "language_loss": 1.03358603, + "learning_rate": 3.864924487621268e-06, + "loss": 1.05535948, + "num_input_tokens_seen": 141645265, + "router_z_loss_clip": 0.81787109, + "router_z_loss_mlp": 0.15332031, + "step": 4967, + "time_per_iteration": 2.6935486793518066 + }, + { + "auxiliary_loss_clip": 0.01132385, + "auxiliary_loss_mlp": 0.01042354, + "balance_loss_clip": 1.05361581, + "balance_loss_mlp": 1.02692211, + "epoch": 0.1441587835877198, + "flos": 13800901789440.0, + "grad_norm": 2.5960158957067025, + "language_loss": 0.84990847, + "learning_rate": 3.864856574925847e-06, + "loss": 0.87165594, + "num_input_tokens_seen": 141656570, + "router_z_loss_clip": 0.78808594, + "router_z_loss_mlp": 0.15454102, + "step": 4968, + "time_per_iteration": 2.5675878524780273 + }, + { + "auxiliary_loss_clip": 0.01129752, + "auxiliary_loss_mlp": 0.0103536, + "balance_loss_clip": 1.05194044, + "balance_loss_mlp": 1.02035785, + "epoch": 0.14418780105623585, + "flos": 14932375182720.0, + "grad_norm": 2.8537862822510722, + "language_loss": 0.92295319, + "learning_rate": 3.864788645759258e-06, + "loss": 0.94460428, + "num_input_tokens_seen": 141670560, + "router_z_loss_clip": 0.77929688, + "router_z_loss_mlp": 0.15014648, + "step": 4969, + "time_per_iteration": 2.4755516052246094 + }, + { + "auxiliary_loss_clip": 0.01135509, + "auxiliary_loss_mlp": 0.01046671, + "balance_loss_clip": 1.05473852, + "balance_loss_mlp": 1.02954614, + "epoch": 0.1442168185247519, + "flos": 27191846678400.0, + "grad_norm": 3.2411613856689208, + "language_loss": 0.97393799, + "learning_rate": 3.864720700122101e-06, + "loss": 0.99575973, + "num_input_tokens_seen": 141687495, + "router_z_loss_clip": 0.80859375, + "router_z_loss_mlp": 0.17126465, + "step": 4970, + "time_per_iteration": 2.6476848125457764 + }, + { + "auxiliary_loss_clip": 0.01126594, + "auxiliary_loss_mlp": 0.01030067, + "balance_loss_clip": 1.05210781, + "balance_loss_mlp": 1.01539242, + "epoch": 0.14424583599326796, + "flos": 24601430574720.0, + "grad_norm": 3.587859685070445, + "language_loss": 0.73328871, + "learning_rate": 3.864652738014977e-06, + "loss": 0.75485533, + "num_input_tokens_seen": 141703160, + "router_z_loss_clip": 0.74462891, + "router_z_loss_mlp": 0.14685059, + "step": 4971, + "time_per_iteration": 2.6223552227020264 + }, + { + "auxiliary_loss_clip": 0.01039249, + "auxiliary_loss_mlp": 0.01000814, + "balance_loss_clip": 1.01866364, + "balance_loss_mlp": 0.99965775, + "epoch": 0.14427485346178398, + "flos": 63243037388160.0, + "grad_norm": 0.6648819865438442, + "language_loss": 0.44714177, + "learning_rate": 3.864584759438485e-06, + "loss": 0.46754238, + "num_input_tokens_seen": 141760610, + "router_z_loss_clip": 0.20605469, + "router_z_loss_mlp": 0.01153564, + "step": 4972, + "time_per_iteration": 3.071688652038574 + }, + { + "auxiliary_loss_clip": 0.01129018, + "auxiliary_loss_mlp": 0.01041813, + "balance_loss_clip": 1.05415869, + "balance_loss_mlp": 1.02644718, + "epoch": 0.14430387093030003, + "flos": 20115469946880.0, + "grad_norm": 1.9776175916854233, + "language_loss": 0.74212849, + "learning_rate": 3.864516764393226e-06, + "loss": 0.76383674, + "num_input_tokens_seen": 141774080, + "router_z_loss_clip": 0.74853516, + "router_z_loss_mlp": 0.15350342, + "step": 4973, + "time_per_iteration": 2.560631275177002 + }, + { + "auxiliary_loss_clip": 0.01127764, + "auxiliary_loss_mlp": 0.010425, + "balance_loss_clip": 1.05196536, + "balance_loss_mlp": 1.02727723, + "epoch": 0.14433288839881608, + "flos": 16172190973440.0, + "grad_norm": 3.162887108818322, + "language_loss": 0.81844854, + "learning_rate": 3.864448752879801e-06, + "loss": 0.84015119, + "num_input_tokens_seen": 141787155, + "router_z_loss_clip": 0.75830078, + "router_z_loss_mlp": 0.15228271, + "step": 4974, + "time_per_iteration": 2.4866273403167725 + }, + { + "auxiliary_loss_clip": 0.01135026, + "auxiliary_loss_mlp": 0.010543, + "balance_loss_clip": 1.05393577, + "balance_loss_mlp": 1.03721142, + "epoch": 0.14436190586733214, + "flos": 31691705869440.0, + "grad_norm": 2.574618646728975, + "language_loss": 0.93198848, + "learning_rate": 3.864380724898809e-06, + "loss": 0.95388174, + "num_input_tokens_seen": 141805525, + "router_z_loss_clip": 0.81103516, + "router_z_loss_mlp": 0.17102051, + "step": 4975, + "time_per_iteration": 2.654111623764038 + }, + { + "auxiliary_loss_clip": 0.01136791, + "auxiliary_loss_mlp": 0.01036496, + "balance_loss_clip": 1.05527341, + "balance_loss_mlp": 1.01974702, + "epoch": 0.1443909233358482, + "flos": 28176445359360.0, + "grad_norm": 2.565817801190906, + "language_loss": 0.72458321, + "learning_rate": 3.864312680450853e-06, + "loss": 0.74631608, + "num_input_tokens_seen": 141821925, + "router_z_loss_clip": 0.81445312, + "router_z_loss_mlp": 0.16748047, + "step": 4976, + "time_per_iteration": 2.597230911254883 + }, + { + "auxiliary_loss_clip": 0.01036222, + "auxiliary_loss_mlp": 0.01005683, + "balance_loss_clip": 1.01585591, + "balance_loss_mlp": 1.00462806, + "epoch": 0.14441994080436424, + "flos": 74777508771840.0, + "grad_norm": 0.6722983167027435, + "language_loss": 0.4989216, + "learning_rate": 3.864244619536532e-06, + "loss": 0.51934063, + "num_input_tokens_seen": 141887380, + "router_z_loss_clip": 0.203125, + "router_z_loss_mlp": 0.01055908, + "step": 4977, + "time_per_iteration": 3.1934051513671875 + }, + { + "auxiliary_loss_clip": 0.01136598, + "auxiliary_loss_mlp": 0.01045506, + "balance_loss_clip": 1.05480742, + "balance_loss_mlp": 1.02852476, + "epoch": 0.14444895827288026, + "flos": 37004688213120.0, + "grad_norm": 2.695034414006044, + "language_loss": 0.90686524, + "learning_rate": 3.8641765421564496e-06, + "loss": 0.92868626, + "num_input_tokens_seen": 141901820, + "router_z_loss_clip": 0.81787109, + "router_z_loss_mlp": 0.16986084, + "step": 4978, + "time_per_iteration": 2.6534688472747803 + }, + { + "auxiliary_loss_clip": 0.01034079, + "auxiliary_loss_mlp": 0.01000787, + "balance_loss_clip": 1.01368976, + "balance_loss_mlp": 0.99970859, + "epoch": 0.14447797574139631, + "flos": 65173558780800.0, + "grad_norm": 0.7317658376424377, + "language_loss": 0.50810772, + "learning_rate": 3.864108448311205e-06, + "loss": 0.52845639, + "num_input_tokens_seen": 141961100, + "router_z_loss_clip": 0.20410156, + "router_z_loss_mlp": 0.01080322, + "step": 4979, + "time_per_iteration": 5.455294847488403 + }, + { + "auxiliary_loss_clip": 0.01134223, + "auxiliary_loss_mlp": 0.01037542, + "balance_loss_clip": 1.05047762, + "balance_loss_mlp": 1.01901686, + "epoch": 0.14450699320991237, + "flos": 15881063241600.0, + "grad_norm": 2.8556057137113955, + "language_loss": 0.92657161, + "learning_rate": 3.8640403380013995e-06, + "loss": 0.94828922, + "num_input_tokens_seen": 141973525, + "router_z_loss_clip": 0.83691406, + "router_z_loss_mlp": 0.18554688, + "step": 4980, + "time_per_iteration": 7.155998706817627 + }, + { + "auxiliary_loss_clip": 0.01127049, + "auxiliary_loss_mlp": 0.01047328, + "balance_loss_clip": 1.05246592, + "balance_loss_mlp": 1.0309962, + "epoch": 0.14453601067842842, + "flos": 25044568663680.0, + "grad_norm": 1.8856586376649285, + "language_loss": 0.69096899, + "learning_rate": 3.863972211227636e-06, + "loss": 0.71271276, + "num_input_tokens_seen": 141989935, + "router_z_loss_clip": 0.74609375, + "router_z_loss_mlp": 0.16345215, + "step": 4981, + "time_per_iteration": 2.5373175144195557 + }, + { + "auxiliary_loss_clip": 0.01032236, + "auxiliary_loss_mlp": 0.00998797, + "balance_loss_clip": 1.01188636, + "balance_loss_mlp": 0.99778986, + "epoch": 0.14456502814694447, + "flos": 64262217888000.0, + "grad_norm": 0.6592329722611846, + "language_loss": 0.5167014, + "learning_rate": 3.863904067990516e-06, + "loss": 0.53701174, + "num_input_tokens_seen": 142052330, + "router_z_loss_clip": 0.203125, + "router_z_loss_mlp": 0.0100708, + "step": 4982, + "time_per_iteration": 5.602055072784424 + }, + { + "auxiliary_loss_clip": 0.01138379, + "auxiliary_loss_mlp": 0.0104279, + "balance_loss_clip": 1.05513263, + "balance_loss_mlp": 1.02534986, + "epoch": 0.14459404561546052, + "flos": 31715262213120.0, + "grad_norm": 2.942273563433019, + "language_loss": 0.83946478, + "learning_rate": 3.86383590829064e-06, + "loss": 0.86127651, + "num_input_tokens_seen": 142067210, + "router_z_loss_clip": 0.83251953, + "router_z_loss_mlp": 0.17449951, + "step": 4983, + "time_per_iteration": 2.6156985759735107 + }, + { + "auxiliary_loss_clip": 0.01124162, + "auxiliary_loss_mlp": 0.01040609, + "balance_loss_clip": 1.04922128, + "balance_loss_mlp": 1.02570152, + "epoch": 0.14462306308397654, + "flos": 48025349498880.0, + "grad_norm": 3.507535463006958, + "language_loss": 0.71842057, + "learning_rate": 3.863767732128612e-06, + "loss": 0.74006826, + "num_input_tokens_seen": 142084600, + "router_z_loss_clip": 0.74951172, + "router_z_loss_mlp": 0.14904785, + "step": 4984, + "time_per_iteration": 2.664716958999634 + }, + { + "auxiliary_loss_clip": 0.01031441, + "auxiliary_loss_mlp": 0.01005735, + "balance_loss_clip": 1.0112021, + "balance_loss_mlp": 1.00477493, + "epoch": 0.1446520805524926, + "flos": 74079692277120.0, + "grad_norm": 0.7614117842545987, + "language_loss": 0.49158692, + "learning_rate": 3.863699539505033e-06, + "loss": 0.51195866, + "num_input_tokens_seen": 142150715, + "router_z_loss_clip": 0.203125, + "router_z_loss_mlp": 0.00958252, + "step": 4985, + "time_per_iteration": 3.3143718242645264 + }, + { + "auxiliary_loss_clip": 0.01133505, + "auxiliary_loss_mlp": 0.01038194, + "balance_loss_clip": 1.0527842, + "balance_loss_mlp": 1.02215481, + "epoch": 0.14468109802100865, + "flos": 19384544090880.0, + "grad_norm": 4.922808689320965, + "language_loss": 0.78247333, + "learning_rate": 3.863631330420505e-06, + "loss": 0.80419028, + "num_input_tokens_seen": 142164305, + "router_z_loss_clip": 0.80664062, + "router_z_loss_mlp": 0.16033936, + "step": 4986, + "time_per_iteration": 2.499889850616455 + }, + { + "auxiliary_loss_clip": 0.01140143, + "auxiliary_loss_mlp": 0.01044717, + "balance_loss_clip": 1.05691123, + "balance_loss_mlp": 1.02796197, + "epoch": 0.1447101154895247, + "flos": 27884563441920.0, + "grad_norm": 2.537390222342741, + "language_loss": 1.15934694, + "learning_rate": 3.863563104875632e-06, + "loss": 1.1811955, + "num_input_tokens_seen": 142184100, + "router_z_loss_clip": 0.83300781, + "router_z_loss_mlp": 0.16741943, + "step": 4987, + "time_per_iteration": 2.57472825050354 + }, + { + "auxiliary_loss_clip": 0.01139338, + "auxiliary_loss_mlp": 0.01047745, + "balance_loss_clip": 1.0537281, + "balance_loss_mlp": 1.0281589, + "epoch": 0.14473913295804075, + "flos": 14093789287680.0, + "grad_norm": 8.597837432761324, + "language_loss": 0.93182588, + "learning_rate": 3.863494862871015e-06, + "loss": 0.95369667, + "num_input_tokens_seen": 142195920, + "router_z_loss_clip": 0.85644531, + "router_z_loss_mlp": 0.19580078, + "step": 4988, + "time_per_iteration": 2.515868902206421 + }, + { + "auxiliary_loss_clip": 0.01133613, + "auxiliary_loss_mlp": 0.01045365, + "balance_loss_clip": 1.05338883, + "balance_loss_mlp": 1.02877736, + "epoch": 0.14476815042655677, + "flos": 19237417983360.0, + "grad_norm": 2.5328968928653177, + "language_loss": 1.02357244, + "learning_rate": 3.863426604407257e-06, + "loss": 1.04536223, + "num_input_tokens_seen": 142209085, + "router_z_loss_clip": 0.80224609, + "router_z_loss_mlp": 0.16583252, + "step": 4989, + "time_per_iteration": 2.534438371658325 + }, + { + "auxiliary_loss_clip": 0.01153006, + "auxiliary_loss_mlp": 0.01045629, + "balance_loss_clip": 1.06006265, + "balance_loss_mlp": 1.02489829, + "epoch": 0.14479716789507283, + "flos": 46783845768960.0, + "grad_norm": 2.600884503255259, + "language_loss": 1.16404808, + "learning_rate": 3.863358329484961e-06, + "loss": 1.18603444, + "num_input_tokens_seen": 142229605, + "router_z_loss_clip": 0.9296875, + "router_z_loss_mlp": 0.20727539, + "step": 4990, + "time_per_iteration": 2.772289991378784 + }, + { + "auxiliary_loss_clip": 0.01135856, + "auxiliary_loss_mlp": 0.0103911, + "balance_loss_clip": 1.05236852, + "balance_loss_mlp": 1.02138925, + "epoch": 0.14482618536358888, + "flos": 19057254341760.0, + "grad_norm": 2.598618395993597, + "language_loss": 0.89709699, + "learning_rate": 3.863290038104731e-06, + "loss": 0.91884667, + "num_input_tokens_seen": 142245485, + "router_z_loss_clip": 0.83447266, + "router_z_loss_mlp": 0.17712402, + "step": 4991, + "time_per_iteration": 2.4957807064056396 + }, + { + "auxiliary_loss_clip": 0.01141382, + "auxiliary_loss_mlp": 0.01035349, + "balance_loss_clip": 1.05624962, + "balance_loss_mlp": 1.0169785, + "epoch": 0.14485520283210493, + "flos": 30438350651520.0, + "grad_norm": 2.1045340225394256, + "language_loss": 0.76872283, + "learning_rate": 3.863221730267169e-06, + "loss": 0.79049015, + "num_input_tokens_seen": 142263885, + "router_z_loss_clip": 0.85107422, + "router_z_loss_mlp": 0.18383789, + "step": 4992, + "time_per_iteration": 2.5999882221221924 + }, + { + "auxiliary_loss_clip": 0.0112973, + "auxiliary_loss_mlp": 0.01040278, + "balance_loss_clip": 1.05225015, + "balance_loss_mlp": 1.02322483, + "epoch": 0.14488422030062098, + "flos": 36204706460160.0, + "grad_norm": 1.9093835217181234, + "language_loss": 0.73535687, + "learning_rate": 3.863153405972879e-06, + "loss": 0.75705689, + "num_input_tokens_seen": 142283255, + "router_z_loss_clip": 0.77490234, + "router_z_loss_mlp": 0.17053223, + "step": 4993, + "time_per_iteration": 2.638684034347534 + }, + { + "auxiliary_loss_clip": 0.01031811, + "auxiliary_loss_mlp": 0.01002657, + "balance_loss_clip": 1.0118525, + "balance_loss_mlp": 1.00175118, + "epoch": 0.14491323776913703, + "flos": 74775892659840.0, + "grad_norm": 0.710349275236597, + "language_loss": 0.49615479, + "learning_rate": 3.863085065222464e-06, + "loss": 0.51649946, + "num_input_tokens_seen": 142343915, + "router_z_loss_clip": 0.19921875, + "router_z_loss_mlp": 0.0090332, + "step": 4994, + "time_per_iteration": 3.1398842334747314 + }, + { + "auxiliary_loss_clip": 0.01134756, + "auxiliary_loss_mlp": 0.01051259, + "balance_loss_clip": 1.05486941, + "balance_loss_mlp": 1.0351541, + "epoch": 0.14494225523765306, + "flos": 26170403621760.0, + "grad_norm": 11.895242346519845, + "language_loss": 0.9200173, + "learning_rate": 3.863016708016527e-06, + "loss": 0.94187742, + "num_input_tokens_seen": 142359890, + "router_z_loss_clip": 0.79785156, + "router_z_loss_mlp": 0.16119385, + "step": 4995, + "time_per_iteration": 2.5498859882354736 + }, + { + "auxiliary_loss_clip": 0.01031291, + "auxiliary_loss_mlp": 0.0100556, + "balance_loss_clip": 1.01138365, + "balance_loss_mlp": 1.00454664, + "epoch": 0.1449712727061691, + "flos": 74781279699840.0, + "grad_norm": 0.6756693397869573, + "language_loss": 0.46523243, + "learning_rate": 3.8629483343556735e-06, + "loss": 0.48560095, + "num_input_tokens_seen": 142426770, + "router_z_loss_clip": 0.19921875, + "router_z_loss_mlp": 0.01013184, + "step": 4996, + "time_per_iteration": 3.2074880599975586 + }, + { + "auxiliary_loss_clip": 0.01031211, + "auxiliary_loss_mlp": 0.0100268, + "balance_loss_clip": 1.0113368, + "balance_loss_mlp": 1.00159478, + "epoch": 0.14500029017468516, + "flos": 57443284909440.0, + "grad_norm": 0.59479694493821, + "language_loss": 0.47211102, + "learning_rate": 3.862879944240506e-06, + "loss": 0.49244991, + "num_input_tokens_seen": 142488090, + "router_z_loss_clip": 0.19921875, + "router_z_loss_mlp": 0.01086426, + "step": 4997, + "time_per_iteration": 3.082418918609619 + }, + { + "auxiliary_loss_clip": 0.01130176, + "auxiliary_loss_mlp": 0.01047957, + "balance_loss_clip": 1.0520463, + "balance_loss_mlp": 1.03162503, + "epoch": 0.1450293076432012, + "flos": 30843890179200.0, + "grad_norm": 1.875896784043469, + "language_loss": 0.79470563, + "learning_rate": 3.86281153767163e-06, + "loss": 0.81648695, + "num_input_tokens_seen": 142506990, + "router_z_loss_clip": 0.78173828, + "router_z_loss_mlp": 0.16326904, + "step": 4998, + "time_per_iteration": 2.5979485511779785 + }, + { + "auxiliary_loss_clip": 0.01136287, + "auxiliary_loss_mlp": 0.0104158, + "balance_loss_clip": 1.05361509, + "balance_loss_mlp": 1.02373409, + "epoch": 0.14505832511171726, + "flos": 16978026643200.0, + "grad_norm": 4.487725872730459, + "language_loss": 0.73549902, + "learning_rate": 3.862743114649647e-06, + "loss": 0.75727767, + "num_input_tokens_seen": 142522395, + "router_z_loss_clip": 0.828125, + "router_z_loss_mlp": 0.17828369, + "step": 4999, + "time_per_iteration": 2.5995590686798096 + }, + { + "auxiliary_loss_clip": 0.01152731, + "auxiliary_loss_mlp": 0.0104999, + "balance_loss_clip": 1.05853748, + "balance_loss_mlp": 1.029248, + "epoch": 0.1450873425802333, + "flos": 15046391928960.0, + "grad_norm": 2.798746126765893, + "language_loss": 0.84143788, + "learning_rate": 3.862674675175164e-06, + "loss": 0.86346507, + "num_input_tokens_seen": 142534995, + "router_z_loss_clip": 0.94287109, + "router_z_loss_mlp": 0.20739746, + "step": 5000, + "time_per_iteration": 2.522448778152466 + }, + { + "auxiliary_loss_clip": 0.0114427, + "auxiliary_loss_mlp": 0.01045433, + "balance_loss_clip": 1.05709195, + "balance_loss_mlp": 1.02742088, + "epoch": 0.14511636004874934, + "flos": 74733873747840.0, + "grad_norm": 2.5768184651048687, + "language_loss": 0.91546565, + "learning_rate": 3.8626062192487845e-06, + "loss": 0.93736273, + "num_input_tokens_seen": 142559210, + "router_z_loss_clip": 0.87109375, + "router_z_loss_mlp": 0.18029785, + "step": 5001, + "time_per_iteration": 2.9869086742401123 + }, + { + "auxiliary_loss_clip": 0.01030802, + "auxiliary_loss_mlp": 0.01003223, + "balance_loss_clip": 1.01102281, + "balance_loss_mlp": 1.00206709, + "epoch": 0.1451453775172654, + "flos": 74493420105600.0, + "grad_norm": 0.6087893776832631, + "language_loss": 0.44876665, + "learning_rate": 3.862537746871113e-06, + "loss": 0.46910691, + "num_input_tokens_seen": 142628375, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.01153564, + "step": 5002, + "time_per_iteration": 3.239098072052002 + }, + { + "auxiliary_loss_clip": 0.01148031, + "auxiliary_loss_mlp": 0.01056877, + "balance_loss_clip": 1.05817294, + "balance_loss_mlp": 1.03697526, + "epoch": 0.14517439498578144, + "flos": 28069072629120.0, + "grad_norm": 2.874375737751596, + "language_loss": 0.96573043, + "learning_rate": 3.862469258042755e-06, + "loss": 0.9877795, + "num_input_tokens_seen": 142645535, + "router_z_loss_clip": 0.89892578, + "router_z_loss_mlp": 0.19885254, + "step": 5003, + "time_per_iteration": 2.6042635440826416 + }, + { + "auxiliary_loss_clip": 0.01030929, + "auxiliary_loss_mlp": 0.01000959, + "balance_loss_clip": 1.01102769, + "balance_loss_mlp": 0.99988598, + "epoch": 0.1452034124542975, + "flos": 56930336737920.0, + "grad_norm": 0.7173463504298548, + "language_loss": 0.50468349, + "learning_rate": 3.862400752764314e-06, + "loss": 0.52500236, + "num_input_tokens_seen": 142704175, + "router_z_loss_clip": 0.19921875, + "router_z_loss_mlp": 0.01074219, + "step": 5004, + "time_per_iteration": 3.0564117431640625 + }, + { + "auxiliary_loss_clip": 0.01031342, + "auxiliary_loss_mlp": 0.00999738, + "balance_loss_clip": 1.01152813, + "balance_loss_mlp": 0.99865305, + "epoch": 0.14523242992281354, + "flos": 66420162241920.0, + "grad_norm": 0.6765114883875202, + "language_loss": 0.51003671, + "learning_rate": 3.862332231036396e-06, + "loss": 0.53034747, + "num_input_tokens_seen": 142769150, + "router_z_loss_clip": 0.19824219, + "router_z_loss_mlp": 0.01086426, + "step": 5005, + "time_per_iteration": 3.132004976272583 + }, + { + "auxiliary_loss_clip": 0.01146292, + "auxiliary_loss_mlp": 0.01047058, + "balance_loss_clip": 1.05916667, + "balance_loss_mlp": 1.02736497, + "epoch": 0.14526144739132957, + "flos": 12267803450880.0, + "grad_norm": 2.601983933655093, + "language_loss": 0.8771531, + "learning_rate": 3.862263692859607e-06, + "loss": 0.89908659, + "num_input_tokens_seen": 142783330, + "router_z_loss_clip": 0.87109375, + "router_z_loss_mlp": 0.19689941, + "step": 5006, + "time_per_iteration": 2.6001877784729004 + }, + { + "auxiliary_loss_clip": 0.01137343, + "auxiliary_loss_mlp": 0.01043648, + "balance_loss_clip": 1.05193424, + "balance_loss_mlp": 1.02503967, + "epoch": 0.14529046485984562, + "flos": 22119464609280.0, + "grad_norm": 2.131126609401256, + "language_loss": 0.85529739, + "learning_rate": 3.86219513823455e-06, + "loss": 0.87710726, + "num_input_tokens_seen": 142798345, + "router_z_loss_clip": 0.85302734, + "router_z_loss_mlp": 0.18603516, + "step": 5007, + "time_per_iteration": 2.6630027294158936 + }, + { + "auxiliary_loss_clip": 0.01136418, + "auxiliary_loss_mlp": 0.01045227, + "balance_loss_clip": 1.05638969, + "balance_loss_mlp": 1.02754784, + "epoch": 0.14531948232836167, + "flos": 16028368917120.0, + "grad_norm": 2.8795125257145147, + "language_loss": 0.81638014, + "learning_rate": 3.8621265671618334e-06, + "loss": 0.83819664, + "num_input_tokens_seen": 142811095, + "router_z_loss_clip": 0.80078125, + "router_z_loss_mlp": 0.17675781, + "step": 5008, + "time_per_iteration": 2.634793996810913 + }, + { + "auxiliary_loss_clip": 0.01137375, + "auxiliary_loss_mlp": 0.01051753, + "balance_loss_clip": 1.05603588, + "balance_loss_mlp": 1.03469992, + "epoch": 0.14534849979687772, + "flos": 20769403000320.0, + "grad_norm": 2.4012246986564856, + "language_loss": 0.74799132, + "learning_rate": 3.862057979642061e-06, + "loss": 0.76988256, + "num_input_tokens_seen": 142823575, + "router_z_loss_clip": 0.81445312, + "router_z_loss_mlp": 0.17059326, + "step": 5009, + "time_per_iteration": 2.5166125297546387 + }, + { + "auxiliary_loss_clip": 0.01136731, + "auxiliary_loss_mlp": 0.01041256, + "balance_loss_clip": 1.05495918, + "balance_loss_mlp": 1.02347636, + "epoch": 0.14537751726539377, + "flos": 48755090206080.0, + "grad_norm": 4.637156155011977, + "language_loss": 0.84323704, + "learning_rate": 3.861989375675839e-06, + "loss": 0.86501694, + "num_input_tokens_seen": 142842025, + "router_z_loss_clip": 0.81787109, + "router_z_loss_mlp": 0.17803955, + "step": 5010, + "time_per_iteration": 2.780186891555786 + }, + { + "auxiliary_loss_clip": 0.01033194, + "auxiliary_loss_mlp": 0.01013362, + "balance_loss_clip": 1.01332426, + "balance_loss_mlp": 1.01233697, + "epoch": 0.14540653473390983, + "flos": 69656574493440.0, + "grad_norm": 0.7588644629180384, + "language_loss": 0.50269377, + "learning_rate": 3.861920755263774e-06, + "loss": 0.52315938, + "num_input_tokens_seen": 142898880, + "router_z_loss_clip": 0.19921875, + "router_z_loss_mlp": 0.01025391, + "step": 5011, + "time_per_iteration": 3.06974196434021 + }, + { + "auxiliary_loss_clip": 0.01139967, + "auxiliary_loss_mlp": 0.01051432, + "balance_loss_clip": 1.05808365, + "balance_loss_mlp": 1.03412247, + "epoch": 0.14543555220242585, + "flos": 26501284730880.0, + "grad_norm": 1.932939185063821, + "language_loss": 0.79482704, + "learning_rate": 3.86185211840647e-06, + "loss": 0.81674105, + "num_input_tokens_seen": 142920875, + "router_z_loss_clip": 0.81787109, + "router_z_loss_mlp": 0.1730957, + "step": 5012, + "time_per_iteration": 2.7091240882873535 + }, + { + "auxiliary_loss_clip": 0.01145223, + "auxiliary_loss_mlp": 0.01038112, + "balance_loss_clip": 1.05765033, + "balance_loss_mlp": 1.01958704, + "epoch": 0.1454645696709419, + "flos": 50323452721920.0, + "grad_norm": 2.517258912510692, + "language_loss": 0.88360399, + "learning_rate": 3.861783465104536e-06, + "loss": 0.90543735, + "num_input_tokens_seen": 142939925, + "router_z_loss_clip": 0.87597656, + "router_z_loss_mlp": 0.18530273, + "step": 5013, + "time_per_iteration": 2.7780404090881348 + }, + { + "auxiliary_loss_clip": 0.01146051, + "auxiliary_loss_mlp": 0.01050316, + "balance_loss_clip": 1.06002879, + "balance_loss_mlp": 1.03267288, + "epoch": 0.14549358713945795, + "flos": 33685285587840.0, + "grad_norm": 2.5381290724380023, + "language_loss": 0.8178916, + "learning_rate": 3.861714795358576e-06, + "loss": 0.83985531, + "num_input_tokens_seen": 142954940, + "router_z_loss_clip": 0.86083984, + "router_z_loss_mlp": 0.1763916, + "step": 5014, + "time_per_iteration": 2.6312174797058105 + }, + { + "auxiliary_loss_clip": 0.01134282, + "auxiliary_loss_mlp": 0.01042414, + "balance_loss_clip": 1.05509984, + "balance_loss_mlp": 1.025635, + "epoch": 0.145522604607974, + "flos": 40763960789760.0, + "grad_norm": 2.120267805832133, + "language_loss": 0.87756735, + "learning_rate": 3.861646109169198e-06, + "loss": 0.89933431, + "num_input_tokens_seen": 142973255, + "router_z_loss_clip": 0.79199219, + "router_z_loss_mlp": 0.16772461, + "step": 5015, + "time_per_iteration": 2.7795186042785645 + }, + { + "auxiliary_loss_clip": 0.01032942, + "auxiliary_loss_mlp": 0.01001903, + "balance_loss_clip": 1.01290798, + "balance_loss_mlp": 1.00097299, + "epoch": 0.14555162207649006, + "flos": 74783901392640.0, + "grad_norm": 0.6406741623766007, + "language_loss": 0.50330925, + "learning_rate": 3.861577406537009e-06, + "loss": 0.52365768, + "num_input_tokens_seen": 143038680, + "router_z_loss_clip": 0.20117188, + "router_z_loss_mlp": 0.00927734, + "step": 5016, + "time_per_iteration": 3.196516990661621 + }, + { + "auxiliary_loss_clip": 0.01135477, + "auxiliary_loss_mlp": 0.01046208, + "balance_loss_clip": 1.05646729, + "balance_loss_mlp": 1.02898169, + "epoch": 0.1455806395450061, + "flos": 30365416085760.0, + "grad_norm": 4.1422021799386926, + "language_loss": 0.99307239, + "learning_rate": 3.861508687462615e-06, + "loss": 1.01488924, + "num_input_tokens_seen": 143055410, + "router_z_loss_clip": 0.79052734, + "router_z_loss_mlp": 0.17224121, + "step": 5017, + "time_per_iteration": 2.6930994987487793 + }, + { + "auxiliary_loss_clip": 0.01142693, + "auxiliary_loss_mlp": 0.01032636, + "balance_loss_clip": 1.06181049, + "balance_loss_mlp": 1.01567268, + "epoch": 0.14560965701352213, + "flos": 25988947090560.0, + "grad_norm": 1.8633425372916703, + "language_loss": 0.77067077, + "learning_rate": 3.861439951946622e-06, + "loss": 0.79242402, + "num_input_tokens_seen": 143071390, + "router_z_loss_clip": 0.80859375, + "router_z_loss_mlp": 0.16967773, + "step": 5018, + "time_per_iteration": 2.5534298419952393 + }, + { + "auxiliary_loss_clip": 0.01031247, + "auxiliary_loss_mlp": 0.01001593, + "balance_loss_clip": 1.01110649, + "balance_loss_mlp": 1.00068688, + "epoch": 0.14563867448203818, + "flos": 61790774607360.0, + "grad_norm": 0.5946353720299196, + "language_loss": 0.49133778, + "learning_rate": 3.861371199989638e-06, + "loss": 0.51166618, + "num_input_tokens_seen": 143138490, + "router_z_loss_clip": 0.20117188, + "router_z_loss_mlp": 0.0090332, + "step": 5019, + "time_per_iteration": 3.245859146118164 + }, + { + "auxiliary_loss_clip": 0.01141638, + "auxiliary_loss_mlp": 0.01047123, + "balance_loss_clip": 1.0581913, + "balance_loss_mlp": 1.02934909, + "epoch": 0.14566769195055423, + "flos": 19930027006080.0, + "grad_norm": 2.8392671082090004, + "language_loss": 0.91299736, + "learning_rate": 3.861302431592271e-06, + "loss": 0.93488503, + "num_input_tokens_seen": 143151160, + "router_z_loss_clip": 0.83349609, + "router_z_loss_mlp": 0.17773438, + "step": 5020, + "time_per_iteration": 2.5810890197753906 + }, + { + "auxiliary_loss_clip": 0.01144214, + "auxiliary_loss_mlp": 0.0105247, + "balance_loss_clip": 1.05778396, + "balance_loss_mlp": 1.03635263, + "epoch": 0.14569670941907029, + "flos": 30984803233920.0, + "grad_norm": 2.1198359497251538, + "language_loss": 0.80373245, + "learning_rate": 3.861233646755127e-06, + "loss": 0.82569933, + "num_input_tokens_seen": 143175920, + "router_z_loss_clip": 0.86474609, + "router_z_loss_mlp": 0.16113281, + "step": 5021, + "time_per_iteration": 2.9617879390716553 + }, + { + "auxiliary_loss_clip": 0.01031576, + "auxiliary_loss_mlp": 0.01005333, + "balance_loss_clip": 1.01154912, + "balance_loss_mlp": 1.00437951, + "epoch": 0.14572572688758634, + "flos": 74770002829440.0, + "grad_norm": 0.6680739617051271, + "language_loss": 0.53060013, + "learning_rate": 3.861164845478815e-06, + "loss": 0.55096924, + "num_input_tokens_seen": 143238255, + "router_z_loss_clip": 0.20117188, + "router_z_loss_mlp": 0.00952148, + "step": 5022, + "time_per_iteration": 3.0968034267425537 + }, + { + "auxiliary_loss_clip": 0.01127421, + "auxiliary_loss_mlp": 0.01041215, + "balance_loss_clip": 1.05430675, + "balance_loss_mlp": 1.02739835, + "epoch": 0.14575474435610236, + "flos": 28326372727680.0, + "grad_norm": 2.4686271705651035, + "language_loss": 0.68104571, + "learning_rate": 3.861096027763942e-06, + "loss": 0.70273209, + "num_input_tokens_seen": 143252165, + "router_z_loss_clip": 0.73046875, + "router_z_loss_mlp": 0.13800049, + "step": 5023, + "time_per_iteration": 2.5099854469299316 + }, + { + "auxiliary_loss_clip": 0.01130866, + "auxiliary_loss_mlp": 0.01047582, + "balance_loss_clip": 1.05331445, + "balance_loss_mlp": 1.03226328, + "epoch": 0.1457837618246184, + "flos": 26754023802240.0, + "grad_norm": 2.0863253335559837, + "language_loss": 0.88753378, + "learning_rate": 3.8610271936111155e-06, + "loss": 0.90931827, + "num_input_tokens_seen": 143267320, + "router_z_loss_clip": 0.77587891, + "router_z_loss_mlp": 0.15319824, + "step": 5024, + "time_per_iteration": 2.570892810821533 + }, + { + "auxiliary_loss_clip": 0.0114038, + "auxiliary_loss_mlp": 0.01046248, + "balance_loss_clip": 1.05905724, + "balance_loss_mlp": 1.02954102, + "epoch": 0.14581277929313446, + "flos": 45069219072000.0, + "grad_norm": 1.990130096667515, + "language_loss": 0.78189749, + "learning_rate": 3.860958343020944e-06, + "loss": 0.80376381, + "num_input_tokens_seen": 143286695, + "router_z_loss_clip": 0.81298828, + "router_z_loss_mlp": 0.16711426, + "step": 5025, + "time_per_iteration": 2.7403883934020996 + }, + { + "auxiliary_loss_clip": 0.01141661, + "auxiliary_loss_mlp": 0.01042799, + "balance_loss_clip": 1.06002235, + "balance_loss_mlp": 1.02608025, + "epoch": 0.14584179676165052, + "flos": 16246669824000.0, + "grad_norm": 2.262365518239563, + "language_loss": 0.77616203, + "learning_rate": 3.860889475994035e-06, + "loss": 0.79800665, + "num_input_tokens_seen": 143299190, + "router_z_loss_clip": 0.81591797, + "router_z_loss_mlp": 0.16705322, + "step": 5026, + "time_per_iteration": 2.5204739570617676 + }, + { + "auxiliary_loss_clip": 0.01031952, + "auxiliary_loss_mlp": 0.0100289, + "balance_loss_clip": 1.01204896, + "balance_loss_mlp": 1.00190663, + "epoch": 0.14587081423016657, + "flos": 48869504979840.0, + "grad_norm": 0.731686021893711, + "language_loss": 0.47973496, + "learning_rate": 3.860820592530997e-06, + "loss": 0.50008339, + "num_input_tokens_seen": 143345855, + "router_z_loss_clip": 0.19921875, + "router_z_loss_mlp": 0.00982666, + "step": 5027, + "time_per_iteration": 2.847520112991333 + }, + { + "auxiliary_loss_clip": 0.01031691, + "auxiliary_loss_mlp": 0.01003471, + "balance_loss_clip": 1.01176894, + "balance_loss_mlp": 1.00242805, + "epoch": 0.14589983169868262, + "flos": 69049757105280.0, + "grad_norm": 1.658419230362658, + "language_loss": 0.46733868, + "learning_rate": 3.860751692632439e-06, + "loss": 0.48769033, + "num_input_tokens_seen": 143406805, + "router_z_loss_clip": 0.19921875, + "router_z_loss_mlp": 0.01043701, + "step": 5028, + "time_per_iteration": 3.0790460109710693 + }, + { + "auxiliary_loss_clip": 0.01140282, + "auxiliary_loss_mlp": 0.01052269, + "balance_loss_clip": 1.05616474, + "balance_loss_mlp": 1.03363705, + "epoch": 0.14592884916719864, + "flos": 33394840214400.0, + "grad_norm": 2.045253182223213, + "language_loss": 0.8348074, + "learning_rate": 3.860682776298968e-06, + "loss": 0.85673296, + "num_input_tokens_seen": 143422145, + "router_z_loss_clip": 0.84130859, + "router_z_loss_mlp": 0.1862793, + "step": 5029, + "time_per_iteration": 2.6848342418670654 + }, + { + "auxiliary_loss_clip": 0.01141977, + "auxiliary_loss_mlp": 0.01046806, + "balance_loss_clip": 1.05855608, + "balance_loss_mlp": 1.02809644, + "epoch": 0.1459578666357147, + "flos": 29236348903680.0, + "grad_norm": 2.0270991218144325, + "language_loss": 0.79838014, + "learning_rate": 3.860613843531196e-06, + "loss": 0.82026803, + "num_input_tokens_seen": 143441885, + "router_z_loss_clip": 0.83447266, + "router_z_loss_mlp": 0.18707275, + "step": 5030, + "time_per_iteration": 2.755824565887451 + }, + { + "auxiliary_loss_clip": 0.01137265, + "auxiliary_loss_mlp": 0.0104383, + "balance_loss_clip": 1.05585265, + "balance_loss_mlp": 1.02625823, + "epoch": 0.14598688410423075, + "flos": 14277328807680.0, + "grad_norm": 2.4798206607364452, + "language_loss": 0.63102806, + "learning_rate": 3.860544894329728e-06, + "loss": 0.65283895, + "num_input_tokens_seen": 143454655, + "router_z_loss_clip": 0.81494141, + "router_z_loss_mlp": 0.17578125, + "step": 5031, + "time_per_iteration": 2.521080255508423 + }, + { + "auxiliary_loss_clip": 0.01031389, + "auxiliary_loss_mlp": 0.01000851, + "balance_loss_clip": 1.01143646, + "balance_loss_mlp": 0.99981958, + "epoch": 0.1460159015727468, + "flos": 64629799718400.0, + "grad_norm": 0.6569779556445702, + "language_loss": 0.49340466, + "learning_rate": 3.860475928695175e-06, + "loss": 0.51372707, + "num_input_tokens_seen": 143517825, + "router_z_loss_clip": 0.19921875, + "router_z_loss_mlp": 0.01031494, + "step": 5032, + "time_per_iteration": 3.0735678672790527 + }, + { + "auxiliary_loss_clip": 0.01154458, + "auxiliary_loss_mlp": 0.01057099, + "balance_loss_clip": 1.06492674, + "balance_loss_mlp": 1.03831148, + "epoch": 0.14604491904126285, + "flos": 21760430215680.0, + "grad_norm": 2.7978038566690824, + "language_loss": 0.82293296, + "learning_rate": 3.860406946628146e-06, + "loss": 0.84504855, + "num_input_tokens_seen": 143535860, + "router_z_loss_clip": 0.89404297, + "router_z_loss_mlp": 0.18786621, + "step": 5033, + "time_per_iteration": 2.6465041637420654 + }, + { + "auxiliary_loss_clip": 0.01141288, + "auxiliary_loss_mlp": 0.01049716, + "balance_loss_clip": 1.05540264, + "balance_loss_mlp": 1.03218603, + "epoch": 0.14607393650977887, + "flos": 26863730916480.0, + "grad_norm": 2.024801554805661, + "language_loss": 0.78374863, + "learning_rate": 3.860337948129249e-06, + "loss": 0.80565864, + "num_input_tokens_seen": 143551355, + "router_z_loss_clip": 0.86035156, + "router_z_loss_mlp": 0.17541504, + "step": 5034, + "time_per_iteration": 2.6544265747070312 + }, + { + "auxiliary_loss_clip": 0.01140657, + "auxiliary_loss_mlp": 0.01048383, + "balance_loss_clip": 1.06107926, + "balance_loss_mlp": 1.03171158, + "epoch": 0.14610295397829492, + "flos": 34709924954880.0, + "grad_norm": 1.9626223235837648, + "language_loss": 0.83445787, + "learning_rate": 3.860268933199095e-06, + "loss": 0.85634828, + "num_input_tokens_seen": 143571900, + "router_z_loss_clip": 0.79638672, + "router_z_loss_mlp": 0.16687012, + "step": 5035, + "time_per_iteration": 2.772928476333618 + }, + { + "auxiliary_loss_clip": 0.01138524, + "auxiliary_loss_mlp": 0.01043383, + "balance_loss_clip": 1.05866337, + "balance_loss_mlp": 1.02589488, + "epoch": 0.14613197144681098, + "flos": 26573501024640.0, + "grad_norm": 1.7806280463276452, + "language_loss": 0.70558715, + "learning_rate": 3.8601999018382935e-06, + "loss": 0.7274062, + "num_input_tokens_seen": 143590530, + "router_z_loss_clip": 0.79882812, + "router_z_loss_mlp": 0.17480469, + "step": 5036, + "time_per_iteration": 2.635138511657715 + }, + { + "auxiliary_loss_clip": 0.01032256, + "auxiliary_loss_mlp": 0.0100073, + "balance_loss_clip": 1.01221442, + "balance_loss_mlp": 0.99979383, + "epoch": 0.14616098891532703, + "flos": 73421449591680.0, + "grad_norm": 0.6893052326554748, + "language_loss": 0.5184896, + "learning_rate": 3.860130854047453e-06, + "loss": 0.53881949, + "num_input_tokens_seen": 143649390, + "router_z_loss_clip": 0.20117188, + "router_z_loss_mlp": 0.00933838, + "step": 5037, + "time_per_iteration": 3.119408369064331 + }, + { + "auxiliary_loss_clip": 0.01134892, + "auxiliary_loss_mlp": 0.01040944, + "balance_loss_clip": 1.05624688, + "balance_loss_mlp": 1.02520227, + "epoch": 0.14619000638384308, + "flos": 23324985889920.0, + "grad_norm": 2.3410543332935214, + "language_loss": 0.77725548, + "learning_rate": 3.860061789827185e-06, + "loss": 0.79901385, + "num_input_tokens_seen": 143664475, + "router_z_loss_clip": 0.78710938, + "router_z_loss_mlp": 0.15722656, + "step": 5038, + "time_per_iteration": 2.5997612476348877 + }, + { + "auxiliary_loss_clip": 0.01142557, + "auxiliary_loss_mlp": 0.010432, + "balance_loss_clip": 1.05836678, + "balance_loss_mlp": 1.02525902, + "epoch": 0.14621902385235913, + "flos": 23033606762880.0, + "grad_norm": 1.893022700588286, + "language_loss": 0.68927836, + "learning_rate": 3.859992709178097e-06, + "loss": 0.71113592, + "num_input_tokens_seen": 143679075, + "router_z_loss_clip": 0.84179688, + "router_z_loss_mlp": 0.17944336, + "step": 5039, + "time_per_iteration": 2.6525521278381348 + }, + { + "auxiliary_loss_clip": 0.01143117, + "auxiliary_loss_mlp": 0.01057267, + "balance_loss_clip": 1.05933845, + "balance_loss_mlp": 1.0378356, + "epoch": 0.14624804132087516, + "flos": 29928347395200.0, + "grad_norm": 2.171330939589758, + "language_loss": 0.78100806, + "learning_rate": 3.859923612100803e-06, + "loss": 0.80301189, + "num_input_tokens_seen": 143695695, + "router_z_loss_clip": 0.83740234, + "router_z_loss_mlp": 0.19421387, + "step": 5040, + "time_per_iteration": 2.6657674312591553 + }, + { + "auxiliary_loss_clip": 0.01154476, + "auxiliary_loss_mlp": 0.01049416, + "balance_loss_clip": 1.06405473, + "balance_loss_mlp": 1.02898324, + "epoch": 0.1462770587893912, + "flos": 16026106360320.0, + "grad_norm": 2.1007390668063226, + "language_loss": 0.86150527, + "learning_rate": 3.859854498595909e-06, + "loss": 0.88354415, + "num_input_tokens_seen": 143708480, + "router_z_loss_clip": 0.90429688, + "router_z_loss_mlp": 0.20458984, + "step": 5041, + "time_per_iteration": 2.5979244709014893 + }, + { + "auxiliary_loss_clip": 0.01139444, + "auxiliary_loss_mlp": 0.01048168, + "balance_loss_clip": 1.05891633, + "balance_loss_mlp": 1.03107882, + "epoch": 0.14630607625790726, + "flos": 26717574476160.0, + "grad_norm": 2.3397623529210256, + "language_loss": 0.61118376, + "learning_rate": 3.859785368664028e-06, + "loss": 0.63305986, + "num_input_tokens_seen": 143726695, + "router_z_loss_clip": 0.80615234, + "router_z_loss_mlp": 0.17108154, + "step": 5042, + "time_per_iteration": 2.746168851852417 + }, + { + "auxiliary_loss_clip": 0.01130183, + "auxiliary_loss_mlp": 0.01034288, + "balance_loss_clip": 1.05102348, + "balance_loss_mlp": 1.01925528, + "epoch": 0.1463350937264233, + "flos": 16321435983360.0, + "grad_norm": 2.9450903125140053, + "language_loss": 0.93437272, + "learning_rate": 3.859716222305771e-06, + "loss": 0.95601743, + "num_input_tokens_seen": 143739125, + "router_z_loss_clip": 0.79248047, + "router_z_loss_mlp": 0.15032959, + "step": 5043, + "time_per_iteration": 2.5398926734924316 + }, + { + "auxiliary_loss_clip": 0.01137928, + "auxiliary_loss_mlp": 0.01037596, + "balance_loss_clip": 1.05758917, + "balance_loss_mlp": 1.01973832, + "epoch": 0.14636411119493936, + "flos": 20368963203840.0, + "grad_norm": 2.2507350211500157, + "language_loss": 0.72673225, + "learning_rate": 3.859647059521747e-06, + "loss": 0.74848747, + "num_input_tokens_seen": 143753705, + "router_z_loss_clip": 0.80273438, + "router_z_loss_mlp": 0.1786499, + "step": 5044, + "time_per_iteration": 2.588841438293457 + }, + { + "auxiliary_loss_clip": 0.01035091, + "auxiliary_loss_mlp": 0.0100033, + "balance_loss_clip": 1.01504076, + "balance_loss_mlp": 0.99933475, + "epoch": 0.1463931286634554, + "flos": 74771403459840.0, + "grad_norm": 0.6885575846988874, + "language_loss": 0.47458005, + "learning_rate": 3.8595778803125675e-06, + "loss": 0.49493426, + "num_input_tokens_seen": 143808190, + "router_z_loss_clip": 0.20117188, + "router_z_loss_mlp": 0.00994873, + "step": 5045, + "time_per_iteration": 3.136564016342163 + }, + { + "auxiliary_loss_clip": 0.01136062, + "auxiliary_loss_mlp": 0.01037993, + "balance_loss_clip": 1.05928743, + "balance_loss_mlp": 1.02195311, + "epoch": 0.14642214613197144, + "flos": 12781110758400.0, + "grad_norm": 3.4806464780371265, + "language_loss": 0.88610816, + "learning_rate": 3.859508684678844e-06, + "loss": 0.90784872, + "num_input_tokens_seen": 143821235, + "router_z_loss_clip": 0.76757812, + "router_z_loss_mlp": 0.1605835, + "step": 5046, + "time_per_iteration": 2.6121041774749756 + }, + { + "auxiliary_loss_clip": 0.01034332, + "auxiliary_loss_mlp": 0.01004407, + "balance_loss_clip": 1.01431501, + "balance_loss_mlp": 1.00347733, + "epoch": 0.1464511636004875, + "flos": 67640910888960.0, + "grad_norm": 0.9078273213580621, + "language_loss": 0.46937883, + "learning_rate": 3.859439472621188e-06, + "loss": 0.48976624, + "num_input_tokens_seen": 143895230, + "router_z_loss_clip": 0.20117188, + "router_z_loss_mlp": 0.00927734, + "step": 5047, + "time_per_iteration": 3.386573314666748 + }, + { + "auxiliary_loss_clip": 0.01033729, + "auxiliary_loss_mlp": 0.01002869, + "balance_loss_clip": 1.01352954, + "balance_loss_mlp": 1.00188541, + "epoch": 0.14648018106900354, + "flos": 72514633812480.0, + "grad_norm": 0.669808735504, + "language_loss": 0.47460744, + "learning_rate": 3.859370244140208e-06, + "loss": 0.49497342, + "num_input_tokens_seen": 143961165, + "router_z_loss_clip": 0.20214844, + "router_z_loss_mlp": 0.00982666, + "step": 5048, + "time_per_iteration": 3.206559658050537 + }, + { + "auxiliary_loss_clip": 0.01033118, + "auxiliary_loss_mlp": 0.01002501, + "balance_loss_clip": 1.01306117, + "balance_loss_mlp": 1.00151157, + "epoch": 0.1465091985375196, + "flos": 67551922944000.0, + "grad_norm": 0.692629377262558, + "language_loss": 0.49857271, + "learning_rate": 3.859300999236519e-06, + "loss": 0.51892889, + "num_input_tokens_seen": 144020665, + "router_z_loss_clip": 0.20117188, + "router_z_loss_mlp": 0.0098877, + "step": 5049, + "time_per_iteration": 3.0775258541107178 + }, + { + "auxiliary_loss_clip": 0.01032287, + "auxiliary_loss_mlp": 0.01004106, + "balance_loss_clip": 1.01223779, + "balance_loss_mlp": 1.00310504, + "epoch": 0.14653821600603564, + "flos": 58866209256960.0, + "grad_norm": 0.6625593419382301, + "language_loss": 0.49210548, + "learning_rate": 3.859231737910732e-06, + "loss": 0.51246941, + "num_input_tokens_seen": 144080745, + "router_z_loss_clip": 0.20117188, + "router_z_loss_mlp": 0.01000977, + "step": 5050, + "time_per_iteration": 7.8272154331207275 + }, + { + "auxiliary_loss_clip": 0.01150521, + "auxiliary_loss_mlp": 0.01048268, + "balance_loss_clip": 1.06064987, + "balance_loss_mlp": 1.0276804, + "epoch": 0.14656723347455167, + "flos": 28250529160320.0, + "grad_norm": 2.1236495558315847, + "language_loss": 0.88767874, + "learning_rate": 3.859162460163457e-06, + "loss": 0.90966654, + "num_input_tokens_seen": 144097910, + "router_z_loss_clip": 0.89892578, + "router_z_loss_mlp": 0.20605469, + "step": 5051, + "time_per_iteration": 4.975882530212402 + }, + { + "auxiliary_loss_clip": 0.01133467, + "auxiliary_loss_mlp": 0.01032964, + "balance_loss_clip": 1.05607057, + "balance_loss_mlp": 1.01714468, + "epoch": 0.14659625094306772, + "flos": 24217295552640.0, + "grad_norm": 2.018419862300059, + "language_loss": 0.74330258, + "learning_rate": 3.859093165995307e-06, + "loss": 0.76496691, + "num_input_tokens_seen": 144115095, + "router_z_loss_clip": 0.77416992, + "router_z_loss_mlp": 0.15808105, + "step": 5052, + "time_per_iteration": 5.07356071472168 + }, + { + "auxiliary_loss_clip": 0.01127441, + "auxiliary_loss_mlp": 0.01029521, + "balance_loss_clip": 1.0536437, + "balance_loss_mlp": 1.01470959, + "epoch": 0.14662526841158377, + "flos": 17120519896320.0, + "grad_norm": 2.664933792403011, + "language_loss": 0.89433378, + "learning_rate": 3.859023855406893e-06, + "loss": 0.91590345, + "num_input_tokens_seen": 144127845, + "router_z_loss_clip": 0.73876953, + "router_z_loss_mlp": 0.14801025, + "step": 5053, + "time_per_iteration": 2.5689241886138916 + }, + { + "auxiliary_loss_clip": 0.011345, + "auxiliary_loss_mlp": 0.01041932, + "balance_loss_clip": 1.05767834, + "balance_loss_mlp": 1.02699494, + "epoch": 0.14665428588009982, + "flos": 46162842508800.0, + "grad_norm": 2.1643481913038634, + "language_loss": 0.79731894, + "learning_rate": 3.858954528398829e-06, + "loss": 0.81908327, + "num_input_tokens_seen": 144147005, + "router_z_loss_clip": 0.76855469, + "router_z_loss_mlp": 0.14929199, + "step": 5054, + "time_per_iteration": 2.826225996017456 + }, + { + "auxiliary_loss_clip": 0.01144399, + "auxiliary_loss_mlp": 0.01050545, + "balance_loss_clip": 1.05844045, + "balance_loss_mlp": 1.03055358, + "epoch": 0.14668330334861587, + "flos": 25548215212800.0, + "grad_norm": 4.686949415317561, + "language_loss": 0.8287701, + "learning_rate": 3.858885184971726e-06, + "loss": 0.85071957, + "num_input_tokens_seen": 144161075, + "router_z_loss_clip": 0.859375, + "router_z_loss_mlp": 0.20007324, + "step": 5055, + "time_per_iteration": 2.6973953247070312 + }, + { + "auxiliary_loss_clip": 0.01030963, + "auxiliary_loss_mlp": 0.01000822, + "balance_loss_clip": 1.01079392, + "balance_loss_mlp": 0.99982113, + "epoch": 0.14671232081713192, + "flos": 74766806519040.0, + "grad_norm": 0.7093520075380695, + "language_loss": 0.45747888, + "learning_rate": 3.858815825126197e-06, + "loss": 0.4777967, + "num_input_tokens_seen": 144225520, + "router_z_loss_clip": 0.20214844, + "router_z_loss_mlp": 0.01000977, + "step": 5056, + "time_per_iteration": 3.372309923171997 + }, + { + "auxiliary_loss_clip": 0.01144532, + "auxiliary_loss_mlp": 0.01048576, + "balance_loss_clip": 1.05845749, + "balance_loss_mlp": 1.02765489, + "epoch": 0.14674133828564795, + "flos": 26572854579840.0, + "grad_norm": 2.3924433354466608, + "language_loss": 0.96814346, + "learning_rate": 3.8587464488628555e-06, + "loss": 0.99007457, + "num_input_tokens_seen": 144244700, + "router_z_loss_clip": 0.86083984, + "router_z_loss_mlp": 0.20910645, + "step": 5057, + "time_per_iteration": 2.642773151397705 + }, + { + "auxiliary_loss_clip": 0.01144641, + "auxiliary_loss_mlp": 0.01043546, + "balance_loss_clip": 1.06036949, + "balance_loss_mlp": 1.02597475, + "epoch": 0.146770355754164, + "flos": 32445505710720.0, + "grad_norm": 2.2584198181735875, + "language_loss": 1.00624406, + "learning_rate": 3.858677056182312e-06, + "loss": 1.02812588, + "num_input_tokens_seen": 144261550, + "router_z_loss_clip": 0.84326172, + "router_z_loss_mlp": 0.17590332, + "step": 5058, + "time_per_iteration": 2.6987972259521484 + }, + { + "auxiliary_loss_clip": 0.01146102, + "auxiliary_loss_mlp": 0.0105013, + "balance_loss_clip": 1.06276441, + "balance_loss_mlp": 1.03054428, + "epoch": 0.14679937322268005, + "flos": 11244169664640.0, + "grad_norm": 3.1465016035188875, + "language_loss": 0.84514302, + "learning_rate": 3.85860764708518e-06, + "loss": 0.86710536, + "num_input_tokens_seen": 144270325, + "router_z_loss_clip": 0.83349609, + "router_z_loss_mlp": 0.19561768, + "step": 5059, + "time_per_iteration": 2.5582077503204346 + }, + { + "auxiliary_loss_clip": 0.01143384, + "auxiliary_loss_mlp": 0.01048831, + "balance_loss_clip": 1.06013286, + "balance_loss_mlp": 1.03211832, + "epoch": 0.1468283906911961, + "flos": 24860598180480.0, + "grad_norm": 2.09268474501403, + "language_loss": 0.78974271, + "learning_rate": 3.858538221572074e-06, + "loss": 0.81166482, + "num_input_tokens_seen": 144287040, + "router_z_loss_clip": 0.83251953, + "router_z_loss_mlp": 0.1673584, + "step": 5060, + "time_per_iteration": 2.6649861335754395 + }, + { + "auxiliary_loss_clip": 0.01145846, + "auxiliary_loss_mlp": 0.01043413, + "balance_loss_clip": 1.06218743, + "balance_loss_mlp": 1.02559137, + "epoch": 0.14685740815971216, + "flos": 17633755376640.0, + "grad_norm": 2.4545314433084777, + "language_loss": 0.99167097, + "learning_rate": 3.858468779643607e-06, + "loss": 1.01356363, + "num_input_tokens_seen": 144301450, + "router_z_loss_clip": 0.8359375, + "router_z_loss_mlp": 0.17822266, + "step": 5061, + "time_per_iteration": 2.520078659057617 + }, + { + "auxiliary_loss_clip": 0.0113304, + "auxiliary_loss_mlp": 0.01048956, + "balance_loss_clip": 1.05773485, + "balance_loss_mlp": 1.03360808, + "epoch": 0.1468864256282282, + "flos": 58459158380160.0, + "grad_norm": 2.594950815640352, + "language_loss": 0.70567572, + "learning_rate": 3.858399321300391e-06, + "loss": 0.72749567, + "num_input_tokens_seen": 144322565, + "router_z_loss_clip": 0.75244141, + "router_z_loss_mlp": 0.15362549, + "step": 5062, + "time_per_iteration": 2.9418556690216064 + }, + { + "auxiliary_loss_clip": 0.01136604, + "auxiliary_loss_mlp": 0.01041968, + "balance_loss_clip": 1.05751216, + "balance_loss_mlp": 1.0251714, + "epoch": 0.14691544309674423, + "flos": 34489361491200.0, + "grad_norm": 4.117340471784735, + "language_loss": 0.81447756, + "learning_rate": 3.85832984654304e-06, + "loss": 0.8362633, + "num_input_tokens_seen": 144340350, + "router_z_loss_clip": 0.79052734, + "router_z_loss_mlp": 0.16784668, + "step": 5063, + "time_per_iteration": 2.695239305496216 + }, + { + "auxiliary_loss_clip": 0.01146972, + "auxiliary_loss_mlp": 0.01046809, + "balance_loss_clip": 1.06280935, + "balance_loss_mlp": 1.02947021, + "epoch": 0.14694446056526028, + "flos": 39451497742080.0, + "grad_norm": 2.247663637629757, + "language_loss": 0.89141947, + "learning_rate": 3.858260355372168e-06, + "loss": 0.91335726, + "num_input_tokens_seen": 144356980, + "router_z_loss_clip": 0.84082031, + "router_z_loss_mlp": 0.17358398, + "step": 5064, + "time_per_iteration": 2.6977405548095703 + }, + { + "auxiliary_loss_clip": 0.01147421, + "auxiliary_loss_mlp": 0.01044577, + "balance_loss_clip": 1.06377411, + "balance_loss_mlp": 1.02614772, + "epoch": 0.14697347803377633, + "flos": 28944000109440.0, + "grad_norm": 1.9940483720267514, + "language_loss": 0.78761911, + "learning_rate": 3.858190847788388e-06, + "loss": 0.80953908, + "num_input_tokens_seen": 144373950, + "router_z_loss_clip": 0.83691406, + "router_z_loss_mlp": 0.18432617, + "step": 5065, + "time_per_iteration": 2.5954906940460205 + }, + { + "auxiliary_loss_clip": 0.01147696, + "auxiliary_loss_mlp": 0.01047442, + "balance_loss_clip": 1.06023788, + "balance_loss_mlp": 1.02770114, + "epoch": 0.14700249550229239, + "flos": 15334251523200.0, + "grad_norm": 2.493613595987266, + "language_loss": 0.72284102, + "learning_rate": 3.858121323792315e-06, + "loss": 0.74479246, + "num_input_tokens_seen": 144386550, + "router_z_loss_clip": 0.87548828, + "router_z_loss_mlp": 0.19750977, + "step": 5066, + "time_per_iteration": 2.5305886268615723 + }, + { + "auxiliary_loss_clip": 0.01033381, + "auxiliary_loss_mlp": 0.01007234, + "balance_loss_clip": 1.01328635, + "balance_loss_mlp": 1.00630462, + "epoch": 0.14703151297080844, + "flos": 59674092001920.0, + "grad_norm": 0.7212200550591076, + "language_loss": 0.52036774, + "learning_rate": 3.858051783384563e-06, + "loss": 0.54077393, + "num_input_tokens_seen": 144449640, + "router_z_loss_clip": 0.20117188, + "router_z_loss_mlp": 0.00927734, + "step": 5067, + "time_per_iteration": 3.1842095851898193 + }, + { + "auxiliary_loss_clip": 0.01147014, + "auxiliary_loss_mlp": 0.01044343, + "balance_loss_clip": 1.05930686, + "balance_loss_mlp": 1.02702832, + "epoch": 0.14706053043932446, + "flos": 16719254087040.0, + "grad_norm": 2.998123911651337, + "language_loss": 0.76121747, + "learning_rate": 3.857982226565745e-06, + "loss": 0.783131, + "num_input_tokens_seen": 144461960, + "router_z_loss_clip": 0.87695312, + "router_z_loss_mlp": 0.1730957, + "step": 5068, + "time_per_iteration": 2.4977290630340576 + }, + { + "auxiliary_loss_clip": 0.01135103, + "auxiliary_loss_mlp": 0.01038482, + "balance_loss_clip": 1.05672812, + "balance_loss_mlp": 1.02122009, + "epoch": 0.1470895479078405, + "flos": 11940621442560.0, + "grad_norm": 2.3230257796568643, + "language_loss": 0.74318898, + "learning_rate": 3.857912653336477e-06, + "loss": 0.76492482, + "num_input_tokens_seen": 144474120, + "router_z_loss_clip": 0.78466797, + "router_z_loss_mlp": 0.17260742, + "step": 5069, + "time_per_iteration": 2.553050994873047 + }, + { + "auxiliary_loss_clip": 0.01138477, + "auxiliary_loss_mlp": 0.01038338, + "balance_loss_clip": 1.05956459, + "balance_loss_mlp": 1.02209556, + "epoch": 0.14711856537635656, + "flos": 15881027328000.0, + "grad_norm": 2.081159910606725, + "language_loss": 0.58257365, + "learning_rate": 3.857843063697372e-06, + "loss": 0.60434175, + "num_input_tokens_seen": 144487805, + "router_z_loss_clip": 0.78857422, + "router_z_loss_mlp": 0.16247559, + "step": 5070, + "time_per_iteration": 2.534796714782715 + }, + { + "auxiliary_loss_clip": 0.01137439, + "auxiliary_loss_mlp": 0.01039307, + "balance_loss_clip": 1.05840182, + "balance_loss_mlp": 1.02206302, + "epoch": 0.14714758284487262, + "flos": 37445851054080.0, + "grad_norm": 2.21306457469956, + "language_loss": 0.81990409, + "learning_rate": 3.857773457649045e-06, + "loss": 0.84167147, + "num_input_tokens_seen": 144505670, + "router_z_loss_clip": 0.78955078, + "router_z_loss_mlp": 0.17260742, + "step": 5071, + "time_per_iteration": 2.7343502044677734 + }, + { + "auxiliary_loss_clip": 0.01157647, + "auxiliary_loss_mlp": 0.01051736, + "balance_loss_clip": 1.06507242, + "balance_loss_mlp": 1.03049302, + "epoch": 0.14717660031338867, + "flos": 30220983498240.0, + "grad_norm": 2.252007055744112, + "language_loss": 0.96440256, + "learning_rate": 3.857703835192112e-06, + "loss": 0.98649639, + "num_input_tokens_seen": 144520500, + "router_z_loss_clip": 0.92626953, + "router_z_loss_mlp": 0.21264648, + "step": 5072, + "time_per_iteration": 2.6323790550231934 + }, + { + "auxiliary_loss_clip": 0.01036384, + "auxiliary_loss_mlp": 0.01003056, + "balance_loss_clip": 1.0159179, + "balance_loss_mlp": 1.00198877, + "epoch": 0.14720561778190472, + "flos": 74781854317440.0, + "grad_norm": 0.6578887833570168, + "language_loss": 0.43380183, + "learning_rate": 3.857634196327187e-06, + "loss": 0.45419624, + "num_input_tokens_seen": 144589870, + "router_z_loss_clip": 0.20507812, + "router_z_loss_mlp": 0.01068115, + "step": 5073, + "time_per_iteration": 3.32662034034729 + }, + { + "auxiliary_loss_clip": 0.01147592, + "auxiliary_loss_mlp": 0.01048538, + "balance_loss_clip": 1.06445098, + "balance_loss_mlp": 1.03158069, + "epoch": 0.14723463525042074, + "flos": 26244774731520.0, + "grad_norm": 2.0826290669498446, + "language_loss": 0.91468489, + "learning_rate": 3.8575645410548845e-06, + "loss": 0.93664616, + "num_input_tokens_seen": 144610505, + "router_z_loss_clip": 0.83154297, + "router_z_loss_mlp": 0.16973877, + "step": 5074, + "time_per_iteration": 2.7186596393585205 + }, + { + "auxiliary_loss_clip": 0.01034759, + "auxiliary_loss_mlp": 0.01001227, + "balance_loss_clip": 1.01427257, + "balance_loss_mlp": 1.00010645, + "epoch": 0.1472636527189368, + "flos": 69956034180480.0, + "grad_norm": 0.6723834742721163, + "language_loss": 0.483643, + "learning_rate": 3.85749486937582e-06, + "loss": 0.50400287, + "num_input_tokens_seen": 144672970, + "router_z_loss_clip": 0.20507812, + "router_z_loss_mlp": 0.01123047, + "step": 5075, + "time_per_iteration": 3.0926544666290283 + }, + { + "auxiliary_loss_clip": 0.01133409, + "auxiliary_loss_mlp": 0.01047385, + "balance_loss_clip": 1.05412316, + "balance_loss_mlp": 1.03192389, + "epoch": 0.14729267018745285, + "flos": 22084487740800.0, + "grad_norm": 2.387772930926959, + "language_loss": 0.90999359, + "learning_rate": 3.85742518129061e-06, + "loss": 0.9318015, + "num_input_tokens_seen": 144687560, + "router_z_loss_clip": 0.79199219, + "router_z_loss_mlp": 0.15466309, + "step": 5076, + "time_per_iteration": 2.568845272064209 + }, + { + "auxiliary_loss_clip": 0.01149699, + "auxiliary_loss_mlp": 0.01043378, + "balance_loss_clip": 1.06217003, + "balance_loss_mlp": 1.02536595, + "epoch": 0.1473216876559689, + "flos": 32043665283840.0, + "grad_norm": 1.893051699054393, + "language_loss": 0.6082508, + "learning_rate": 3.857355476799868e-06, + "loss": 0.63018155, + "num_input_tokens_seen": 144704625, + "router_z_loss_clip": 0.87451172, + "router_z_loss_mlp": 0.18023682, + "step": 5077, + "time_per_iteration": 2.687882423400879 + }, + { + "auxiliary_loss_clip": 0.01033881, + "auxiliary_loss_mlp": 0.01001348, + "balance_loss_clip": 1.01370859, + "balance_loss_mlp": 1.00037658, + "epoch": 0.14735070512448495, + "flos": 63870217787520.0, + "grad_norm": 0.6557164322245365, + "language_loss": 0.47630072, + "learning_rate": 3.857285755904212e-06, + "loss": 0.49665299, + "num_input_tokens_seen": 144762385, + "router_z_loss_clip": 0.20214844, + "router_z_loss_mlp": 0.00970459, + "step": 5078, + "time_per_iteration": 3.033813238143921 + }, + { + "auxiliary_loss_clip": 0.01033613, + "auxiliary_loss_mlp": 0.01006724, + "balance_loss_clip": 1.01340175, + "balance_loss_mlp": 1.00562692, + "epoch": 0.147379722593001, + "flos": 74776323623040.0, + "grad_norm": 0.7242019481662155, + "language_loss": 0.52961016, + "learning_rate": 3.857216018604256e-06, + "loss": 0.55001354, + "num_input_tokens_seen": 144823505, + "router_z_loss_clip": 0.20214844, + "router_z_loss_mlp": 0.01098633, + "step": 5079, + "time_per_iteration": 3.0892319679260254 + }, + { + "auxiliary_loss_clip": 0.01032859, + "auxiliary_loss_mlp": 0.01006681, + "balance_loss_clip": 1.01258159, + "balance_loss_mlp": 1.00560844, + "epoch": 0.14740874006151702, + "flos": 54627385178880.0, + "grad_norm": 0.7253121819856231, + "language_loss": 0.48006293, + "learning_rate": 3.857146264900617e-06, + "loss": 0.50045836, + "num_input_tokens_seen": 144879920, + "router_z_loss_clip": 0.203125, + "router_z_loss_mlp": 0.01074219, + "step": 5080, + "time_per_iteration": 2.9464471340179443 + }, + { + "auxiliary_loss_clip": 0.01136514, + "auxiliary_loss_mlp": 0.0104562, + "balance_loss_clip": 1.05915177, + "balance_loss_mlp": 1.03088605, + "epoch": 0.14743775753003308, + "flos": 68421029443200.0, + "grad_norm": 1.9764505858203623, + "language_loss": 0.61830628, + "learning_rate": 3.857076494793911e-06, + "loss": 0.64012766, + "num_input_tokens_seen": 144901945, + "router_z_loss_clip": 0.77294922, + "router_z_loss_mlp": 0.14733887, + "step": 5081, + "time_per_iteration": 2.917659044265747 + }, + { + "auxiliary_loss_clip": 0.01134747, + "auxiliary_loss_mlp": 0.01043259, + "balance_loss_clip": 1.05440021, + "balance_loss_mlp": 1.02715397, + "epoch": 0.14746677499854913, + "flos": 10151659549440.0, + "grad_norm": 3.6253048660051403, + "language_loss": 0.8652668, + "learning_rate": 3.857006708284753e-06, + "loss": 0.88704693, + "num_input_tokens_seen": 144913195, + "router_z_loss_clip": 0.80322266, + "router_z_loss_mlp": 0.16101074, + "step": 5082, + "time_per_iteration": 2.590149402618408 + }, + { + "auxiliary_loss_clip": 0.01140544, + "auxiliary_loss_mlp": 0.01038465, + "balance_loss_clip": 1.0579288, + "balance_loss_mlp": 1.02115631, + "epoch": 0.14749579246706518, + "flos": 40038673368960.0, + "grad_norm": 2.029059796647758, + "language_loss": 0.6993947, + "learning_rate": 3.856936905373761e-06, + "loss": 0.72118473, + "num_input_tokens_seen": 144933445, + "router_z_loss_clip": 0.82568359, + "router_z_loss_mlp": 0.1730957, + "step": 5083, + "time_per_iteration": 2.716479539871216 + }, + { + "auxiliary_loss_clip": 0.0114502, + "auxiliary_loss_mlp": 0.01055464, + "balance_loss_clip": 1.05919945, + "balance_loss_mlp": 1.03625941, + "epoch": 0.14752480993558123, + "flos": 27338362254720.0, + "grad_norm": 1.7700696753957172, + "language_loss": 0.80675668, + "learning_rate": 3.85686708606155e-06, + "loss": 0.82876146, + "num_input_tokens_seen": 144950185, + "router_z_loss_clip": 0.85888672, + "router_z_loss_mlp": 0.19189453, + "step": 5084, + "time_per_iteration": 2.681401014328003 + }, + { + "auxiliary_loss_clip": 0.01133939, + "auxiliary_loss_mlp": 0.0104403, + "balance_loss_clip": 1.05469525, + "balance_loss_mlp": 1.02756715, + "epoch": 0.14755382740409725, + "flos": 21428040735360.0, + "grad_norm": 3.1481170208599787, + "language_loss": 0.81577402, + "learning_rate": 3.856797250348738e-06, + "loss": 0.83755374, + "num_input_tokens_seen": 144965775, + "router_z_loss_clip": 0.79248047, + "router_z_loss_mlp": 0.16461182, + "step": 5085, + "time_per_iteration": 2.6678943634033203 + }, + { + "auxiliary_loss_clip": 0.01139372, + "auxiliary_loss_mlp": 0.01056335, + "balance_loss_clip": 1.05872226, + "balance_loss_mlp": 1.0400269, + "epoch": 0.1475828448726133, + "flos": 22704485420160.0, + "grad_norm": 2.4148193721794797, + "language_loss": 0.76202345, + "learning_rate": 3.856727398235941e-06, + "loss": 0.78398055, + "num_input_tokens_seen": 144980110, + "router_z_loss_clip": 0.80664062, + "router_z_loss_mlp": 0.16314697, + "step": 5086, + "time_per_iteration": 2.6241893768310547 + }, + { + "auxiliary_loss_clip": 0.01142434, + "auxiliary_loss_mlp": 0.01053134, + "balance_loss_clip": 1.05792654, + "balance_loss_mlp": 1.0346086, + "epoch": 0.14761186234112936, + "flos": 74732257635840.0, + "grad_norm": 1.9862738387842567, + "language_loss": 0.88616043, + "learning_rate": 3.856657529723777e-06, + "loss": 0.9081161, + "num_input_tokens_seen": 145003225, + "router_z_loss_clip": 0.84667969, + "router_z_loss_mlp": 0.18518066, + "step": 5087, + "time_per_iteration": 3.0004029273986816 + }, + { + "auxiliary_loss_clip": 0.01133803, + "auxiliary_loss_mlp": 0.01040033, + "balance_loss_clip": 1.05734861, + "balance_loss_mlp": 1.02559686, + "epoch": 0.1476408798096454, + "flos": 34307797219200.0, + "grad_norm": 3.416914854794175, + "language_loss": 0.95239949, + "learning_rate": 3.856587644812862e-06, + "loss": 0.97413784, + "num_input_tokens_seen": 145021380, + "router_z_loss_clip": 0.76513672, + "router_z_loss_mlp": 0.14440918, + "step": 5088, + "time_per_iteration": 2.700488805770874 + }, + { + "auxiliary_loss_clip": 0.01136426, + "auxiliary_loss_mlp": 0.01040371, + "balance_loss_clip": 1.05573416, + "balance_loss_mlp": 1.02408075, + "epoch": 0.14766989727816146, + "flos": 14824212353280.0, + "grad_norm": 2.3894196743569642, + "language_loss": 0.75181639, + "learning_rate": 3.8565177435038134e-06, + "loss": 0.77358443, + "num_input_tokens_seen": 145035575, + "router_z_loss_clip": 0.80664062, + "router_z_loss_mlp": 0.16278076, + "step": 5089, + "time_per_iteration": 2.5748679637908936 + }, + { + "auxiliary_loss_clip": 0.01139898, + "auxiliary_loss_mlp": 0.01045186, + "balance_loss_clip": 1.05759263, + "balance_loss_mlp": 1.0283004, + "epoch": 0.1476989147466775, + "flos": 23508309928320.0, + "grad_norm": 2.222545062119466, + "language_loss": 0.88802546, + "learning_rate": 3.85644782579725e-06, + "loss": 0.90987635, + "num_input_tokens_seen": 145052955, + "router_z_loss_clip": 0.82275391, + "router_z_loss_mlp": 0.16876221, + "step": 5090, + "time_per_iteration": 2.666891098022461 + }, + { + "auxiliary_loss_clip": 0.01137122, + "auxiliary_loss_mlp": 0.0104225, + "balance_loss_clip": 1.05973053, + "balance_loss_mlp": 1.02634144, + "epoch": 0.14772793221519354, + "flos": 36459420779520.0, + "grad_norm": 2.6674921919868613, + "language_loss": 0.7761423, + "learning_rate": 3.8563778916937865e-06, + "loss": 0.79793602, + "num_input_tokens_seen": 145068495, + "router_z_loss_clip": 0.7734375, + "router_z_loss_mlp": 0.15911865, + "step": 5091, + "time_per_iteration": 2.807559013366699 + }, + { + "auxiliary_loss_clip": 0.01140367, + "auxiliary_loss_mlp": 0.01044607, + "balance_loss_clip": 1.06000137, + "balance_loss_mlp": 1.02842402, + "epoch": 0.1477569496837096, + "flos": 31276290101760.0, + "grad_norm": 4.498437048368639, + "language_loss": 1.00263774, + "learning_rate": 3.856307941194042e-06, + "loss": 1.0244875, + "num_input_tokens_seen": 145082205, + "router_z_loss_clip": 0.8046875, + "router_z_loss_mlp": 0.16174316, + "step": 5092, + "time_per_iteration": 2.64577317237854 + }, + { + "auxiliary_loss_clip": 0.01130267, + "auxiliary_loss_mlp": 0.01044465, + "balance_loss_clip": 1.0546416, + "balance_loss_mlp": 1.02979088, + "epoch": 0.14778596715222564, + "flos": 13181155505280.0, + "grad_norm": 2.404105624235698, + "language_loss": 0.86121827, + "learning_rate": 3.856237974298636e-06, + "loss": 0.88296562, + "num_input_tokens_seen": 145094755, + "router_z_loss_clip": 0.75537109, + "router_z_loss_mlp": 0.14685059, + "step": 5093, + "time_per_iteration": 2.714067220687866 + }, + { + "auxiliary_loss_clip": 0.01137277, + "auxiliary_loss_mlp": 0.01050964, + "balance_loss_clip": 1.05809689, + "balance_loss_mlp": 1.03334451, + "epoch": 0.1478149846207417, + "flos": 12815764404480.0, + "grad_norm": 2.879082309763347, + "language_loss": 0.87050533, + "learning_rate": 3.856167991008185e-06, + "loss": 0.89238775, + "num_input_tokens_seen": 145105365, + "router_z_loss_clip": 0.79248047, + "router_z_loss_mlp": 0.17626953, + "step": 5094, + "time_per_iteration": 2.607658624649048 + }, + { + "auxiliary_loss_clip": 0.01036498, + "auxiliary_loss_mlp": 0.01005432, + "balance_loss_clip": 1.0160445, + "balance_loss_mlp": 1.00452602, + "epoch": 0.14784400208925774, + "flos": 60161831804160.0, + "grad_norm": 0.747861822293773, + "language_loss": 0.47701484, + "learning_rate": 3.856097991323307e-06, + "loss": 0.49743414, + "num_input_tokens_seen": 145167355, + "router_z_loss_clip": 0.20507812, + "router_z_loss_mlp": 0.0090332, + "step": 5095, + "time_per_iteration": 3.182983636856079 + }, + { + "auxiliary_loss_clip": 0.01036805, + "auxiliary_loss_mlp": 0.01001557, + "balance_loss_clip": 1.01642418, + "balance_loss_mlp": 1.00060964, + "epoch": 0.14787301955777377, + "flos": 74795321917440.0, + "grad_norm": 0.6752730953560321, + "language_loss": 0.46061429, + "learning_rate": 3.856027975244621e-06, + "loss": 0.48099792, + "num_input_tokens_seen": 145232100, + "router_z_loss_clip": 0.20410156, + "router_z_loss_mlp": 0.00946045, + "step": 5096, + "time_per_iteration": 3.3234362602233887 + }, + { + "auxiliary_loss_clip": 0.01036715, + "auxiliary_loss_mlp": 0.01001553, + "balance_loss_clip": 1.0165422, + "balance_loss_mlp": 1.00065899, + "epoch": 0.14790203702628982, + "flos": 74773342794240.0, + "grad_norm": 0.6448141892349561, + "language_loss": 0.4780975, + "learning_rate": 3.855957942772743e-06, + "loss": 0.49848023, + "num_input_tokens_seen": 145298040, + "router_z_loss_clip": 0.20214844, + "router_z_loss_mlp": 0.00891113, + "step": 5097, + "time_per_iteration": 3.228984832763672 + }, + { + "auxiliary_loss_clip": 0.01143319, + "auxiliary_loss_mlp": 0.01054574, + "balance_loss_clip": 1.0626061, + "balance_loss_mlp": 1.0371815, + "epoch": 0.14793105449480587, + "flos": 12524780327040.0, + "grad_norm": 2.746652212063072, + "language_loss": 0.7821365, + "learning_rate": 3.855887893908295e-06, + "loss": 0.80411547, + "num_input_tokens_seen": 145310535, + "router_z_loss_clip": 0.80664062, + "router_z_loss_mlp": 0.17388916, + "step": 5098, + "time_per_iteration": 2.729001998901367 + }, + { + "auxiliary_loss_clip": 0.01139184, + "auxiliary_loss_mlp": 0.01047667, + "balance_loss_clip": 1.05873442, + "balance_loss_mlp": 1.02972555, + "epoch": 0.14796007196332192, + "flos": 32700435511680.0, + "grad_norm": 1.7339921773293956, + "language_loss": 0.73076552, + "learning_rate": 3.855817828651894e-06, + "loss": 0.75263405, + "num_input_tokens_seen": 145327840, + "router_z_loss_clip": 0.80371094, + "router_z_loss_mlp": 0.17926025, + "step": 5099, + "time_per_iteration": 2.640950918197632 + }, + { + "auxiliary_loss_clip": 0.01140252, + "auxiliary_loss_mlp": 0.01062189, + "balance_loss_clip": 1.05965722, + "balance_loss_mlp": 1.04383063, + "epoch": 0.14798908943183797, + "flos": 13692307996800.0, + "grad_norm": 2.528607635424786, + "language_loss": 0.71293163, + "learning_rate": 3.855747747004159e-06, + "loss": 0.73495603, + "num_input_tokens_seen": 145339350, + "router_z_loss_clip": 0.80615234, + "router_z_loss_mlp": 0.18359375, + "step": 5100, + "time_per_iteration": 2.519562244415283 + }, + { + "auxiliary_loss_clip": 0.01137295, + "auxiliary_loss_mlp": 0.01059106, + "balance_loss_clip": 1.05857289, + "balance_loss_mlp": 1.04394233, + "epoch": 0.14801810690035402, + "flos": 15881278723200.0, + "grad_norm": 7.386230196954845, + "language_loss": 0.51615775, + "learning_rate": 3.855677648965709e-06, + "loss": 0.5381217, + "num_input_tokens_seen": 145352450, + "router_z_loss_clip": 0.78808594, + "router_z_loss_mlp": 0.15161133, + "step": 5101, + "time_per_iteration": 2.469560146331787 + }, + { + "auxiliary_loss_clip": 0.01142953, + "auxiliary_loss_mlp": 0.01055128, + "balance_loss_clip": 1.05808973, + "balance_loss_mlp": 1.03744948, + "epoch": 0.14804712436887005, + "flos": 41896044714240.0, + "grad_norm": 3.218069885431459, + "language_loss": 0.85706878, + "learning_rate": 3.855607534537162e-06, + "loss": 0.8790496, + "num_input_tokens_seen": 145371780, + "router_z_loss_clip": 0.84863281, + "router_z_loss_mlp": 0.17669678, + "step": 5102, + "time_per_iteration": 2.719784736633301 + }, + { + "auxiliary_loss_clip": 0.01138362, + "auxiliary_loss_mlp": 0.01052172, + "balance_loss_clip": 1.05600464, + "balance_loss_mlp": 1.03604245, + "epoch": 0.1480761418373861, + "flos": 18475393927680.0, + "grad_norm": 2.382355526829062, + "language_loss": 0.88809109, + "learning_rate": 3.8555374037191395e-06, + "loss": 0.90999645, + "num_input_tokens_seen": 145385675, + "router_z_loss_clip": 0.82470703, + "router_z_loss_mlp": 0.16143799, + "step": 5103, + "time_per_iteration": 2.649247407913208 + }, + { + "auxiliary_loss_clip": 0.01144076, + "auxiliary_loss_mlp": 0.01071742, + "balance_loss_clip": 1.05836892, + "balance_loss_mlp": 1.05494523, + "epoch": 0.14810515930590215, + "flos": 52075821634560.0, + "grad_norm": 61.0844105994863, + "language_loss": 0.74595547, + "learning_rate": 3.855467256512259e-06, + "loss": 0.76811361, + "num_input_tokens_seen": 145404905, + "router_z_loss_clip": 0.85742188, + "router_z_loss_mlp": 0.16796875, + "step": 5104, + "time_per_iteration": 2.822138786315918 + }, + { + "auxiliary_loss_clip": 0.01145499, + "auxiliary_loss_mlp": 0.01065223, + "balance_loss_clip": 1.06078696, + "balance_loss_mlp": 1.04724073, + "epoch": 0.1481341767744182, + "flos": 21572221927680.0, + "grad_norm": 2.3997236370100765, + "language_loss": 0.88244247, + "learning_rate": 3.8553970929171414e-06, + "loss": 0.90454978, + "num_input_tokens_seen": 145417545, + "router_z_loss_clip": 0.84716797, + "router_z_loss_mlp": 0.17974854, + "step": 5105, + "time_per_iteration": 2.5803382396698 + }, + { + "auxiliary_loss_clip": 0.01129541, + "auxiliary_loss_mlp": 0.01062015, + "balance_loss_clip": 1.05226517, + "balance_loss_mlp": 1.04607701, + "epoch": 0.14816319424293425, + "flos": 20953373483520.0, + "grad_norm": 2.5005618219712624, + "language_loss": 0.78542, + "learning_rate": 3.855326912934406e-06, + "loss": 0.80733562, + "num_input_tokens_seen": 145431050, + "router_z_loss_clip": 0.7734375, + "router_z_loss_mlp": 0.15930176, + "step": 5106, + "time_per_iteration": 2.5329813957214355 + }, + { + "auxiliary_loss_clip": 0.01133326, + "auxiliary_loss_mlp": 0.01051072, + "balance_loss_clip": 1.05374837, + "balance_loss_mlp": 1.03578901, + "epoch": 0.1481922117114503, + "flos": 35913219592320.0, + "grad_norm": 2.187144882120241, + "language_loss": 0.60788316, + "learning_rate": 3.855256716564672e-06, + "loss": 0.62972713, + "num_input_tokens_seen": 145447895, + "router_z_loss_clip": 0.79492188, + "router_z_loss_mlp": 0.15301514, + "step": 5107, + "time_per_iteration": 2.6940813064575195 + }, + { + "auxiliary_loss_clip": 0.01140089, + "auxiliary_loss_mlp": 0.01052982, + "balance_loss_clip": 1.05640984, + "balance_loss_mlp": 1.03637612, + "epoch": 0.14822122917996633, + "flos": 11102358769920.0, + "grad_norm": 3.0866604190412374, + "language_loss": 0.7981925, + "learning_rate": 3.8551865038085605e-06, + "loss": 0.8201232, + "num_input_tokens_seen": 145459170, + "router_z_loss_clip": 0.83740234, + "router_z_loss_mlp": 0.16619873, + "step": 5108, + "time_per_iteration": 2.572124719619751 + }, + { + "auxiliary_loss_clip": 0.01130366, + "auxiliary_loss_mlp": 0.01049619, + "balance_loss_clip": 1.05523515, + "balance_loss_mlp": 1.03717375, + "epoch": 0.14825024664848238, + "flos": 22740575610240.0, + "grad_norm": 3.0790632955591444, + "language_loss": 0.79464865, + "learning_rate": 3.8551162746666904e-06, + "loss": 0.81644857, + "num_input_tokens_seen": 145471930, + "router_z_loss_clip": 0.75195312, + "router_z_loss_mlp": 0.12463379, + "step": 5109, + "time_per_iteration": 2.5517773628234863 + }, + { + "auxiliary_loss_clip": 0.01034603, + "auxiliary_loss_mlp": 0.01052904, + "balance_loss_clip": 1.01358318, + "balance_loss_mlp": 1.05199194, + "epoch": 0.14827926411699843, + "flos": 60798059452800.0, + "grad_norm": 0.7295854878227152, + "language_loss": 0.5670138, + "learning_rate": 3.855046029139683e-06, + "loss": 0.5878889, + "num_input_tokens_seen": 145532300, + "router_z_loss_clip": 0.2109375, + "router_z_loss_mlp": 0.00909424, + "step": 5110, + "time_per_iteration": 3.0546860694885254 + }, + { + "auxiliary_loss_clip": 0.01139893, + "auxiliary_loss_mlp": 0.01043587, + "balance_loss_clip": 1.05636048, + "balance_loss_mlp": 1.02814341, + "epoch": 0.14830828158551448, + "flos": 23872049003520.0, + "grad_norm": 1.7826244331016103, + "language_loss": 0.69452071, + "learning_rate": 3.854975767228159e-06, + "loss": 0.7163555, + "num_input_tokens_seen": 145547105, + "router_z_loss_clip": 0.83544922, + "router_z_loss_mlp": 0.15447998, + "step": 5111, + "time_per_iteration": 2.691267251968384 + }, + { + "auxiliary_loss_clip": 0.01147705, + "auxiliary_loss_mlp": 0.01052335, + "balance_loss_clip": 1.06034136, + "balance_loss_mlp": 1.03379226, + "epoch": 0.14833729905403054, + "flos": 46237860063360.0, + "grad_norm": 2.38072125600475, + "language_loss": 0.8479284, + "learning_rate": 3.854905488932738e-06, + "loss": 0.86992878, + "num_input_tokens_seen": 145568290, + "router_z_loss_clip": 0.87451172, + "router_z_loss_mlp": 0.18554688, + "step": 5112, + "time_per_iteration": 2.764328956604004 + }, + { + "auxiliary_loss_clip": 0.01142652, + "auxiliary_loss_mlp": 0.01061001, + "balance_loss_clip": 1.05821931, + "balance_loss_mlp": 1.04224992, + "epoch": 0.14836631652254656, + "flos": 21863206005120.0, + "grad_norm": 6.551848457855167, + "language_loss": 0.78340077, + "learning_rate": 3.854835194254041e-06, + "loss": 0.80543733, + "num_input_tokens_seen": 145581085, + "router_z_loss_clip": 0.84423828, + "router_z_loss_mlp": 0.18737793, + "step": 5113, + "time_per_iteration": 2.536684513092041 + }, + { + "auxiliary_loss_clip": 0.01143808, + "auxiliary_loss_mlp": 0.01057148, + "balance_loss_clip": 1.058617, + "balance_loss_mlp": 1.03838491, + "epoch": 0.1483953339910626, + "flos": 30592156688640.0, + "grad_norm": 2.1488875141638872, + "language_loss": 0.92097354, + "learning_rate": 3.854764883192689e-06, + "loss": 0.94298309, + "num_input_tokens_seen": 145598440, + "router_z_loss_clip": 0.85302734, + "router_z_loss_mlp": 0.18737793, + "step": 5114, + "time_per_iteration": 2.5990149974823 + }, + { + "auxiliary_loss_clip": 0.01144857, + "auxiliary_loss_mlp": 0.01048456, + "balance_loss_clip": 1.06108069, + "balance_loss_mlp": 1.03114653, + "epoch": 0.14842435145957866, + "flos": 22960133493120.0, + "grad_norm": 2.487200229742816, + "language_loss": 0.85524631, + "learning_rate": 3.854694555749303e-06, + "loss": 0.87717944, + "num_input_tokens_seen": 145615035, + "router_z_loss_clip": 0.83691406, + "router_z_loss_mlp": 0.17321777, + "step": 5115, + "time_per_iteration": 2.609090805053711 + }, + { + "auxiliary_loss_clip": 0.01141459, + "auxiliary_loss_mlp": 0.01045948, + "balance_loss_clip": 1.05854583, + "balance_loss_mlp": 1.02959216, + "epoch": 0.14845336892809471, + "flos": 28432129345920.0, + "grad_norm": 2.3739929960917445, + "language_loss": 0.7887907, + "learning_rate": 3.854624211924504e-06, + "loss": 0.81066477, + "num_input_tokens_seen": 145630095, + "router_z_loss_clip": 0.828125, + "router_z_loss_mlp": 0.16357422, + "step": 5116, + "time_per_iteration": 2.5910396575927734 + }, + { + "auxiliary_loss_clip": 0.01143208, + "auxiliary_loss_mlp": 0.01049622, + "balance_loss_clip": 1.06319284, + "balance_loss_mlp": 1.03363585, + "epoch": 0.14848238639661077, + "flos": 24856863166080.0, + "grad_norm": 3.420123509312257, + "language_loss": 1.10276556, + "learning_rate": 3.854553851718913e-06, + "loss": 1.12469387, + "num_input_tokens_seen": 145648145, + "router_z_loss_clip": 0.79980469, + "router_z_loss_mlp": 0.15979004, + "step": 5117, + "time_per_iteration": 2.6830787658691406 + }, + { + "auxiliary_loss_clip": 0.01039179, + "auxiliary_loss_mlp": 0.01006591, + "balance_loss_clip": 1.01843357, + "balance_loss_mlp": 1.00563776, + "epoch": 0.14851140386512682, + "flos": 62044267864320.0, + "grad_norm": 0.7147922268358723, + "language_loss": 0.49531502, + "learning_rate": 3.854483475133153e-06, + "loss": 0.5157727, + "num_input_tokens_seen": 145707785, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.00952148, + "step": 5118, + "time_per_iteration": 3.1047585010528564 + }, + { + "auxiliary_loss_clip": 0.01136284, + "auxiliary_loss_mlp": 0.01041624, + "balance_loss_clip": 1.0580821, + "balance_loss_mlp": 1.02583516, + "epoch": 0.14854042133364284, + "flos": 22593557243520.0, + "grad_norm": 1.895515064394933, + "language_loss": 0.82823497, + "learning_rate": 3.854413082167844e-06, + "loss": 0.85001409, + "num_input_tokens_seen": 145724840, + "router_z_loss_clip": 0.78125, + "router_z_loss_mlp": 0.15802002, + "step": 5119, + "time_per_iteration": 2.612544298171997 + }, + { + "auxiliary_loss_clip": 0.01143172, + "auxiliary_loss_mlp": 0.0106115, + "balance_loss_clip": 1.05893505, + "balance_loss_mlp": 1.04200459, + "epoch": 0.1485694388021589, + "flos": 16138327426560.0, + "grad_norm": 2.793811271153475, + "language_loss": 0.76419163, + "learning_rate": 3.8543426728236086e-06, + "loss": 0.78623474, + "num_input_tokens_seen": 145738475, + "router_z_loss_clip": 0.84130859, + "router_z_loss_mlp": 0.19152832, + "step": 5120, + "time_per_iteration": 2.5234262943267822 + }, + { + "auxiliary_loss_clip": 0.0114327, + "auxiliary_loss_mlp": 0.01040952, + "balance_loss_clip": 1.05952835, + "balance_loss_mlp": 1.02393484, + "epoch": 0.14859845627067494, + "flos": 13436588096640.0, + "grad_norm": 2.3799108104316566, + "language_loss": 0.86564153, + "learning_rate": 3.8542722471010674e-06, + "loss": 0.88748378, + "num_input_tokens_seen": 145751075, + "router_z_loss_clip": 0.83691406, + "router_z_loss_mlp": 0.16998291, + "step": 5121, + "time_per_iteration": 7.2657790184021 + }, + { + "auxiliary_loss_clip": 0.01134808, + "auxiliary_loss_mlp": 0.01049661, + "balance_loss_clip": 1.05697763, + "balance_loss_mlp": 1.0310111, + "epoch": 0.148627473739191, + "flos": 31169276507520.0, + "grad_norm": 2.526497909430887, + "language_loss": 0.91243738, + "learning_rate": 3.8542018050008445e-06, + "loss": 0.93428212, + "num_input_tokens_seen": 145766705, + "router_z_loss_clip": 0.77880859, + "router_z_loss_mlp": 0.1864624, + "step": 5122, + "time_per_iteration": 4.960013151168823 + }, + { + "auxiliary_loss_clip": 0.01147663, + "auxiliary_loss_mlp": 0.01052405, + "balance_loss_clip": 1.06456399, + "balance_loss_mlp": 1.03481519, + "epoch": 0.14865649120770705, + "flos": 15992278727040.0, + "grad_norm": 2.259919233245947, + "language_loss": 0.70697463, + "learning_rate": 3.8541313465235605e-06, + "loss": 0.7289753, + "num_input_tokens_seen": 145779545, + "router_z_loss_clip": 0.83105469, + "router_z_loss_mlp": 0.17590332, + "step": 5123, + "time_per_iteration": 4.912460803985596 + }, + { + "auxiliary_loss_clip": 0.01142531, + "auxiliary_loss_mlp": 0.01043245, + "balance_loss_clip": 1.06460309, + "balance_loss_mlp": 1.02773571, + "epoch": 0.1486855086762231, + "flos": 13352379638400.0, + "grad_norm": 2.7459237243439722, + "language_loss": 0.68534225, + "learning_rate": 3.854060871669838e-06, + "loss": 0.70719999, + "num_input_tokens_seen": 145795485, + "router_z_loss_clip": 0.77929688, + "router_z_loss_mlp": 0.15527344, + "step": 5124, + "time_per_iteration": 2.5700767040252686 + }, + { + "auxiliary_loss_clip": 0.01130338, + "auxiliary_loss_mlp": 0.01053176, + "balance_loss_clip": 1.05561686, + "balance_loss_mlp": 1.03676128, + "epoch": 0.14871452614473912, + "flos": 19642382893440.0, + "grad_norm": 4.739571492834234, + "language_loss": 0.89153045, + "learning_rate": 3.8539903804403e-06, + "loss": 0.9133656, + "num_input_tokens_seen": 145810060, + "router_z_loss_clip": 0.74707031, + "router_z_loss_mlp": 0.1640625, + "step": 5125, + "time_per_iteration": 2.486717462539673 + }, + { + "auxiliary_loss_clip": 0.01039009, + "auxiliary_loss_mlp": 0.01006242, + "balance_loss_clip": 1.01902258, + "balance_loss_mlp": 1.00532961, + "epoch": 0.14874354361325517, + "flos": 71733431894400.0, + "grad_norm": 0.6818507415818397, + "language_loss": 0.48865199, + "learning_rate": 3.853919872835568e-06, + "loss": 0.50910449, + "num_input_tokens_seen": 145867190, + "router_z_loss_clip": 0.20019531, + "router_z_loss_mlp": 0.00909424, + "step": 5126, + "time_per_iteration": 3.0066990852355957 + }, + { + "auxiliary_loss_clip": 0.01140485, + "auxiliary_loss_mlp": 0.01038001, + "balance_loss_clip": 1.06129503, + "balance_loss_mlp": 1.02258158, + "epoch": 0.14877256108177123, + "flos": 32298487344000.0, + "grad_norm": 2.101069420952156, + "language_loss": 0.82627398, + "learning_rate": 3.853849348856267e-06, + "loss": 0.84805882, + "num_input_tokens_seen": 145881805, + "router_z_loss_clip": 0.79296875, + "router_z_loss_mlp": 0.15423584, + "step": 5127, + "time_per_iteration": 2.5925350189208984 + }, + { + "auxiliary_loss_clip": 0.01155902, + "auxiliary_loss_mlp": 0.01055639, + "balance_loss_clip": 1.0674504, + "balance_loss_mlp": 1.03706598, + "epoch": 0.14880157855028728, + "flos": 14534269770240.0, + "grad_norm": 3.350110371376704, + "language_loss": 0.82698488, + "learning_rate": 3.853778808503017e-06, + "loss": 0.84910023, + "num_input_tokens_seen": 145895125, + "router_z_loss_clip": 0.88378906, + "router_z_loss_mlp": 0.18591309, + "step": 5128, + "time_per_iteration": 2.5530622005462646 + }, + { + "auxiliary_loss_clip": 0.01036378, + "auxiliary_loss_mlp": 0.01005372, + "balance_loss_clip": 1.01609254, + "balance_loss_mlp": 1.00443649, + "epoch": 0.14883059601880333, + "flos": 70211969562240.0, + "grad_norm": 0.7147073938178949, + "language_loss": 0.51059163, + "learning_rate": 3.8537082517764425e-06, + "loss": 0.53100914, + "num_input_tokens_seen": 145961350, + "router_z_loss_clip": 0.203125, + "router_z_loss_mlp": 0.00933838, + "step": 5129, + "time_per_iteration": 3.2046873569488525 + }, + { + "auxiliary_loss_clip": 0.01035311, + "auxiliary_loss_mlp": 0.01003997, + "balance_loss_clip": 1.01500154, + "balance_loss_mlp": 1.00306749, + "epoch": 0.14885961348731935, + "flos": 72426831016320.0, + "grad_norm": 0.671997696260568, + "language_loss": 0.5053789, + "learning_rate": 3.853637678677167e-06, + "loss": 0.52577198, + "num_input_tokens_seen": 146022580, + "router_z_loss_clip": 0.203125, + "router_z_loss_mlp": 0.00927734, + "step": 5130, + "time_per_iteration": 3.1237869262695312 + }, + { + "auxiliary_loss_clip": 0.01143522, + "auxiliary_loss_mlp": 0.01048623, + "balance_loss_clip": 1.06312037, + "balance_loss_mlp": 1.03279245, + "epoch": 0.1488886309558354, + "flos": 11539391546880.0, + "grad_norm": 2.5754467863001196, + "language_loss": 0.6794883, + "learning_rate": 3.853567089205813e-06, + "loss": 0.70140976, + "num_input_tokens_seen": 146034575, + "router_z_loss_clip": 0.80371094, + "router_z_loss_mlp": 0.15844727, + "step": 5131, + "time_per_iteration": 2.501668930053711 + }, + { + "auxiliary_loss_clip": 0.01144144, + "auxiliary_loss_mlp": 0.01040009, + "balance_loss_clip": 1.06052399, + "balance_loss_mlp": 1.02235389, + "epoch": 0.14891764842435146, + "flos": 25368662102400.0, + "grad_norm": 1.801850986077652, + "language_loss": 0.70204926, + "learning_rate": 3.853496483363005e-06, + "loss": 0.72389072, + "num_input_tokens_seen": 146050335, + "router_z_loss_clip": 0.83642578, + "router_z_loss_mlp": 0.17663574, + "step": 5132, + "time_per_iteration": 2.6040916442871094 + }, + { + "auxiliary_loss_clip": 0.01137578, + "auxiliary_loss_mlp": 0.01035419, + "balance_loss_clip": 1.06106257, + "balance_loss_mlp": 1.02113724, + "epoch": 0.1489466658928675, + "flos": 20446243315200.0, + "grad_norm": 2.1628589694764924, + "language_loss": 0.69441426, + "learning_rate": 3.853425861149366e-06, + "loss": 0.71614426, + "num_input_tokens_seen": 146070985, + "router_z_loss_clip": 0.76513672, + "router_z_loss_mlp": 0.14276123, + "step": 5133, + "time_per_iteration": 2.669585943222046 + }, + { + "auxiliary_loss_clip": 0.01142305, + "auxiliary_loss_mlp": 0.01041896, + "balance_loss_clip": 1.06205463, + "balance_loss_mlp": 1.02561212, + "epoch": 0.14897568336138356, + "flos": 17705576620800.0, + "grad_norm": 2.262695461759579, + "language_loss": 0.71081114, + "learning_rate": 3.85335522256552e-06, + "loss": 0.73265314, + "num_input_tokens_seen": 146084475, + "router_z_loss_clip": 0.80175781, + "router_z_loss_mlp": 0.16296387, + "step": 5134, + "time_per_iteration": 2.499950647354126 + }, + { + "auxiliary_loss_clip": 0.01139411, + "auxiliary_loss_mlp": 0.01062626, + "balance_loss_clip": 1.05897617, + "balance_loss_mlp": 1.04514432, + "epoch": 0.1490047008298996, + "flos": 30330080081280.0, + "grad_norm": 2.135395174686455, + "language_loss": 0.9433831, + "learning_rate": 3.853284567612089e-06, + "loss": 0.96540344, + "num_input_tokens_seen": 146102110, + "router_z_loss_clip": 0.80419922, + "router_z_loss_mlp": 0.17486572, + "step": 5135, + "time_per_iteration": 2.6420483589172363 + }, + { + "auxiliary_loss_clip": 0.01033591, + "auxiliary_loss_mlp": 0.0100183, + "balance_loss_clip": 1.01348805, + "balance_loss_mlp": 1.00082898, + "epoch": 0.14903371829841564, + "flos": 64591698366720.0, + "grad_norm": 0.7255173584489462, + "language_loss": 0.50069392, + "learning_rate": 3.8532138962897e-06, + "loss": 0.52104813, + "num_input_tokens_seen": 146160200, + "router_z_loss_clip": 0.20117188, + "router_z_loss_mlp": 0.01000977, + "step": 5136, + "time_per_iteration": 3.041069984436035 + }, + { + "auxiliary_loss_clip": 0.01140054, + "auxiliary_loss_mlp": 0.01041888, + "balance_loss_clip": 1.06047678, + "balance_loss_mlp": 1.02378011, + "epoch": 0.1490627357669317, + "flos": 29417661780480.0, + "grad_norm": 2.3799213769378063, + "language_loss": 0.78493309, + "learning_rate": 3.8531432085989764e-06, + "loss": 0.8067525, + "num_input_tokens_seen": 146175330, + "router_z_loss_clip": 0.79589844, + "router_z_loss_mlp": 0.18103027, + "step": 5137, + "time_per_iteration": 2.621640205383301 + }, + { + "auxiliary_loss_clip": 0.01142148, + "auxiliary_loss_mlp": 0.01041401, + "balance_loss_clip": 1.05943847, + "balance_loss_mlp": 1.02366269, + "epoch": 0.14909175323544774, + "flos": 74746874471040.0, + "grad_norm": 2.1742531955716795, + "language_loss": 0.85424244, + "learning_rate": 3.8530725045405415e-06, + "loss": 0.87607789, + "num_input_tokens_seen": 146201930, + "router_z_loss_clip": 0.82617188, + "router_z_loss_mlp": 0.17736816, + "step": 5138, + "time_per_iteration": 2.92166805267334 + }, + { + "auxiliary_loss_clip": 0.01034682, + "auxiliary_loss_mlp": 0.01002009, + "balance_loss_clip": 1.01458478, + "balance_loss_mlp": 1.00096023, + "epoch": 0.1491207707039638, + "flos": 74773091399040.0, + "grad_norm": 0.6838366582844236, + "language_loss": 0.52426827, + "learning_rate": 3.853001784115021e-06, + "loss": 0.54463518, + "num_input_tokens_seen": 146264080, + "router_z_loss_clip": 0.20117188, + "router_z_loss_mlp": 0.01049805, + "step": 5139, + "time_per_iteration": 3.1239235401153564 + }, + { + "auxiliary_loss_clip": 0.01143539, + "auxiliary_loss_mlp": 0.01055856, + "balance_loss_clip": 1.06061614, + "balance_loss_mlp": 1.03839779, + "epoch": 0.14914978817247984, + "flos": 13145173056000.0, + "grad_norm": 2.4741877223331437, + "language_loss": 0.83942229, + "learning_rate": 3.8529310473230385e-06, + "loss": 0.86141628, + "num_input_tokens_seen": 146276855, + "router_z_loss_clip": 0.83007812, + "router_z_loss_mlp": 0.17449951, + "step": 5140, + "time_per_iteration": 2.5408852100372314 + }, + { + "auxiliary_loss_clip": 0.01139145, + "auxiliary_loss_mlp": 0.01037156, + "balance_loss_clip": 1.05869448, + "balance_loss_mlp": 1.0209496, + "epoch": 0.1491788056409959, + "flos": 55252371870720.0, + "grad_norm": 1.609916425598035, + "language_loss": 0.81588185, + "learning_rate": 3.852860294165219e-06, + "loss": 0.83764493, + "num_input_tokens_seen": 146300830, + "router_z_loss_clip": 0.8046875, + "router_z_loss_mlp": 0.16223145, + "step": 5141, + "time_per_iteration": 2.8767249584198 + }, + { + "auxiliary_loss_clip": 0.01130741, + "auxiliary_loss_mlp": 0.0103672, + "balance_loss_clip": 1.05629361, + "balance_loss_mlp": 1.02222466, + "epoch": 0.14920782310951192, + "flos": 33433552097280.0, + "grad_norm": 2.295037201772184, + "language_loss": 0.95345992, + "learning_rate": 3.852789524642188e-06, + "loss": 0.97513461, + "num_input_tokens_seen": 146317850, + "router_z_loss_clip": 0.74365234, + "router_z_loss_mlp": 0.14501953, + "step": 5142, + "time_per_iteration": 2.6423027515411377 + }, + { + "auxiliary_loss_clip": 0.01139459, + "auxiliary_loss_mlp": 0.01042333, + "balance_loss_clip": 1.05934072, + "balance_loss_mlp": 1.0268054, + "epoch": 0.14923684057802797, + "flos": 71740432068480.0, + "grad_norm": 1.835345218023888, + "language_loss": 0.67960232, + "learning_rate": 3.8527187387545695e-06, + "loss": 0.70142031, + "num_input_tokens_seen": 146350005, + "router_z_loss_clip": 0.80126953, + "router_z_loss_mlp": 0.15527344, + "step": 5143, + "time_per_iteration": 2.981362819671631 + }, + { + "auxiliary_loss_clip": 0.01147223, + "auxiliary_loss_mlp": 0.01045129, + "balance_loss_clip": 1.05938148, + "balance_loss_mlp": 1.02731872, + "epoch": 0.14926585804654402, + "flos": 36648957870720.0, + "grad_norm": 1.896523416752065, + "language_loss": 0.77444178, + "learning_rate": 3.85264793650299e-06, + "loss": 0.79636526, + "num_input_tokens_seen": 146370360, + "router_z_loss_clip": 0.87890625, + "router_z_loss_mlp": 0.17797852, + "step": 5144, + "time_per_iteration": 2.6960628032684326 + }, + { + "auxiliary_loss_clip": 0.01140228, + "auxiliary_loss_mlp": 0.01043112, + "balance_loss_clip": 1.05989206, + "balance_loss_mlp": 1.02756691, + "epoch": 0.14929487551506007, + "flos": 30584722573440.0, + "grad_norm": 2.517985724750972, + "language_loss": 0.79879594, + "learning_rate": 3.8525771178880735e-06, + "loss": 0.82062936, + "num_input_tokens_seen": 146386680, + "router_z_loss_clip": 0.80273438, + "router_z_loss_mlp": 0.15545654, + "step": 5145, + "time_per_iteration": 2.5837674140930176 + }, + { + "auxiliary_loss_clip": 0.01133445, + "auxiliary_loss_mlp": 0.01044838, + "balance_loss_clip": 1.05339253, + "balance_loss_mlp": 1.02786839, + "epoch": 0.14932389298357612, + "flos": 21829270631040.0, + "grad_norm": 2.2886831810080692, + "language_loss": 0.84716809, + "learning_rate": 3.852506282910447e-06, + "loss": 0.86895096, + "num_input_tokens_seen": 146403400, + "router_z_loss_clip": 0.80029297, + "router_z_loss_mlp": 0.16967773, + "step": 5146, + "time_per_iteration": 2.5653488636016846 + }, + { + "auxiliary_loss_clip": 0.01037174, + "auxiliary_loss_mlp": 0.01008809, + "balance_loss_clip": 1.01702154, + "balance_loss_mlp": 1.00787914, + "epoch": 0.14935291045209215, + "flos": 50869046355840.0, + "grad_norm": 0.6822989884424799, + "language_loss": 0.50926501, + "learning_rate": 3.852435431570735e-06, + "loss": 0.52972484, + "num_input_tokens_seen": 146456605, + "router_z_loss_clip": 0.20117188, + "router_z_loss_mlp": 0.00927734, + "step": 5147, + "time_per_iteration": 2.9100372791290283 + }, + { + "auxiliary_loss_clip": 0.0114603, + "auxiliary_loss_mlp": 0.01046824, + "balance_loss_clip": 1.0596137, + "balance_loss_mlp": 1.02744079, + "epoch": 0.1493819279206082, + "flos": 16829859041280.0, + "grad_norm": 3.3368693700589898, + "language_loss": 0.78464496, + "learning_rate": 3.852364563869564e-06, + "loss": 0.80657351, + "num_input_tokens_seen": 146470630, + "router_z_loss_clip": 0.86376953, + "router_z_loss_mlp": 0.19384766, + "step": 5148, + "time_per_iteration": 2.6181435585021973 + }, + { + "auxiliary_loss_clip": 0.01137002, + "auxiliary_loss_mlp": 0.01046501, + "balance_loss_clip": 1.05547714, + "balance_loss_mlp": 1.02997875, + "epoch": 0.14941094538912425, + "flos": 55881276122880.0, + "grad_norm": 2.1401997292925437, + "language_loss": 0.78257298, + "learning_rate": 3.8522936798075595e-06, + "loss": 0.80440795, + "num_input_tokens_seen": 146491775, + "router_z_loss_clip": 0.81542969, + "router_z_loss_mlp": 0.16534424, + "step": 5149, + "time_per_iteration": 2.7492926120758057 + }, + { + "auxiliary_loss_clip": 0.01035702, + "auxiliary_loss_mlp": 0.01005917, + "balance_loss_clip": 1.01536119, + "balance_loss_mlp": 1.00492191, + "epoch": 0.1494399628576403, + "flos": 57581432616960.0, + "grad_norm": 0.6565121729525983, + "language_loss": 0.45494747, + "learning_rate": 3.852222779385347e-06, + "loss": 0.47536367, + "num_input_tokens_seen": 146547960, + "router_z_loss_clip": 0.203125, + "router_z_loss_mlp": 0.00994873, + "step": 5150, + "time_per_iteration": 3.1241369247436523 + }, + { + "auxiliary_loss_clip": 0.01034397, + "auxiliary_loss_mlp": 0.01001892, + "balance_loss_clip": 1.01433253, + "balance_loss_mlp": 1.00096226, + "epoch": 0.14946898032615635, + "flos": 74666185545600.0, + "grad_norm": 0.6768370944432265, + "language_loss": 0.46928495, + "learning_rate": 3.852151862603554e-06, + "loss": 0.48964787, + "num_input_tokens_seen": 146615260, + "router_z_loss_clip": 0.20117188, + "router_z_loss_mlp": 0.00927734, + "step": 5151, + "time_per_iteration": 3.2396554946899414 + }, + { + "auxiliary_loss_clip": 0.01032636, + "auxiliary_loss_mlp": 0.0100086, + "balance_loss_clip": 1.01261365, + "balance_loss_mlp": 0.9998228, + "epoch": 0.1494979977946724, + "flos": 72325312202880.0, + "grad_norm": 0.8049230297670924, + "language_loss": 0.5333401, + "learning_rate": 3.852080929462807e-06, + "loss": 0.55367506, + "num_input_tokens_seen": 146672820, + "router_z_loss_clip": 0.20019531, + "router_z_loss_mlp": 0.01037598, + "step": 5152, + "time_per_iteration": 3.030339241027832 + }, + { + "auxiliary_loss_clip": 0.01137525, + "auxiliary_loss_mlp": 0.01047922, + "balance_loss_clip": 1.06099725, + "balance_loss_mlp": 1.03239465, + "epoch": 0.14952701526318843, + "flos": 16209681793920.0, + "grad_norm": 2.99552790061186, + "language_loss": 0.85652089, + "learning_rate": 3.852009979963731e-06, + "loss": 0.87837529, + "num_input_tokens_seen": 146685325, + "router_z_loss_clip": 0.765625, + "router_z_loss_mlp": 0.15539551, + "step": 5153, + "time_per_iteration": 2.5596208572387695 + }, + { + "auxiliary_loss_clip": 0.01030884, + "auxiliary_loss_mlp": 0.01000966, + "balance_loss_clip": 1.01076937, + "balance_loss_mlp": 0.99997061, + "epoch": 0.14955603273170448, + "flos": 64126512305280.0, + "grad_norm": 0.6249422451403868, + "language_loss": 0.46507469, + "learning_rate": 3.851939014106954e-06, + "loss": 0.48539317, + "num_input_tokens_seen": 146747815, + "router_z_loss_clip": 0.20117188, + "router_z_loss_mlp": 0.00994873, + "step": 5154, + "time_per_iteration": 3.092503309249878 + }, + { + "auxiliary_loss_clip": 0.01148981, + "auxiliary_loss_mlp": 0.01051522, + "balance_loss_clip": 1.05963564, + "balance_loss_mlp": 1.03268719, + "epoch": 0.14958505020022053, + "flos": 28471272192000.0, + "grad_norm": 3.1868746519133033, + "language_loss": 1.20004189, + "learning_rate": 3.851868031893101e-06, + "loss": 1.22204685, + "num_input_tokens_seen": 146762335, + "router_z_loss_clip": 0.89404297, + "router_z_loss_mlp": 0.18835449, + "step": 5155, + "time_per_iteration": 2.6399619579315186 + }, + { + "auxiliary_loss_clip": 0.01141424, + "auxiliary_loss_mlp": 0.01051693, + "balance_loss_clip": 1.06015372, + "balance_loss_mlp": 1.03509927, + "epoch": 0.14961406766873658, + "flos": 58642231023360.0, + "grad_norm": 2.5264498798746864, + "language_loss": 0.91220909, + "learning_rate": 3.851797033322801e-06, + "loss": 0.93414021, + "num_input_tokens_seen": 146780740, + "router_z_loss_clip": 0.81396484, + "router_z_loss_mlp": 0.16589355, + "step": 5156, + "time_per_iteration": 2.8025784492492676 + }, + { + "auxiliary_loss_clip": 0.01147601, + "auxiliary_loss_mlp": 0.01055881, + "balance_loss_clip": 1.06076181, + "balance_loss_mlp": 1.0388515, + "epoch": 0.14964308513725263, + "flos": 16539162272640.0, + "grad_norm": 2.370590149494664, + "language_loss": 0.94898593, + "learning_rate": 3.85172601839668e-06, + "loss": 0.9710207, + "num_input_tokens_seen": 146794685, + "router_z_loss_clip": 0.86767578, + "router_z_loss_mlp": 0.17022705, + "step": 5157, + "time_per_iteration": 2.5451130867004395 + }, + { + "auxiliary_loss_clip": 0.01136503, + "auxiliary_loss_mlp": 0.01042016, + "balance_loss_clip": 1.05726135, + "balance_loss_mlp": 1.02610159, + "epoch": 0.1496721026057687, + "flos": 23434621176960.0, + "grad_norm": 2.3843216420415776, + "language_loss": 0.91255409, + "learning_rate": 3.851654987115365e-06, + "loss": 0.93433928, + "num_input_tokens_seen": 146809910, + "router_z_loss_clip": 0.79199219, + "router_z_loss_mlp": 0.15905762, + "step": 5158, + "time_per_iteration": 2.5670506954193115 + }, + { + "auxiliary_loss_clip": 0.01151567, + "auxiliary_loss_mlp": 0.01054502, + "balance_loss_clip": 1.06234419, + "balance_loss_mlp": 1.03641188, + "epoch": 0.1497011200742847, + "flos": 26751330282240.0, + "grad_norm": 2.4648718459296837, + "language_loss": 0.91913307, + "learning_rate": 3.851583939479485e-06, + "loss": 0.94119376, + "num_input_tokens_seen": 146821945, + "router_z_loss_clip": 0.89306641, + "router_z_loss_mlp": 0.18096924, + "step": 5159, + "time_per_iteration": 2.5742177963256836 + }, + { + "auxiliary_loss_clip": 0.01138995, + "auxiliary_loss_mlp": 0.01044432, + "balance_loss_clip": 1.06002593, + "balance_loss_mlp": 1.02761161, + "epoch": 0.14973013754280076, + "flos": 13217676658560.0, + "grad_norm": 3.1586180262127432, + "language_loss": 0.83469892, + "learning_rate": 3.851512875489666e-06, + "loss": 0.85653317, + "num_input_tokens_seen": 146835940, + "router_z_loss_clip": 0.79003906, + "router_z_loss_mlp": 0.16821289, + "step": 5160, + "time_per_iteration": 2.5715901851654053 + }, + { + "auxiliary_loss_clip": 0.01139143, + "auxiliary_loss_mlp": 0.01050031, + "balance_loss_clip": 1.0580529, + "balance_loss_mlp": 1.03266215, + "epoch": 0.1497591550113168, + "flos": 23469849440640.0, + "grad_norm": 2.0312991603261343, + "language_loss": 0.98348927, + "learning_rate": 3.851441795146535e-06, + "loss": 1.00538111, + "num_input_tokens_seen": 146850805, + "router_z_loss_clip": 0.81103516, + "router_z_loss_mlp": 0.17370605, + "step": 5161, + "time_per_iteration": 2.5571672916412354 + }, + { + "auxiliary_loss_clip": 0.01036294, + "auxiliary_loss_mlp": 0.0100662, + "balance_loss_clip": 1.01591408, + "balance_loss_mlp": 1.00566602, + "epoch": 0.14978817247983287, + "flos": 60294807953280.0, + "grad_norm": 0.6376942767779469, + "language_loss": 0.4616729, + "learning_rate": 3.851370698450722e-06, + "loss": 0.48210204, + "num_input_tokens_seen": 146912870, + "router_z_loss_clip": 0.20410156, + "router_z_loss_mlp": 0.00952148, + "step": 5162, + "time_per_iteration": 3.1111061573028564 + }, + { + "auxiliary_loss_clip": 0.01035966, + "auxiliary_loss_mlp": 0.01001267, + "balance_loss_clip": 1.01570559, + "balance_loss_mlp": 1.00033116, + "epoch": 0.14981718994834892, + "flos": 61066887816960.0, + "grad_norm": 0.6190266805764533, + "language_loss": 0.47702563, + "learning_rate": 3.851299585402854e-06, + "loss": 0.49739796, + "num_input_tokens_seen": 146977635, + "router_z_loss_clip": 0.203125, + "router_z_loss_mlp": 0.00933838, + "step": 5163, + "time_per_iteration": 3.1565020084381104 + }, + { + "auxiliary_loss_clip": 0.01150195, + "auxiliary_loss_mlp": 0.01054511, + "balance_loss_clip": 1.06381726, + "balance_loss_mlp": 1.03754735, + "epoch": 0.14984620741686494, + "flos": 33103748396160.0, + "grad_norm": 2.4979529046004396, + "language_loss": 1.07666564, + "learning_rate": 3.851228456003558e-06, + "loss": 1.09871268, + "num_input_tokens_seen": 146995155, + "router_z_loss_clip": 0.86328125, + "router_z_loss_mlp": 0.16955566, + "step": 5164, + "time_per_iteration": 2.6213228702545166 + }, + { + "auxiliary_loss_clip": 0.01136657, + "auxiliary_loss_mlp": 0.01038865, + "balance_loss_clip": 1.05700731, + "balance_loss_mlp": 1.0233798, + "epoch": 0.149875224885381, + "flos": 21973451823360.0, + "grad_norm": 2.068289171394689, + "language_loss": 0.90712845, + "learning_rate": 3.8511573102534645e-06, + "loss": 0.92888367, + "num_input_tokens_seen": 147010330, + "router_z_loss_clip": 0.79638672, + "router_z_loss_mlp": 0.15496826, + "step": 5165, + "time_per_iteration": 2.515033721923828 + }, + { + "auxiliary_loss_clip": 0.01144015, + "auxiliary_loss_mlp": 0.01044986, + "balance_loss_clip": 1.06203532, + "balance_loss_mlp": 1.02677095, + "epoch": 0.14990424235389704, + "flos": 17777792914560.0, + "grad_norm": 2.274807179397159, + "language_loss": 0.80354512, + "learning_rate": 3.851086148153199e-06, + "loss": 0.8254351, + "num_input_tokens_seen": 147025005, + "router_z_loss_clip": 0.81933594, + "router_z_loss_mlp": 0.18206787, + "step": 5166, + "time_per_iteration": 2.54461932182312 + }, + { + "auxiliary_loss_clip": 0.01145356, + "auxiliary_loss_mlp": 0.01045382, + "balance_loss_clip": 1.06040788, + "balance_loss_mlp": 1.02880597, + "epoch": 0.1499332598224131, + "flos": 16390527793920.0, + "grad_norm": 2.563163418680644, + "language_loss": 0.88399452, + "learning_rate": 3.851014969703393e-06, + "loss": 0.90590191, + "num_input_tokens_seen": 147037045, + "router_z_loss_clip": 0.84912109, + "router_z_loss_mlp": 0.16583252, + "step": 5167, + "time_per_iteration": 2.4704058170318604 + }, + { + "auxiliary_loss_clip": 0.01035312, + "auxiliary_loss_mlp": 0.01010416, + "balance_loss_clip": 1.01512742, + "balance_loss_mlp": 1.00957, + "epoch": 0.14996227729092915, + "flos": 60977648476800.0, + "grad_norm": 0.7269512780293447, + "language_loss": 0.47864881, + "learning_rate": 3.850943774904672e-06, + "loss": 0.49910611, + "num_input_tokens_seen": 147094255, + "router_z_loss_clip": 0.20214844, + "router_z_loss_mlp": 0.00848389, + "step": 5168, + "time_per_iteration": 2.9164340496063232 + }, + { + "auxiliary_loss_clip": 0.01033385, + "auxiliary_loss_mlp": 0.01009147, + "balance_loss_clip": 1.01317978, + "balance_loss_mlp": 1.0083189, + "epoch": 0.1499912947594452, + "flos": 62914921626240.0, + "grad_norm": 0.684149434426648, + "language_loss": 0.50120997, + "learning_rate": 3.850872563757669e-06, + "loss": 0.52163529, + "num_input_tokens_seen": 147153650, + "router_z_loss_clip": 0.20214844, + "router_z_loss_mlp": 0.00830078, + "step": 5169, + "time_per_iteration": 2.9635424613952637 + }, + { + "auxiliary_loss_clip": 0.01136626, + "auxiliary_loss_mlp": 0.01046273, + "balance_loss_clip": 1.0569458, + "balance_loss_mlp": 1.03099048, + "epoch": 0.15002031222796122, + "flos": 22887306668160.0, + "grad_norm": 1.9734432292010637, + "language_loss": 0.66232169, + "learning_rate": 3.850801336263008e-06, + "loss": 0.6841507, + "num_input_tokens_seen": 147170410, + "router_z_loss_clip": 0.796875, + "router_z_loss_mlp": 0.15283203, + "step": 5170, + "time_per_iteration": 2.5497238636016846 + }, + { + "auxiliary_loss_clip": 0.01143393, + "auxiliary_loss_mlp": 0.01058068, + "balance_loss_clip": 1.05694556, + "balance_loss_mlp": 1.04061532, + "epoch": 0.15004932969647727, + "flos": 25258093061760.0, + "grad_norm": 2.177844062960069, + "language_loss": 0.76572847, + "learning_rate": 3.850730092421322e-06, + "loss": 0.78774315, + "num_input_tokens_seen": 147186820, + "router_z_loss_clip": 0.86376953, + "router_z_loss_mlp": 0.17437744, + "step": 5171, + "time_per_iteration": 2.573671579360962 + }, + { + "auxiliary_loss_clip": 0.01145122, + "auxiliary_loss_mlp": 0.01045678, + "balance_loss_clip": 1.05946887, + "balance_loss_mlp": 1.02813053, + "epoch": 0.15007834716499333, + "flos": 24270513552000.0, + "grad_norm": 2.4043820009315042, + "language_loss": 0.98217225, + "learning_rate": 3.850658832233239e-06, + "loss": 1.0040803, + "num_input_tokens_seen": 147202715, + "router_z_loss_clip": 0.85693359, + "router_z_loss_mlp": 0.17541504, + "step": 5172, + "time_per_iteration": 2.5889885425567627 + }, + { + "auxiliary_loss_clip": 0.01144548, + "auxiliary_loss_mlp": 0.01055037, + "balance_loss_clip": 1.05863619, + "balance_loss_mlp": 1.03767967, + "epoch": 0.15010736463350938, + "flos": 16283370545280.0, + "grad_norm": 2.63032852493492, + "language_loss": 0.81987631, + "learning_rate": 3.850587555699388e-06, + "loss": 0.8418721, + "num_input_tokens_seen": 147217940, + "router_z_loss_clip": 0.85986328, + "router_z_loss_mlp": 0.17358398, + "step": 5173, + "time_per_iteration": 2.4997918605804443 + }, + { + "auxiliary_loss_clip": 0.01036562, + "auxiliary_loss_mlp": 0.01001676, + "balance_loss_clip": 1.01632261, + "balance_loss_mlp": 1.00076413, + "epoch": 0.15013638210202543, + "flos": 74786487171840.0, + "grad_norm": 0.6597123385521191, + "language_loss": 0.49321562, + "learning_rate": 3.8505162628203986e-06, + "loss": 0.51359797, + "num_input_tokens_seen": 147285345, + "router_z_loss_clip": 0.203125, + "router_z_loss_mlp": 0.00909424, + "step": 5174, + "time_per_iteration": 3.2431652545928955 + }, + { + "auxiliary_loss_clip": 0.01139719, + "auxiliary_loss_mlp": 0.01054453, + "balance_loss_clip": 1.06134701, + "balance_loss_mlp": 1.03941476, + "epoch": 0.15016539957054145, + "flos": 13253048576640.0, + "grad_norm": 3.486836321024246, + "language_loss": 1.01373148, + "learning_rate": 3.850444953596902e-06, + "loss": 1.03567326, + "num_input_tokens_seen": 147297045, + "router_z_loss_clip": 0.78369141, + "router_z_loss_mlp": 0.15026855, + "step": 5175, + "time_per_iteration": 2.5160136222839355 + }, + { + "auxiliary_loss_clip": 0.0112901, + "auxiliary_loss_mlp": 0.01035217, + "balance_loss_clip": 1.05325198, + "balance_loss_mlp": 1.02224088, + "epoch": 0.1501944170390575, + "flos": 26862438026880.0, + "grad_norm": 2.4241896650463652, + "language_loss": 0.92738003, + "learning_rate": 3.850373628029525e-06, + "loss": 0.94902229, + "num_input_tokens_seen": 147311040, + "router_z_loss_clip": 0.7578125, + "router_z_loss_mlp": 0.12969971, + "step": 5176, + "time_per_iteration": 2.5742430686950684 + }, + { + "auxiliary_loss_clip": 0.01140011, + "auxiliary_loss_mlp": 0.01043431, + "balance_loss_clip": 1.0568223, + "balance_loss_mlp": 1.0254904, + "epoch": 0.15022343450757356, + "flos": 25184512051200.0, + "grad_norm": 2.167825247374606, + "language_loss": 0.75355518, + "learning_rate": 3.850302286118901e-06, + "loss": 0.77538967, + "num_input_tokens_seen": 147326720, + "router_z_loss_clip": 0.83203125, + "router_z_loss_mlp": 0.17956543, + "step": 5177, + "time_per_iteration": 2.589125156402588 + }, + { + "auxiliary_loss_clip": 0.01130552, + "auxiliary_loss_mlp": 0.01033691, + "balance_loss_clip": 1.05633688, + "balance_loss_mlp": 1.02008951, + "epoch": 0.1502524519760896, + "flos": 22012810151040.0, + "grad_norm": 2.339475143967954, + "language_loss": 0.82969564, + "learning_rate": 3.8502309278656576e-06, + "loss": 0.85133809, + "num_input_tokens_seen": 147344865, + "router_z_loss_clip": 0.74121094, + "router_z_loss_mlp": 0.13604736, + "step": 5178, + "time_per_iteration": 2.5556485652923584 + }, + { + "auxiliary_loss_clip": 0.01131104, + "auxiliary_loss_mlp": 0.01039068, + "balance_loss_clip": 1.05591679, + "balance_loss_mlp": 1.02536535, + "epoch": 0.15028146944460566, + "flos": 26353727660160.0, + "grad_norm": 2.299179684906937, + "language_loss": 0.75416726, + "learning_rate": 3.8501595532704256e-06, + "loss": 0.77586901, + "num_input_tokens_seen": 147359405, + "router_z_loss_clip": 0.75195312, + "router_z_loss_mlp": 0.13702393, + "step": 5179, + "time_per_iteration": 2.589338541030884 + }, + { + "auxiliary_loss_clip": 0.01133055, + "auxiliary_loss_mlp": 0.01032522, + "balance_loss_clip": 1.05573583, + "balance_loss_mlp": 1.01728106, + "epoch": 0.1503104869131217, + "flos": 19747600807680.0, + "grad_norm": 4.737255583075905, + "language_loss": 0.75121325, + "learning_rate": 3.850088162333837e-06, + "loss": 0.77286899, + "num_input_tokens_seen": 147369800, + "router_z_loss_clip": 0.7734375, + "router_z_loss_mlp": 0.15240479, + "step": 5180, + "time_per_iteration": 2.5111727714538574 + }, + { + "auxiliary_loss_clip": 0.011412, + "auxiliary_loss_mlp": 0.01047132, + "balance_loss_clip": 1.05872238, + "balance_loss_mlp": 1.03015065, + "epoch": 0.15033950438163773, + "flos": 35727992133120.0, + "grad_norm": 2.119974363563617, + "language_loss": 0.89554763, + "learning_rate": 3.8500167550565194e-06, + "loss": 0.917431, + "num_input_tokens_seen": 147389580, + "router_z_loss_clip": 0.82617188, + "router_z_loss_mlp": 0.16986084, + "step": 5181, + "time_per_iteration": 2.760464906692505 + }, + { + "auxiliary_loss_clip": 0.01137097, + "auxiliary_loss_mlp": 0.01036034, + "balance_loss_clip": 1.05847168, + "balance_loss_mlp": 1.02013147, + "epoch": 0.15036852185015379, + "flos": 30803598097920.0, + "grad_norm": 2.1543449839253412, + "language_loss": 0.73024464, + "learning_rate": 3.8499453314391065e-06, + "loss": 0.75197601, + "num_input_tokens_seen": 147405005, + "router_z_loss_clip": 0.78662109, + "router_z_loss_mlp": 0.15899658, + "step": 5182, + "time_per_iteration": 2.5519583225250244 + }, + { + "auxiliary_loss_clip": 0.01150716, + "auxiliary_loss_mlp": 0.01058901, + "balance_loss_clip": 1.05772376, + "balance_loss_mlp": 1.0384686, + "epoch": 0.15039753931866984, + "flos": 26863515434880.0, + "grad_norm": 3.587098851078222, + "language_loss": 0.90971422, + "learning_rate": 3.849873891482227e-06, + "loss": 0.93181038, + "num_input_tokens_seen": 147421210, + "router_z_loss_clip": 0.9296875, + "router_z_loss_mlp": 0.20440674, + "step": 5183, + "time_per_iteration": 2.577629804611206 + }, + { + "auxiliary_loss_clip": 0.011347, + "auxiliary_loss_mlp": 0.0104114, + "balance_loss_clip": 1.05834687, + "balance_loss_mlp": 1.02709103, + "epoch": 0.1504265567871859, + "flos": 12451594366080.0, + "grad_norm": 2.272802075599215, + "language_loss": 0.78640389, + "learning_rate": 3.849802435186513e-06, + "loss": 0.80816233, + "num_input_tokens_seen": 147433300, + "router_z_loss_clip": 0.76416016, + "router_z_loss_mlp": 0.14038086, + "step": 5184, + "time_per_iteration": 2.4388210773468018 + }, + { + "auxiliary_loss_clip": 0.01038034, + "auxiliary_loss_mlp": 0.01004518, + "balance_loss_clip": 1.01741672, + "balance_loss_mlp": 1.00367761, + "epoch": 0.15045557425570194, + "flos": 74769751434240.0, + "grad_norm": 0.683712367276209, + "language_loss": 0.47933057, + "learning_rate": 3.849730962552596e-06, + "loss": 0.49975604, + "num_input_tokens_seen": 147493310, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.00842285, + "step": 5185, + "time_per_iteration": 3.110523223876953 + }, + { + "auxiliary_loss_clip": 0.01138296, + "auxiliary_loss_mlp": 0.01037144, + "balance_loss_clip": 1.05641484, + "balance_loss_mlp": 1.02117562, + "epoch": 0.150484591724218, + "flos": 16429131936000.0, + "grad_norm": 2.2544756699154864, + "language_loss": 0.74221611, + "learning_rate": 3.849659473581106e-06, + "loss": 0.76397049, + "num_input_tokens_seen": 147507690, + "router_z_loss_clip": 0.81835938, + "router_z_loss_mlp": 0.15966797, + "step": 5186, + "time_per_iteration": 2.466397762298584 + }, + { + "auxiliary_loss_clip": 0.0113297, + "auxiliary_loss_mlp": 0.01038441, + "balance_loss_clip": 1.05563486, + "balance_loss_mlp": 1.02250862, + "epoch": 0.15051360919273402, + "flos": 14238940147200.0, + "grad_norm": 3.0450689344850606, + "language_loss": 0.75428414, + "learning_rate": 3.849587968272675e-06, + "loss": 0.77599829, + "num_input_tokens_seen": 147521145, + "router_z_loss_clip": 0.77441406, + "router_z_loss_mlp": 0.15936279, + "step": 5187, + "time_per_iteration": 2.5372629165649414 + }, + { + "auxiliary_loss_clip": 0.01131271, + "auxiliary_loss_mlp": 0.01035487, + "balance_loss_clip": 1.05839729, + "balance_loss_mlp": 1.02230787, + "epoch": 0.15054262666125007, + "flos": 40509390124800.0, + "grad_norm": 2.0093840781874897, + "language_loss": 0.72225761, + "learning_rate": 3.849516446627935e-06, + "loss": 0.74392521, + "num_input_tokens_seen": 147544010, + "router_z_loss_clip": 0.72851562, + "router_z_loss_mlp": 0.13183594, + "step": 5188, + "time_per_iteration": 2.7625701427459717 + }, + { + "auxiliary_loss_clip": 0.01036462, + "auxiliary_loss_mlp": 0.01005991, + "balance_loss_clip": 1.01594472, + "balance_loss_mlp": 1.0050137, + "epoch": 0.15057164412976612, + "flos": 66602768008320.0, + "grad_norm": 0.6811929391359446, + "language_loss": 0.51858199, + "learning_rate": 3.849444908647517e-06, + "loss": 0.53900653, + "num_input_tokens_seen": 147603900, + "router_z_loss_clip": 0.20507812, + "router_z_loss_mlp": 0.00976562, + "step": 5189, + "time_per_iteration": 3.186828374862671 + }, + { + "auxiliary_loss_clip": 0.01138492, + "auxiliary_loss_mlp": 0.01036803, + "balance_loss_clip": 1.05727887, + "balance_loss_mlp": 1.02049494, + "epoch": 0.15060066159828217, + "flos": 9201570860160.0, + "grad_norm": 3.0932077611992184, + "language_loss": 0.83048904, + "learning_rate": 3.8493733543320535e-06, + "loss": 0.85224199, + "num_input_tokens_seen": 147614415, + "router_z_loss_clip": 0.81152344, + "router_z_loss_mlp": 0.16296387, + "step": 5190, + "time_per_iteration": 2.5909793376922607 + }, + { + "auxiliary_loss_clip": 0.0114799, + "auxiliary_loss_mlp": 0.01057337, + "balance_loss_clip": 1.06283879, + "balance_loss_mlp": 1.03948557, + "epoch": 0.15062967906679822, + "flos": 14531935386240.0, + "grad_norm": 2.3636222265866063, + "language_loss": 0.8812654, + "learning_rate": 3.849301783682176e-06, + "loss": 0.90331864, + "num_input_tokens_seen": 147626310, + "router_z_loss_clip": 0.8515625, + "router_z_loss_mlp": 0.17871094, + "step": 5191, + "time_per_iteration": 2.601731300354004 + }, + { + "auxiliary_loss_clip": 0.0103829, + "auxiliary_loss_mlp": 0.01010253, + "balance_loss_clip": 1.01792455, + "balance_loss_mlp": 1.00935292, + "epoch": 0.15065869653531425, + "flos": 67946401082880.0, + "grad_norm": 0.6841092200213076, + "language_loss": 0.51288855, + "learning_rate": 3.849230196698516e-06, + "loss": 0.53337395, + "num_input_tokens_seen": 147687625, + "router_z_loss_clip": 0.20410156, + "router_z_loss_mlp": 0.00897217, + "step": 5192, + "time_per_iteration": 5.696733713150024 + }, + { + "auxiliary_loss_clip": 0.01139428, + "auxiliary_loss_mlp": 0.01058268, + "balance_loss_clip": 1.05677354, + "balance_loss_mlp": 1.03996968, + "epoch": 0.1506877140038303, + "flos": 12159496967040.0, + "grad_norm": 3.7182826794490813, + "language_loss": 0.83032131, + "learning_rate": 3.849158593381707e-06, + "loss": 0.85229832, + "num_input_tokens_seen": 147699515, + "router_z_loss_clip": 0.82617188, + "router_z_loss_mlp": 0.18310547, + "step": 5193, + "time_per_iteration": 2.4915640354156494 + }, + { + "auxiliary_loss_clip": 0.01144477, + "auxiliary_loss_mlp": 0.01046048, + "balance_loss_clip": 1.05920625, + "balance_loss_mlp": 1.02666426, + "epoch": 0.15071673147234635, + "flos": 31571081020800.0, + "grad_norm": 2.260163080651906, + "language_loss": 0.89471662, + "learning_rate": 3.849086973732382e-06, + "loss": 0.9166218, + "num_input_tokens_seen": 147719435, + "router_z_loss_clip": 0.85351562, + "router_z_loss_mlp": 0.19390869, + "step": 5194, + "time_per_iteration": 7.402156829833984 + }, + { + "auxiliary_loss_clip": 0.01146217, + "auxiliary_loss_mlp": 0.01052765, + "balance_loss_clip": 1.06126511, + "balance_loss_mlp": 1.03469288, + "epoch": 0.1507457489408624, + "flos": 22739570029440.0, + "grad_norm": 2.261740681367729, + "language_loss": 0.74054354, + "learning_rate": 3.8490153377511725e-06, + "loss": 0.76253343, + "num_input_tokens_seen": 147735075, + "router_z_loss_clip": 0.84960938, + "router_z_loss_mlp": 0.18078613, + "step": 5195, + "time_per_iteration": 2.5580198764801025 + }, + { + "auxiliary_loss_clip": 0.01038361, + "auxiliary_loss_mlp": 0.01006331, + "balance_loss_clip": 1.01791799, + "balance_loss_mlp": 1.00558043, + "epoch": 0.15077476640937845, + "flos": 61960990181760.0, + "grad_norm": 0.675686757579364, + "language_loss": 0.49451104, + "learning_rate": 3.84894368543871e-06, + "loss": 0.51495796, + "num_input_tokens_seen": 147793500, + "router_z_loss_clip": 0.20507812, + "router_z_loss_mlp": 0.00750732, + "step": 5196, + "time_per_iteration": 3.0072591304779053 + }, + { + "auxiliary_loss_clip": 0.01142474, + "auxiliary_loss_mlp": 0.01043794, + "balance_loss_clip": 1.06135452, + "balance_loss_mlp": 1.02640152, + "epoch": 0.1508037838778945, + "flos": 35403826867200.0, + "grad_norm": 2.035338376934247, + "language_loss": 0.97108668, + "learning_rate": 3.84887201679563e-06, + "loss": 0.99294931, + "num_input_tokens_seen": 147813355, + "router_z_loss_clip": 0.81005859, + "router_z_loss_mlp": 0.17407227, + "step": 5197, + "time_per_iteration": 2.722702741622925 + }, + { + "auxiliary_loss_clip": 0.01037082, + "auxiliary_loss_mlp": 0.01001523, + "balance_loss_clip": 1.0168457, + "balance_loss_mlp": 1.00066507, + "epoch": 0.15083280134641053, + "flos": 55841489809920.0, + "grad_norm": 0.676387983078475, + "language_loss": 0.51831019, + "learning_rate": 3.848800331822563e-06, + "loss": 0.53869629, + "num_input_tokens_seen": 147871345, + "router_z_loss_clip": 0.203125, + "router_z_loss_mlp": 0.00860596, + "step": 5198, + "time_per_iteration": 2.988968849182129 + }, + { + "auxiliary_loss_clip": 0.01037556, + "auxiliary_loss_mlp": 0.0099904, + "balance_loss_clip": 1.01721406, + "balance_loss_mlp": 0.99813432, + "epoch": 0.15086181881492658, + "flos": 56897550599040.0, + "grad_norm": 0.6742723442742792, + "language_loss": 0.49581385, + "learning_rate": 3.848728630520144e-06, + "loss": 0.5161798, + "num_input_tokens_seen": 147927610, + "router_z_loss_clip": 0.203125, + "router_z_loss_mlp": 0.0090332, + "step": 5199, + "time_per_iteration": 2.969204902648926 + }, + { + "auxiliary_loss_clip": 0.01133404, + "auxiliary_loss_mlp": 0.01045242, + "balance_loss_clip": 1.05611634, + "balance_loss_mlp": 1.03022718, + "epoch": 0.15089083628344263, + "flos": 14676798936960.0, + "grad_norm": 2.2519424968030792, + "language_loss": 0.73955202, + "learning_rate": 3.8486569128890065e-06, + "loss": 0.76133847, + "num_input_tokens_seen": 147940990, + "router_z_loss_clip": 0.77294922, + "router_z_loss_mlp": 0.14996338, + "step": 5200, + "time_per_iteration": 2.5245630741119385 + }, + { + "auxiliary_loss_clip": 0.01148258, + "auxiliary_loss_mlp": 0.01057896, + "balance_loss_clip": 1.06125259, + "balance_loss_mlp": 1.03921592, + "epoch": 0.15091985375195868, + "flos": 23871402558720.0, + "grad_norm": 2.5913211490898695, + "language_loss": 0.95451826, + "learning_rate": 3.848585178929782e-06, + "loss": 0.97657979, + "num_input_tokens_seen": 147954550, + "router_z_loss_clip": 0.87011719, + "router_z_loss_mlp": 0.18676758, + "step": 5201, + "time_per_iteration": 2.574634313583374 + }, + { + "auxiliary_loss_clip": 0.01139987, + "auxiliary_loss_mlp": 0.01044083, + "balance_loss_clip": 1.05776954, + "balance_loss_mlp": 1.02622569, + "epoch": 0.15094887122047473, + "flos": 26788785189120.0, + "grad_norm": 3.200467949770581, + "language_loss": 0.92592186, + "learning_rate": 3.848513428643105e-06, + "loss": 0.94776261, + "num_input_tokens_seen": 147967735, + "router_z_loss_clip": 0.82128906, + "router_z_loss_mlp": 0.17877197, + "step": 5202, + "time_per_iteration": 2.612804889678955 + }, + { + "auxiliary_loss_clip": 0.01136725, + "auxiliary_loss_mlp": 0.01046087, + "balance_loss_clip": 1.05716288, + "balance_loss_mlp": 1.03006566, + "epoch": 0.15097788868899079, + "flos": 24527310860160.0, + "grad_norm": 3.6225741609703723, + "language_loss": 0.8451342, + "learning_rate": 3.84844166202961e-06, + "loss": 0.86696225, + "num_input_tokens_seen": 147982070, + "router_z_loss_clip": 0.79589844, + "router_z_loss_mlp": 0.16027832, + "step": 5203, + "time_per_iteration": 2.517533540725708 + }, + { + "auxiliary_loss_clip": 0.01137594, + "auxiliary_loss_mlp": 0.01058718, + "balance_loss_clip": 1.05576992, + "balance_loss_mlp": 1.04211783, + "epoch": 0.1510069061575068, + "flos": 18946469819520.0, + "grad_norm": 2.416279959466016, + "language_loss": 0.7174437, + "learning_rate": 3.8483698790899295e-06, + "loss": 0.73940682, + "num_input_tokens_seen": 147997340, + "router_z_loss_clip": 0.81689453, + "router_z_loss_mlp": 0.16595459, + "step": 5204, + "time_per_iteration": 2.5149450302124023 + }, + { + "auxiliary_loss_clip": 0.01038912, + "auxiliary_loss_mlp": 0.01010066, + "balance_loss_clip": 1.0184691, + "balance_loss_mlp": 1.00913, + "epoch": 0.15103592362602286, + "flos": 74774061066240.0, + "grad_norm": 0.6768312895130941, + "language_loss": 0.46715933, + "learning_rate": 3.8482980798247e-06, + "loss": 0.48764914, + "num_input_tokens_seen": 148055800, + "router_z_loss_clip": 0.20507812, + "router_z_loss_mlp": 0.00933838, + "step": 5205, + "time_per_iteration": 3.1276705265045166 + }, + { + "auxiliary_loss_clip": 0.01152973, + "auxiliary_loss_mlp": 0.01052162, + "balance_loss_clip": 1.06327033, + "balance_loss_mlp": 1.03289771, + "epoch": 0.1510649410945389, + "flos": 44854581352320.0, + "grad_norm": 1.9532359163839783, + "language_loss": 0.80582047, + "learning_rate": 3.848226264234552e-06, + "loss": 0.82787192, + "num_input_tokens_seen": 148077505, + "router_z_loss_clip": 0.89794922, + "router_z_loss_mlp": 0.19287109, + "step": 5206, + "time_per_iteration": 2.7614150047302246 + }, + { + "auxiliary_loss_clip": 0.01038271, + "auxiliary_loss_mlp": 0.01005788, + "balance_loss_clip": 1.01811194, + "balance_loss_mlp": 1.00481069, + "epoch": 0.15109395856305496, + "flos": 58059152524800.0, + "grad_norm": 0.7069674647308699, + "language_loss": 0.49315062, + "learning_rate": 3.848154432320122e-06, + "loss": 0.51359117, + "num_input_tokens_seen": 148136010, + "router_z_loss_clip": 0.20117188, + "router_z_loss_mlp": 0.00976562, + "step": 5207, + "time_per_iteration": 2.9663712978363037 + }, + { + "auxiliary_loss_clip": 0.01038895, + "auxiliary_loss_mlp": 0.01001171, + "balance_loss_clip": 1.01872098, + "balance_loss_mlp": 1.00033081, + "epoch": 0.15112297603157102, + "flos": 68587084995840.0, + "grad_norm": 0.6536349158326507, + "language_loss": 0.50348556, + "learning_rate": 3.8480825840820444e-06, + "loss": 0.5238862, + "num_input_tokens_seen": 148205460, + "router_z_loss_clip": 0.20214844, + "router_z_loss_mlp": 0.00842285, + "step": 5208, + "time_per_iteration": 3.221834421157837 + }, + { + "auxiliary_loss_clip": 0.01037385, + "auxiliary_loss_mlp": 0.01000312, + "balance_loss_clip": 1.01720071, + "balance_loss_mlp": 0.99947149, + "epoch": 0.15115199350008704, + "flos": 66642485472000.0, + "grad_norm": 0.6522249028636328, + "language_loss": 0.49991405, + "learning_rate": 3.848010719520954e-06, + "loss": 0.52029097, + "num_input_tokens_seen": 148269135, + "router_z_loss_clip": 0.20117188, + "router_z_loss_mlp": 0.00842285, + "step": 5209, + "time_per_iteration": 3.1452088356018066 + }, + { + "auxiliary_loss_clip": 0.01143358, + "auxiliary_loss_mlp": 0.01043858, + "balance_loss_clip": 1.06045604, + "balance_loss_mlp": 1.02912354, + "epoch": 0.1511810109686031, + "flos": 20007558512640.0, + "grad_norm": 1.8433191681415655, + "language_loss": 0.70092809, + "learning_rate": 3.847938838637485e-06, + "loss": 0.72280025, + "num_input_tokens_seen": 148284100, + "router_z_loss_clip": 0.82910156, + "router_z_loss_mlp": 0.14727783, + "step": 5210, + "time_per_iteration": 2.524643659591675 + }, + { + "auxiliary_loss_clip": 0.01126416, + "auxiliary_loss_mlp": 0.01042029, + "balance_loss_clip": 1.05352175, + "balance_loss_mlp": 1.02621639, + "epoch": 0.15121002843711914, + "flos": 18473310938880.0, + "grad_norm": 2.09349099076629, + "language_loss": 0.87392616, + "learning_rate": 3.8478669414322725e-06, + "loss": 0.89561057, + "num_input_tokens_seen": 148299795, + "router_z_loss_clip": 0.72949219, + "router_z_loss_mlp": 0.15802002, + "step": 5211, + "time_per_iteration": 2.5062971115112305 + }, + { + "auxiliary_loss_clip": 0.01138885, + "auxiliary_loss_mlp": 0.01041065, + "balance_loss_clip": 1.05832124, + "balance_loss_mlp": 1.0261457, + "epoch": 0.1512390459056352, + "flos": 15954141461760.0, + "grad_norm": 1.9823586277622753, + "language_loss": 0.69858813, + "learning_rate": 3.847795027905951e-06, + "loss": 0.72038764, + "num_input_tokens_seen": 148312880, + "router_z_loss_clip": 0.80566406, + "router_z_loss_mlp": 0.14923096, + "step": 5212, + "time_per_iteration": 2.477139472961426 + }, + { + "auxiliary_loss_clip": 0.01143528, + "auxiliary_loss_mlp": 0.0105065, + "balance_loss_clip": 1.05724883, + "balance_loss_mlp": 1.03320932, + "epoch": 0.15126806337415125, + "flos": 17014080919680.0, + "grad_norm": 3.2682828467897265, + "language_loss": 0.8579154, + "learning_rate": 3.847723098059156e-06, + "loss": 0.87985718, + "num_input_tokens_seen": 148326010, + "router_z_loss_clip": 0.86230469, + "router_z_loss_mlp": 0.17419434, + "step": 5213, + "time_per_iteration": 2.53586745262146 + }, + { + "auxiliary_loss_clip": 0.01034372, + "auxiliary_loss_mlp": 0.01009705, + "balance_loss_clip": 1.01440477, + "balance_loss_mlp": 1.00882936, + "epoch": 0.1512970808426673, + "flos": 63422985548160.0, + "grad_norm": 0.7249471863174849, + "language_loss": 0.48913723, + "learning_rate": 3.847651151892524e-06, + "loss": 0.50957805, + "num_input_tokens_seen": 148385245, + "router_z_loss_clip": 0.20019531, + "router_z_loss_mlp": 0.00878906, + "step": 5214, + "time_per_iteration": 3.043626546859741 + }, + { + "auxiliary_loss_clip": 0.01033985, + "auxiliary_loss_mlp": 0.01008536, + "balance_loss_clip": 1.01404762, + "balance_loss_mlp": 1.00770175, + "epoch": 0.15132609831118332, + "flos": 66274939555200.0, + "grad_norm": 0.6589039382405705, + "language_loss": 0.50773263, + "learning_rate": 3.847579189406688e-06, + "loss": 0.52815783, + "num_input_tokens_seen": 148452720, + "router_z_loss_clip": 0.19921875, + "router_z_loss_mlp": 0.00836182, + "step": 5215, + "time_per_iteration": 3.1470680236816406 + }, + { + "auxiliary_loss_clip": 0.01139443, + "auxiliary_loss_mlp": 0.01044755, + "balance_loss_clip": 1.05543447, + "balance_loss_mlp": 1.02708244, + "epoch": 0.15135511577969937, + "flos": 30111024988800.0, + "grad_norm": 2.022993183465572, + "language_loss": 0.94268906, + "learning_rate": 3.847507210602286e-06, + "loss": 0.96453106, + "num_input_tokens_seen": 148476500, + "router_z_loss_clip": 0.83984375, + "router_z_loss_mlp": 0.17669678, + "step": 5216, + "time_per_iteration": 2.7692301273345947 + }, + { + "auxiliary_loss_clip": 0.01033087, + "auxiliary_loss_mlp": 0.01001779, + "balance_loss_clip": 1.01309204, + "balance_loss_mlp": 1.00092685, + "epoch": 0.15138413324821542, + "flos": 64221351189120.0, + "grad_norm": 0.6772612470363577, + "language_loss": 0.45847714, + "learning_rate": 3.847435215479952e-06, + "loss": 0.47882581, + "num_input_tokens_seen": 148531920, + "router_z_loss_clip": 0.20019531, + "router_z_loss_mlp": 0.00854492, + "step": 5217, + "time_per_iteration": 2.9854326248168945 + }, + { + "auxiliary_loss_clip": 0.01144534, + "auxiliary_loss_mlp": 0.01054413, + "balance_loss_clip": 1.0611639, + "balance_loss_mlp": 1.03622127, + "epoch": 0.15141315071673148, + "flos": 34819093365120.0, + "grad_norm": 2.8746209945140127, + "language_loss": 0.78292418, + "learning_rate": 3.847363204040323e-06, + "loss": 0.80491364, + "num_input_tokens_seen": 148553070, + "router_z_loss_clip": 0.83349609, + "router_z_loss_mlp": 0.18200684, + "step": 5218, + "time_per_iteration": 2.6397197246551514 + }, + { + "auxiliary_loss_clip": 0.01032673, + "auxiliary_loss_mlp": 0.01002207, + "balance_loss_clip": 1.01269376, + "balance_loss_mlp": 1.00132525, + "epoch": 0.15144216818524753, + "flos": 64919131770240.0, + "grad_norm": 0.6966925068623347, + "language_loss": 0.49380419, + "learning_rate": 3.8472911762840345e-06, + "loss": 0.514153, + "num_input_tokens_seen": 148614505, + "router_z_loss_clip": 0.19921875, + "router_z_loss_mlp": 0.0088501, + "step": 5219, + "time_per_iteration": 3.1260581016540527 + }, + { + "auxiliary_loss_clip": 0.01122475, + "auxiliary_loss_mlp": 0.01036981, + "balance_loss_clip": 1.05057621, + "balance_loss_mlp": 1.02163267, + "epoch": 0.15147118565376358, + "flos": 24274320393600.0, + "grad_norm": 2.009317354700812, + "language_loss": 0.79093719, + "learning_rate": 3.847219132211723e-06, + "loss": 0.81253183, + "num_input_tokens_seen": 148631590, + "router_z_loss_clip": 0.71826172, + "router_z_loss_mlp": 0.15344238, + "step": 5220, + "time_per_iteration": 2.554292678833008 + }, + { + "auxiliary_loss_clip": 0.01140998, + "auxiliary_loss_mlp": 0.01053167, + "balance_loss_clip": 1.05601513, + "balance_loss_mlp": 1.03457057, + "epoch": 0.1515002031222796, + "flos": 27118948026240.0, + "grad_norm": 1.7586596381222985, + "language_loss": 0.81582856, + "learning_rate": 3.847147071824024e-06, + "loss": 0.83777022, + "num_input_tokens_seen": 148647015, + "router_z_loss_clip": 0.84863281, + "router_z_loss_mlp": 0.18591309, + "step": 5221, + "time_per_iteration": 2.5626723766326904 + }, + { + "auxiliary_loss_clip": 0.0103291, + "auxiliary_loss_mlp": 0.01006758, + "balance_loss_clip": 1.01274729, + "balance_loss_mlp": 1.00579882, + "epoch": 0.15152922059079565, + "flos": 64278235353600.0, + "grad_norm": 0.6340496865217037, + "language_loss": 0.46237993, + "learning_rate": 3.8470749951215755e-06, + "loss": 0.48277664, + "num_input_tokens_seen": 148712555, + "router_z_loss_clip": 0.20117188, + "router_z_loss_mlp": 0.00958252, + "step": 5222, + "time_per_iteration": 3.1443166732788086 + }, + { + "auxiliary_loss_clip": 0.01127002, + "auxiliary_loss_mlp": 0.01038153, + "balance_loss_clip": 1.05228233, + "balance_loss_mlp": 1.02428901, + "epoch": 0.1515582380593117, + "flos": 15626672144640.0, + "grad_norm": 1.9907475945887334, + "language_loss": 0.80279881, + "learning_rate": 3.847002902105013e-06, + "loss": 0.82445037, + "num_input_tokens_seen": 148727460, + "router_z_loss_clip": 0.74755859, + "router_z_loss_mlp": 0.13861084, + "step": 5223, + "time_per_iteration": 2.488556385040283 + }, + { + "auxiliary_loss_clip": 0.01129113, + "auxiliary_loss_mlp": 0.01042457, + "balance_loss_clip": 1.05276895, + "balance_loss_mlp": 1.02589929, + "epoch": 0.15158725552782776, + "flos": 11137730688000.0, + "grad_norm": 2.1672221228583717, + "language_loss": 0.71517271, + "learning_rate": 3.846930792774973e-06, + "loss": 0.73688841, + "num_input_tokens_seen": 148739400, + "router_z_loss_clip": 0.76367188, + "router_z_loss_mlp": 0.16552734, + "step": 5224, + "time_per_iteration": 2.5004019737243652 + }, + { + "auxiliary_loss_clip": 0.01033625, + "auxiliary_loss_mlp": 0.01010383, + "balance_loss_clip": 1.01342535, + "balance_loss_mlp": 1.0095427, + "epoch": 0.1516162729963438, + "flos": 68461468156800.0, + "grad_norm": 0.6834608968099081, + "language_loss": 0.46836632, + "learning_rate": 3.846858667132093e-06, + "loss": 0.48880637, + "num_input_tokens_seen": 148802720, + "router_z_loss_clip": 0.20214844, + "router_z_loss_mlp": 0.00842285, + "step": 5225, + "time_per_iteration": 3.066650390625 + }, + { + "auxiliary_loss_clip": 0.01136433, + "auxiliary_loss_mlp": 0.01045906, + "balance_loss_clip": 1.05572236, + "balance_loss_mlp": 1.029562, + "epoch": 0.15164529046485983, + "flos": 25256979740160.0, + "grad_norm": 2.720593904267967, + "language_loss": 0.86743343, + "learning_rate": 3.84678652517701e-06, + "loss": 0.88925672, + "num_input_tokens_seen": 148816750, + "router_z_loss_clip": 0.80664062, + "router_z_loss_mlp": 0.16351318, + "step": 5226, + "time_per_iteration": 2.6001837253570557 + }, + { + "auxiliary_loss_clip": 0.01131414, + "auxiliary_loss_mlp": 0.01042712, + "balance_loss_clip": 1.05521274, + "balance_loss_mlp": 1.02789378, + "epoch": 0.15167430793337588, + "flos": 30731992335360.0, + "grad_norm": 1.6232421561608952, + "language_loss": 0.68010306, + "learning_rate": 3.846714366910361e-06, + "loss": 0.70184433, + "num_input_tokens_seen": 148837610, + "router_z_loss_clip": 0.76123047, + "router_z_loss_mlp": 0.14831543, + "step": 5227, + "time_per_iteration": 2.586850166320801 + }, + { + "auxiliary_loss_clip": 0.01134635, + "auxiliary_loss_mlp": 0.01052689, + "balance_loss_clip": 1.0539521, + "balance_loss_mlp": 1.03411603, + "epoch": 0.15170332540189194, + "flos": 16135921215360.0, + "grad_norm": 3.303630996713529, + "language_loss": 0.747697, + "learning_rate": 3.8466421923327835e-06, + "loss": 0.76957035, + "num_input_tokens_seen": 148849635, + "router_z_loss_clip": 0.80712891, + "router_z_loss_mlp": 0.18579102, + "step": 5228, + "time_per_iteration": 2.44169020652771 + }, + { + "auxiliary_loss_clip": 0.01128706, + "auxiliary_loss_mlp": 0.01049461, + "balance_loss_clip": 1.05299127, + "balance_loss_mlp": 1.03347516, + "epoch": 0.151732342870408, + "flos": 16063992230400.0, + "grad_norm": 2.427379841059134, + "language_loss": 0.76199341, + "learning_rate": 3.846570001444915e-06, + "loss": 0.78377509, + "num_input_tokens_seen": 148861775, + "router_z_loss_clip": 0.75683594, + "router_z_loss_mlp": 0.16009521, + "step": 5229, + "time_per_iteration": 2.492640733718872 + }, + { + "auxiliary_loss_clip": 0.01138163, + "auxiliary_loss_mlp": 0.01053432, + "balance_loss_clip": 1.05903721, + "balance_loss_mlp": 1.03516281, + "epoch": 0.15176136033892404, + "flos": 33176682961920.0, + "grad_norm": 1.7697965545938772, + "language_loss": 0.76578969, + "learning_rate": 3.846497794247393e-06, + "loss": 0.78770566, + "num_input_tokens_seen": 148879175, + "router_z_loss_clip": 0.79248047, + "router_z_loss_mlp": 0.18273926, + "step": 5230, + "time_per_iteration": 2.6304852962493896 + }, + { + "auxiliary_loss_clip": 0.01034354, + "auxiliary_loss_mlp": 0.01009537, + "balance_loss_clip": 1.01395369, + "balance_loss_mlp": 1.0087266, + "epoch": 0.1517903778074401, + "flos": 74776790499840.0, + "grad_norm": 0.6859299684670036, + "language_loss": 0.48384923, + "learning_rate": 3.846425570740855e-06, + "loss": 0.50428814, + "num_input_tokens_seen": 148939010, + "router_z_loss_clip": 0.20410156, + "router_z_loss_mlp": 0.00811768, + "step": 5231, + "time_per_iteration": 3.1388955116271973 + }, + { + "auxiliary_loss_clip": 0.01034222, + "auxiliary_loss_mlp": 0.01011814, + "balance_loss_clip": 1.01375985, + "balance_loss_mlp": 1.01096749, + "epoch": 0.15181939527595611, + "flos": 60606403459200.0, + "grad_norm": 0.6787633403643956, + "language_loss": 0.49843597, + "learning_rate": 3.846353330925939e-06, + "loss": 0.51889634, + "num_input_tokens_seen": 148990485, + "router_z_loss_clip": 0.20507812, + "router_z_loss_mlp": 0.00848389, + "step": 5232, + "time_per_iteration": 2.9335672855377197 + }, + { + "auxiliary_loss_clip": 0.01033201, + "auxiliary_loss_mlp": 0.01001799, + "balance_loss_clip": 1.01320529, + "balance_loss_mlp": 1.00092912, + "epoch": 0.15184841274447217, + "flos": 74780345946240.0, + "grad_norm": 0.6565834391762347, + "language_loss": 0.52325594, + "learning_rate": 3.846281074803283e-06, + "loss": 0.54360592, + "num_input_tokens_seen": 149054945, + "router_z_loss_clip": 0.20019531, + "router_z_loss_mlp": 0.00872803, + "step": 5233, + "time_per_iteration": 3.1690821647644043 + }, + { + "auxiliary_loss_clip": 0.01134872, + "auxiliary_loss_mlp": 0.01044475, + "balance_loss_clip": 1.05414557, + "balance_loss_mlp": 1.02727878, + "epoch": 0.15187743021298822, + "flos": 30183241282560.0, + "grad_norm": 2.44469376587926, + "language_loss": 0.70821488, + "learning_rate": 3.846208802373527e-06, + "loss": 0.7300083, + "num_input_tokens_seen": 149069900, + "router_z_loss_clip": 0.80712891, + "router_z_loss_mlp": 0.17199707, + "step": 5234, + "time_per_iteration": 2.60302472114563 + }, + { + "auxiliary_loss_clip": 0.01131708, + "auxiliary_loss_mlp": 0.01041433, + "balance_loss_clip": 1.05219507, + "balance_loss_mlp": 1.0253036, + "epoch": 0.15190644768150427, + "flos": 15115447825920.0, + "grad_norm": 2.424483695020966, + "language_loss": 0.82385027, + "learning_rate": 3.846136513637307e-06, + "loss": 0.84558165, + "num_input_tokens_seen": 149082220, + "router_z_loss_clip": 0.79589844, + "router_z_loss_mlp": 0.16137695, + "step": 5235, + "time_per_iteration": 2.483222007751465 + }, + { + "auxiliary_loss_clip": 0.01134803, + "auxiliary_loss_mlp": 0.01043791, + "balance_loss_clip": 1.05610347, + "balance_loss_mlp": 1.02884221, + "epoch": 0.15193546515002032, + "flos": 28505638529280.0, + "grad_norm": 1.749330716821453, + "language_loss": 0.79370332, + "learning_rate": 3.846064208595262e-06, + "loss": 0.81548929, + "num_input_tokens_seen": 149102100, + "router_z_loss_clip": 0.78808594, + "router_z_loss_mlp": 0.14953613, + "step": 5236, + "time_per_iteration": 2.6952033042907715 + }, + { + "auxiliary_loss_clip": 0.01135061, + "auxiliary_loss_mlp": 0.01047756, + "balance_loss_clip": 1.05655456, + "balance_loss_mlp": 1.03224707, + "epoch": 0.15196448261853635, + "flos": 16068984220800.0, + "grad_norm": 1.9206269104274614, + "language_loss": 0.71930587, + "learning_rate": 3.845991887248031e-06, + "loss": 0.74113399, + "num_input_tokens_seen": 149116850, + "router_z_loss_clip": 0.78515625, + "router_z_loss_mlp": 0.1550293, + "step": 5237, + "time_per_iteration": 2.50480055809021 + }, + { + "auxiliary_loss_clip": 0.0112948, + "auxiliary_loss_mlp": 0.01043225, + "balance_loss_clip": 1.05491304, + "balance_loss_mlp": 1.02796578, + "epoch": 0.1519935000870524, + "flos": 22227411957120.0, + "grad_norm": 2.137713163925137, + "language_loss": 0.78357035, + "learning_rate": 3.845919549596252e-06, + "loss": 0.80529737, + "num_input_tokens_seen": 149129200, + "router_z_loss_clip": 0.74658203, + "router_z_loss_mlp": 0.15252686, + "step": 5238, + "time_per_iteration": 2.545311689376831 + }, + { + "auxiliary_loss_clip": 0.01130955, + "auxiliary_loss_mlp": 0.01041525, + "balance_loss_clip": 1.05424607, + "balance_loss_mlp": 1.02461553, + "epoch": 0.15202251755556845, + "flos": 25441919890560.0, + "grad_norm": 1.880316594688878, + "language_loss": 0.81898147, + "learning_rate": 3.845847195640566e-06, + "loss": 0.84070623, + "num_input_tokens_seen": 149148845, + "router_z_loss_clip": 0.76806641, + "router_z_loss_mlp": 0.16906738, + "step": 5239, + "time_per_iteration": 2.6032352447509766 + }, + { + "auxiliary_loss_clip": 0.01137736, + "auxiliary_loss_mlp": 0.01053445, + "balance_loss_clip": 1.0585475, + "balance_loss_mlp": 1.03670764, + "epoch": 0.1520515350240845, + "flos": 24383524717440.0, + "grad_norm": 1.8470512921696236, + "language_loss": 0.88706851, + "learning_rate": 3.84577482538161e-06, + "loss": 0.90898025, + "num_input_tokens_seen": 149166385, + "router_z_loss_clip": 0.79150391, + "router_z_loss_mlp": 0.1673584, + "step": 5240, + "time_per_iteration": 2.581406593322754 + }, + { + "auxiliary_loss_clip": 0.01133334, + "auxiliary_loss_mlp": 0.01038322, + "balance_loss_clip": 1.05403757, + "balance_loss_mlp": 1.02373052, + "epoch": 0.15208055249260055, + "flos": 14640205956480.0, + "grad_norm": 2.651452832549706, + "language_loss": 0.83131492, + "learning_rate": 3.845702438820023e-06, + "loss": 0.85303152, + "num_input_tokens_seen": 149178420, + "router_z_loss_clip": 0.79296875, + "router_z_loss_mlp": 0.14593506, + "step": 5241, + "time_per_iteration": 2.484959840774536 + }, + { + "auxiliary_loss_clip": 0.01129192, + "auxiliary_loss_mlp": 0.01049433, + "balance_loss_clip": 1.05160522, + "balance_loss_mlp": 1.0315938, + "epoch": 0.1521095699611166, + "flos": 35513426240640.0, + "grad_norm": 5.9050667757458255, + "language_loss": 0.83349603, + "learning_rate": 3.845630035956447e-06, + "loss": 0.85528231, + "num_input_tokens_seen": 149196130, + "router_z_loss_clip": 0.77636719, + "router_z_loss_mlp": 0.17840576, + "step": 5242, + "time_per_iteration": 2.6609203815460205 + }, + { + "auxiliary_loss_clip": 0.01125564, + "auxiliary_loss_mlp": 0.01044413, + "balance_loss_clip": 1.05081129, + "balance_loss_mlp": 1.02935648, + "epoch": 0.15213858742963263, + "flos": 25081736261760.0, + "grad_norm": 2.2224095287326575, + "language_loss": 0.68547958, + "learning_rate": 3.845557616791517e-06, + "loss": 0.70717931, + "num_input_tokens_seen": 149213565, + "router_z_loss_clip": 0.74755859, + "router_z_loss_mlp": 0.15057373, + "step": 5243, + "time_per_iteration": 2.5275280475616455 + }, + { + "auxiliary_loss_clip": 0.01137872, + "auxiliary_loss_mlp": 0.0105096, + "balance_loss_clip": 1.05241776, + "balance_loss_mlp": 1.03177881, + "epoch": 0.15216760489814868, + "flos": 28870490926080.0, + "grad_norm": 2.288962907622499, + "language_loss": 0.9125607, + "learning_rate": 3.8454851813258775e-06, + "loss": 0.93444902, + "num_input_tokens_seen": 149229375, + "router_z_loss_clip": 0.85351562, + "router_z_loss_mlp": 0.19165039, + "step": 5244, + "time_per_iteration": 2.6694304943084717 + }, + { + "auxiliary_loss_clip": 0.01038669, + "auxiliary_loss_mlp": 0.01017975, + "balance_loss_clip": 1.01838088, + "balance_loss_mlp": 1.01696777, + "epoch": 0.15219662236666473, + "flos": 74771295719040.0, + "grad_norm": 0.6927545908286381, + "language_loss": 0.49400333, + "learning_rate": 3.845412729560165e-06, + "loss": 0.51456976, + "num_input_tokens_seen": 149294480, + "router_z_loss_clip": 0.203125, + "router_z_loss_mlp": 0.0100708, + "step": 5245, + "time_per_iteration": 3.1553409099578857 + }, + { + "auxiliary_loss_clip": 0.01126425, + "auxiliary_loss_mlp": 0.01035954, + "balance_loss_clip": 1.05015731, + "balance_loss_mlp": 1.02015316, + "epoch": 0.15222563983518078, + "flos": 14968357632000.0, + "grad_norm": 2.735792989165445, + "language_loss": 0.79332513, + "learning_rate": 3.845340261495021e-06, + "loss": 0.81494892, + "num_input_tokens_seen": 149307540, + "router_z_loss_clip": 0.76220703, + "router_z_loss_mlp": 0.15808105, + "step": 5246, + "time_per_iteration": 2.4723994731903076 + }, + { + "auxiliary_loss_clip": 0.01128292, + "auxiliary_loss_mlp": 0.01044377, + "balance_loss_clip": 1.05220556, + "balance_loss_mlp": 1.02752638, + "epoch": 0.15225465730369683, + "flos": 27593327969280.0, + "grad_norm": 1.8981162811002865, + "language_loss": 0.60888636, + "learning_rate": 3.845267777131086e-06, + "loss": 0.63061303, + "num_input_tokens_seen": 149325240, + "router_z_loss_clip": 0.76171875, + "router_z_loss_mlp": 0.16845703, + "step": 5247, + "time_per_iteration": 2.513746976852417 + }, + { + "auxiliary_loss_clip": 0.01038031, + "auxiliary_loss_mlp": 0.01002478, + "balance_loss_clip": 1.01796937, + "balance_loss_mlp": 1.00151241, + "epoch": 0.15228367477221288, + "flos": 69736224902400.0, + "grad_norm": 0.7145444908801231, + "language_loss": 0.49042326, + "learning_rate": 3.845195276468998e-06, + "loss": 0.51082838, + "num_input_tokens_seen": 149383125, + "router_z_loss_clip": 0.20117188, + "router_z_loss_mlp": 0.00964355, + "step": 5248, + "time_per_iteration": 3.048424243927002 + }, + { + "auxiliary_loss_clip": 0.0113273, + "auxiliary_loss_mlp": 0.01047015, + "balance_loss_clip": 1.05345225, + "balance_loss_mlp": 1.03207803, + "epoch": 0.1523126922407289, + "flos": 13947453279360.0, + "grad_norm": 2.1569559451511244, + "language_loss": 0.6318714, + "learning_rate": 3.845122759509399e-06, + "loss": 0.65366888, + "num_input_tokens_seen": 149395645, + "router_z_loss_clip": 0.79248047, + "router_z_loss_mlp": 0.14941406, + "step": 5249, + "time_per_iteration": 2.462993860244751 + }, + { + "auxiliary_loss_clip": 0.01122366, + "auxiliary_loss_mlp": 0.01048991, + "balance_loss_clip": 1.04820418, + "balance_loss_mlp": 1.03449488, + "epoch": 0.15234170970924496, + "flos": 26935300765440.0, + "grad_norm": 2.636781455161887, + "language_loss": 0.84276503, + "learning_rate": 3.845050226252929e-06, + "loss": 0.86447859, + "num_input_tokens_seen": 149410345, + "router_z_loss_clip": 0.74072266, + "router_z_loss_mlp": 0.14489746, + "step": 5250, + "time_per_iteration": 2.492490768432617 + }, + { + "auxiliary_loss_clip": 0.01134724, + "auxiliary_loss_mlp": 0.01047109, + "balance_loss_clip": 1.05427837, + "balance_loss_mlp": 1.03030682, + "epoch": 0.152370727177761, + "flos": 14605372742400.0, + "grad_norm": 2.3514686741121666, + "language_loss": 0.75488174, + "learning_rate": 3.844977676700229e-06, + "loss": 0.77670008, + "num_input_tokens_seen": 149423675, + "router_z_loss_clip": 0.8046875, + "router_z_loss_mlp": 0.16802979, + "step": 5251, + "time_per_iteration": 2.440573215484619 + }, + { + "auxiliary_loss_clip": 0.01137225, + "auxiliary_loss_mlp": 0.01060209, + "balance_loss_clip": 1.05416882, + "balance_loss_mlp": 1.04319191, + "epoch": 0.15239974464627706, + "flos": 14785428643200.0, + "grad_norm": 2.9449909926969506, + "language_loss": 1.04398537, + "learning_rate": 3.844905110851939e-06, + "loss": 1.06595969, + "num_input_tokens_seen": 149436075, + "router_z_loss_clip": 0.82958984, + "router_z_loss_mlp": 0.17016602, + "step": 5252, + "time_per_iteration": 2.5065717697143555 + }, + { + "auxiliary_loss_clip": 0.01136655, + "auxiliary_loss_mlp": 0.01054031, + "balance_loss_clip": 1.05350935, + "balance_loss_mlp": 1.03625727, + "epoch": 0.15242876211479311, + "flos": 16575216549120.0, + "grad_norm": 4.94441004458011, + "language_loss": 0.73435807, + "learning_rate": 3.844832528708702e-06, + "loss": 0.75626493, + "num_input_tokens_seen": 149450525, + "router_z_loss_clip": 0.83251953, + "router_z_loss_mlp": 0.17773438, + "step": 5253, + "time_per_iteration": 2.4634642601013184 + }, + { + "auxiliary_loss_clip": 0.0113714, + "auxiliary_loss_mlp": 0.01060374, + "balance_loss_clip": 1.05486369, + "balance_loss_mlp": 1.04379833, + "epoch": 0.15245777958330914, + "flos": 36606223664640.0, + "grad_norm": 1.8021944233411549, + "language_loss": 0.74298567, + "learning_rate": 3.844759930271156e-06, + "loss": 0.76496077, + "num_input_tokens_seen": 149470005, + "router_z_loss_clip": 0.82275391, + "router_z_loss_mlp": 0.16577148, + "step": 5254, + "time_per_iteration": 2.606661796569824 + }, + { + "auxiliary_loss_clip": 0.01139657, + "auxiliary_loss_mlp": 0.01053747, + "balance_loss_clip": 1.05685806, + "balance_loss_mlp": 1.03782701, + "epoch": 0.1524867970518252, + "flos": 16685426453760.0, + "grad_norm": 2.4093138687148787, + "language_loss": 0.78099942, + "learning_rate": 3.844687315539944e-06, + "loss": 0.80293351, + "num_input_tokens_seen": 149487030, + "router_z_loss_clip": 0.82861328, + "router_z_loss_mlp": 0.15917969, + "step": 5255, + "time_per_iteration": 2.4586105346679688 + }, + { + "auxiliary_loss_clip": 0.0104446, + "auxiliary_loss_mlp": 0.01018929, + "balance_loss_clip": 1.02328539, + "balance_loss_mlp": 1.01808894, + "epoch": 0.15251581452034124, + "flos": 57780414984960.0, + "grad_norm": 0.6321621834423443, + "language_loss": 0.51554644, + "learning_rate": 3.844614684515708e-06, + "loss": 0.53618032, + "num_input_tokens_seen": 149548885, + "router_z_loss_clip": 0.21191406, + "router_z_loss_mlp": 0.00842285, + "step": 5256, + "time_per_iteration": 3.129047393798828 + }, + { + "auxiliary_loss_clip": 0.01148151, + "auxiliary_loss_mlp": 0.01066387, + "balance_loss_clip": 1.05770946, + "balance_loss_mlp": 1.0478977, + "epoch": 0.1525448319888573, + "flos": 35184771774720.0, + "grad_norm": 3.590263376884526, + "language_loss": 0.82739395, + "learning_rate": 3.844542037199088e-06, + "loss": 0.84953928, + "num_input_tokens_seen": 149563085, + "router_z_loss_clip": 0.90429688, + "router_z_loss_mlp": 0.18481445, + "step": 5257, + "time_per_iteration": 2.626493453979492 + }, + { + "auxiliary_loss_clip": 0.01046376, + "auxiliary_loss_mlp": 0.01015004, + "balance_loss_clip": 1.02535796, + "balance_loss_mlp": 1.01412141, + "epoch": 0.15257384945737335, + "flos": 59051400802560.0, + "grad_norm": 0.6383053531967289, + "language_loss": 0.47572583, + "learning_rate": 3.844469373590727e-06, + "loss": 0.49633959, + "num_input_tokens_seen": 149622295, + "router_z_loss_clip": 0.2109375, + "router_z_loss_mlp": 0.0088501, + "step": 5258, + "time_per_iteration": 2.9905691146850586 + }, + { + "auxiliary_loss_clip": 0.01124026, + "auxiliary_loss_mlp": 0.01050618, + "balance_loss_clip": 1.05205607, + "balance_loss_mlp": 1.03657556, + "epoch": 0.1526028669258894, + "flos": 12780751622400.0, + "grad_norm": 2.6808236517008743, + "language_loss": 0.82141572, + "learning_rate": 3.844396693691265e-06, + "loss": 0.84316218, + "num_input_tokens_seen": 149634485, + "router_z_loss_clip": 0.72021484, + "router_z_loss_mlp": 0.14044189, + "step": 5259, + "time_per_iteration": 2.498765707015991 + }, + { + "auxiliary_loss_clip": 0.01134926, + "auxiliary_loss_mlp": 0.01055368, + "balance_loss_clip": 1.05281663, + "balance_loss_mlp": 1.03901231, + "epoch": 0.15263188439440542, + "flos": 17268651584640.0, + "grad_norm": 2.2098137204866295, + "language_loss": 0.94744855, + "learning_rate": 3.8443239975013456e-06, + "loss": 0.96935147, + "num_input_tokens_seen": 149649105, + "router_z_loss_clip": 0.82128906, + "router_z_loss_mlp": 0.16357422, + "step": 5260, + "time_per_iteration": 2.465637445449829 + }, + { + "auxiliary_loss_clip": 0.01127848, + "auxiliary_loss_mlp": 0.01043541, + "balance_loss_clip": 1.05229378, + "balance_loss_mlp": 1.02827632, + "epoch": 0.15266090186292147, + "flos": 26097217660800.0, + "grad_norm": 2.5513002144316292, + "language_loss": 0.86639774, + "learning_rate": 3.84425128502161e-06, + "loss": 0.88811165, + "num_input_tokens_seen": 149665025, + "router_z_loss_clip": 0.75585938, + "router_z_loss_mlp": 0.15258789, + "step": 5261, + "time_per_iteration": 2.598184585571289 + }, + { + "auxiliary_loss_clip": 0.01129295, + "auxiliary_loss_mlp": 0.01037505, + "balance_loss_clip": 1.05269289, + "balance_loss_mlp": 1.0224669, + "epoch": 0.15268991933143752, + "flos": 33149822567040.0, + "grad_norm": 2.2567327575542495, + "language_loss": 0.96831006, + "learning_rate": 3.844178556252702e-06, + "loss": 0.98997802, + "num_input_tokens_seen": 149686610, + "router_z_loss_clip": 0.76538086, + "router_z_loss_mlp": 0.15039062, + "step": 5262, + "time_per_iteration": 2.6445846557617188 + }, + { + "auxiliary_loss_clip": 0.01129527, + "auxiliary_loss_mlp": 0.01041663, + "balance_loss_clip": 1.05307722, + "balance_loss_mlp": 1.0242821, + "epoch": 0.15271893679995358, + "flos": 26098690118400.0, + "grad_norm": 1.9784262439498361, + "language_loss": 0.90043128, + "learning_rate": 3.844105811195262e-06, + "loss": 0.92214322, + "num_input_tokens_seen": 149702500, + "router_z_loss_clip": 0.76416016, + "router_z_loss_mlp": 0.17382812, + "step": 5263, + "time_per_iteration": 7.394378423690796 + }, + { + "auxiliary_loss_clip": 0.01125651, + "auxiliary_loss_mlp": 0.01037253, + "balance_loss_clip": 1.04945946, + "balance_loss_mlp": 1.0216186, + "epoch": 0.15274795426846963, + "flos": 22594670565120.0, + "grad_norm": 4.29457150089696, + "language_loss": 0.92533386, + "learning_rate": 3.844033049849933e-06, + "loss": 0.94696295, + "num_input_tokens_seen": 149716190, + "router_z_loss_clip": 0.76171875, + "router_z_loss_mlp": 0.15649414, + "step": 5264, + "time_per_iteration": 2.5068204402923584 + }, + { + "auxiliary_loss_clip": 0.01134653, + "auxiliary_loss_mlp": 0.01042236, + "balance_loss_clip": 1.05049658, + "balance_loss_mlp": 1.02511144, + "epoch": 0.15277697173698568, + "flos": 45915346823040.0, + "grad_norm": 2.1609934880097827, + "language_loss": 0.81073821, + "learning_rate": 3.843960272217358e-06, + "loss": 0.83250713, + "num_input_tokens_seen": 149739165, + "router_z_loss_clip": 0.84130859, + "router_z_loss_mlp": 0.17102051, + "step": 5265, + "time_per_iteration": 7.629128456115723 + }, + { + "auxiliary_loss_clip": 0.0113547, + "auxiliary_loss_mlp": 0.01047308, + "balance_loss_clip": 1.05300295, + "balance_loss_mlp": 1.02958179, + "epoch": 0.1528059892055017, + "flos": 31972921447680.0, + "grad_norm": 2.8140831385312643, + "language_loss": 0.80246091, + "learning_rate": 3.8438874782981804e-06, + "loss": 0.82428861, + "num_input_tokens_seen": 149756160, + "router_z_loss_clip": 0.82470703, + "router_z_loss_mlp": 0.17712402, + "step": 5266, + "time_per_iteration": 2.556633234024048 + }, + { + "auxiliary_loss_clip": 0.01123222, + "auxiliary_loss_mlp": 0.01037913, + "balance_loss_clip": 1.04781747, + "balance_loss_mlp": 1.02172422, + "epoch": 0.15283500667401775, + "flos": 16721337075840.0, + "grad_norm": 2.8898112659549184, + "language_loss": 0.85441393, + "learning_rate": 3.843814668093041e-06, + "loss": 0.87602532, + "num_input_tokens_seen": 149768480, + "router_z_loss_clip": 0.75439453, + "router_z_loss_mlp": 0.16204834, + "step": 5267, + "time_per_iteration": 2.461806297302246 + }, + { + "auxiliary_loss_clip": 0.011473, + "auxiliary_loss_mlp": 0.01053542, + "balance_loss_clip": 1.0545404, + "balance_loss_mlp": 1.03277612, + "epoch": 0.1528640241425338, + "flos": 74733442784640.0, + "grad_norm": 2.0996612731748003, + "language_loss": 0.80286622, + "learning_rate": 3.843741841602585e-06, + "loss": 0.82487464, + "num_input_tokens_seen": 149791540, + "router_z_loss_clip": 0.92773438, + "router_z_loss_mlp": 0.20776367, + "step": 5268, + "time_per_iteration": 2.909882068634033 + }, + { + "auxiliary_loss_clip": 0.01127382, + "auxiliary_loss_mlp": 0.01034998, + "balance_loss_clip": 1.05168223, + "balance_loss_mlp": 1.01945305, + "epoch": 0.15289304161104986, + "flos": 20187434845440.0, + "grad_norm": 1.8227563709439656, + "language_loss": 0.75439966, + "learning_rate": 3.843668998827455e-06, + "loss": 0.77602339, + "num_input_tokens_seen": 149808655, + "router_z_loss_clip": 0.7578125, + "router_z_loss_mlp": 0.15539551, + "step": 5269, + "time_per_iteration": 2.534590005874634 + }, + { + "auxiliary_loss_clip": 0.01126254, + "auxiliary_loss_mlp": 0.01034533, + "balance_loss_clip": 1.05118942, + "balance_loss_mlp": 1.01882744, + "epoch": 0.1529220590795659, + "flos": 35291031183360.0, + "grad_norm": 2.847112779221673, + "language_loss": 0.78526491, + "learning_rate": 3.843596139768295e-06, + "loss": 0.80687284, + "num_input_tokens_seen": 149824870, + "router_z_loss_clip": 0.75048828, + "router_z_loss_mlp": 0.15716553, + "step": 5270, + "time_per_iteration": 2.7064287662506104 + }, + { + "auxiliary_loss_clip": 0.01138659, + "auxiliary_loss_mlp": 0.01050871, + "balance_loss_clip": 1.05499244, + "balance_loss_mlp": 1.03365684, + "epoch": 0.15295107654808193, + "flos": 15480731185920.0, + "grad_norm": 4.732126461492529, + "language_loss": 0.86535597, + "learning_rate": 3.843523264425747e-06, + "loss": 0.88725126, + "num_input_tokens_seen": 149838865, + "router_z_loss_clip": 0.83691406, + "router_z_loss_mlp": 0.17211914, + "step": 5271, + "time_per_iteration": 2.460716962814331 + }, + { + "auxiliary_loss_clip": 0.01127284, + "auxiliary_loss_mlp": 0.01039022, + "balance_loss_clip": 1.05230904, + "balance_loss_mlp": 1.02385855, + "epoch": 0.15298009401659798, + "flos": 33248180983680.0, + "grad_norm": 2.0582995477069814, + "language_loss": 0.68627095, + "learning_rate": 3.843450372800456e-06, + "loss": 0.70793402, + "num_input_tokens_seen": 149855390, + "router_z_loss_clip": 0.75048828, + "router_z_loss_mlp": 0.1517334, + "step": 5272, + "time_per_iteration": 2.5765397548675537 + }, + { + "auxiliary_loss_clip": 0.01127875, + "auxiliary_loss_mlp": 0.01036732, + "balance_loss_clip": 1.05098724, + "balance_loss_mlp": 1.01992893, + "epoch": 0.15300911148511404, + "flos": 40471899304320.0, + "grad_norm": 2.2445852455864754, + "language_loss": 0.86337364, + "learning_rate": 3.843377464893066e-06, + "loss": 0.88501966, + "num_input_tokens_seen": 149872310, + "router_z_loss_clip": 0.76855469, + "router_z_loss_mlp": 0.16784668, + "step": 5273, + "time_per_iteration": 2.672610282897949 + }, + { + "auxiliary_loss_clip": 0.01039011, + "auxiliary_loss_mlp": 0.01010181, + "balance_loss_clip": 1.01877499, + "balance_loss_mlp": 1.00916755, + "epoch": 0.1530381289536301, + "flos": 63025849802880.0, + "grad_norm": 1.6486596694218947, + "language_loss": 0.49887216, + "learning_rate": 3.843304540704219e-06, + "loss": 0.51936406, + "num_input_tokens_seen": 149935145, + "router_z_loss_clip": 0.203125, + "router_z_loss_mlp": 0.01013184, + "step": 5274, + "time_per_iteration": 3.1064505577087402 + }, + { + "auxiliary_loss_clip": 0.01039963, + "auxiliary_loss_mlp": 0.01006339, + "balance_loss_clip": 1.01967311, + "balance_loss_mlp": 1.00536144, + "epoch": 0.15306714642214614, + "flos": 55769991788160.0, + "grad_norm": 0.6863933517579899, + "language_loss": 0.48977348, + "learning_rate": 3.8432316002345605e-06, + "loss": 0.5102365, + "num_input_tokens_seen": 149995325, + "router_z_loss_clip": 0.203125, + "router_z_loss_mlp": 0.00976562, + "step": 5275, + "time_per_iteration": 3.0675103664398193 + }, + { + "auxiliary_loss_clip": 0.01120186, + "auxiliary_loss_mlp": 0.01036969, + "balance_loss_clip": 1.04839134, + "balance_loss_mlp": 1.02291989, + "epoch": 0.1530961638906622, + "flos": 12962567289600.0, + "grad_norm": 3.4098217063229335, + "language_loss": 0.89951992, + "learning_rate": 3.843158643484736e-06, + "loss": 0.9210915, + "num_input_tokens_seen": 150005595, + "router_z_loss_clip": 0.71826172, + "router_z_loss_mlp": 0.14056396, + "step": 5276, + "time_per_iteration": 2.4604785442352295 + }, + { + "auxiliary_loss_clip": 0.01125, + "auxiliary_loss_mlp": 0.01047154, + "balance_loss_clip": 1.0503267, + "balance_loss_mlp": 1.0326339, + "epoch": 0.15312518135917821, + "flos": 15587960261760.0, + "grad_norm": 3.4865695984635434, + "language_loss": 0.85764658, + "learning_rate": 3.8430856704553865e-06, + "loss": 0.87936819, + "num_input_tokens_seen": 150018230, + "router_z_loss_clip": 0.74707031, + "router_z_loss_mlp": 0.14508057, + "step": 5277, + "time_per_iteration": 2.4539759159088135 + }, + { + "auxiliary_loss_clip": 0.01140228, + "auxiliary_loss_mlp": 0.01043179, + "balance_loss_clip": 1.05838513, + "balance_loss_mlp": 1.02725244, + "epoch": 0.15315419882769427, + "flos": 13838464437120.0, + "grad_norm": 2.3076601919962862, + "language_loss": 0.86996549, + "learning_rate": 3.843012681147159e-06, + "loss": 0.89179957, + "num_input_tokens_seen": 150030560, + "router_z_loss_clip": 0.81835938, + "router_z_loss_mlp": 0.15930176, + "step": 5278, + "time_per_iteration": 2.4697060585021973 + }, + { + "auxiliary_loss_clip": 0.01130334, + "auxiliary_loss_mlp": 0.01041307, + "balance_loss_clip": 1.05207741, + "balance_loss_mlp": 1.02458251, + "epoch": 0.15318321629621032, + "flos": 27996245804160.0, + "grad_norm": 2.734562337274577, + "language_loss": 0.93546587, + "learning_rate": 3.8429396755606995e-06, + "loss": 0.95718223, + "num_input_tokens_seen": 150047200, + "router_z_loss_clip": 0.78271484, + "router_z_loss_mlp": 0.16729736, + "step": 5279, + "time_per_iteration": 2.571852207183838 + }, + { + "auxiliary_loss_clip": 0.01136401, + "auxiliary_loss_mlp": 0.01043808, + "balance_loss_clip": 1.05652213, + "balance_loss_mlp": 1.02909112, + "epoch": 0.15321223376472637, + "flos": 35730470171520.0, + "grad_norm": 2.1161950881337743, + "language_loss": 1.01907468, + "learning_rate": 3.842866653696649e-06, + "loss": 1.04087675, + "num_input_tokens_seen": 150067710, + "router_z_loss_clip": 0.79833984, + "router_z_loss_mlp": 0.14715576, + "step": 5280, + "time_per_iteration": 2.6360182762145996 + }, + { + "auxiliary_loss_clip": 0.01038932, + "auxiliary_loss_mlp": 0.0101062, + "balance_loss_clip": 1.01886499, + "balance_loss_mlp": 1.00972581, + "epoch": 0.15324125123324242, + "flos": 66631208601600.0, + "grad_norm": 0.6746275737866734, + "language_loss": 0.49454802, + "learning_rate": 3.842793615555656e-06, + "loss": 0.51504356, + "num_input_tokens_seen": 150126730, + "router_z_loss_clip": 0.20117188, + "router_z_loss_mlp": 0.00891113, + "step": 5281, + "time_per_iteration": 3.0454649925231934 + }, + { + "auxiliary_loss_clip": 0.01037015, + "auxiliary_loss_mlp": 0.01004388, + "balance_loss_clip": 1.01718009, + "balance_loss_mlp": 1.00351167, + "epoch": 0.15327026870175847, + "flos": 74776287709440.0, + "grad_norm": 0.7284816473386503, + "language_loss": 0.46195042, + "learning_rate": 3.842720561138363e-06, + "loss": 0.48236442, + "num_input_tokens_seen": 150188880, + "router_z_loss_clip": 0.19824219, + "router_z_loss_mlp": 0.00878906, + "step": 5282, + "time_per_iteration": 3.1276679039001465 + }, + { + "auxiliary_loss_clip": 0.01143423, + "auxiliary_loss_mlp": 0.01051678, + "balance_loss_clip": 1.0596658, + "balance_loss_mlp": 1.03585315, + "epoch": 0.1532992861702745, + "flos": 60180755293440.0, + "grad_norm": 2.2659887398359215, + "language_loss": 1.03508341, + "learning_rate": 3.842647490445417e-06, + "loss": 1.05703437, + "num_input_tokens_seen": 150216475, + "router_z_loss_clip": 0.83789062, + "router_z_loss_mlp": 0.1583252, + "step": 5283, + "time_per_iteration": 3.0151851177215576 + }, + { + "auxiliary_loss_clip": 0.01151149, + "auxiliary_loss_mlp": 0.01049577, + "balance_loss_clip": 1.05977726, + "balance_loss_mlp": 1.03131425, + "epoch": 0.15332830363879055, + "flos": 11063036355840.0, + "grad_norm": 2.7237288164492033, + "language_loss": 0.98037308, + "learning_rate": 3.842574403477463e-06, + "loss": 1.00238037, + "num_input_tokens_seen": 150227945, + "router_z_loss_clip": 0.9140625, + "router_z_loss_mlp": 0.18261719, + "step": 5284, + "time_per_iteration": 2.474346160888672 + }, + { + "auxiliary_loss_clip": 0.01138973, + "auxiliary_loss_mlp": 0.01056062, + "balance_loss_clip": 1.05464995, + "balance_loss_mlp": 1.03895545, + "epoch": 0.1533573211073066, + "flos": 44998762544640.0, + "grad_norm": 2.4586132713444004, + "language_loss": 0.8653248, + "learning_rate": 3.842501300235146e-06, + "loss": 0.88727516, + "num_input_tokens_seen": 150249560, + "router_z_loss_clip": 0.84375, + "router_z_loss_mlp": 0.17108154, + "step": 5285, + "time_per_iteration": 2.705761671066284 + }, + { + "auxiliary_loss_clip": 0.01131575, + "auxiliary_loss_mlp": 0.01037956, + "balance_loss_clip": 1.05617106, + "balance_loss_mlp": 1.02227378, + "epoch": 0.15338633857582265, + "flos": 19273436346240.0, + "grad_norm": 3.1419730614518633, + "language_loss": 0.83059192, + "learning_rate": 3.842428180719111e-06, + "loss": 0.85228723, + "num_input_tokens_seen": 150262115, + "router_z_loss_clip": 0.75488281, + "router_z_loss_mlp": 0.15679932, + "step": 5286, + "time_per_iteration": 2.4944963455200195 + }, + { + "auxiliary_loss_clip": 0.01133359, + "auxiliary_loss_mlp": 0.01040133, + "balance_loss_clip": 1.05246198, + "balance_loss_mlp": 1.02325273, + "epoch": 0.1534153560443387, + "flos": 27124658288640.0, + "grad_norm": 1.9082984284281643, + "language_loss": 0.65796119, + "learning_rate": 3.8423550449300056e-06, + "loss": 0.67969608, + "num_input_tokens_seen": 150280535, + "router_z_loss_clip": 0.80810547, + "router_z_loss_mlp": 0.16870117, + "step": 5287, + "time_per_iteration": 2.583449125289917 + }, + { + "auxiliary_loss_clip": 0.01035431, + "auxiliary_loss_mlp": 0.0100334, + "balance_loss_clip": 1.0158267, + "balance_loss_mlp": 1.00249982, + "epoch": 0.15344437351285473, + "flos": 74768781767040.0, + "grad_norm": 0.7192925262192797, + "language_loss": 0.49771023, + "learning_rate": 3.842281892868474e-06, + "loss": 0.518098, + "num_input_tokens_seen": 150334525, + "router_z_loss_clip": 0.19628906, + "router_z_loss_mlp": 0.00842285, + "step": 5288, + "time_per_iteration": 3.0977916717529297 + }, + { + "auxiliary_loss_clip": 0.01140128, + "auxiliary_loss_mlp": 0.01046481, + "balance_loss_clip": 1.05757117, + "balance_loss_mlp": 1.03054893, + "epoch": 0.15347339098137078, + "flos": 18469719578880.0, + "grad_norm": 2.9045536438161728, + "language_loss": 0.81755644, + "learning_rate": 3.842208724535164e-06, + "loss": 0.83942246, + "num_input_tokens_seen": 150346590, + "router_z_loss_clip": 0.82519531, + "router_z_loss_mlp": 0.15942383, + "step": 5289, + "time_per_iteration": 2.4887430667877197 + }, + { + "auxiliary_loss_clip": 0.01125548, + "auxiliary_loss_mlp": 0.01038881, + "balance_loss_clip": 1.0531708, + "balance_loss_mlp": 1.02615499, + "epoch": 0.15350240844988683, + "flos": 23470639539840.0, + "grad_norm": 1.9568804956758372, + "language_loss": 0.83753228, + "learning_rate": 3.842135539930721e-06, + "loss": 0.85917658, + "num_input_tokens_seen": 150367315, + "router_z_loss_clip": 0.72460938, + "router_z_loss_mlp": 0.12731934, + "step": 5290, + "time_per_iteration": 2.6894969940185547 + }, + { + "auxiliary_loss_clip": 0.01126761, + "auxiliary_loss_mlp": 0.01050784, + "balance_loss_clip": 1.05234933, + "balance_loss_mlp": 1.03540564, + "epoch": 0.15353142591840288, + "flos": 18179633341440.0, + "grad_norm": 3.109107991475149, + "language_loss": 0.93527007, + "learning_rate": 3.842062339055791e-06, + "loss": 0.9570455, + "num_input_tokens_seen": 150380365, + "router_z_loss_clip": 0.74414062, + "router_z_loss_mlp": 0.15362549, + "step": 5291, + "time_per_iteration": 2.4649996757507324 + }, + { + "auxiliary_loss_clip": 0.01033074, + "auxiliary_loss_mlp": 0.01002165, + "balance_loss_clip": 1.01339519, + "balance_loss_mlp": 1.00119901, + "epoch": 0.15356044338691893, + "flos": 70367176229760.0, + "grad_norm": 0.7219925194029114, + "language_loss": 0.52330661, + "learning_rate": 3.8419891219110225e-06, + "loss": 0.54365897, + "num_input_tokens_seen": 150442250, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.00964355, + "step": 5292, + "time_per_iteration": 3.1500155925750732 + }, + { + "auxiliary_loss_clip": 0.01130993, + "auxiliary_loss_mlp": 0.01045303, + "balance_loss_clip": 1.05376482, + "balance_loss_mlp": 1.02793443, + "epoch": 0.15358946085543498, + "flos": 17119011525120.0, + "grad_norm": 3.0718746941470116, + "language_loss": 0.88766885, + "learning_rate": 3.84191588849706e-06, + "loss": 0.90943182, + "num_input_tokens_seen": 150453265, + "router_z_loss_clip": 0.77294922, + "router_z_loss_mlp": 0.17370605, + "step": 5293, + "time_per_iteration": 2.4736053943634033 + }, + { + "auxiliary_loss_clip": 0.01135991, + "auxiliary_loss_mlp": 0.01041687, + "balance_loss_clip": 1.05512583, + "balance_loss_mlp": 1.02539754, + "epoch": 0.153618478323951, + "flos": 29635208501760.0, + "grad_norm": 1.8367620170408792, + "language_loss": 0.77243733, + "learning_rate": 3.84184263881455e-06, + "loss": 0.79421413, + "num_input_tokens_seen": 150470255, + "router_z_loss_clip": 0.80810547, + "router_z_loss_mlp": 0.16296387, + "step": 5294, + "time_per_iteration": 2.569260597229004 + }, + { + "auxiliary_loss_clip": 0.01034425, + "auxiliary_loss_mlp": 0.01000876, + "balance_loss_clip": 1.01470578, + "balance_loss_mlp": 1.00005949, + "epoch": 0.15364749579246706, + "flos": 64780732667520.0, + "grad_norm": 0.7326108252, + "language_loss": 0.46380854, + "learning_rate": 3.841769372864141e-06, + "loss": 0.48416156, + "num_input_tokens_seen": 150530675, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.00817871, + "step": 5295, + "time_per_iteration": 3.037269353866577 + }, + { + "auxiliary_loss_clip": 0.0103313, + "auxiliary_loss_mlp": 0.01002692, + "balance_loss_clip": 1.01347423, + "balance_loss_mlp": 1.00185728, + "epoch": 0.1536765132609831, + "flos": 68545248629760.0, + "grad_norm": 0.6289762544136724, + "language_loss": 0.47801709, + "learning_rate": 3.841696090646481e-06, + "loss": 0.4983753, + "num_input_tokens_seen": 150596975, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.00836182, + "step": 5296, + "time_per_iteration": 3.1510932445526123 + }, + { + "auxiliary_loss_clip": 0.01032766, + "auxiliary_loss_mlp": 0.01002552, + "balance_loss_clip": 1.01309013, + "balance_loss_mlp": 1.00175881, + "epoch": 0.15370553072949916, + "flos": 60930460984320.0, + "grad_norm": 0.7040216510453975, + "language_loss": 0.50074708, + "learning_rate": 3.841622792162214e-06, + "loss": 0.52110028, + "num_input_tokens_seen": 150655175, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.00793457, + "step": 5297, + "time_per_iteration": 3.020752429962158 + }, + { + "auxiliary_loss_clip": 0.01136684, + "auxiliary_loss_mlp": 0.01043778, + "balance_loss_clip": 1.05759096, + "balance_loss_mlp": 1.02739239, + "epoch": 0.15373454819801521, + "flos": 53429510517120.0, + "grad_norm": 2.3864362092163796, + "language_loss": 0.78155482, + "learning_rate": 3.84154947741199e-06, + "loss": 0.80335951, + "num_input_tokens_seen": 150678155, + "router_z_loss_clip": 0.79052734, + "router_z_loss_mlp": 0.16387939, + "step": 5298, + "time_per_iteration": 2.7491745948791504 + }, + { + "auxiliary_loss_clip": 0.01030933, + "auxiliary_loss_mlp": 0.01000538, + "balance_loss_clip": 1.01133251, + "balance_loss_mlp": 0.99965566, + "epoch": 0.15376356566653127, + "flos": 60183628381440.0, + "grad_norm": 1.4607465488513196, + "language_loss": 0.46889067, + "learning_rate": 3.8414761463964555e-06, + "loss": 0.48920539, + "num_input_tokens_seen": 150738410, + "router_z_loss_clip": 0.19628906, + "router_z_loss_mlp": 0.0088501, + "step": 5299, + "time_per_iteration": 3.0741124153137207 + }, + { + "auxiliary_loss_clip": 0.01120097, + "auxiliary_loss_mlp": 0.01036399, + "balance_loss_clip": 1.0501132, + "balance_loss_mlp": 1.02332187, + "epoch": 0.1537925831350473, + "flos": 42627078311040.0, + "grad_norm": 1.8972066515080743, + "language_loss": 0.6031335, + "learning_rate": 3.841402799116259e-06, + "loss": 0.62469852, + "num_input_tokens_seen": 150756725, + "router_z_loss_clip": 0.69970703, + "router_z_loss_mlp": 0.13085938, + "step": 5300, + "time_per_iteration": 2.662673234939575 + }, + { + "auxiliary_loss_clip": 0.01030608, + "auxiliary_loss_mlp": 0.0100107, + "balance_loss_clip": 1.01096821, + "balance_loss_mlp": 1.00022411, + "epoch": 0.15382160060356334, + "flos": 63219804266880.0, + "grad_norm": 0.6426217440067145, + "language_loss": 0.54311752, + "learning_rate": 3.841329435572048e-06, + "loss": 0.5634343, + "num_input_tokens_seen": 150824370, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.00848389, + "step": 5301, + "time_per_iteration": 3.1900887489318848 + }, + { + "auxiliary_loss_clip": 0.01131912, + "auxiliary_loss_mlp": 0.01045971, + "balance_loss_clip": 1.05436885, + "balance_loss_mlp": 1.03096855, + "epoch": 0.1538506180720794, + "flos": 34234216208640.0, + "grad_norm": 2.2532112787907113, + "language_loss": 0.84383959, + "learning_rate": 3.84125605576447e-06, + "loss": 0.86561841, + "num_input_tokens_seen": 150843555, + "router_z_loss_clip": 0.77636719, + "router_z_loss_mlp": 0.15002441, + "step": 5302, + "time_per_iteration": 2.6787664890289307 + }, + { + "auxiliary_loss_clip": 0.01149841, + "auxiliary_loss_mlp": 0.01056063, + "balance_loss_clip": 1.05841756, + "balance_loss_mlp": 1.03499866, + "epoch": 0.15387963554059544, + "flos": 41130824348160.0, + "grad_norm": 1.5605442195835604, + "language_loss": 0.79161882, + "learning_rate": 3.841182659694174e-06, + "loss": 0.81367785, + "num_input_tokens_seen": 150865280, + "router_z_loss_clip": 0.91503906, + "router_z_loss_mlp": 0.21069336, + "step": 5303, + "time_per_iteration": 2.693274974822998 + }, + { + "auxiliary_loss_clip": 0.01143886, + "auxiliary_loss_mlp": 0.01042492, + "balance_loss_clip": 1.05807185, + "balance_loss_mlp": 1.02520025, + "epoch": 0.1539086530091115, + "flos": 16501060920960.0, + "grad_norm": 3.031757379402319, + "language_loss": 0.84686595, + "learning_rate": 3.8411092473618065e-06, + "loss": 0.86872971, + "num_input_tokens_seen": 150879405, + "router_z_loss_clip": 0.85791016, + "router_z_loss_mlp": 0.17272949, + "step": 5304, + "time_per_iteration": 2.4948201179504395 + }, + { + "auxiliary_loss_clip": 0.01029806, + "auxiliary_loss_mlp": 0.0100282, + "balance_loss_clip": 1.01035416, + "balance_loss_mlp": 1.00202715, + "epoch": 0.15393767047762752, + "flos": 74775353955840.0, + "grad_norm": 0.6578967302816465, + "language_loss": 0.4863539, + "learning_rate": 3.841035818768018e-06, + "loss": 0.50668013, + "num_input_tokens_seen": 150943615, + "router_z_loss_clip": 0.19433594, + "router_z_loss_mlp": 0.00793457, + "step": 5305, + "time_per_iteration": 3.1556549072265625 + }, + { + "auxiliary_loss_clip": 0.0113184, + "auxiliary_loss_mlp": 0.01033976, + "balance_loss_clip": 1.05242896, + "balance_loss_mlp": 1.01841879, + "epoch": 0.15396668794614357, + "flos": 42592029615360.0, + "grad_norm": 2.2320027586246636, + "language_loss": 0.73987663, + "learning_rate": 3.8409623739134555e-06, + "loss": 0.76153481, + "num_input_tokens_seen": 150963870, + "router_z_loss_clip": 0.79394531, + "router_z_loss_mlp": 0.15557861, + "step": 5306, + "time_per_iteration": 2.7008798122406006 + }, + { + "auxiliary_loss_clip": 0.01134638, + "auxiliary_loss_mlp": 0.01045745, + "balance_loss_clip": 1.05312991, + "balance_loss_mlp": 1.02666569, + "epoch": 0.15399570541465962, + "flos": 37772602099200.0, + "grad_norm": 4.282135902253765, + "language_loss": 0.96045822, + "learning_rate": 3.840888912798769e-06, + "loss": 0.98226213, + "num_input_tokens_seen": 150979985, + "router_z_loss_clip": 0.81494141, + "router_z_loss_mlp": 0.19091797, + "step": 5307, + "time_per_iteration": 2.668147087097168 + }, + { + "auxiliary_loss_clip": 0.01129936, + "auxiliary_loss_mlp": 0.01047633, + "balance_loss_clip": 1.05350447, + "balance_loss_mlp": 1.03167725, + "epoch": 0.15402472288317567, + "flos": 32702051623680.0, + "grad_norm": 1.8331725888097705, + "language_loss": 0.8472361, + "learning_rate": 3.8408154354246065e-06, + "loss": 0.86901176, + "num_input_tokens_seen": 151000335, + "router_z_loss_clip": 0.76416016, + "router_z_loss_mlp": 0.15942383, + "step": 5308, + "time_per_iteration": 2.582292318344116 + }, + { + "auxiliary_loss_clip": 0.01030141, + "auxiliary_loss_mlp": 0.01002584, + "balance_loss_clip": 1.01060891, + "balance_loss_mlp": 1.0018034, + "epoch": 0.15405374035169173, + "flos": 64339928962560.0, + "grad_norm": 0.6738384188671427, + "language_loss": 0.49631181, + "learning_rate": 3.8407419417916174e-06, + "loss": 0.51663905, + "num_input_tokens_seen": 151063600, + "router_z_loss_clip": 0.1953125, + "router_z_loss_mlp": 0.0078125, + "step": 5309, + "time_per_iteration": 3.121812343597412 + }, + { + "auxiliary_loss_clip": 0.01030218, + "auxiliary_loss_mlp": 0.01000673, + "balance_loss_clip": 1.01069176, + "balance_loss_mlp": 0.99987471, + "epoch": 0.15408275782020778, + "flos": 64117821214080.0, + "grad_norm": 0.6520047120595134, + "language_loss": 0.46666709, + "learning_rate": 3.84066843190045e-06, + "loss": 0.486976, + "num_input_tokens_seen": 151128035, + "router_z_loss_clip": 0.1953125, + "router_z_loss_mlp": 0.00799561, + "step": 5310, + "time_per_iteration": 3.122509717941284 + }, + { + "auxiliary_loss_clip": 0.01124988, + "auxiliary_loss_mlp": 0.01042564, + "balance_loss_clip": 1.05347633, + "balance_loss_mlp": 1.02824688, + "epoch": 0.1541117752887238, + "flos": 21390514001280.0, + "grad_norm": 2.178817761644034, + "language_loss": 0.90093076, + "learning_rate": 3.840594905751754e-06, + "loss": 0.92260623, + "num_input_tokens_seen": 151144270, + "router_z_loss_clip": 0.71484375, + "router_z_loss_mlp": 0.14312744, + "step": 5311, + "time_per_iteration": 2.616276741027832 + }, + { + "auxiliary_loss_clip": 0.01138669, + "auxiliary_loss_mlp": 0.01040898, + "balance_loss_clip": 1.05916405, + "balance_loss_mlp": 1.02454805, + "epoch": 0.15414079275723985, + "flos": 30695147959680.0, + "grad_norm": 1.6033095660666645, + "language_loss": 0.83259964, + "learning_rate": 3.84052136334618e-06, + "loss": 0.85439527, + "num_input_tokens_seen": 151164740, + "router_z_loss_clip": 0.79492188, + "router_z_loss_mlp": 0.16357422, + "step": 5312, + "time_per_iteration": 2.6031296253204346 + }, + { + "auxiliary_loss_clip": 0.01029656, + "auxiliary_loss_mlp": 0.00999764, + "balance_loss_clip": 1.00992131, + "balance_loss_mlp": 0.99895364, + "epoch": 0.1541698102257559, + "flos": 74779555847040.0, + "grad_norm": 0.6124800558804994, + "language_loss": 0.43840459, + "learning_rate": 3.840447804684376e-06, + "loss": 0.45869878, + "num_input_tokens_seen": 151233285, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.00811768, + "step": 5313, + "time_per_iteration": 3.1578369140625 + }, + { + "auxiliary_loss_clip": 0.01128387, + "auxiliary_loss_mlp": 0.01046308, + "balance_loss_clip": 1.05235171, + "balance_loss_mlp": 1.03062034, + "epoch": 0.15419882769427196, + "flos": 31388511168000.0, + "grad_norm": 1.9947114301488842, + "language_loss": 0.92035866, + "learning_rate": 3.840374229766993e-06, + "loss": 0.94210559, + "num_input_tokens_seen": 151254855, + "router_z_loss_clip": 0.75927734, + "router_z_loss_mlp": 0.15673828, + "step": 5314, + "time_per_iteration": 2.666417121887207 + }, + { + "auxiliary_loss_clip": 0.01029761, + "auxiliary_loss_mlp": 0.01000799, + "balance_loss_clip": 1.01008368, + "balance_loss_mlp": 1.00000668, + "epoch": 0.154227845162788, + "flos": 67257706642560.0, + "grad_norm": 0.6861965834666338, + "language_loss": 0.47928032, + "learning_rate": 3.840300638594678e-06, + "loss": 0.49958593, + "num_input_tokens_seen": 151309375, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.00793457, + "step": 5315, + "time_per_iteration": 3.0118215084075928 + }, + { + "auxiliary_loss_clip": 0.01130124, + "auxiliary_loss_mlp": 0.01042385, + "balance_loss_clip": 1.05240333, + "balance_loss_mlp": 1.02649999, + "epoch": 0.15425686263130403, + "flos": 25841425933440.0, + "grad_norm": 2.190171445793498, + "language_loss": 0.78834093, + "learning_rate": 3.840227031168086e-06, + "loss": 0.81006598, + "num_input_tokens_seen": 151322415, + "router_z_loss_clip": 0.77734375, + "router_z_loss_mlp": 0.15899658, + "step": 5316, + "time_per_iteration": 2.5194106101989746 + }, + { + "auxiliary_loss_clip": 0.01134673, + "auxiliary_loss_mlp": 0.01043362, + "balance_loss_clip": 1.05532444, + "balance_loss_mlp": 1.02830565, + "epoch": 0.15428588009982008, + "flos": 26279141068800.0, + "grad_norm": 2.3837275770961006, + "language_loss": 0.97008407, + "learning_rate": 3.8401534074878615e-06, + "loss": 0.99186444, + "num_input_tokens_seen": 151337855, + "router_z_loss_clip": 0.79394531, + "router_z_loss_mlp": 0.15045166, + "step": 5317, + "time_per_iteration": 2.5331215858459473 + }, + { + "auxiliary_loss_clip": 0.01129815, + "auxiliary_loss_mlp": 0.01041023, + "balance_loss_clip": 1.05474746, + "balance_loss_mlp": 1.02604997, + "epoch": 0.15431489756833613, + "flos": 35662060719360.0, + "grad_norm": 2.0043622515051216, + "language_loss": 0.75841546, + "learning_rate": 3.840079767554659e-06, + "loss": 0.78012383, + "num_input_tokens_seen": 151358955, + "router_z_loss_clip": 0.75146484, + "router_z_loss_mlp": 0.14978027, + "step": 5318, + "time_per_iteration": 2.655388593673706 + }, + { + "auxiliary_loss_clip": 0.01127662, + "auxiliary_loss_mlp": 0.01045731, + "balance_loss_clip": 1.05170548, + "balance_loss_mlp": 1.03014421, + "epoch": 0.1543439150368522, + "flos": 29021818924800.0, + "grad_norm": 1.6884456264149177, + "language_loss": 0.90601242, + "learning_rate": 3.840006111369127e-06, + "loss": 0.92774636, + "num_input_tokens_seen": 151379815, + "router_z_loss_clip": 0.76074219, + "router_z_loss_mlp": 0.15582275, + "step": 5319, + "time_per_iteration": 2.633739471435547 + }, + { + "auxiliary_loss_clip": 0.01030971, + "auxiliary_loss_mlp": 0.01002803, + "balance_loss_clip": 1.01120138, + "balance_loss_mlp": 1.00200415, + "epoch": 0.15437293250536824, + "flos": 70147043729280.0, + "grad_norm": 0.6667085701978559, + "language_loss": 0.51496732, + "learning_rate": 3.839932438931916e-06, + "loss": 0.53530508, + "num_input_tokens_seen": 151444045, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.00799561, + "step": 5320, + "time_per_iteration": 3.1603424549102783 + }, + { + "auxiliary_loss_clip": 0.01124181, + "auxiliary_loss_mlp": 0.01038719, + "balance_loss_clip": 1.04824924, + "balance_loss_mlp": 1.02340686, + "epoch": 0.1544019499738843, + "flos": 24749777744640.0, + "grad_norm": 2.309431276460688, + "language_loss": 0.78054041, + "learning_rate": 3.839858750243678e-06, + "loss": 0.80216944, + "num_input_tokens_seen": 151461275, + "router_z_loss_clip": 0.75927734, + "router_z_loss_mlp": 0.15313721, + "step": 5321, + "time_per_iteration": 2.6513803005218506 + }, + { + "auxiliary_loss_clip": 0.01124429, + "auxiliary_loss_mlp": 0.01039239, + "balance_loss_clip": 1.05363071, + "balance_loss_mlp": 1.02505851, + "epoch": 0.1544309674424003, + "flos": 48756562663680.0, + "grad_norm": 2.5425942191022637, + "language_loss": 0.8181026, + "learning_rate": 3.839785045305062e-06, + "loss": 0.8397392, + "num_input_tokens_seen": 151484520, + "router_z_loss_clip": 0.70751953, + "router_z_loss_mlp": 0.14178467, + "step": 5322, + "time_per_iteration": 2.7387001514434814 + }, + { + "auxiliary_loss_clip": 0.01142456, + "auxiliary_loss_mlp": 0.01050691, + "balance_loss_clip": 1.0593822, + "balance_loss_mlp": 1.03245187, + "epoch": 0.15445998491091636, + "flos": 34272856264320.0, + "grad_norm": 2.632181762876561, + "language_loss": 0.88356364, + "learning_rate": 3.839711324116721e-06, + "loss": 0.90549505, + "num_input_tokens_seen": 151499665, + "router_z_loss_clip": 0.83056641, + "router_z_loss_mlp": 0.18237305, + "step": 5323, + "time_per_iteration": 2.629486083984375 + }, + { + "auxiliary_loss_clip": 0.01032331, + "auxiliary_loss_mlp": 0.01000464, + "balance_loss_clip": 1.01244807, + "balance_loss_mlp": 0.99967766, + "epoch": 0.15448900237943242, + "flos": 64925991267840.0, + "grad_norm": 0.6522359055724609, + "language_loss": 0.54742587, + "learning_rate": 3.8396375866793046e-06, + "loss": 0.56775379, + "num_input_tokens_seen": 151567395, + "router_z_loss_clip": 0.19921875, + "router_z_loss_mlp": 0.00787354, + "step": 5324, + "time_per_iteration": 3.1396570205688477 + }, + { + "auxiliary_loss_clip": 0.01130887, + "auxiliary_loss_mlp": 0.01042479, + "balance_loss_clip": 1.05091381, + "balance_loss_mlp": 1.02620661, + "epoch": 0.15451801984794847, + "flos": 48541278499200.0, + "grad_norm": 2.5748684128505355, + "language_loss": 0.62579441, + "learning_rate": 3.839563832993465e-06, + "loss": 0.64752805, + "num_input_tokens_seen": 151586705, + "router_z_loss_clip": 0.79931641, + "router_z_loss_mlp": 0.16271973, + "step": 5325, + "time_per_iteration": 2.6677427291870117 + }, + { + "auxiliary_loss_clip": 0.01032849, + "auxiliary_loss_mlp": 0.01001039, + "balance_loss_clip": 1.01293564, + "balance_loss_mlp": 1.0002104, + "epoch": 0.15454703731646452, + "flos": 72359642626560.0, + "grad_norm": 0.6568053946395285, + "language_loss": 0.46377423, + "learning_rate": 3.8394900630598525e-06, + "loss": 0.48411313, + "num_input_tokens_seen": 151640880, + "router_z_loss_clip": 0.19921875, + "router_z_loss_mlp": 0.00830078, + "step": 5326, + "time_per_iteration": 2.999924659729004 + }, + { + "auxiliary_loss_clip": 0.01147667, + "auxiliary_loss_mlp": 0.01060557, + "balance_loss_clip": 1.05807209, + "balance_loss_mlp": 1.04017186, + "epoch": 0.15457605478498057, + "flos": 20151201000960.0, + "grad_norm": 2.6315257369780243, + "language_loss": 1.02391851, + "learning_rate": 3.8394162768791205e-06, + "loss": 1.04600072, + "num_input_tokens_seen": 151654885, + "router_z_loss_clip": 0.89599609, + "router_z_loss_mlp": 0.20385742, + "step": 5327, + "time_per_iteration": 2.5334320068359375 + }, + { + "auxiliary_loss_clip": 0.01147955, + "auxiliary_loss_mlp": 0.01049689, + "balance_loss_clip": 1.05495012, + "balance_loss_mlp": 1.02998376, + "epoch": 0.1546050722534966, + "flos": 22703946716160.0, + "grad_norm": 2.7763575432970318, + "language_loss": 0.92536163, + "learning_rate": 3.839342474451919e-06, + "loss": 0.9473381, + "num_input_tokens_seen": 151667895, + "router_z_loss_clip": 0.9296875, + "router_z_loss_mlp": 0.19677734, + "step": 5328, + "time_per_iteration": 2.489354372024536 + }, + { + "auxiliary_loss_clip": 0.01129177, + "auxiliary_loss_mlp": 0.01035732, + "balance_loss_clip": 1.05170178, + "balance_loss_mlp": 1.02052069, + "epoch": 0.15463408972201265, + "flos": 16027291509120.0, + "grad_norm": 2.705074501549078, + "language_loss": 0.72428012, + "learning_rate": 3.8392686557789e-06, + "loss": 0.74592924, + "num_input_tokens_seen": 151680935, + "router_z_loss_clip": 0.77441406, + "router_z_loss_mlp": 0.15203857, + "step": 5329, + "time_per_iteration": 2.525947093963623 + }, + { + "auxiliary_loss_clip": 0.01130321, + "auxiliary_loss_mlp": 0.01037464, + "balance_loss_clip": 1.05170918, + "balance_loss_mlp": 1.02250302, + "epoch": 0.1546631071905287, + "flos": 15994397629440.0, + "grad_norm": 2.3067038301649725, + "language_loss": 0.63601458, + "learning_rate": 3.839194820860716e-06, + "loss": 0.65769237, + "num_input_tokens_seen": 151697605, + "router_z_loss_clip": 0.78662109, + "router_z_loss_mlp": 0.14971924, + "step": 5330, + "time_per_iteration": 2.5335447788238525 + }, + { + "auxiliary_loss_clip": 0.0103449, + "auxiliary_loss_mlp": 0.01000789, + "balance_loss_clip": 1.01473618, + "balance_loss_mlp": 0.99997854, + "epoch": 0.15469212465904475, + "flos": 74775174387840.0, + "grad_norm": 0.6694995315060639, + "language_loss": 0.43760532, + "learning_rate": 3.83912096969802e-06, + "loss": 0.4579581, + "num_input_tokens_seen": 151759735, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.00811768, + "step": 5331, + "time_per_iteration": 3.118387222290039 + }, + { + "auxiliary_loss_clip": 0.01138997, + "auxiliary_loss_mlp": 0.01050456, + "balance_loss_clip": 1.05618, + "balance_loss_mlp": 1.03286111, + "epoch": 0.1547211421275608, + "flos": 30074001045120.0, + "grad_norm": 1.996327154423788, + "language_loss": 0.748142, + "learning_rate": 3.839047102291463e-06, + "loss": 0.77003658, + "num_input_tokens_seen": 151775845, + "router_z_loss_clip": 0.82910156, + "router_z_loss_mlp": 0.17602539, + "step": 5332, + "time_per_iteration": 2.590968370437622 + }, + { + "auxiliary_loss_clip": 0.01138283, + "auxiliary_loss_mlp": 0.01044784, + "balance_loss_clip": 1.05611372, + "balance_loss_mlp": 1.02789831, + "epoch": 0.15475015959607683, + "flos": 49811258736000.0, + "grad_norm": 2.3187545191012267, + "language_loss": 0.95185477, + "learning_rate": 3.838973218641698e-06, + "loss": 0.97368544, + "num_input_tokens_seen": 151795645, + "router_z_loss_clip": 0.82128906, + "router_z_loss_mlp": 0.16876221, + "step": 5333, + "time_per_iteration": 2.7585203647613525 + }, + { + "auxiliary_loss_clip": 0.01134433, + "auxiliary_loss_mlp": 0.01043079, + "balance_loss_clip": 1.05530643, + "balance_loss_mlp": 1.02564454, + "epoch": 0.15477917706459288, + "flos": 15918338580480.0, + "grad_norm": 1.8947881967543558, + "language_loss": 0.75100493, + "learning_rate": 3.838899318749377e-06, + "loss": 0.77278, + "num_input_tokens_seen": 151811520, + "router_z_loss_clip": 0.79150391, + "router_z_loss_mlp": 0.17449951, + "step": 5334, + "time_per_iteration": 7.312507629394531 + }, + { + "auxiliary_loss_clip": 0.01131685, + "auxiliary_loss_mlp": 0.01037119, + "balance_loss_clip": 1.0568614, + "balance_loss_mlp": 1.02221179, + "epoch": 0.15480819453310893, + "flos": 47258153884800.0, + "grad_norm": 2.112536922634934, + "language_loss": 0.72543406, + "learning_rate": 3.838825402615153e-06, + "loss": 0.74712217, + "num_input_tokens_seen": 151829315, + "router_z_loss_clip": 0.74951172, + "router_z_loss_mlp": 0.14904785, + "step": 5335, + "time_per_iteration": 2.6579010486602783 + }, + { + "auxiliary_loss_clip": 0.01133662, + "auxiliary_loss_mlp": 0.01042624, + "balance_loss_clip": 1.05451393, + "balance_loss_mlp": 1.02643585, + "epoch": 0.15483721200162498, + "flos": 23546375366400.0, + "grad_norm": 2.330566262801999, + "language_loss": 0.90729129, + "learning_rate": 3.838751470239679e-06, + "loss": 0.92905414, + "num_input_tokens_seen": 151847055, + "router_z_loss_clip": 0.79052734, + "router_z_loss_mlp": 0.16192627, + "step": 5336, + "time_per_iteration": 7.395626068115234 + }, + { + "auxiliary_loss_clip": 0.01132323, + "auxiliary_loss_mlp": 0.01041067, + "balance_loss_clip": 1.05633128, + "balance_loss_mlp": 1.02502131, + "epoch": 0.15486622947014103, + "flos": 22265656963200.0, + "grad_norm": 2.4656069656654798, + "language_loss": 0.61294556, + "learning_rate": 3.838677521623608e-06, + "loss": 0.6346795, + "num_input_tokens_seen": 151860460, + "router_z_loss_clip": 0.75976562, + "router_z_loss_mlp": 0.1605835, + "step": 5337, + "time_per_iteration": 2.499537467956543 + }, + { + "auxiliary_loss_clip": 0.01037602, + "auxiliary_loss_mlp": 0.01002221, + "balance_loss_clip": 1.01785755, + "balance_loss_mlp": 1.00139272, + "epoch": 0.15489524693865708, + "flos": 60903492848640.0, + "grad_norm": 0.6925853256661973, + "language_loss": 0.49519855, + "learning_rate": 3.838603556767593e-06, + "loss": 0.51559681, + "num_input_tokens_seen": 151919815, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.00830078, + "step": 5338, + "time_per_iteration": 3.108438491821289 + }, + { + "auxiliary_loss_clip": 0.01035956, + "auxiliary_loss_mlp": 0.01002796, + "balance_loss_clip": 1.016119, + "balance_loss_mlp": 1.00193179, + "epoch": 0.1549242644071731, + "flos": 74776790499840.0, + "grad_norm": 0.6478092543671115, + "language_loss": 0.44125134, + "learning_rate": 3.8385295756722875e-06, + "loss": 0.46163887, + "num_input_tokens_seen": 151988620, + "router_z_loss_clip": 0.19824219, + "router_z_loss_mlp": 0.00866699, + "step": 5339, + "time_per_iteration": 3.201784133911133 + }, + { + "auxiliary_loss_clip": 0.0103667, + "auxiliary_loss_mlp": 0.01002486, + "balance_loss_clip": 1.01674294, + "balance_loss_mlp": 1.00163972, + "epoch": 0.15495328187568916, + "flos": 74777365117440.0, + "grad_norm": 0.6642309285343092, + "language_loss": 0.49268812, + "learning_rate": 3.838455578338345e-06, + "loss": 0.51307964, + "num_input_tokens_seen": 152054610, + "router_z_loss_clip": 0.19921875, + "router_z_loss_mlp": 0.00848389, + "step": 5340, + "time_per_iteration": 3.161362409591675 + }, + { + "auxiliary_loss_clip": 0.01035256, + "auxiliary_loss_mlp": 0.01002316, + "balance_loss_clip": 1.01545715, + "balance_loss_mlp": 1.00148201, + "epoch": 0.1549822993442052, + "flos": 74775569437440.0, + "grad_norm": 0.6133849061037135, + "language_loss": 0.44726527, + "learning_rate": 3.838381564766418e-06, + "loss": 0.467641, + "num_input_tokens_seen": 152119915, + "router_z_loss_clip": 0.19824219, + "router_z_loss_mlp": 0.00836182, + "step": 5341, + "time_per_iteration": 3.3055529594421387 + }, + { + "auxiliary_loss_clip": 0.01136924, + "auxiliary_loss_mlp": 0.01047149, + "balance_loss_clip": 1.05481756, + "balance_loss_mlp": 1.0304296, + "epoch": 0.15501131681272126, + "flos": 23907815971200.0, + "grad_norm": 2.0992901293081356, + "language_loss": 0.77349079, + "learning_rate": 3.838307534957162e-06, + "loss": 0.79533154, + "num_input_tokens_seen": 152137190, + "router_z_loss_clip": 0.82128906, + "router_z_loss_mlp": 0.16705322, + "step": 5342, + "time_per_iteration": 2.5406365394592285 + }, + { + "auxiliary_loss_clip": 0.01033757, + "auxiliary_loss_mlp": 0.01002476, + "balance_loss_clip": 1.01394641, + "balance_loss_mlp": 1.00162351, + "epoch": 0.1550403342812373, + "flos": 58790437516800.0, + "grad_norm": 0.7112974639178287, + "language_loss": 0.50657064, + "learning_rate": 3.83823348891123e-06, + "loss": 0.52693301, + "num_input_tokens_seen": 152191495, + "router_z_loss_clip": 0.19824219, + "router_z_loss_mlp": 0.00854492, + "step": 5343, + "time_per_iteration": 2.9565157890319824 + }, + { + "auxiliary_loss_clip": 0.01138573, + "auxiliary_loss_mlp": 0.0104513, + "balance_loss_clip": 1.05928063, + "balance_loss_mlp": 1.02884018, + "epoch": 0.15506935174975336, + "flos": 25769425121280.0, + "grad_norm": 3.6977244089106707, + "language_loss": 0.64308178, + "learning_rate": 3.838159426629276e-06, + "loss": 0.66491878, + "num_input_tokens_seen": 152204460, + "router_z_loss_clip": 0.79296875, + "router_z_loss_mlp": 0.16290283, + "step": 5344, + "time_per_iteration": 2.532663106918335 + }, + { + "auxiliary_loss_clip": 0.01125867, + "auxiliary_loss_mlp": 0.01041629, + "balance_loss_clip": 1.05172539, + "balance_loss_mlp": 1.02414703, + "epoch": 0.1550983692182694, + "flos": 19312974241920.0, + "grad_norm": 2.4499835795846043, + "language_loss": 0.78773761, + "learning_rate": 3.8380853481119536e-06, + "loss": 0.80941254, + "num_input_tokens_seen": 152218755, + "router_z_loss_clip": 0.74267578, + "router_z_loss_mlp": 0.17462158, + "step": 5345, + "time_per_iteration": 2.5537261962890625 + }, + { + "auxiliary_loss_clip": 0.01032012, + "auxiliary_loss_mlp": 0.01004697, + "balance_loss_clip": 1.0122683, + "balance_loss_mlp": 1.0038743, + "epoch": 0.15512738668678544, + "flos": 74773342794240.0, + "grad_norm": 0.6622150274610339, + "language_loss": 0.45556992, + "learning_rate": 3.838011253359918e-06, + "loss": 0.47593698, + "num_input_tokens_seen": 152281510, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.00823975, + "step": 5346, + "time_per_iteration": 3.2566068172454834 + }, + { + "auxiliary_loss_clip": 0.01032011, + "auxiliary_loss_mlp": 0.01001651, + "balance_loss_clip": 1.01231718, + "balance_loss_mlp": 1.00079858, + "epoch": 0.1551564041553015, + "flos": 74778981229440.0, + "grad_norm": 0.6298887941524327, + "language_loss": 0.5085547, + "learning_rate": 3.837937142373823e-06, + "loss": 0.52889132, + "num_input_tokens_seen": 152346185, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.00854492, + "step": 5347, + "time_per_iteration": 3.136042356491089 + }, + { + "auxiliary_loss_clip": 0.01126166, + "auxiliary_loss_mlp": 0.01046206, + "balance_loss_clip": 1.05336022, + "balance_loss_mlp": 1.03162646, + "epoch": 0.15518542162381754, + "flos": 11463907115520.0, + "grad_norm": 2.831717088616351, + "language_loss": 0.71000892, + "learning_rate": 3.837863015154324e-06, + "loss": 0.73173261, + "num_input_tokens_seen": 152355605, + "router_z_loss_clip": 0.72753906, + "router_z_loss_mlp": 0.14556885, + "step": 5348, + "time_per_iteration": 2.487212657928467 + }, + { + "auxiliary_loss_clip": 0.01031631, + "auxiliary_loss_mlp": 0.00999995, + "balance_loss_clip": 1.01196313, + "balance_loss_mlp": 0.99916667, + "epoch": 0.1552144390923336, + "flos": 60649748196480.0, + "grad_norm": 0.7373506868756965, + "language_loss": 0.5119338, + "learning_rate": 3.837788871702074e-06, + "loss": 0.53225005, + "num_input_tokens_seen": 152414360, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.00830078, + "step": 5349, + "time_per_iteration": 3.000680685043335 + }, + { + "auxiliary_loss_clip": 0.01032326, + "auxiliary_loss_mlp": 0.0100202, + "balance_loss_clip": 1.01258469, + "balance_loss_mlp": 1.00114965, + "epoch": 0.15524345656084962, + "flos": 54450633329280.0, + "grad_norm": 0.7161849249923358, + "language_loss": 0.50809515, + "learning_rate": 3.837714712017731e-06, + "loss": 0.52843863, + "num_input_tokens_seen": 152473815, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.00872803, + "step": 5350, + "time_per_iteration": 3.023343324661255 + }, + { + "auxiliary_loss_clip": 0.01124238, + "auxiliary_loss_mlp": 0.0103932, + "balance_loss_clip": 1.05142975, + "balance_loss_mlp": 1.02533114, + "epoch": 0.15527247402936567, + "flos": 14676870764160.0, + "grad_norm": 2.972687422947881, + "language_loss": 0.8131302, + "learning_rate": 3.837640536101946e-06, + "loss": 0.83476579, + "num_input_tokens_seen": 152487350, + "router_z_loss_clip": 0.72802734, + "router_z_loss_mlp": 0.13989258, + "step": 5351, + "time_per_iteration": 2.4683568477630615 + }, + { + "auxiliary_loss_clip": 0.01144491, + "auxiliary_loss_mlp": 0.01062497, + "balance_loss_clip": 1.0582726, + "balance_loss_mlp": 1.04333448, + "epoch": 0.15530149149788172, + "flos": 36755648242560.0, + "grad_norm": 1.7309659688510552, + "language_loss": 0.85871077, + "learning_rate": 3.837566343955377e-06, + "loss": 0.88078064, + "num_input_tokens_seen": 152510030, + "router_z_loss_clip": 0.86132812, + "router_z_loss_mlp": 0.19177246, + "step": 5352, + "time_per_iteration": 2.651000738143921 + }, + { + "auxiliary_loss_clip": 0.01132707, + "auxiliary_loss_mlp": 0.01044554, + "balance_loss_clip": 1.05269229, + "balance_loss_mlp": 1.02922344, + "epoch": 0.15533050896639777, + "flos": 37264214954880.0, + "grad_norm": 2.254951490040788, + "language_loss": 0.77988249, + "learning_rate": 3.8374921355786786e-06, + "loss": 0.80165511, + "num_input_tokens_seen": 152530320, + "router_z_loss_clip": 0.80029297, + "router_z_loss_mlp": 0.15319824, + "step": 5353, + "time_per_iteration": 2.622380495071411 + }, + { + "auxiliary_loss_clip": 0.01125888, + "auxiliary_loss_mlp": 0.01040618, + "balance_loss_clip": 1.05068088, + "balance_loss_mlp": 1.02597344, + "epoch": 0.15535952643491382, + "flos": 16610049763200.0, + "grad_norm": 2.80246524424736, + "language_loss": 0.86934221, + "learning_rate": 3.8374179109725055e-06, + "loss": 0.89100718, + "num_input_tokens_seen": 152543035, + "router_z_loss_clip": 0.75195312, + "router_z_loss_mlp": 0.1463623, + "step": 5354, + "time_per_iteration": 2.465114116668701 + }, + { + "auxiliary_loss_clip": 0.0113047, + "auxiliary_loss_mlp": 0.01044523, + "balance_loss_clip": 1.05300188, + "balance_loss_mlp": 1.02961588, + "epoch": 0.15538854390342988, + "flos": 23652634775040.0, + "grad_norm": 2.2212321494887313, + "language_loss": 0.88701987, + "learning_rate": 3.837343670137515e-06, + "loss": 0.90876979, + "num_input_tokens_seen": 152557730, + "router_z_loss_clip": 0.77392578, + "router_z_loss_mlp": 0.14910889, + "step": 5355, + "time_per_iteration": 2.5129482746124268 + }, + { + "auxiliary_loss_clip": 0.01130352, + "auxiliary_loss_mlp": 0.01038607, + "balance_loss_clip": 1.05439568, + "balance_loss_mlp": 1.02386689, + "epoch": 0.1554175613719459, + "flos": 36533576407680.0, + "grad_norm": 1.8181019719710676, + "language_loss": 0.45032474, + "learning_rate": 3.837269413074361e-06, + "loss": 0.47201434, + "num_input_tokens_seen": 152575735, + "router_z_loss_clip": 0.76025391, + "router_z_loss_mlp": 0.1473999, + "step": 5356, + "time_per_iteration": 2.5830652713775635 + }, + { + "auxiliary_loss_clip": 0.01130919, + "auxiliary_loss_mlp": 0.01043158, + "balance_loss_clip": 1.05323982, + "balance_loss_mlp": 1.02638507, + "epoch": 0.15544657884046195, + "flos": 30699098455680.0, + "grad_norm": 2.1917962859887923, + "language_loss": 0.93807119, + "learning_rate": 3.837195139783699e-06, + "loss": 0.95981205, + "num_input_tokens_seen": 152593720, + "router_z_loss_clip": 0.77685547, + "router_z_loss_mlp": 0.16784668, + "step": 5357, + "time_per_iteration": 2.543229103088379 + }, + { + "auxiliary_loss_clip": 0.01035503, + "auxiliary_loss_mlp": 0.01009361, + "balance_loss_clip": 1.01576185, + "balance_loss_mlp": 1.00854993, + "epoch": 0.155475596308978, + "flos": 69226724436480.0, + "grad_norm": 0.6430125364089643, + "language_loss": 0.47630194, + "learning_rate": 3.837120850266188e-06, + "loss": 0.49675059, + "num_input_tokens_seen": 152656455, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.00811768, + "step": 5358, + "time_per_iteration": 3.057549238204956 + }, + { + "auxiliary_loss_clip": 0.0113244, + "auxiliary_loss_mlp": 0.01044758, + "balance_loss_clip": 1.0551393, + "balance_loss_mlp": 1.02951694, + "epoch": 0.15550461377749406, + "flos": 18507389967360.0, + "grad_norm": 2.1123940181786502, + "language_loss": 0.8490091, + "learning_rate": 3.837046544522481e-06, + "loss": 0.87078106, + "num_input_tokens_seen": 152672630, + "router_z_loss_clip": 0.77246094, + "router_z_loss_mlp": 0.15240479, + "step": 5359, + "time_per_iteration": 2.633798360824585 + }, + { + "auxiliary_loss_clip": 0.0113908, + "auxiliary_loss_mlp": 0.01039924, + "balance_loss_clip": 1.05665874, + "balance_loss_mlp": 1.02361667, + "epoch": 0.1555336312460101, + "flos": 14314029528960.0, + "grad_norm": 2.1194474165348196, + "language_loss": 0.79236162, + "learning_rate": 3.836972222553236e-06, + "loss": 0.81415164, + "num_input_tokens_seen": 152686995, + "router_z_loss_clip": 0.82421875, + "router_z_loss_mlp": 0.16308594, + "step": 5360, + "time_per_iteration": 2.4771904945373535 + }, + { + "auxiliary_loss_clip": 0.01127107, + "auxiliary_loss_mlp": 0.01030137, + "balance_loss_clip": 1.05419326, + "balance_loss_mlp": 1.01655936, + "epoch": 0.15556264871452616, + "flos": 22230823749120.0, + "grad_norm": 1.7930489456806606, + "language_loss": 0.79059678, + "learning_rate": 3.836897884359109e-06, + "loss": 0.81216919, + "num_input_tokens_seen": 152702335, + "router_z_loss_clip": 0.72900391, + "router_z_loss_mlp": 0.13592529, + "step": 5361, + "time_per_iteration": 2.5827016830444336 + }, + { + "auxiliary_loss_clip": 0.01142449, + "auxiliary_loss_mlp": 0.0104588, + "balance_loss_clip": 1.05788708, + "balance_loss_mlp": 1.02601957, + "epoch": 0.15559166618304218, + "flos": 37260479940480.0, + "grad_norm": 2.9766180976184264, + "language_loss": 0.77985775, + "learning_rate": 3.836823529940757e-06, + "loss": 0.80174106, + "num_input_tokens_seen": 152717690, + "router_z_loss_clip": 0.84619141, + "router_z_loss_mlp": 0.19866943, + "step": 5362, + "time_per_iteration": 2.6834635734558105 + }, + { + "auxiliary_loss_clip": 0.01036225, + "auxiliary_loss_mlp": 0.01003956, + "balance_loss_clip": 1.01660514, + "balance_loss_mlp": 1.00315118, + "epoch": 0.15562068365155823, + "flos": 74768494458240.0, + "grad_norm": 0.6376137863474787, + "language_loss": 0.4746089, + "learning_rate": 3.836749159298835e-06, + "loss": 0.49501073, + "num_input_tokens_seen": 152781315, + "router_z_loss_clip": 0.19628906, + "router_z_loss_mlp": 0.00805664, + "step": 5363, + "time_per_iteration": 3.140406370162964 + }, + { + "auxiliary_loss_clip": 0.01131815, + "auxiliary_loss_mlp": 0.01042971, + "balance_loss_clip": 1.05312097, + "balance_loss_mlp": 1.02630568, + "epoch": 0.15564970112007429, + "flos": 16872916469760.0, + "grad_norm": 1.8990882238623192, + "language_loss": 0.7142936, + "learning_rate": 3.836674772434002e-06, + "loss": 0.73604149, + "num_input_tokens_seen": 152795310, + "router_z_loss_clip": 0.78710938, + "router_z_loss_mlp": 0.16662598, + "step": 5364, + "time_per_iteration": 2.4870681762695312 + }, + { + "auxiliary_loss_clip": 0.01037305, + "auxiliary_loss_mlp": 0.01000116, + "balance_loss_clip": 1.01765108, + "balance_loss_mlp": 0.99932909, + "epoch": 0.15567871858859034, + "flos": 58751079189120.0, + "grad_norm": 0.7359788681614516, + "language_loss": 0.4700222, + "learning_rate": 3.836600369346915e-06, + "loss": 0.49039644, + "num_input_tokens_seen": 152845250, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.00787354, + "step": 5365, + "time_per_iteration": 2.873237133026123 + }, + { + "auxiliary_loss_clip": 0.01037178, + "auxiliary_loss_mlp": 0.00999322, + "balance_loss_clip": 1.01747346, + "balance_loss_mlp": 0.99842787, + "epoch": 0.1557077360571064, + "flos": 74777939735040.0, + "grad_norm": 0.6555847403546965, + "language_loss": 0.48649403, + "learning_rate": 3.836525950038229e-06, + "loss": 0.50685906, + "num_input_tokens_seen": 152910570, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.00891113, + "step": 5366, + "time_per_iteration": 3.1576457023620605 + }, + { + "auxiliary_loss_clip": 0.01036695, + "auxiliary_loss_mlp": 0.00999417, + "balance_loss_clip": 1.0169313, + "balance_loss_mlp": 0.99853462, + "epoch": 0.1557367535256224, + "flos": 69325513816320.0, + "grad_norm": 0.7083077854081096, + "language_loss": 0.47813821, + "learning_rate": 3.836451514508603e-06, + "loss": 0.49849936, + "num_input_tokens_seen": 152967555, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.0088501, + "step": 5367, + "time_per_iteration": 3.0519330501556396 + }, + { + "auxiliary_loss_clip": 0.01123558, + "auxiliary_loss_mlp": 0.0104337, + "balance_loss_clip": 1.05207217, + "balance_loss_mlp": 1.0279448, + "epoch": 0.15576577099413846, + "flos": 22486866871680.0, + "grad_norm": 2.0913695578734424, + "language_loss": 0.72061479, + "learning_rate": 3.8363770627586944e-06, + "loss": 0.74228406, + "num_input_tokens_seen": 152983750, + "router_z_loss_clip": 0.71484375, + "router_z_loss_mlp": 0.1541748, + "step": 5368, + "time_per_iteration": 2.5287859439849854 + }, + { + "auxiliary_loss_clip": 0.01143398, + "auxiliary_loss_mlp": 0.01054959, + "balance_loss_clip": 1.060848, + "balance_loss_mlp": 1.03804278, + "epoch": 0.15579478846265452, + "flos": 33174097182720.0, + "grad_norm": 2.5476942896250736, + "language_loss": 0.7709446, + "learning_rate": 3.83630259478916e-06, + "loss": 0.79292816, + "num_input_tokens_seen": 152997715, + "router_z_loss_clip": 0.82617188, + "router_z_loss_mlp": 0.16931152, + "step": 5369, + "time_per_iteration": 2.6428658962249756 + }, + { + "auxiliary_loss_clip": 0.01130843, + "auxiliary_loss_mlp": 0.01041576, + "balance_loss_clip": 1.05673242, + "balance_loss_mlp": 1.02740812, + "epoch": 0.15582380593117057, + "flos": 16025603569920.0, + "grad_norm": 2.33103395747152, + "language_loss": 0.76664793, + "learning_rate": 3.8362281106006585e-06, + "loss": 0.7883721, + "num_input_tokens_seen": 153009495, + "router_z_loss_clip": 0.74169922, + "router_z_loss_mlp": 0.14172363, + "step": 5370, + "time_per_iteration": 2.503392457962036 + }, + { + "auxiliary_loss_clip": 0.01143317, + "auxiliary_loss_mlp": 0.01040513, + "balance_loss_clip": 1.06140161, + "balance_loss_mlp": 1.02518833, + "epoch": 0.15585282339968662, + "flos": 10590631660800.0, + "grad_norm": 3.4116285151505075, + "language_loss": 0.81213713, + "learning_rate": 3.836153610193848e-06, + "loss": 0.83397543, + "num_input_tokens_seen": 153020275, + "router_z_loss_clip": 0.8203125, + "router_z_loss_mlp": 0.15338135, + "step": 5371, + "time_per_iteration": 2.4858901500701904 + }, + { + "auxiliary_loss_clip": 0.010331, + "auxiliary_loss_mlp": 0.010009, + "balance_loss_clip": 1.01360798, + "balance_loss_mlp": 1.00004721, + "epoch": 0.15588184086820267, + "flos": 60968741904000.0, + "grad_norm": 0.619152454825739, + "language_loss": 0.40727803, + "learning_rate": 3.836079093569384e-06, + "loss": 0.42761803, + "num_input_tokens_seen": 153081410, + "router_z_loss_clip": 0.1953125, + "router_z_loss_mlp": 0.00854492, + "step": 5372, + "time_per_iteration": 3.3351917266845703 + }, + { + "auxiliary_loss_clip": 0.01143984, + "auxiliary_loss_mlp": 0.01052414, + "balance_loss_clip": 1.05967593, + "balance_loss_mlp": 1.03309655, + "epoch": 0.1559108583367187, + "flos": 30766789635840.0, + "grad_norm": 17.305579870317647, + "language_loss": 0.87940079, + "learning_rate": 3.836004560727927e-06, + "loss": 0.9013648, + "num_input_tokens_seen": 153096350, + "router_z_loss_clip": 0.84326172, + "router_z_loss_mlp": 0.19311523, + "step": 5373, + "time_per_iteration": 2.722625255584717 + }, + { + "auxiliary_loss_clip": 0.01031613, + "auxiliary_loss_mlp": 0.01003477, + "balance_loss_clip": 1.01202929, + "balance_loss_mlp": 1.00264835, + "epoch": 0.15593987580523475, + "flos": 65330812523520.0, + "grad_norm": 0.6219154016260231, + "language_loss": 0.44643056, + "learning_rate": 3.835930011670136e-06, + "loss": 0.46678144, + "num_input_tokens_seen": 153163015, + "router_z_loss_clip": 0.1953125, + "router_z_loss_mlp": 0.00830078, + "step": 5374, + "time_per_iteration": 3.3586559295654297 + }, + { + "auxiliary_loss_clip": 0.01124256, + "auxiliary_loss_mlp": 0.01031809, + "balance_loss_clip": 1.0540328, + "balance_loss_mlp": 1.01791561, + "epoch": 0.1559688932737508, + "flos": 10772483241600.0, + "grad_norm": 2.5834287401409646, + "language_loss": 0.80946249, + "learning_rate": 3.835855446396667e-06, + "loss": 0.83102316, + "num_input_tokens_seen": 153173570, + "router_z_loss_clip": 0.70214844, + "router_z_loss_mlp": 0.13891602, + "step": 5375, + "time_per_iteration": 2.6224420070648193 + }, + { + "auxiliary_loss_clip": 0.01032639, + "auxiliary_loss_mlp": 0.01002549, + "balance_loss_clip": 1.01294804, + "balance_loss_mlp": 1.00173223, + "epoch": 0.15599791074226685, + "flos": 65945207681280.0, + "grad_norm": 0.6524556429497428, + "language_loss": 0.47530293, + "learning_rate": 3.83578086490818e-06, + "loss": 0.49565482, + "num_input_tokens_seen": 153236170, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.00817871, + "step": 5376, + "time_per_iteration": 3.1179473400115967 + }, + { + "auxiliary_loss_clip": 0.01141699, + "auxiliary_loss_mlp": 0.01039092, + "balance_loss_clip": 1.05867219, + "balance_loss_mlp": 1.02180636, + "epoch": 0.1560269282107829, + "flos": 29525249992320.0, + "grad_norm": 2.6043592017979194, + "language_loss": 0.87658143, + "learning_rate": 3.835706267205334e-06, + "loss": 0.89838934, + "num_input_tokens_seen": 153249690, + "router_z_loss_clip": 0.83007812, + "router_z_loss_mlp": 0.17285156, + "step": 5377, + "time_per_iteration": 2.6000704765319824 + }, + { + "auxiliary_loss_clip": 0.01032039, + "auxiliary_loss_mlp": 0.01004836, + "balance_loss_clip": 1.012393, + "balance_loss_mlp": 1.00400186, + "epoch": 0.15605594567929892, + "flos": 68763513623040.0, + "grad_norm": 0.6507715852072635, + "language_loss": 0.52900875, + "learning_rate": 3.835631653288787e-06, + "loss": 0.5493775, + "num_input_tokens_seen": 153315500, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.00836182, + "step": 5378, + "time_per_iteration": 3.146939754486084 + }, + { + "auxiliary_loss_clip": 0.01130967, + "auxiliary_loss_mlp": 0.01036727, + "balance_loss_clip": 1.05484843, + "balance_loss_mlp": 1.02194524, + "epoch": 0.15608496314781498, + "flos": 24057994734720.0, + "grad_norm": 2.4696789468369666, + "language_loss": 0.97710067, + "learning_rate": 3.835557023159199e-06, + "loss": 0.99877757, + "num_input_tokens_seen": 153330165, + "router_z_loss_clip": 0.76025391, + "router_z_loss_mlp": 0.14782715, + "step": 5379, + "time_per_iteration": 2.5551013946533203 + }, + { + "auxiliary_loss_clip": 0.01135164, + "auxiliary_loss_mlp": 0.01033915, + "balance_loss_clip": 1.05691814, + "balance_loss_mlp": 1.01871562, + "epoch": 0.15611398061633103, + "flos": 44449544615040.0, + "grad_norm": 2.8397063431739364, + "language_loss": 0.71529639, + "learning_rate": 3.835482376817228e-06, + "loss": 0.73698723, + "num_input_tokens_seen": 153349285, + "router_z_loss_clip": 0.78173828, + "router_z_loss_mlp": 0.15197754, + "step": 5380, + "time_per_iteration": 2.694152355194092 + }, + { + "auxiliary_loss_clip": 0.01133616, + "auxiliary_loss_mlp": 0.01043813, + "balance_loss_clip": 1.05599093, + "balance_loss_mlp": 1.02748156, + "epoch": 0.15614299808484708, + "flos": 12049789852800.0, + "grad_norm": 2.9664788846209516, + "language_loss": 0.86904931, + "learning_rate": 3.8354077142635335e-06, + "loss": 0.8908236, + "num_input_tokens_seen": 153360750, + "router_z_loss_clip": 0.77636719, + "router_z_loss_mlp": 0.16351318, + "step": 5381, + "time_per_iteration": 2.535336971282959 + }, + { + "auxiliary_loss_clip": 0.01138708, + "auxiliary_loss_mlp": 0.01040914, + "balance_loss_clip": 1.05864358, + "balance_loss_mlp": 1.0235157, + "epoch": 0.15617201555336313, + "flos": 26754634333440.0, + "grad_norm": 2.5855228233394927, + "language_loss": 0.83211434, + "learning_rate": 3.835333035498776e-06, + "loss": 0.85391057, + "num_input_tokens_seen": 153374720, + "router_z_loss_clip": 0.80126953, + "router_z_loss_mlp": 0.17401123, + "step": 5382, + "time_per_iteration": 2.562551975250244 + }, + { + "auxiliary_loss_clip": 0.01132347, + "auxiliary_loss_mlp": 0.01042946, + "balance_loss_clip": 1.05491245, + "balance_loss_mlp": 1.02672195, + "epoch": 0.15620103302187918, + "flos": 31424888666880.0, + "grad_norm": 2.1732267675400307, + "language_loss": 0.73683929, + "learning_rate": 3.835258340523614e-06, + "loss": 0.75859225, + "num_input_tokens_seen": 153391245, + "router_z_loss_clip": 0.77490234, + "router_z_loss_mlp": 0.16229248, + "step": 5383, + "time_per_iteration": 2.620377779006958 + }, + { + "auxiliary_loss_clip": 0.01136873, + "auxiliary_loss_mlp": 0.01035505, + "balance_loss_clip": 1.05863678, + "balance_loss_mlp": 1.01916718, + "epoch": 0.1562300504903952, + "flos": 16172873331840.0, + "grad_norm": 3.030239321115061, + "language_loss": 0.71993202, + "learning_rate": 3.835183629338709e-06, + "loss": 0.74165583, + "num_input_tokens_seen": 153404575, + "router_z_loss_clip": 0.78173828, + "router_z_loss_mlp": 0.16326904, + "step": 5384, + "time_per_iteration": 2.451935291290283 + }, + { + "auxiliary_loss_clip": 0.01136903, + "auxiliary_loss_mlp": 0.01043797, + "balance_loss_clip": 1.05905342, + "balance_loss_mlp": 1.02812696, + "epoch": 0.15625906795891126, + "flos": 28285434201600.0, + "grad_norm": 2.959293239161058, + "language_loss": 1.02484584, + "learning_rate": 3.835108901944719e-06, + "loss": 1.04665279, + "num_input_tokens_seen": 153418335, + "router_z_loss_clip": 0.77880859, + "router_z_loss_mlp": 0.15679932, + "step": 5385, + "time_per_iteration": 2.624805450439453 + }, + { + "auxiliary_loss_clip": 0.01135938, + "auxiliary_loss_mlp": 0.01038505, + "balance_loss_clip": 1.05794859, + "balance_loss_mlp": 1.02163672, + "epoch": 0.1562880854274273, + "flos": 26717682216960.0, + "grad_norm": 2.564726505056935, + "language_loss": 0.71637869, + "learning_rate": 3.835034158342303e-06, + "loss": 0.73812312, + "num_input_tokens_seen": 153434940, + "router_z_loss_clip": 0.77978516, + "router_z_loss_mlp": 0.16870117, + "step": 5386, + "time_per_iteration": 2.6230218410491943 + }, + { + "auxiliary_loss_clip": 0.01136051, + "auxiliary_loss_mlp": 0.01036416, + "balance_loss_clip": 1.05915868, + "balance_loss_mlp": 1.02090657, + "epoch": 0.15631710289594336, + "flos": 34964244224640.0, + "grad_norm": 2.018799541721008, + "language_loss": 0.98151869, + "learning_rate": 3.834959398532125e-06, + "loss": 1.00324333, + "num_input_tokens_seen": 153454355, + "router_z_loss_clip": 0.76904297, + "router_z_loss_mlp": 0.15509033, + "step": 5387, + "time_per_iteration": 2.7112410068511963 + }, + { + "auxiliary_loss_clip": 0.01131852, + "auxiliary_loss_mlp": 0.01047568, + "balance_loss_clip": 1.05551684, + "balance_loss_mlp": 1.03219056, + "epoch": 0.1563461203644594, + "flos": 41273568996480.0, + "grad_norm": 2.89012194595793, + "language_loss": 1.14806199, + "learning_rate": 3.834884622514842e-06, + "loss": 1.16985619, + "num_input_tokens_seen": 153468645, + "router_z_loss_clip": 0.76367188, + "router_z_loss_mlp": 0.15380859, + "step": 5388, + "time_per_iteration": 2.7762045860290527 + }, + { + "auxiliary_loss_clip": 0.01036394, + "auxiliary_loss_mlp": 0.01005545, + "balance_loss_clip": 1.01663423, + "balance_loss_mlp": 1.00472212, + "epoch": 0.15637513783297546, + "flos": 66740377011840.0, + "grad_norm": 0.6917738593111651, + "language_loss": 0.47060525, + "learning_rate": 3.834809830291115e-06, + "loss": 0.49102464, + "num_input_tokens_seen": 153519480, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.00823975, + "step": 5389, + "time_per_iteration": 3.020371198654175 + }, + { + "auxiliary_loss_clip": 0.01132985, + "auxiliary_loss_mlp": 0.01043308, + "balance_loss_clip": 1.05474448, + "balance_loss_mlp": 1.02697062, + "epoch": 0.1564041553014915, + "flos": 30959307555840.0, + "grad_norm": 2.943649106073498, + "language_loss": 0.90136027, + "learning_rate": 3.834735021861605e-06, + "loss": 0.92312324, + "num_input_tokens_seen": 153538725, + "router_z_loss_clip": 0.78173828, + "router_z_loss_mlp": 0.16345215, + "step": 5390, + "time_per_iteration": 2.67710018157959 + }, + { + "auxiliary_loss_clip": 0.01138725, + "auxiliary_loss_mlp": 0.01051918, + "balance_loss_clip": 1.05709732, + "balance_loss_mlp": 1.03305888, + "epoch": 0.15643317277000754, + "flos": 50143396821120.0, + "grad_norm": 2.9550671946685303, + "language_loss": 0.88775635, + "learning_rate": 3.834660197226974e-06, + "loss": 0.90966284, + "num_input_tokens_seen": 153559940, + "router_z_loss_clip": 0.81640625, + "router_z_loss_mlp": 0.18859863, + "step": 5391, + "time_per_iteration": 2.775428056716919 + }, + { + "auxiliary_loss_clip": 0.01036043, + "auxiliary_loss_mlp": 0.01005095, + "balance_loss_clip": 1.01633549, + "balance_loss_mlp": 1.00424242, + "epoch": 0.1564621902385236, + "flos": 66595728942720.0, + "grad_norm": 0.6980832045022178, + "language_loss": 0.49388868, + "learning_rate": 3.834585356387881e-06, + "loss": 0.51430005, + "num_input_tokens_seen": 153616820, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.00854492, + "step": 5392, + "time_per_iteration": 3.005084753036499 + }, + { + "auxiliary_loss_clip": 0.01133076, + "auxiliary_loss_mlp": 0.01036335, + "balance_loss_clip": 1.05915082, + "balance_loss_mlp": 1.02142155, + "epoch": 0.15649120770703964, + "flos": 40734622356480.0, + "grad_norm": 2.072647138501389, + "language_loss": 0.72657025, + "learning_rate": 3.8345104993449884e-06, + "loss": 0.74826431, + "num_input_tokens_seen": 153643095, + "router_z_loss_clip": 0.73925781, + "router_z_loss_mlp": 0.14898682, + "step": 5393, + "time_per_iteration": 2.6948726177215576 + }, + { + "auxiliary_loss_clip": 0.01036343, + "auxiliary_loss_mlp": 0.0100271, + "balance_loss_clip": 1.01653731, + "balance_loss_mlp": 1.0018878, + "epoch": 0.1565202251755557, + "flos": 74787349098240.0, + "grad_norm": 0.6276589618316537, + "language_loss": 0.4543072, + "learning_rate": 3.834435626098956e-06, + "loss": 0.47469771, + "num_input_tokens_seen": 153708760, + "router_z_loss_clip": 0.19824219, + "router_z_loss_mlp": 0.00823975, + "step": 5394, + "time_per_iteration": 3.2695775032043457 + }, + { + "auxiliary_loss_clip": 0.01138639, + "auxiliary_loss_mlp": 0.01044852, + "balance_loss_clip": 1.06276774, + "balance_loss_mlp": 1.02871144, + "epoch": 0.15654924264407172, + "flos": 23323908481920.0, + "grad_norm": 4.590587823935936, + "language_loss": 0.80276275, + "learning_rate": 3.834360736650447e-06, + "loss": 0.8245976, + "num_input_tokens_seen": 153723440, + "router_z_loss_clip": 0.75927734, + "router_z_loss_mlp": 0.16125488, + "step": 5395, + "time_per_iteration": 2.5312697887420654 + }, + { + "auxiliary_loss_clip": 0.01131219, + "auxiliary_loss_mlp": 0.01047275, + "balance_loss_clip": 1.05865264, + "balance_loss_mlp": 1.03230238, + "epoch": 0.15657826011258777, + "flos": 10115964408960.0, + "grad_norm": 1.8107001446695965, + "language_loss": 0.60422552, + "learning_rate": 3.83428583100012e-06, + "loss": 0.62601048, + "num_input_tokens_seen": 153737020, + "router_z_loss_clip": 0.72460938, + "router_z_loss_mlp": 0.14978027, + "step": 5396, + "time_per_iteration": 2.5002806186676025 + }, + { + "auxiliary_loss_clip": 0.01129241, + "auxiliary_loss_mlp": 0.01037593, + "balance_loss_clip": 1.05614054, + "balance_loss_mlp": 1.02321017, + "epoch": 0.15660727758110382, + "flos": 16719361827840.0, + "grad_norm": 2.4567220532036154, + "language_loss": 0.88109618, + "learning_rate": 3.834210909148639e-06, + "loss": 0.90276456, + "num_input_tokens_seen": 153749515, + "router_z_loss_clip": 0.73046875, + "router_z_loss_mlp": 0.1439209, + "step": 5397, + "time_per_iteration": 2.4858508110046387 + }, + { + "auxiliary_loss_clip": 0.01131466, + "auxiliary_loss_mlp": 0.01038202, + "balance_loss_clip": 1.05694366, + "balance_loss_mlp": 1.02338982, + "epoch": 0.15663629504961987, + "flos": 30182451183360.0, + "grad_norm": 2.7464117955376026, + "language_loss": 0.73141158, + "learning_rate": 3.8341359710966655e-06, + "loss": 0.75310826, + "num_input_tokens_seen": 153768435, + "router_z_loss_clip": 0.74560547, + "router_z_loss_mlp": 0.14819336, + "step": 5398, + "time_per_iteration": 2.7488489151000977 + }, + { + "auxiliary_loss_clip": 0.01138659, + "auxiliary_loss_mlp": 0.01048703, + "balance_loss_clip": 1.05985034, + "balance_loss_mlp": 1.02976072, + "epoch": 0.15666531251813592, + "flos": 27852172352640.0, + "grad_norm": 1.6827365335221536, + "language_loss": 0.87966871, + "learning_rate": 3.834061016844861e-06, + "loss": 0.90154225, + "num_input_tokens_seen": 153797585, + "router_z_loss_clip": 0.78759766, + "router_z_loss_mlp": 0.1895752, + "step": 5399, + "time_per_iteration": 2.8877272605895996 + }, + { + "auxiliary_loss_clip": 0.01036478, + "auxiliary_loss_mlp": 0.01004164, + "balance_loss_clip": 1.01642525, + "balance_loss_mlp": 1.00328207, + "epoch": 0.15669432998665198, + "flos": 74763538381440.0, + "grad_norm": 0.6881854288023411, + "language_loss": 0.46657282, + "learning_rate": 3.833986046393886e-06, + "loss": 0.48697922, + "num_input_tokens_seen": 153855665, + "router_z_loss_clip": 0.20117188, + "router_z_loss_mlp": 0.0088501, + "step": 5400, + "time_per_iteration": 3.03609037399292 + }, + { + "auxiliary_loss_clip": 0.0103575, + "auxiliary_loss_mlp": 0.01004409, + "balance_loss_clip": 1.01545048, + "balance_loss_mlp": 1.003497, + "epoch": 0.156723347455168, + "flos": 66532742444160.0, + "grad_norm": 0.6328918767374072, + "language_loss": 0.47950262, + "learning_rate": 3.833911059744405e-06, + "loss": 0.49990422, + "num_input_tokens_seen": 153924695, + "router_z_loss_clip": 0.203125, + "router_z_loss_mlp": 0.00909424, + "step": 5401, + "time_per_iteration": 3.330226421356201 + }, + { + "auxiliary_loss_clip": 0.0103648, + "auxiliary_loss_mlp": 0.0099902, + "balance_loss_clip": 1.01648164, + "balance_loss_mlp": 0.99814951, + "epoch": 0.15675236492368405, + "flos": 74005859871360.0, + "grad_norm": 0.670966306392432, + "language_loss": 0.47895515, + "learning_rate": 3.83383605689708e-06, + "loss": 0.49931014, + "num_input_tokens_seen": 153984580, + "router_z_loss_clip": 0.20019531, + "router_z_loss_mlp": 0.00872803, + "step": 5402, + "time_per_iteration": 3.090700626373291 + }, + { + "auxiliary_loss_clip": 0.01035859, + "auxiliary_loss_mlp": 0.01000602, + "balance_loss_clip": 1.01596618, + "balance_loss_mlp": 0.99972028, + "epoch": 0.1567813823922001, + "flos": 66038107230720.0, + "grad_norm": 0.7135797023722562, + "language_loss": 0.48374179, + "learning_rate": 3.833761037852572e-06, + "loss": 0.5041064, + "num_input_tokens_seen": 154037205, + "router_z_loss_clip": 0.19921875, + "router_z_loss_mlp": 0.0088501, + "step": 5403, + "time_per_iteration": 2.9021894931793213 + }, + { + "auxiliary_loss_clip": 0.01034655, + "auxiliary_loss_mlp": 0.01008832, + "balance_loss_clip": 1.01478243, + "balance_loss_mlp": 1.00796175, + "epoch": 0.15681039986071615, + "flos": 71165829179520.0, + "grad_norm": 0.6442214729177203, + "language_loss": 0.50926328, + "learning_rate": 3.833686002611545e-06, + "loss": 0.52969813, + "num_input_tokens_seen": 154100915, + "router_z_loss_clip": 0.19921875, + "router_z_loss_mlp": 0.00872803, + "step": 5404, + "time_per_iteration": 3.213735580444336 + }, + { + "auxiliary_loss_clip": 0.01129316, + "auxiliary_loss_mlp": 0.01052227, + "balance_loss_clip": 1.05599558, + "balance_loss_mlp": 1.03717649, + "epoch": 0.1568394173292322, + "flos": 14384701537920.0, + "grad_norm": 2.139738985417246, + "language_loss": 0.76514077, + "learning_rate": 3.833610951174661e-06, + "loss": 0.78695619, + "num_input_tokens_seen": 154114080, + "router_z_loss_clip": 0.73339844, + "router_z_loss_mlp": 0.1505127, + "step": 5405, + "time_per_iteration": 7.307062149047852 + }, + { + "auxiliary_loss_clip": 0.01139899, + "auxiliary_loss_mlp": 0.01062071, + "balance_loss_clip": 1.05979156, + "balance_loss_mlp": 1.04593062, + "epoch": 0.15686843479774826, + "flos": 32883759550080.0, + "grad_norm": 2.2888266464906164, + "language_loss": 0.86493158, + "learning_rate": 3.8335358835425835e-06, + "loss": 0.88695133, + "num_input_tokens_seen": 154130755, + "router_z_loss_clip": 0.80175781, + "router_z_loss_mlp": 0.16137695, + "step": 5406, + "time_per_iteration": 2.6465325355529785 + }, + { + "auxiliary_loss_clip": 0.0103115, + "auxiliary_loss_mlp": 0.01021578, + "balance_loss_clip": 1.01097822, + "balance_loss_mlp": 1.0206244, + "epoch": 0.15689745226626428, + "flos": 61126857573120.0, + "grad_norm": 0.6954557777591294, + "language_loss": 0.49971676, + "learning_rate": 3.8334607997159745e-06, + "loss": 0.520244, + "num_input_tokens_seen": 154193105, + "router_z_loss_clip": 0.20117188, + "router_z_loss_mlp": 0.00952148, + "step": 5407, + "time_per_iteration": 7.89682674407959 + }, + { + "auxiliary_loss_clip": 0.01030171, + "auxiliary_loss_mlp": 0.01025295, + "balance_loss_clip": 1.00998437, + "balance_loss_mlp": 1.02438343, + "epoch": 0.15692646973478033, + "flos": 70061036613120.0, + "grad_norm": 0.6531500809617531, + "language_loss": 0.48746023, + "learning_rate": 3.833385699695497e-06, + "loss": 0.50801492, + "num_input_tokens_seen": 154254745, + "router_z_loss_clip": 0.20214844, + "router_z_loss_mlp": 0.00909424, + "step": 5408, + "time_per_iteration": 3.2345430850982666 + }, + { + "auxiliary_loss_clip": 0.01029117, + "auxiliary_loss_mlp": 0.01010256, + "balance_loss_clip": 1.00907493, + "balance_loss_mlp": 1.00939775, + "epoch": 0.15695548720329638, + "flos": 63132216952320.0, + "grad_norm": 0.677878447450452, + "language_loss": 0.43342805, + "learning_rate": 3.833310583481817e-06, + "loss": 0.45382178, + "num_input_tokens_seen": 154309580, + "router_z_loss_clip": 0.20117188, + "router_z_loss_mlp": 0.00860596, + "step": 5409, + "time_per_iteration": 2.974522829055786 + }, + { + "auxiliary_loss_clip": 0.01135983, + "auxiliary_loss_mlp": 0.01043812, + "balance_loss_clip": 1.05533934, + "balance_loss_mlp": 1.02735567, + "epoch": 0.15698450467181244, + "flos": 41093369441280.0, + "grad_norm": 2.1058392225981666, + "language_loss": 0.9162333, + "learning_rate": 3.833235451075596e-06, + "loss": 0.9380312, + "num_input_tokens_seen": 154327585, + "router_z_loss_clip": 0.80615234, + "router_z_loss_mlp": 0.16455078, + "step": 5410, + "time_per_iteration": 2.7083640098571777 + }, + { + "auxiliary_loss_clip": 0.01137177, + "auxiliary_loss_mlp": 0.01055751, + "balance_loss_clip": 1.06180751, + "balance_loss_mlp": 1.04109979, + "epoch": 0.1570135221403285, + "flos": 40034686959360.0, + "grad_norm": 1.6592099190066645, + "language_loss": 0.73182809, + "learning_rate": 3.833160302477496e-06, + "loss": 0.75375736, + "num_input_tokens_seen": 154346230, + "router_z_loss_clip": 0.75390625, + "router_z_loss_mlp": 0.14648438, + "step": 5411, + "time_per_iteration": 2.6820309162139893 + }, + { + "auxiliary_loss_clip": 0.01127062, + "auxiliary_loss_mlp": 0.01048859, + "balance_loss_clip": 1.05715489, + "balance_loss_mlp": 1.03619266, + "epoch": 0.1570425396088445, + "flos": 18982775491200.0, + "grad_norm": 2.0961221762866464, + "language_loss": 0.80002046, + "learning_rate": 3.833085137688183e-06, + "loss": 0.82177961, + "num_input_tokens_seen": 154360660, + "router_z_loss_clip": 0.69921875, + "router_z_loss_mlp": 0.12670898, + "step": 5412, + "time_per_iteration": 2.6119420528411865 + }, + { + "auxiliary_loss_clip": 0.01133516, + "auxiliary_loss_mlp": 0.01034694, + "balance_loss_clip": 1.05589914, + "balance_loss_mlp": 1.01916718, + "epoch": 0.15707155707736056, + "flos": 19128536881920.0, + "grad_norm": 3.214959414395661, + "language_loss": 0.84293413, + "learning_rate": 3.833009956708321e-06, + "loss": 0.86461622, + "num_input_tokens_seen": 154373305, + "router_z_loss_clip": 0.77685547, + "router_z_loss_mlp": 0.15509033, + "step": 5413, + "time_per_iteration": 2.5230767726898193 + }, + { + "auxiliary_loss_clip": 0.01128818, + "auxiliary_loss_mlp": 0.01042265, + "balance_loss_clip": 1.05614984, + "balance_loss_mlp": 1.02801394, + "epoch": 0.15710057454587661, + "flos": 15479761518720.0, + "grad_norm": 2.70539872879735, + "language_loss": 0.97641963, + "learning_rate": 3.832934759538573e-06, + "loss": 0.9981305, + "num_input_tokens_seen": 154385315, + "router_z_loss_clip": 0.72558594, + "router_z_loss_mlp": 0.14257812, + "step": 5414, + "time_per_iteration": 2.4942028522491455 + }, + { + "auxiliary_loss_clip": 0.01134803, + "auxiliary_loss_mlp": 0.01039917, + "balance_loss_clip": 1.05478072, + "balance_loss_mlp": 1.02470553, + "epoch": 0.15712959201439267, + "flos": 59189761013760.0, + "grad_norm": 1.6425169650035645, + "language_loss": 0.86016285, + "learning_rate": 3.832859546179604e-06, + "loss": 0.88191003, + "num_input_tokens_seen": 154413220, + "router_z_loss_clip": 0.79980469, + "router_z_loss_mlp": 0.15197754, + "step": 5415, + "time_per_iteration": 2.8702359199523926 + }, + { + "auxiliary_loss_clip": 0.01130997, + "auxiliary_loss_mlp": 0.01046022, + "balance_loss_clip": 1.05260837, + "balance_loss_mlp": 1.02961302, + "epoch": 0.15715860948290872, + "flos": 31608356359680.0, + "grad_norm": 2.018395891313177, + "language_loss": 0.82367873, + "learning_rate": 3.8327843166320766e-06, + "loss": 0.84544885, + "num_input_tokens_seen": 154430595, + "router_z_loss_clip": 0.78417969, + "router_z_loss_mlp": 0.16412354, + "step": 5416, + "time_per_iteration": 2.626342296600342 + }, + { + "auxiliary_loss_clip": 0.01120703, + "auxiliary_loss_mlp": 0.01033215, + "balance_loss_clip": 1.05318809, + "balance_loss_mlp": 1.02165818, + "epoch": 0.15718762695142477, + "flos": 15225298594560.0, + "grad_norm": 2.2846673059438625, + "language_loss": 0.65717578, + "learning_rate": 3.832709070896657e-06, + "loss": 0.67871499, + "num_input_tokens_seen": 154444470, + "router_z_loss_clip": 0.67529297, + "router_z_loss_mlp": 0.11566162, + "step": 5417, + "time_per_iteration": 2.477731943130493 + }, + { + "auxiliary_loss_clip": 0.01121471, + "auxiliary_loss_mlp": 0.01056646, + "balance_loss_clip": 1.05221415, + "balance_loss_mlp": 1.0416193, + "epoch": 0.1572166444199408, + "flos": 30787799091840.0, + "grad_norm": 2.2575514871378726, + "language_loss": 1.05966592, + "learning_rate": 3.832633808974009e-06, + "loss": 1.08144712, + "num_input_tokens_seen": 154466245, + "router_z_loss_clip": 0.69189453, + "router_z_loss_mlp": 0.15020752, + "step": 5418, + "time_per_iteration": 2.6867551803588867 + }, + { + "auxiliary_loss_clip": 0.0103165, + "auxiliary_loss_mlp": 0.01069304, + "balance_loss_clip": 1.01108098, + "balance_loss_mlp": 1.06828451, + "epoch": 0.15724566188845684, + "flos": 61232614191360.0, + "grad_norm": 0.6900335687675133, + "language_loss": 0.48412433, + "learning_rate": 3.832558530864798e-06, + "loss": 0.50513387, + "num_input_tokens_seen": 154526850, + "router_z_loss_clip": 0.20605469, + "router_z_loss_mlp": 0.01019287, + "step": 5419, + "time_per_iteration": 3.0914132595062256 + }, + { + "auxiliary_loss_clip": 0.01125169, + "auxiliary_loss_mlp": 0.0103327, + "balance_loss_clip": 1.0529685, + "balance_loss_mlp": 1.01909041, + "epoch": 0.1572746793569729, + "flos": 24927571088640.0, + "grad_norm": 2.3138147199736077, + "language_loss": 0.86264992, + "learning_rate": 3.832483236569689e-06, + "loss": 0.88423437, + "num_input_tokens_seen": 154540125, + "router_z_loss_clip": 0.72265625, + "router_z_loss_mlp": 0.14178467, + "step": 5420, + "time_per_iteration": 2.5298006534576416 + }, + { + "auxiliary_loss_clip": 0.01132039, + "auxiliary_loss_mlp": 0.01048231, + "balance_loss_clip": 1.05362904, + "balance_loss_mlp": 1.03205979, + "epoch": 0.15730369682548895, + "flos": 11283240683520.0, + "grad_norm": 2.57591954808045, + "language_loss": 0.79038072, + "learning_rate": 3.832407926089345e-06, + "loss": 0.81218338, + "num_input_tokens_seen": 154551175, + "router_z_loss_clip": 0.78466797, + "router_z_loss_mlp": 0.16149902, + "step": 5421, + "time_per_iteration": 2.5318093299865723 + }, + { + "auxiliary_loss_clip": 0.01128113, + "auxiliary_loss_mlp": 0.01040834, + "balance_loss_clip": 1.05307937, + "balance_loss_mlp": 1.02540874, + "epoch": 0.157332714294005, + "flos": 74735525773440.0, + "grad_norm": 2.2236257573167793, + "language_loss": 0.64673781, + "learning_rate": 3.8323325994244346e-06, + "loss": 0.66842729, + "num_input_tokens_seen": 154579510, + "router_z_loss_clip": 0.75048828, + "router_z_loss_mlp": 0.15423584, + "step": 5422, + "time_per_iteration": 3.0098488330841064 + }, + { + "auxiliary_loss_clip": 0.01139713, + "auxiliary_loss_mlp": 0.01039129, + "balance_loss_clip": 1.05864608, + "balance_loss_mlp": 1.02296484, + "epoch": 0.15736173176252105, + "flos": 25404752292480.0, + "grad_norm": 3.222172375830204, + "language_loss": 0.884664, + "learning_rate": 3.8322572565756195e-06, + "loss": 0.90645242, + "num_input_tokens_seen": 154597015, + "router_z_loss_clip": 0.81005859, + "router_z_loss_mlp": 0.16168213, + "step": 5423, + "time_per_iteration": 2.610970973968506 + }, + { + "auxiliary_loss_clip": 0.0112681, + "auxiliary_loss_mlp": 0.01037105, + "balance_loss_clip": 1.05231309, + "balance_loss_mlp": 1.02294302, + "epoch": 0.15739074923103707, + "flos": 13947489192960.0, + "grad_norm": 4.029220644300714, + "language_loss": 0.67423606, + "learning_rate": 3.832181897543568e-06, + "loss": 0.69587523, + "num_input_tokens_seen": 154609670, + "router_z_loss_clip": 0.74560547, + "router_z_loss_mlp": 0.14154053, + "step": 5424, + "time_per_iteration": 2.4658167362213135 + }, + { + "auxiliary_loss_clip": 0.01126995, + "auxiliary_loss_mlp": 0.01043914, + "balance_loss_clip": 1.05319667, + "balance_loss_mlp": 1.02979922, + "epoch": 0.15741976669955313, + "flos": 25698681285120.0, + "grad_norm": 3.1544352495522543, + "language_loss": 0.7376349, + "learning_rate": 3.832106522328944e-06, + "loss": 0.75934398, + "num_input_tokens_seen": 154625150, + "router_z_loss_clip": 0.73779297, + "router_z_loss_mlp": 0.14117432, + "step": 5425, + "time_per_iteration": 2.5685441493988037 + }, + { + "auxiliary_loss_clip": 0.01125634, + "auxiliary_loss_mlp": 0.01037672, + "balance_loss_clip": 1.05548215, + "balance_loss_mlp": 1.0242784, + "epoch": 0.15744878416806918, + "flos": 19713665433600.0, + "grad_norm": 1.9300747257763209, + "language_loss": 0.69681001, + "learning_rate": 3.832031130932415e-06, + "loss": 0.71844304, + "num_input_tokens_seen": 154640635, + "router_z_loss_clip": 0.70214844, + "router_z_loss_mlp": 0.1340332, + "step": 5426, + "time_per_iteration": 2.5276598930358887 + }, + { + "auxiliary_loss_clip": 0.01046352, + "auxiliary_loss_mlp": 0.01010397, + "balance_loss_clip": 1.02614677, + "balance_loss_mlp": 1.00956845, + "epoch": 0.15747780163658523, + "flos": 69085775468160.0, + "grad_norm": 0.7064911158179747, + "language_loss": 0.46186197, + "learning_rate": 3.8319557233546446e-06, + "loss": 0.48242944, + "num_input_tokens_seen": 154699175, + "router_z_loss_clip": 0.20214844, + "router_z_loss_mlp": 0.00830078, + "step": 5427, + "time_per_iteration": 3.117354393005371 + }, + { + "auxiliary_loss_clip": 0.01132309, + "auxiliary_loss_mlp": 0.01043207, + "balance_loss_clip": 1.05804324, + "balance_loss_mlp": 1.02919412, + "epoch": 0.15750681910510128, + "flos": 24200739383040.0, + "grad_norm": 3.674214085439103, + "language_loss": 0.82917941, + "learning_rate": 3.8318802995963e-06, + "loss": 0.85093457, + "num_input_tokens_seen": 154713945, + "router_z_loss_clip": 0.74267578, + "router_z_loss_mlp": 0.14013672, + "step": 5428, + "time_per_iteration": 2.555626630783081 + }, + { + "auxiliary_loss_clip": 0.010448, + "auxiliary_loss_mlp": 0.01012506, + "balance_loss_clip": 1.02433705, + "balance_loss_mlp": 1.01155269, + "epoch": 0.1575358365736173, + "flos": 74785373850240.0, + "grad_norm": 0.6266887568846407, + "language_loss": 0.46353582, + "learning_rate": 3.831804859658047e-06, + "loss": 0.48410887, + "num_input_tokens_seen": 154781365, + "router_z_loss_clip": 0.20507812, + "router_z_loss_mlp": 0.00952148, + "step": 5429, + "time_per_iteration": 3.2321197986602783 + }, + { + "auxiliary_loss_clip": 0.01141837, + "auxiliary_loss_mlp": 0.0104812, + "balance_loss_clip": 1.05992949, + "balance_loss_mlp": 1.03284335, + "epoch": 0.15756485404213336, + "flos": 17862794449920.0, + "grad_norm": 2.7877329280231593, + "language_loss": 0.71064371, + "learning_rate": 3.831729403540553e-06, + "loss": 0.73254335, + "num_input_tokens_seen": 154796105, + "router_z_loss_clip": 0.8203125, + "router_z_loss_mlp": 0.152771, + "step": 5430, + "time_per_iteration": 2.5150387287139893 + }, + { + "auxiliary_loss_clip": 0.01132231, + "auxiliary_loss_mlp": 0.01040644, + "balance_loss_clip": 1.05676889, + "balance_loss_mlp": 1.02571273, + "epoch": 0.1575938715106494, + "flos": 14677409468160.0, + "grad_norm": 5.6302116987382815, + "language_loss": 0.74168319, + "learning_rate": 3.831653931244483e-06, + "loss": 0.76341188, + "num_input_tokens_seen": 154807700, + "router_z_loss_clip": 0.75585938, + "router_z_loss_mlp": 0.14935303, + "step": 5431, + "time_per_iteration": 2.474668025970459 + }, + { + "auxiliary_loss_clip": 0.0104064, + "auxiliary_loss_mlp": 0.01004, + "balance_loss_clip": 1.02067125, + "balance_loss_mlp": 1.00307035, + "epoch": 0.15762288897916546, + "flos": 63542928038400.0, + "grad_norm": 0.6271407451299519, + "language_loss": 0.47166526, + "learning_rate": 3.831578442770505e-06, + "loss": 0.49211168, + "num_input_tokens_seen": 154873925, + "router_z_loss_clip": 0.20019531, + "router_z_loss_mlp": 0.00927734, + "step": 5432, + "time_per_iteration": 3.1189262866973877 + }, + { + "auxiliary_loss_clip": 0.01129527, + "auxiliary_loss_mlp": 0.01046514, + "balance_loss_clip": 1.0545013, + "balance_loss_mlp": 1.03161311, + "epoch": 0.1576519064476815, + "flos": 10588800067200.0, + "grad_norm": 3.670722836108644, + "language_loss": 1.0927577, + "learning_rate": 3.831502938119284e-06, + "loss": 1.11451817, + "num_input_tokens_seen": 154884125, + "router_z_loss_clip": 0.75097656, + "router_z_loss_mlp": 0.14886475, + "step": 5433, + "time_per_iteration": 2.4524621963500977 + }, + { + "auxiliary_loss_clip": 0.01133887, + "auxiliary_loss_mlp": 0.01049534, + "balance_loss_clip": 1.05704355, + "balance_loss_mlp": 1.03329158, + "epoch": 0.15768092391619756, + "flos": 11361274980480.0, + "grad_norm": 3.535852524287255, + "language_loss": 1.00150752, + "learning_rate": 3.831427417291489e-06, + "loss": 1.02334177, + "num_input_tokens_seen": 154894315, + "router_z_loss_clip": 0.76708984, + "router_z_loss_mlp": 0.16229248, + "step": 5434, + "time_per_iteration": 2.5035667419433594 + }, + { + "auxiliary_loss_clip": 0.01134698, + "auxiliary_loss_mlp": 0.01042081, + "balance_loss_clip": 1.05639219, + "balance_loss_mlp": 1.02510536, + "epoch": 0.1577099413847136, + "flos": 13474509880320.0, + "grad_norm": 2.0785544731477468, + "language_loss": 0.66717148, + "learning_rate": 3.831351880287786e-06, + "loss": 0.68893921, + "num_input_tokens_seen": 154906480, + "router_z_loss_clip": 0.78369141, + "router_z_loss_mlp": 0.16967773, + "step": 5435, + "time_per_iteration": 2.5426812171936035 + }, + { + "auxiliary_loss_clip": 0.01132524, + "auxiliary_loss_mlp": 0.01046658, + "balance_loss_clip": 1.0554719, + "balance_loss_mlp": 1.03100014, + "epoch": 0.15773895885322964, + "flos": 19931427636480.0, + "grad_norm": 2.253793424423292, + "language_loss": 0.72074914, + "learning_rate": 3.8312763271088415e-06, + "loss": 0.74254096, + "num_input_tokens_seen": 154921025, + "router_z_loss_clip": 0.77001953, + "router_z_loss_mlp": 0.15673828, + "step": 5436, + "time_per_iteration": 2.5315823554992676 + }, + { + "auxiliary_loss_clip": 0.01035339, + "auxiliary_loss_mlp": 0.01003117, + "balance_loss_clip": 1.01556516, + "balance_loss_mlp": 1.00228238, + "epoch": 0.1577679763217457, + "flos": 69954238500480.0, + "grad_norm": 0.6250464011914499, + "language_loss": 0.48541123, + "learning_rate": 3.831200757755323e-06, + "loss": 0.50579578, + "num_input_tokens_seen": 154981375, + "router_z_loss_clip": 0.19824219, + "router_z_loss_mlp": 0.00836182, + "step": 5437, + "time_per_iteration": 3.089236259460449 + }, + { + "auxiliary_loss_clip": 0.01140364, + "auxiliary_loss_mlp": 0.01043645, + "balance_loss_clip": 1.05597711, + "balance_loss_mlp": 1.02665162, + "epoch": 0.15779699379026174, + "flos": 39159041207040.0, + "grad_norm": 4.161964801410933, + "language_loss": 0.88629079, + "learning_rate": 3.831125172227899e-06, + "loss": 0.90813088, + "num_input_tokens_seen": 154997115, + "router_z_loss_clip": 0.84375, + "router_z_loss_mlp": 0.17004395, + "step": 5438, + "time_per_iteration": 2.6669375896453857 + }, + { + "auxiliary_loss_clip": 0.01127596, + "auxiliary_loss_mlp": 0.01036601, + "balance_loss_clip": 1.05526054, + "balance_loss_mlp": 1.02327895, + "epoch": 0.1578260112587778, + "flos": 27593974414080.0, + "grad_norm": 1.8997651282858043, + "language_loss": 0.75294209, + "learning_rate": 3.831049570527236e-06, + "loss": 0.77458411, + "num_input_tokens_seen": 155016990, + "router_z_loss_clip": 0.72314453, + "router_z_loss_mlp": 0.13323975, + "step": 5439, + "time_per_iteration": 2.663916826248169 + }, + { + "auxiliary_loss_clip": 0.01132251, + "auxiliary_loss_mlp": 0.01033095, + "balance_loss_clip": 1.05464256, + "balance_loss_mlp": 1.01695395, + "epoch": 0.15785502872729384, + "flos": 27703070997120.0, + "grad_norm": 2.379478064108156, + "language_loss": 0.87008584, + "learning_rate": 3.830973952654002e-06, + "loss": 0.89173925, + "num_input_tokens_seen": 155031860, + "router_z_loss_clip": 0.77490234, + "router_z_loss_mlp": 0.16137695, + "step": 5440, + "time_per_iteration": 2.6315295696258545 + }, + { + "auxiliary_loss_clip": 0.01115453, + "auxiliary_loss_mlp": 0.01033734, + "balance_loss_clip": 1.05001438, + "balance_loss_mlp": 1.02227831, + "epoch": 0.15788404619580987, + "flos": 23471142330240.0, + "grad_norm": 2.933510862361546, + "language_loss": 0.63652056, + "learning_rate": 3.830898318608867e-06, + "loss": 0.65801245, + "num_input_tokens_seen": 155046005, + "router_z_loss_clip": 0.65478516, + "router_z_loss_mlp": 0.11468506, + "step": 5441, + "time_per_iteration": 2.5392298698425293 + }, + { + "auxiliary_loss_clip": 0.01040182, + "auxiliary_loss_mlp": 0.01002248, + "balance_loss_clip": 1.02024174, + "balance_loss_mlp": 1.00146163, + "epoch": 0.15791306366432592, + "flos": 62982792374400.0, + "grad_norm": 0.7040538640407469, + "language_loss": 0.47772574, + "learning_rate": 3.830822668392496e-06, + "loss": 0.49815005, + "num_input_tokens_seen": 155101105, + "router_z_loss_clip": 0.19921875, + "router_z_loss_mlp": 0.00787354, + "step": 5442, + "time_per_iteration": 2.9493722915649414 + }, + { + "auxiliary_loss_clip": 0.01135076, + "auxiliary_loss_mlp": 0.01043814, + "balance_loss_clip": 1.0557332, + "balance_loss_mlp": 1.02686286, + "epoch": 0.15794208113284197, + "flos": 10442176750080.0, + "grad_norm": 2.340423688670631, + "language_loss": 0.8610661, + "learning_rate": 3.830747002005559e-06, + "loss": 0.88285494, + "num_input_tokens_seen": 155113315, + "router_z_loss_clip": 0.79492188, + "router_z_loss_mlp": 0.16943359, + "step": 5443, + "time_per_iteration": 2.529848337173462 + }, + { + "auxiliary_loss_clip": 0.0104414, + "auxiliary_loss_mlp": 0.01004543, + "balance_loss_clip": 1.02417588, + "balance_loss_mlp": 1.00370872, + "epoch": 0.15797109860135802, + "flos": 63133725323520.0, + "grad_norm": 0.6424685570027331, + "language_loss": 0.50991839, + "learning_rate": 3.830671319448722e-06, + "loss": 0.53040522, + "num_input_tokens_seen": 155174625, + "router_z_loss_clip": 0.19921875, + "router_z_loss_mlp": 0.00836182, + "step": 5444, + "time_per_iteration": 3.0753870010375977 + }, + { + "auxiliary_loss_clip": 0.01125685, + "auxiliary_loss_mlp": 0.01040284, + "balance_loss_clip": 1.0540874, + "balance_loss_mlp": 1.02631855, + "epoch": 0.15800011606987407, + "flos": 11649565537920.0, + "grad_norm": 2.599724048837719, + "language_loss": 0.71485293, + "learning_rate": 3.830595620722656e-06, + "loss": 0.73651254, + "num_input_tokens_seen": 155187260, + "router_z_loss_clip": 0.71630859, + "router_z_loss_mlp": 0.1395874, + "step": 5445, + "time_per_iteration": 2.5273590087890625 + }, + { + "auxiliary_loss_clip": 0.01144294, + "auxiliary_loss_mlp": 0.01049832, + "balance_loss_clip": 1.06134653, + "balance_loss_mlp": 1.03128266, + "epoch": 0.1580291335383901, + "flos": 24200057024640.0, + "grad_norm": 3.206317553969673, + "language_loss": 0.99142671, + "learning_rate": 3.8305199058280294e-06, + "loss": 1.01336789, + "num_input_tokens_seen": 155200975, + "router_z_loss_clip": 0.82958984, + "router_z_loss_mlp": 0.18530273, + "step": 5446, + "time_per_iteration": 2.556485176086426 + }, + { + "auxiliary_loss_clip": 0.01136748, + "auxiliary_loss_mlp": 0.01038477, + "balance_loss_clip": 1.05712771, + "balance_loss_mlp": 1.02191854, + "epoch": 0.15805815100690615, + "flos": 26864449188480.0, + "grad_norm": 2.209788736293607, + "language_loss": 0.93150991, + "learning_rate": 3.8304441747655096e-06, + "loss": 0.95326221, + "num_input_tokens_seen": 155219505, + "router_z_loss_clip": 0.79638672, + "router_z_loss_mlp": 0.16546631, + "step": 5447, + "time_per_iteration": 2.5690956115722656 + }, + { + "auxiliary_loss_clip": 0.01045049, + "auxiliary_loss_mlp": 0.01004438, + "balance_loss_clip": 1.02516413, + "balance_loss_mlp": 1.00365162, + "epoch": 0.1580871684754222, + "flos": 66126340990080.0, + "grad_norm": 1.8231013522236539, + "language_loss": 0.524849, + "learning_rate": 3.830368427535766e-06, + "loss": 0.54534388, + "num_input_tokens_seen": 155274495, + "router_z_loss_clip": 0.19921875, + "router_z_loss_mlp": 0.00787354, + "step": 5448, + "time_per_iteration": 2.9876856803894043 + }, + { + "auxiliary_loss_clip": 0.01131814, + "auxiliary_loss_mlp": 0.01046615, + "balance_loss_clip": 1.05357051, + "balance_loss_mlp": 1.03022361, + "epoch": 0.15811618594393825, + "flos": 25951599924480.0, + "grad_norm": 1.9818811435240795, + "language_loss": 0.80806792, + "learning_rate": 3.830292664139468e-06, + "loss": 0.82985222, + "num_input_tokens_seen": 155291645, + "router_z_loss_clip": 0.78173828, + "router_z_loss_mlp": 0.16387939, + "step": 5449, + "time_per_iteration": 2.6536381244659424 + }, + { + "auxiliary_loss_clip": 0.01131042, + "auxiliary_loss_mlp": 0.01043132, + "balance_loss_clip": 1.05312288, + "balance_loss_mlp": 1.02823687, + "epoch": 0.1581452034124543, + "flos": 22885008197760.0, + "grad_norm": 2.2920217280718544, + "language_loss": 0.74668717, + "learning_rate": 3.830216884577284e-06, + "loss": 0.76842892, + "num_input_tokens_seen": 155305145, + "router_z_loss_clip": 0.77929688, + "router_z_loss_mlp": 0.14892578, + "step": 5450, + "time_per_iteration": 2.529839277267456 + }, + { + "auxiliary_loss_clip": 0.01139508, + "auxiliary_loss_mlp": 0.01042012, + "balance_loss_clip": 1.05760849, + "balance_loss_mlp": 1.0259068, + "epoch": 0.15817422088097036, + "flos": 20184525843840.0, + "grad_norm": 2.8416438254408374, + "language_loss": 0.86085975, + "learning_rate": 3.830141088849885e-06, + "loss": 0.88267493, + "num_input_tokens_seen": 155317720, + "router_z_loss_clip": 0.81835938, + "router_z_loss_mlp": 0.16101074, + "step": 5451, + "time_per_iteration": 2.4904274940490723 + }, + { + "auxiliary_loss_clip": 0.01135817, + "auxiliary_loss_mlp": 0.01045448, + "balance_loss_clip": 1.05578351, + "balance_loss_mlp": 1.02871108, + "epoch": 0.15820323834948638, + "flos": 10484946869760.0, + "grad_norm": 2.6736498751097777, + "language_loss": 0.82101375, + "learning_rate": 3.830065276957939e-06, + "loss": 0.84282637, + "num_input_tokens_seen": 155331865, + "router_z_loss_clip": 0.80078125, + "router_z_loss_mlp": 0.16748047, + "step": 5452, + "time_per_iteration": 2.475883722305298 + }, + { + "auxiliary_loss_clip": 0.01129594, + "auxiliary_loss_mlp": 0.01048657, + "balance_loss_clip": 1.05500078, + "balance_loss_mlp": 1.03357697, + "epoch": 0.15823225581800243, + "flos": 29195589945600.0, + "grad_norm": 2.7193419301303448, + "language_loss": 0.79297483, + "learning_rate": 3.829989448902116e-06, + "loss": 0.81475735, + "num_input_tokens_seen": 155345575, + "router_z_loss_clip": 0.74609375, + "router_z_loss_mlp": 0.15081787, + "step": 5453, + "time_per_iteration": 2.5646090507507324 + }, + { + "auxiliary_loss_clip": 0.01042251, + "auxiliary_loss_mlp": 0.01001967, + "balance_loss_clip": 1.02254117, + "balance_loss_mlp": 1.00116849, + "epoch": 0.15826127328651848, + "flos": 61496953355520.0, + "grad_norm": 0.6836536681169376, + "language_loss": 0.47683048, + "learning_rate": 3.829913604683085e-06, + "loss": 0.49727264, + "num_input_tokens_seen": 155407290, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.00799561, + "step": 5454, + "time_per_iteration": 3.087345600128174 + }, + { + "auxiliary_loss_clip": 0.01130852, + "auxiliary_loss_mlp": 0.01044248, + "balance_loss_clip": 1.05411053, + "balance_loss_mlp": 1.02816117, + "epoch": 0.15829029075503454, + "flos": 42077537159040.0, + "grad_norm": 2.241205536855726, + "language_loss": 0.9321475, + "learning_rate": 3.8298377443015165e-06, + "loss": 0.95389849, + "num_input_tokens_seen": 155427570, + "router_z_loss_clip": 0.76757812, + "router_z_loss_mlp": 0.16088867, + "step": 5455, + "time_per_iteration": 2.6912002563476562 + }, + { + "auxiliary_loss_clip": 0.01128756, + "auxiliary_loss_mlp": 0.01041784, + "balance_loss_clip": 1.05426669, + "balance_loss_mlp": 1.02758574, + "epoch": 0.1583193082235506, + "flos": 22776234837120.0, + "grad_norm": 2.372880456423171, + "language_loss": 0.91685951, + "learning_rate": 3.829761867758081e-06, + "loss": 0.9385649, + "num_input_tokens_seen": 155440070, + "router_z_loss_clip": 0.74511719, + "router_z_loss_mlp": 0.14196777, + "step": 5456, + "time_per_iteration": 2.5401768684387207 + }, + { + "auxiliary_loss_clip": 0.01131488, + "auxiliary_loss_mlp": 0.01040821, + "balance_loss_clip": 1.0536592, + "balance_loss_mlp": 1.02549052, + "epoch": 0.1583483256920666, + "flos": 12412631088000.0, + "grad_norm": 2.5863995802364617, + "language_loss": 0.89452583, + "learning_rate": 3.829685975053448e-06, + "loss": 0.91624892, + "num_input_tokens_seen": 155455055, + "router_z_loss_clip": 0.77929688, + "router_z_loss_mlp": 0.15344238, + "step": 5457, + "time_per_iteration": 2.4648213386535645 + }, + { + "auxiliary_loss_clip": 0.01134322, + "auxiliary_loss_mlp": 0.01044082, + "balance_loss_clip": 1.05643833, + "balance_loss_mlp": 1.02824521, + "epoch": 0.15837734316058266, + "flos": 26900611205760.0, + "grad_norm": 5.171022220648844, + "language_loss": 0.99110281, + "learning_rate": 3.829610066188288e-06, + "loss": 1.01288676, + "num_input_tokens_seen": 155469645, + "router_z_loss_clip": 0.77880859, + "router_z_loss_mlp": 0.15844727, + "step": 5458, + "time_per_iteration": 2.5748491287231445 + }, + { + "auxiliary_loss_clip": 0.01125566, + "auxiliary_loss_mlp": 0.01037664, + "balance_loss_clip": 1.04879463, + "balance_loss_mlp": 1.02198148, + "epoch": 0.1584063606290987, + "flos": 15406791039360.0, + "grad_norm": 2.1448142300058968, + "language_loss": 0.66806287, + "learning_rate": 3.829534141163273e-06, + "loss": 0.68969518, + "num_input_tokens_seen": 155484005, + "router_z_loss_clip": 0.76806641, + "router_z_loss_mlp": 0.15689087, + "step": 5459, + "time_per_iteration": 2.489161252975464 + }, + { + "auxiliary_loss_clip": 0.01038408, + "auxiliary_loss_mlp": 0.01000965, + "balance_loss_clip": 1.01849341, + "balance_loss_mlp": 1.00001132, + "epoch": 0.15843537809761477, + "flos": 61964078751360.0, + "grad_norm": 0.6838993799922035, + "language_loss": 0.45587683, + "learning_rate": 3.82945819997907e-06, + "loss": 0.47627056, + "num_input_tokens_seen": 155540640, + "router_z_loss_clip": 0.19921875, + "router_z_loss_mlp": 0.00952148, + "step": 5460, + "time_per_iteration": 3.0070621967315674 + }, + { + "auxiliary_loss_clip": 0.01123465, + "auxiliary_loss_mlp": 0.01031135, + "balance_loss_clip": 1.04928255, + "balance_loss_mlp": 1.0161798, + "epoch": 0.15846439556613082, + "flos": 20515299212160.0, + "grad_norm": 2.0145561101361573, + "language_loss": 0.70023501, + "learning_rate": 3.829382242636354e-06, + "loss": 0.72178102, + "num_input_tokens_seen": 155554915, + "router_z_loss_clip": 0.74169922, + "router_z_loss_mlp": 0.1494751, + "step": 5461, + "time_per_iteration": 2.4965710639953613 + }, + { + "auxiliary_loss_clip": 0.01037672, + "auxiliary_loss_mlp": 0.01004619, + "balance_loss_clip": 1.01777589, + "balance_loss_mlp": 1.00371873, + "epoch": 0.15849341303464687, + "flos": 64667434193280.0, + "grad_norm": 0.6511460397086706, + "language_loss": 0.5088858, + "learning_rate": 3.829306269135792e-06, + "loss": 0.52930874, + "num_input_tokens_seen": 155615830, + "router_z_loss_clip": 0.19921875, + "router_z_loss_mlp": 0.00897217, + "step": 5462, + "time_per_iteration": 3.0646140575408936 + }, + { + "auxiliary_loss_clip": 0.01131042, + "auxiliary_loss_mlp": 0.0103682, + "balance_loss_clip": 1.05392098, + "balance_loss_mlp": 1.0210191, + "epoch": 0.1585224305031629, + "flos": 12706380512640.0, + "grad_norm": 2.6783954174908855, + "language_loss": 0.78923851, + "learning_rate": 3.829230279478058e-06, + "loss": 0.81091714, + "num_input_tokens_seen": 155628010, + "router_z_loss_clip": 0.77148438, + "router_z_loss_mlp": 0.15802002, + "step": 5463, + "time_per_iteration": 2.499701499938965 + }, + { + "auxiliary_loss_clip": 0.01133574, + "auxiliary_loss_mlp": 0.01037489, + "balance_loss_clip": 1.05458188, + "balance_loss_mlp": 1.02144372, + "epoch": 0.15855144797167894, + "flos": 13254413293440.0, + "grad_norm": 2.8625502765701865, + "language_loss": 0.78191471, + "learning_rate": 3.829154273663821e-06, + "loss": 0.80362535, + "num_input_tokens_seen": 155640200, + "router_z_loss_clip": 0.79003906, + "router_z_loss_mlp": 0.1605835, + "step": 5464, + "time_per_iteration": 2.504032611846924 + }, + { + "auxiliary_loss_clip": 0.01127385, + "auxiliary_loss_mlp": 0.01043353, + "balance_loss_clip": 1.05190837, + "balance_loss_mlp": 1.02827919, + "epoch": 0.158580465440195, + "flos": 32192192021760.0, + "grad_norm": 1.858170333254141, + "language_loss": 0.93918741, + "learning_rate": 3.829078251693753e-06, + "loss": 0.96089482, + "num_input_tokens_seen": 155661665, + "router_z_loss_clip": 0.75439453, + "router_z_loss_mlp": 0.15087891, + "step": 5465, + "time_per_iteration": 2.6827123165130615 + }, + { + "auxiliary_loss_clip": 0.01127876, + "auxiliary_loss_mlp": 0.01038637, + "balance_loss_clip": 1.05439329, + "balance_loss_mlp": 1.02391481, + "epoch": 0.15860948290871105, + "flos": 38683476115200.0, + "grad_norm": 2.231172947033533, + "language_loss": 0.82772827, + "learning_rate": 3.829002213568526e-06, + "loss": 0.84939343, + "num_input_tokens_seen": 155676705, + "router_z_loss_clip": 0.734375, + "router_z_loss_mlp": 0.14715576, + "step": 5466, + "time_per_iteration": 2.6715688705444336 + }, + { + "auxiliary_loss_clip": 0.01126473, + "auxiliary_loss_mlp": 0.010499, + "balance_loss_clip": 1.05359018, + "balance_loss_mlp": 1.03603923, + "epoch": 0.1586385003772271, + "flos": 24964882341120.0, + "grad_norm": 3.1253394305507314, + "language_loss": 0.94484442, + "learning_rate": 3.828926159288812e-06, + "loss": 0.96660817, + "num_input_tokens_seen": 155691290, + "router_z_loss_clip": 0.72949219, + "router_z_loss_mlp": 0.13858032, + "step": 5467, + "time_per_iteration": 2.5302937030792236 + }, + { + "auxiliary_loss_clip": 0.01122929, + "auxiliary_loss_mlp": 0.01042237, + "balance_loss_clip": 1.05176663, + "balance_loss_mlp": 1.0272522, + "epoch": 0.15866751784574315, + "flos": 12667560888960.0, + "grad_norm": 4.770514437697455, + "language_loss": 0.78155982, + "learning_rate": 3.828850088855282e-06, + "loss": 0.80321157, + "num_input_tokens_seen": 155701230, + "router_z_loss_clip": 0.7109375, + "router_z_loss_mlp": 0.14978027, + "step": 5468, + "time_per_iteration": 2.4719390869140625 + }, + { + "auxiliary_loss_clip": 0.01039692, + "auxiliary_loss_mlp": 0.01005985, + "balance_loss_clip": 1.01995063, + "balance_loss_mlp": 1.00510335, + "epoch": 0.15869653531425917, + "flos": 55791716538240.0, + "grad_norm": 0.6948030831883409, + "language_loss": 0.46770194, + "learning_rate": 3.828774002268608e-06, + "loss": 0.4881587, + "num_input_tokens_seen": 155754945, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.0088501, + "step": 5469, + "time_per_iteration": 2.982644557952881 + }, + { + "auxiliary_loss_clip": 0.01037075, + "auxiliary_loss_mlp": 0.01004118, + "balance_loss_clip": 1.01757514, + "balance_loss_mlp": 1.00331926, + "epoch": 0.15872555278277523, + "flos": 74771654855040.0, + "grad_norm": 0.6299230055440539, + "language_loss": 0.47309136, + "learning_rate": 3.828697899529461e-06, + "loss": 0.49350327, + "num_input_tokens_seen": 155822315, + "router_z_loss_clip": 0.1953125, + "router_z_loss_mlp": 0.00799561, + "step": 5470, + "time_per_iteration": 3.1892595291137695 + }, + { + "auxiliary_loss_clip": 0.01124313, + "auxiliary_loss_mlp": 0.01042466, + "balance_loss_clip": 1.05217409, + "balance_loss_mlp": 1.02821422, + "epoch": 0.15875457025129128, + "flos": 49372717587840.0, + "grad_norm": 2.74321617768457, + "language_loss": 0.67298955, + "learning_rate": 3.828621780638515e-06, + "loss": 0.69465733, + "num_input_tokens_seen": 155838145, + "router_z_loss_clip": 0.72119141, + "router_z_loss_mlp": 0.14245605, + "step": 5471, + "time_per_iteration": 2.780327796936035 + }, + { + "auxiliary_loss_clip": 0.01037716, + "auxiliary_loss_mlp": 0.01001912, + "balance_loss_clip": 1.01782262, + "balance_loss_mlp": 1.00100636, + "epoch": 0.15878358771980733, + "flos": 58352363245440.0, + "grad_norm": 0.7017644964383737, + "language_loss": 0.46364111, + "learning_rate": 3.828545645596442e-06, + "loss": 0.4840374, + "num_input_tokens_seen": 155891600, + "router_z_loss_clip": 0.19921875, + "router_z_loss_mlp": 0.0090332, + "step": 5472, + "time_per_iteration": 2.902592420578003 + }, + { + "auxiliary_loss_clip": 0.01132812, + "auxiliary_loss_mlp": 0.01039719, + "balance_loss_clip": 1.05506253, + "balance_loss_mlp": 1.02373302, + "epoch": 0.15881260518832338, + "flos": 12489013359360.0, + "grad_norm": 2.4055739125692597, + "language_loss": 0.80985808, + "learning_rate": 3.828469494403913e-06, + "loss": 0.83158338, + "num_input_tokens_seen": 155904770, + "router_z_loss_clip": 0.77587891, + "router_z_loss_mlp": 0.15966797, + "step": 5473, + "time_per_iteration": 2.5236997604370117 + }, + { + "auxiliary_loss_clip": 0.01133368, + "auxiliary_loss_mlp": 0.01037808, + "balance_loss_clip": 1.0562346, + "balance_loss_mlp": 1.02310312, + "epoch": 0.1588416226568394, + "flos": 40401227295360.0, + "grad_norm": 2.160724250662645, + "language_loss": 0.79262507, + "learning_rate": 3.828393327061602e-06, + "loss": 0.81433684, + "num_input_tokens_seen": 155921130, + "router_z_loss_clip": 0.77099609, + "router_z_loss_mlp": 0.14697266, + "step": 5474, + "time_per_iteration": 2.6929941177368164 + }, + { + "auxiliary_loss_clip": 0.01036562, + "auxiliary_loss_mlp": 0.01000316, + "balance_loss_clip": 1.01673293, + "balance_loss_mlp": 0.99948162, + "epoch": 0.15887064012535546, + "flos": 57043850693760.0, + "grad_norm": 0.6413681636796779, + "language_loss": 0.45273191, + "learning_rate": 3.8283171435701805e-06, + "loss": 0.47310066, + "num_input_tokens_seen": 155980125, + "router_z_loss_clip": 0.19921875, + "router_z_loss_mlp": 0.00836182, + "step": 5475, + "time_per_iteration": 2.976377248764038 + }, + { + "auxiliary_loss_clip": 0.01139416, + "auxiliary_loss_mlp": 0.01046026, + "balance_loss_clip": 1.05793977, + "balance_loss_mlp": 1.02935457, + "epoch": 0.1588996575938715, + "flos": 30330331476480.0, + "grad_norm": 2.548693620686972, + "language_loss": 0.88437021, + "learning_rate": 3.828240943930323e-06, + "loss": 0.90622461, + "num_input_tokens_seen": 155995950, + "router_z_loss_clip": 0.81494141, + "router_z_loss_mlp": 0.16662598, + "step": 5476, + "time_per_iteration": 5.075418710708618 + }, + { + "auxiliary_loss_clip": 0.01131355, + "auxiliary_loss_mlp": 0.01039784, + "balance_loss_clip": 1.05580914, + "balance_loss_mlp": 1.02568114, + "epoch": 0.15892867506238756, + "flos": 32051314880640.0, + "grad_norm": 1.6324428830215654, + "language_loss": 0.83405459, + "learning_rate": 3.828164728142701e-06, + "loss": 0.855766, + "num_input_tokens_seen": 156017995, + "router_z_loss_clip": 0.75537109, + "router_z_loss_mlp": 0.14111328, + "step": 5477, + "time_per_iteration": 4.922019958496094 + }, + { + "auxiliary_loss_clip": 0.01035983, + "auxiliary_loss_mlp": 0.01005788, + "balance_loss_clip": 1.01615667, + "balance_loss_mlp": 1.00491798, + "epoch": 0.1589576925309036, + "flos": 65185194787200.0, + "grad_norm": 0.7086097389256376, + "language_loss": 0.46433657, + "learning_rate": 3.8280884962079885e-06, + "loss": 0.48475426, + "num_input_tokens_seen": 156078535, + "router_z_loss_clip": 0.19824219, + "router_z_loss_mlp": 0.00872803, + "step": 5478, + "time_per_iteration": 5.563570022583008 + }, + { + "auxiliary_loss_clip": 0.0111896, + "auxiliary_loss_mlp": 0.01036784, + "balance_loss_clip": 1.04797351, + "balance_loss_mlp": 1.023278, + "epoch": 0.15898670999941966, + "flos": 12486930370560.0, + "grad_norm": 2.170464535754478, + "language_loss": 0.7181831, + "learning_rate": 3.828012248126859e-06, + "loss": 0.73974061, + "num_input_tokens_seen": 156090020, + "router_z_loss_clip": 0.71044922, + "router_z_loss_mlp": 0.13513184, + "step": 5479, + "time_per_iteration": 2.4712908267974854 + }, + { + "auxiliary_loss_clip": 0.0112953, + "auxiliary_loss_mlp": 0.01044509, + "balance_loss_clip": 1.05028367, + "balance_loss_mlp": 1.02743828, + "epoch": 0.15901572746793569, + "flos": 19457191347840.0, + "grad_norm": 3.4793962748986957, + "language_loss": 0.81695557, + "learning_rate": 3.827935983899985e-06, + "loss": 0.83869594, + "num_input_tokens_seen": 156103295, + "router_z_loss_clip": 0.79101562, + "router_z_loss_mlp": 0.17059326, + "step": 5480, + "time_per_iteration": 2.4966917037963867 + }, + { + "auxiliary_loss_clip": 0.0113189, + "auxiliary_loss_mlp": 0.01043705, + "balance_loss_clip": 1.05602932, + "balance_loss_mlp": 1.02854776, + "epoch": 0.15904474493645174, + "flos": 13180365406080.0, + "grad_norm": 4.392438463463502, + "language_loss": 0.75668639, + "learning_rate": 3.827859703528042e-06, + "loss": 0.77844238, + "num_input_tokens_seen": 156113795, + "router_z_loss_clip": 0.75927734, + "router_z_loss_mlp": 0.15148926, + "step": 5481, + "time_per_iteration": 2.4827206134796143 + }, + { + "auxiliary_loss_clip": 0.01036243, + "auxiliary_loss_mlp": 0.0100473, + "balance_loss_clip": 1.01654291, + "balance_loss_mlp": 1.00390744, + "epoch": 0.1590737624049678, + "flos": 63815955315840.0, + "grad_norm": 0.6512436385149234, + "language_loss": 0.46140301, + "learning_rate": 3.827783407011701e-06, + "loss": 0.48181275, + "num_input_tokens_seen": 156174840, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.00823975, + "step": 5482, + "time_per_iteration": 3.105558395385742 + }, + { + "auxiliary_loss_clip": 0.01138378, + "auxiliary_loss_mlp": 0.01049104, + "balance_loss_clip": 1.05394053, + "balance_loss_mlp": 1.03056717, + "epoch": 0.15910277987348384, + "flos": 29235199668480.0, + "grad_norm": 2.4072895496180533, + "language_loss": 0.89873135, + "learning_rate": 3.8277070943516384e-06, + "loss": 0.92060614, + "num_input_tokens_seen": 156199035, + "router_z_loss_clip": 0.84472656, + "router_z_loss_mlp": 0.18554688, + "step": 5483, + "time_per_iteration": 2.953305959701538 + }, + { + "auxiliary_loss_clip": 0.01037322, + "auxiliary_loss_mlp": 0.01001156, + "balance_loss_clip": 1.01750731, + "balance_loss_mlp": 1.00030375, + "epoch": 0.1591317973419999, + "flos": 67368922128000.0, + "grad_norm": 0.6873775184445053, + "language_loss": 0.5006671, + "learning_rate": 3.827630765548527e-06, + "loss": 0.52105188, + "num_input_tokens_seen": 156259805, + "router_z_loss_clip": 0.19824219, + "router_z_loss_mlp": 0.00854492, + "step": 5484, + "time_per_iteration": 3.1502933502197266 + }, + { + "auxiliary_loss_clip": 0.01035246, + "auxiliary_loss_mlp": 0.01000067, + "balance_loss_clip": 1.01515579, + "balance_loss_mlp": 0.99925679, + "epoch": 0.15916081481051594, + "flos": 59636277959040.0, + "grad_norm": 0.6857170430346863, + "language_loss": 0.53856868, + "learning_rate": 3.827554420603041e-06, + "loss": 0.55892175, + "num_input_tokens_seen": 156319440, + "router_z_loss_clip": 0.20117188, + "router_z_loss_mlp": 0.00811768, + "step": 5485, + "time_per_iteration": 3.1897966861724854 + }, + { + "auxiliary_loss_clip": 0.01034397, + "auxiliary_loss_mlp": 0.00999668, + "balance_loss_clip": 1.01447296, + "balance_loss_mlp": 0.99883986, + "epoch": 0.15918983227903197, + "flos": 72404244339840.0, + "grad_norm": 0.6747038283356471, + "language_loss": 0.48435229, + "learning_rate": 3.827478059515854e-06, + "loss": 0.50469291, + "num_input_tokens_seen": 156390040, + "router_z_loss_clip": 0.19921875, + "router_z_loss_mlp": 0.00830078, + "step": 5486, + "time_per_iteration": 3.2095282077789307 + }, + { + "auxiliary_loss_clip": 0.01130843, + "auxiliary_loss_mlp": 0.01033365, + "balance_loss_clip": 1.05303633, + "balance_loss_mlp": 1.01561439, + "epoch": 0.15921884974754802, + "flos": 24459080976000.0, + "grad_norm": 2.0371617295096422, + "language_loss": 0.88311058, + "learning_rate": 3.827401682287642e-06, + "loss": 0.90475273, + "num_input_tokens_seen": 156407655, + "router_z_loss_clip": 0.77929688, + "router_z_loss_mlp": 0.17730713, + "step": 5487, + "time_per_iteration": 2.563894748687744 + }, + { + "auxiliary_loss_clip": 0.01030623, + "auxiliary_loss_mlp": 0.01001747, + "balance_loss_clip": 1.01089954, + "balance_loss_mlp": 1.00095451, + "epoch": 0.15924786721606407, + "flos": 66129537300480.0, + "grad_norm": 0.639366683494452, + "language_loss": 0.46018445, + "learning_rate": 3.827325288919079e-06, + "loss": 0.48050815, + "num_input_tokens_seen": 156468495, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.00793457, + "step": 5488, + "time_per_iteration": 3.119767427444458 + }, + { + "auxiliary_loss_clip": 0.01128832, + "auxiliary_loss_mlp": 0.0104232, + "balance_loss_clip": 1.049088, + "balance_loss_mlp": 1.02343762, + "epoch": 0.15927688468458012, + "flos": 23032960318080.0, + "grad_norm": 2.625442055457572, + "language_loss": 0.96573085, + "learning_rate": 3.827248879410839e-06, + "loss": 0.98744243, + "num_input_tokens_seen": 156484090, + "router_z_loss_clip": 0.79833984, + "router_z_loss_mlp": 0.18884277, + "step": 5489, + "time_per_iteration": 2.545555591583252 + }, + { + "auxiliary_loss_clip": 0.01029466, + "auxiliary_loss_mlp": 0.01005679, + "balance_loss_clip": 1.00979233, + "balance_loss_mlp": 1.00488043, + "epoch": 0.15930590215309617, + "flos": 74778693920640.0, + "grad_norm": 0.636187684370326, + "language_loss": 0.4759188, + "learning_rate": 3.827172453763598e-06, + "loss": 0.4962703, + "num_input_tokens_seen": 156551520, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.00799561, + "step": 5490, + "time_per_iteration": 3.15484356880188 + }, + { + "auxiliary_loss_clip": 0.01028372, + "auxiliary_loss_mlp": 0.0100624, + "balance_loss_clip": 1.00875854, + "balance_loss_mlp": 1.00548291, + "epoch": 0.1593349196216122, + "flos": 73422993876480.0, + "grad_norm": 0.6697210022534478, + "language_loss": 0.48554271, + "learning_rate": 3.82709601197803e-06, + "loss": 0.50588882, + "num_input_tokens_seen": 156613155, + "router_z_loss_clip": 0.19628906, + "router_z_loss_mlp": 0.00756836, + "step": 5491, + "time_per_iteration": 3.0761468410491943 + }, + { + "auxiliary_loss_clip": 0.01028078, + "auxiliary_loss_mlp": 0.01004718, + "balance_loss_clip": 1.00838757, + "balance_loss_mlp": 1.00393128, + "epoch": 0.15936393709012825, + "flos": 71421728647680.0, + "grad_norm": 0.6455164586251364, + "language_loss": 0.50237095, + "learning_rate": 3.827019554054811e-06, + "loss": 0.52269888, + "num_input_tokens_seen": 156675235, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.00787354, + "step": 5492, + "time_per_iteration": 3.095367908477783 + }, + { + "auxiliary_loss_clip": 0.01028392, + "auxiliary_loss_mlp": 0.01002403, + "balance_loss_clip": 1.00871468, + "balance_loss_mlp": 1.0016551, + "epoch": 0.1593929545586443, + "flos": 63939201857280.0, + "grad_norm": 0.687886884298702, + "language_loss": 0.48759139, + "learning_rate": 3.826943079994616e-06, + "loss": 0.50789934, + "num_input_tokens_seen": 156736975, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.00747681, + "step": 5493, + "time_per_iteration": 3.0947189331054688 + }, + { + "auxiliary_loss_clip": 0.01027927, + "auxiliary_loss_mlp": 0.01004676, + "balance_loss_clip": 1.00837469, + "balance_loss_mlp": 1.00390708, + "epoch": 0.15942197202716035, + "flos": 74774743424640.0, + "grad_norm": 0.6532349031347662, + "language_loss": 0.47952533, + "learning_rate": 3.826866589798121e-06, + "loss": 0.49985138, + "num_input_tokens_seen": 156798550, + "router_z_loss_clip": 0.1953125, + "router_z_loss_mlp": 0.00765991, + "step": 5494, + "time_per_iteration": 3.081015110015869 + }, + { + "auxiliary_loss_clip": 0.01028076, + "auxiliary_loss_mlp": 0.01002451, + "balance_loss_clip": 1.00847697, + "balance_loss_mlp": 1.00167584, + "epoch": 0.1594509894956764, + "flos": 63092714970240.0, + "grad_norm": 0.7168764797316816, + "language_loss": 0.50722873, + "learning_rate": 3.826790083466e-06, + "loss": 0.52753401, + "num_input_tokens_seen": 156858150, + "router_z_loss_clip": 0.19628906, + "router_z_loss_mlp": 0.00775146, + "step": 5495, + "time_per_iteration": 3.0039663314819336 + }, + { + "auxiliary_loss_clip": 0.0102684, + "auxiliary_loss_mlp": 0.01004203, + "balance_loss_clip": 1.00723445, + "balance_loss_mlp": 1.00346363, + "epoch": 0.15948000696419246, + "flos": 74775533523840.0, + "grad_norm": 0.6292457522856018, + "language_loss": 0.46045011, + "learning_rate": 3.826713560998931e-06, + "loss": 0.48076051, + "num_input_tokens_seen": 156918615, + "router_z_loss_clip": 0.19628906, + "router_z_loss_mlp": 0.00738525, + "step": 5496, + "time_per_iteration": 3.132258653640747 + }, + { + "auxiliary_loss_clip": 0.01026224, + "auxiliary_loss_mlp": 0.01003734, + "balance_loss_clip": 1.00680017, + "balance_loss_mlp": 1.00291181, + "epoch": 0.15950902443270848, + "flos": 74792053779840.0, + "grad_norm": 0.6894927835789799, + "language_loss": 0.50923133, + "learning_rate": 3.826637022397588e-06, + "loss": 0.52953088, + "num_input_tokens_seen": 156991100, + "router_z_loss_clip": 0.19433594, + "router_z_loss_mlp": 0.00823975, + "step": 5497, + "time_per_iteration": 3.3186185359954834 + }, + { + "auxiliary_loss_clip": 0.01025237, + "auxiliary_loss_mlp": 0.01002751, + "balance_loss_clip": 1.00595188, + "balance_loss_mlp": 1.00194633, + "epoch": 0.15953804190122453, + "flos": 65322947445120.0, + "grad_norm": 0.599092529140002, + "language_loss": 0.45715025, + "learning_rate": 3.826560467662647e-06, + "loss": 0.47743014, + "num_input_tokens_seen": 157055700, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.00805664, + "step": 5498, + "time_per_iteration": 3.1238341331481934 + }, + { + "auxiliary_loss_clip": 0.01123206, + "auxiliary_loss_mlp": 0.01036703, + "balance_loss_clip": 1.04973674, + "balance_loss_mlp": 1.02185583, + "epoch": 0.15956705936974058, + "flos": 29308421543040.0, + "grad_norm": 3.0514954245321704, + "language_loss": 0.89877701, + "learning_rate": 3.826483896794785e-06, + "loss": 0.92037618, + "num_input_tokens_seen": 157070395, + "router_z_loss_clip": 0.73535156, + "router_z_loss_mlp": 0.14849854, + "step": 5499, + "time_per_iteration": 2.5752222537994385 + }, + { + "auxiliary_loss_clip": 0.01024115, + "auxiliary_loss_mlp": 0.01000034, + "balance_loss_clip": 1.0048908, + "balance_loss_mlp": 0.99924165, + "epoch": 0.15959607683825663, + "flos": 67413164705280.0, + "grad_norm": 0.6260618021148551, + "language_loss": 0.50113809, + "learning_rate": 3.826407309794678e-06, + "loss": 0.52137959, + "num_input_tokens_seen": 157134590, + "router_z_loss_clip": 0.19238281, + "router_z_loss_mlp": 0.00793457, + "step": 5500, + "time_per_iteration": 3.184091806411743 + }, + { + "auxiliary_loss_clip": 0.01124855, + "auxiliary_loss_mlp": 0.0103809, + "balance_loss_clip": 1.0486145, + "balance_loss_mlp": 1.02278316, + "epoch": 0.15962509430677269, + "flos": 11536877594880.0, + "grad_norm": 3.331241856364172, + "language_loss": 0.86216402, + "learning_rate": 3.8263307066630035e-06, + "loss": 0.88379347, + "num_input_tokens_seen": 157145410, + "router_z_loss_clip": 0.76269531, + "router_z_loss_mlp": 0.15307617, + "step": 5501, + "time_per_iteration": 2.5101113319396973 + }, + { + "auxiliary_loss_clip": 0.01124644, + "auxiliary_loss_mlp": 0.01044923, + "balance_loss_clip": 1.04945469, + "balance_loss_mlp": 1.02877021, + "epoch": 0.15965411177528874, + "flos": 13838536264320.0, + "grad_norm": 3.669256124643598, + "language_loss": 0.85792416, + "learning_rate": 3.826254087400437e-06, + "loss": 0.87961978, + "num_input_tokens_seen": 157157190, + "router_z_loss_clip": 0.75097656, + "router_z_loss_mlp": 0.16149902, + "step": 5502, + "time_per_iteration": 2.478933811187744 + }, + { + "auxiliary_loss_clip": 0.01130775, + "auxiliary_loss_mlp": 0.01047455, + "balance_loss_clip": 1.05059516, + "balance_loss_mlp": 1.03163624, + "epoch": 0.15968312924380476, + "flos": 25915473820800.0, + "grad_norm": 2.0549479895897482, + "language_loss": 0.76324862, + "learning_rate": 3.8261774520076545e-06, + "loss": 0.7850309, + "num_input_tokens_seen": 157174015, + "router_z_loss_clip": 0.80126953, + "router_z_loss_mlp": 0.15808105, + "step": 5503, + "time_per_iteration": 2.581132411956787 + }, + { + "auxiliary_loss_clip": 0.01122425, + "auxiliary_loss_mlp": 0.01035576, + "balance_loss_clip": 1.04826474, + "balance_loss_mlp": 1.01989448, + "epoch": 0.1597121467123208, + "flos": 32482960617600.0, + "grad_norm": 2.184554511161964, + "language_loss": 0.88915575, + "learning_rate": 3.826100800485335e-06, + "loss": 0.91073567, + "num_input_tokens_seen": 157194165, + "router_z_loss_clip": 0.74169922, + "router_z_loss_mlp": 0.15679932, + "step": 5504, + "time_per_iteration": 2.607227087020874 + }, + { + "auxiliary_loss_clip": 0.01027919, + "auxiliary_loss_mlp": 0.01002037, + "balance_loss_clip": 1.00843954, + "balance_loss_mlp": 1.00125623, + "epoch": 0.15974116418083686, + "flos": 67291534275840.0, + "grad_norm": 0.7083985039549583, + "language_loss": 0.49111173, + "learning_rate": 3.826024132834153e-06, + "loss": 0.51141131, + "num_input_tokens_seen": 157254490, + "router_z_loss_clip": 0.1953125, + "router_z_loss_mlp": 0.0078125, + "step": 5505, + "time_per_iteration": 3.2652018070220947 + }, + { + "auxiliary_loss_clip": 0.01129893, + "auxiliary_loss_mlp": 0.01046431, + "balance_loss_clip": 1.05173481, + "balance_loss_mlp": 1.03074336, + "epoch": 0.15977018164935292, + "flos": 34600648803840.0, + "grad_norm": 2.1655803904236497, + "language_loss": 0.98086214, + "learning_rate": 3.8259474490547875e-06, + "loss": 1.00262535, + "num_input_tokens_seen": 157272890, + "router_z_loss_clip": 0.78125, + "router_z_loss_mlp": 0.15686035, + "step": 5506, + "time_per_iteration": 2.61228609085083 + }, + { + "auxiliary_loss_clip": 0.01026812, + "auxiliary_loss_mlp": 0.01000604, + "balance_loss_clip": 1.00737965, + "balance_loss_mlp": 0.99986821, + "epoch": 0.15979919911786897, + "flos": 63656513821440.0, + "grad_norm": 0.6425589624176481, + "language_loss": 0.48273188, + "learning_rate": 3.825870749147915e-06, + "loss": 0.5030061, + "num_input_tokens_seen": 157340100, + "router_z_loss_clip": 0.19433594, + "router_z_loss_mlp": 0.00735474, + "step": 5507, + "time_per_iteration": 3.150979995727539 + }, + { + "auxiliary_loss_clip": 0.01126563, + "auxiliary_loss_mlp": 0.01043168, + "balance_loss_clip": 1.0487628, + "balance_loss_mlp": 1.02633595, + "epoch": 0.159828216586385, + "flos": 31425104148480.0, + "grad_norm": 1.8621765318770147, + "language_loss": 0.87470186, + "learning_rate": 3.825794033114214e-06, + "loss": 0.8963992, + "num_input_tokens_seen": 157360340, + "router_z_loss_clip": 0.77856445, + "router_z_loss_mlp": 0.16845703, + "step": 5508, + "time_per_iteration": 2.623990774154663 + }, + { + "auxiliary_loss_clip": 0.01027515, + "auxiliary_loss_mlp": 0.01001673, + "balance_loss_clip": 1.00803745, + "balance_loss_mlp": 1.0008986, + "epoch": 0.15985723405490104, + "flos": 70277793235200.0, + "grad_norm": 0.7819294016069294, + "language_loss": 0.49892429, + "learning_rate": 3.825717300954361e-06, + "loss": 0.51921618, + "num_input_tokens_seen": 157418565, + "router_z_loss_clip": 0.1953125, + "router_z_loss_mlp": 0.00772095, + "step": 5509, + "time_per_iteration": 3.036402463912964 + }, + { + "auxiliary_loss_clip": 0.01026488, + "auxiliary_loss_mlp": 0.01002282, + "balance_loss_clip": 1.0069567, + "balance_loss_mlp": 1.00147772, + "epoch": 0.1598862515234171, + "flos": 63269107683840.0, + "grad_norm": 0.7711687139166976, + "language_loss": 0.5176574, + "learning_rate": 3.825640552669034e-06, + "loss": 0.53794503, + "num_input_tokens_seen": 157474625, + "router_z_loss_clip": 0.1953125, + "router_z_loss_mlp": 0.00805664, + "step": 5510, + "time_per_iteration": 2.9476962089538574 + }, + { + "auxiliary_loss_clip": 0.01025987, + "auxiliary_loss_mlp": 0.0100253, + "balance_loss_clip": 1.00651574, + "balance_loss_mlp": 1.0017314, + "epoch": 0.15991526899193315, + "flos": 65075559500160.0, + "grad_norm": 0.664787919495942, + "language_loss": 0.45300129, + "learning_rate": 3.825563788258911e-06, + "loss": 0.47328645, + "num_input_tokens_seen": 157536375, + "router_z_loss_clip": 0.1953125, + "router_z_loss_mlp": 0.00799561, + "step": 5511, + "time_per_iteration": 3.1976819038391113 + }, + { + "auxiliary_loss_clip": 0.01137961, + "auxiliary_loss_mlp": 0.010435, + "balance_loss_clip": 1.05487156, + "balance_loss_mlp": 1.0253917, + "epoch": 0.1599442864604492, + "flos": 27153960808320.0, + "grad_norm": 2.2669965515970114, + "language_loss": 0.95522779, + "learning_rate": 3.825487007724669e-06, + "loss": 0.97704238, + "num_input_tokens_seen": 157550625, + "router_z_loss_clip": 0.83056641, + "router_z_loss_mlp": 0.18109131, + "step": 5512, + "time_per_iteration": 2.5562994480133057 + }, + { + "auxiliary_loss_clip": 0.01024955, + "auxiliary_loss_mlp": 0.0100322, + "balance_loss_clip": 1.00547445, + "balance_loss_mlp": 1.00246286, + "epoch": 0.15997330392896525, + "flos": 74768566285440.0, + "grad_norm": 0.6790318621506036, + "language_loss": 0.51471508, + "learning_rate": 3.825410211066987e-06, + "loss": 0.53499681, + "num_input_tokens_seen": 157612330, + "router_z_loss_clip": 0.1953125, + "router_z_loss_mlp": 0.00756836, + "step": 5513, + "time_per_iteration": 3.0917420387268066 + }, + { + "auxiliary_loss_clip": 0.01024288, + "auxiliary_loss_mlp": 0.00999691, + "balance_loss_clip": 1.00500929, + "balance_loss_mlp": 0.99881506, + "epoch": 0.16000232139748127, + "flos": 68976176094720.0, + "grad_norm": 0.6738550583775749, + "language_loss": 0.50837278, + "learning_rate": 3.825333398286544e-06, + "loss": 0.52861261, + "num_input_tokens_seen": 157678455, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.00878906, + "step": 5514, + "time_per_iteration": 3.2758421897888184 + }, + { + "auxiliary_loss_clip": 0.01134143, + "auxiliary_loss_mlp": 0.01039618, + "balance_loss_clip": 1.05197883, + "balance_loss_mlp": 1.02243972, + "epoch": 0.16003133886599732, + "flos": 29600123892480.0, + "grad_norm": 2.0195821459731333, + "language_loss": 0.65434968, + "learning_rate": 3.825256569384018e-06, + "loss": 0.67608732, + "num_input_tokens_seen": 157695005, + "router_z_loss_clip": 0.82226562, + "router_z_loss_mlp": 0.17193604, + "step": 5515, + "time_per_iteration": 2.58931303024292 + }, + { + "auxiliary_loss_clip": 0.01118639, + "auxiliary_loss_mlp": 0.01047713, + "balance_loss_clip": 1.04949486, + "balance_loss_mlp": 1.03427219, + "epoch": 0.16006035633451338, + "flos": 36278467038720.0, + "grad_norm": 1.8821363093218726, + "language_loss": 0.86313194, + "learning_rate": 3.825179724360087e-06, + "loss": 0.88479543, + "num_input_tokens_seen": 157712575, + "router_z_loss_clip": 0.69140625, + "router_z_loss_mlp": 0.13439941, + "step": 5516, + "time_per_iteration": 2.624206781387329 + }, + { + "auxiliary_loss_clip": 0.01129074, + "auxiliary_loss_mlp": 0.01046052, + "balance_loss_clip": 1.05073726, + "balance_loss_mlp": 1.02808714, + "epoch": 0.16008937380302943, + "flos": 41130501125760.0, + "grad_norm": 1.8042128629123408, + "language_loss": 0.76124769, + "learning_rate": 3.825102863215431e-06, + "loss": 0.78299892, + "num_input_tokens_seen": 157733665, + "router_z_loss_clip": 0.78466797, + "router_z_loss_mlp": 0.17962646, + "step": 5517, + "time_per_iteration": 2.6721105575561523 + }, + { + "auxiliary_loss_clip": 0.01133541, + "auxiliary_loss_mlp": 0.01042524, + "balance_loss_clip": 1.05170226, + "balance_loss_mlp": 1.02491713, + "epoch": 0.16011839127154548, + "flos": 33069382058880.0, + "grad_norm": 1.707146699818618, + "language_loss": 0.93810713, + "learning_rate": 3.825025985950727e-06, + "loss": 0.95986784, + "num_input_tokens_seen": 157759790, + "router_z_loss_clip": 0.81787109, + "router_z_loss_mlp": 0.17602539, + "step": 5518, + "time_per_iteration": 2.665297508239746 + }, + { + "auxiliary_loss_clip": 0.01126601, + "auxiliary_loss_mlp": 0.01043041, + "balance_loss_clip": 1.05135322, + "balance_loss_mlp": 1.02697766, + "epoch": 0.1601474087400615, + "flos": 28686951406080.0, + "grad_norm": 2.872088480777988, + "language_loss": 0.9071843, + "learning_rate": 3.824949092566655e-06, + "loss": 0.92888069, + "num_input_tokens_seen": 157774880, + "router_z_loss_clip": 0.75097656, + "router_z_loss_mlp": 0.1605835, + "step": 5519, + "time_per_iteration": 2.5719404220581055 + }, + { + "auxiliary_loss_clip": 0.01130192, + "auxiliary_loss_mlp": 0.01041378, + "balance_loss_clip": 1.04947948, + "balance_loss_mlp": 1.0241766, + "epoch": 0.16017642620857755, + "flos": 27265607256960.0, + "grad_norm": 1.9206405685263739, + "language_loss": 0.79899955, + "learning_rate": 3.824872183063894e-06, + "loss": 0.82071519, + "num_input_tokens_seen": 157791620, + "router_z_loss_clip": 0.80810547, + "router_z_loss_mlp": 0.17205811, + "step": 5520, + "time_per_iteration": 2.5821375846862793 + }, + { + "auxiliary_loss_clip": 0.0112854, + "auxiliary_loss_mlp": 0.01044704, + "balance_loss_clip": 1.05077314, + "balance_loss_mlp": 1.02918911, + "epoch": 0.1602054436770936, + "flos": 17778870322560.0, + "grad_norm": 3.3390861219362664, + "language_loss": 0.86294764, + "learning_rate": 3.824795257443125e-06, + "loss": 0.88468009, + "num_input_tokens_seen": 157805055, + "router_z_loss_clip": 0.77685547, + "router_z_loss_mlp": 0.15515137, + "step": 5521, + "time_per_iteration": 2.522184133529663 + }, + { + "auxiliary_loss_clip": 0.01031159, + "auxiliary_loss_mlp": 0.01018943, + "balance_loss_clip": 1.01161599, + "balance_loss_mlp": 1.01809072, + "epoch": 0.16023446114560966, + "flos": 74763897517440.0, + "grad_norm": 0.6762218330801024, + "language_loss": 0.4308871, + "learning_rate": 3.824718315705023e-06, + "loss": 0.45138812, + "num_input_tokens_seen": 157860530, + "router_z_loss_clip": 0.1953125, + "router_z_loss_mlp": 0.00854492, + "step": 5522, + "time_per_iteration": 3.0418519973754883 + }, + { + "auxiliary_loss_clip": 0.01031456, + "auxiliary_loss_mlp": 0.01015765, + "balance_loss_clip": 1.01206326, + "balance_loss_mlp": 1.01494265, + "epoch": 0.1602634786141257, + "flos": 74771618941440.0, + "grad_norm": 0.6716773307593346, + "language_loss": 0.46967334, + "learning_rate": 3.824641357850273e-06, + "loss": 0.4901455, + "num_input_tokens_seen": 157924425, + "router_z_loss_clip": 0.19433594, + "router_z_loss_mlp": 0.00823975, + "step": 5523, + "time_per_iteration": 3.2860443592071533 + }, + { + "auxiliary_loss_clip": 0.0113984, + "auxiliary_loss_mlp": 0.01048132, + "balance_loss_clip": 1.05444515, + "balance_loss_mlp": 1.02914786, + "epoch": 0.16029249608264176, + "flos": 36385336978560.0, + "grad_norm": 2.3864621123638923, + "language_loss": 0.87933344, + "learning_rate": 3.824564383879551e-06, + "loss": 0.90121311, + "num_input_tokens_seen": 157942305, + "router_z_loss_clip": 0.85400391, + "router_z_loss_mlp": 0.1897583, + "step": 5524, + "time_per_iteration": 2.6465141773223877 + }, + { + "auxiliary_loss_clip": 0.01133, + "auxiliary_loss_mlp": 0.01052021, + "balance_loss_clip": 1.05305171, + "balance_loss_mlp": 1.03656602, + "epoch": 0.16032151355115778, + "flos": 24467125622400.0, + "grad_norm": 2.1640692221196307, + "language_loss": 0.96701115, + "learning_rate": 3.8244873937935385e-06, + "loss": 0.98886144, + "num_input_tokens_seen": 157962280, + "router_z_loss_clip": 0.79980469, + "router_z_loss_mlp": 0.15454102, + "step": 5525, + "time_per_iteration": 2.6274986267089844 + }, + { + "auxiliary_loss_clip": 0.01130191, + "auxiliary_loss_mlp": 0.01049436, + "balance_loss_clip": 1.05010545, + "balance_loss_mlp": 1.03226376, + "epoch": 0.16035053101967384, + "flos": 34708093361280.0, + "grad_norm": 1.9314349943360025, + "language_loss": 0.83219773, + "learning_rate": 3.8244103875929144e-06, + "loss": 0.85399401, + "num_input_tokens_seen": 157980460, + "router_z_loss_clip": 0.80126953, + "router_z_loss_mlp": 0.17169189, + "step": 5526, + "time_per_iteration": 2.6988399028778076 + }, + { + "auxiliary_loss_clip": 0.01131505, + "auxiliary_loss_mlp": 0.01045557, + "balance_loss_clip": 1.05342567, + "balance_loss_mlp": 1.03116858, + "epoch": 0.1603795484881899, + "flos": 21974744712960.0, + "grad_norm": 1.9841603857807197, + "language_loss": 0.61784148, + "learning_rate": 3.82433336527836e-06, + "loss": 0.63961202, + "num_input_tokens_seen": 158000630, + "router_z_loss_clip": 0.77978516, + "router_z_loss_mlp": 0.1439209, + "step": 5527, + "time_per_iteration": 2.7213993072509766 + }, + { + "auxiliary_loss_clip": 0.01133473, + "auxiliary_loss_mlp": 0.01047541, + "balance_loss_clip": 1.0503931, + "balance_loss_mlp": 1.0300355, + "epoch": 0.16040856595670594, + "flos": 26863838657280.0, + "grad_norm": 2.557964724159373, + "language_loss": 1.02269971, + "learning_rate": 3.824256326850555e-06, + "loss": 1.04450989, + "num_input_tokens_seen": 158015625, + "router_z_loss_clip": 0.83056641, + "router_z_loss_mlp": 0.17504883, + "step": 5528, + "time_per_iteration": 2.5490453243255615 + }, + { + "auxiliary_loss_clip": 0.01118906, + "auxiliary_loss_mlp": 0.01043944, + "balance_loss_clip": 1.04863262, + "balance_loss_mlp": 1.03054523, + "epoch": 0.160437583425222, + "flos": 20406202629120.0, + "grad_norm": 2.1365311650104206, + "language_loss": 0.80146486, + "learning_rate": 3.824179272310181e-06, + "loss": 0.82309341, + "num_input_tokens_seen": 158028275, + "router_z_loss_clip": 0.70214844, + "router_z_loss_mlp": 0.13409424, + "step": 5529, + "time_per_iteration": 2.531968116760254 + }, + { + "auxiliary_loss_clip": 0.01029554, + "auxiliary_loss_mlp": 0.0104614, + "balance_loss_clip": 1.01001167, + "balance_loss_mlp": 1.04534185, + "epoch": 0.16046660089373804, + "flos": 58059978537600.0, + "grad_norm": 0.6765827960765298, + "language_loss": 0.46915877, + "learning_rate": 3.824102201657916e-06, + "loss": 0.48991573, + "num_input_tokens_seen": 158082425, + "router_z_loss_clip": 0.1953125, + "router_z_loss_mlp": 0.00799561, + "step": 5530, + "time_per_iteration": 2.936666250228882 + }, + { + "auxiliary_loss_clip": 0.01118344, + "auxiliary_loss_mlp": 0.01065759, + "balance_loss_clip": 1.04842091, + "balance_loss_mlp": 1.05056, + "epoch": 0.16049561836225407, + "flos": 19676282353920.0, + "grad_norm": 3.780521829351801, + "language_loss": 0.8141712, + "learning_rate": 3.824025114894443e-06, + "loss": 0.83601218, + "num_input_tokens_seen": 158096030, + "router_z_loss_clip": 0.69921875, + "router_z_loss_mlp": 0.15185547, + "step": 5531, + "time_per_iteration": 2.5773797035217285 + }, + { + "auxiliary_loss_clip": 0.01032142, + "auxiliary_loss_mlp": 0.01030249, + "balance_loss_clip": 1.01262283, + "balance_loss_mlp": 1.02948654, + "epoch": 0.16052463583077012, + "flos": 63395478708480.0, + "grad_norm": 0.6892380956919407, + "language_loss": 0.49038023, + "learning_rate": 3.823948012020441e-06, + "loss": 0.51100415, + "num_input_tokens_seen": 158155745, + "router_z_loss_clip": 0.1953125, + "router_z_loss_mlp": 0.00762939, + "step": 5532, + "time_per_iteration": 3.0658843517303467 + }, + { + "auxiliary_loss_clip": 0.01119444, + "auxiliary_loss_mlp": 0.01037495, + "balance_loss_clip": 1.0494082, + "balance_loss_mlp": 1.02334476, + "epoch": 0.16055365329928617, + "flos": 32993574405120.0, + "grad_norm": 2.0017454716867915, + "language_loss": 0.70445216, + "learning_rate": 3.823870893036594e-06, + "loss": 0.72602153, + "num_input_tokens_seen": 158185620, + "router_z_loss_clip": 0.70068359, + "router_z_loss_mlp": 0.14147949, + "step": 5533, + "time_per_iteration": 2.8592662811279297 + }, + { + "auxiliary_loss_clip": 0.01132474, + "auxiliary_loss_mlp": 0.01038185, + "balance_loss_clip": 1.05466437, + "balance_loss_mlp": 1.02269983, + "epoch": 0.16058267076780222, + "flos": 14385024760320.0, + "grad_norm": 2.6244601103640215, + "language_loss": 0.91632748, + "learning_rate": 3.823793757943579e-06, + "loss": 0.93803412, + "num_input_tokens_seen": 158197945, + "router_z_loss_clip": 0.77832031, + "router_z_loss_mlp": 0.15484619, + "step": 5534, + "time_per_iteration": 2.46390962600708 + }, + { + "auxiliary_loss_clip": 0.01139559, + "auxiliary_loss_mlp": 0.01043746, + "balance_loss_clip": 1.05900311, + "balance_loss_mlp": 1.02648401, + "epoch": 0.16061168823631827, + "flos": 18875043624960.0, + "grad_norm": 2.224650929194634, + "language_loss": 0.90347171, + "learning_rate": 3.82371660674208e-06, + "loss": 0.92530477, + "num_input_tokens_seen": 158211695, + "router_z_loss_clip": 0.8046875, + "router_z_loss_mlp": 0.17254639, + "step": 5535, + "time_per_iteration": 2.5134992599487305 + }, + { + "auxiliary_loss_clip": 0.01123184, + "auxiliary_loss_mlp": 0.01038702, + "balance_loss_clip": 1.04892766, + "balance_loss_mlp": 1.02367592, + "epoch": 0.1606407057048343, + "flos": 34414343936640.0, + "grad_norm": 2.606330123311984, + "language_loss": 0.90963382, + "learning_rate": 3.8236394394327785e-06, + "loss": 0.93125272, + "num_input_tokens_seen": 158227980, + "router_z_loss_clip": 0.74267578, + "router_z_loss_mlp": 0.15026855, + "step": 5536, + "time_per_iteration": 2.639345169067383 + }, + { + "auxiliary_loss_clip": 0.0113914, + "auxiliary_loss_mlp": 0.01040556, + "balance_loss_clip": 1.0569036, + "balance_loss_mlp": 1.02256727, + "epoch": 0.16066972317335035, + "flos": 16537366592640.0, + "grad_norm": 2.143082502046312, + "language_loss": 0.78785437, + "learning_rate": 3.823562256016357e-06, + "loss": 0.80965132, + "num_input_tokens_seen": 158241780, + "router_z_loss_clip": 0.82177734, + "router_z_loss_mlp": 0.17980957, + "step": 5537, + "time_per_iteration": 2.4901950359344482 + }, + { + "auxiliary_loss_clip": 0.01042796, + "auxiliary_loss_mlp": 0.01011097, + "balance_loss_clip": 1.02262306, + "balance_loss_mlp": 1.01022077, + "epoch": 0.1606987406418664, + "flos": 70514589646080.0, + "grad_norm": 0.6529505785869698, + "language_loss": 0.48691788, + "learning_rate": 3.823485056493493e-06, + "loss": 0.50745678, + "num_input_tokens_seen": 158307870, + "router_z_loss_clip": 0.20117188, + "router_z_loss_mlp": 0.00878906, + "step": 5538, + "time_per_iteration": 3.1623334884643555 + }, + { + "auxiliary_loss_clip": 0.01130281, + "auxiliary_loss_mlp": 0.01048688, + "balance_loss_clip": 1.05406046, + "balance_loss_mlp": 1.03282142, + "epoch": 0.16072775811038245, + "flos": 16576258043520.0, + "grad_norm": 2.966834524572379, + "language_loss": 0.93724275, + "learning_rate": 3.823407840864873e-06, + "loss": 0.95903242, + "num_input_tokens_seen": 158323055, + "router_z_loss_clip": 0.76171875, + "router_z_loss_mlp": 0.15869141, + "step": 5539, + "time_per_iteration": 2.475829601287842 + }, + { + "auxiliary_loss_clip": 0.0113209, + "auxiliary_loss_mlp": 0.010438, + "balance_loss_clip": 1.05261207, + "balance_loss_mlp": 1.02805269, + "epoch": 0.1607567755788985, + "flos": 22267057593600.0, + "grad_norm": 2.078908117922943, + "language_loss": 0.82620466, + "learning_rate": 3.8233306091311765e-06, + "loss": 0.84796357, + "num_input_tokens_seen": 158336765, + "router_z_loss_clip": 0.79541016, + "router_z_loss_mlp": 0.1574707, + "step": 5540, + "time_per_iteration": 2.4933745861053467 + }, + { + "auxiliary_loss_clip": 0.01125853, + "auxiliary_loss_mlp": 0.01036559, + "balance_loss_clip": 1.05443943, + "balance_loss_mlp": 1.02295685, + "epoch": 0.16078579304741455, + "flos": 18361520835840.0, + "grad_norm": 2.6126553267586594, + "language_loss": 0.92753565, + "learning_rate": 3.823253361293086e-06, + "loss": 0.9491598, + "num_input_tokens_seen": 158350190, + "router_z_loss_clip": 0.71435547, + "router_z_loss_mlp": 0.13598633, + "step": 5541, + "time_per_iteration": 2.49755859375 + }, + { + "auxiliary_loss_clip": 0.01138903, + "auxiliary_loss_mlp": 0.01047283, + "balance_loss_clip": 1.05477941, + "balance_loss_mlp": 1.03029561, + "epoch": 0.16081481051593058, + "flos": 16064064057600.0, + "grad_norm": 2.430899357519343, + "language_loss": 0.89157152, + "learning_rate": 3.823176097351284e-06, + "loss": 0.91343343, + "num_input_tokens_seen": 158363585, + "router_z_loss_clip": 0.84179688, + "router_z_loss_mlp": 0.16992188, + "step": 5542, + "time_per_iteration": 2.4786221981048584 + }, + { + "auxiliary_loss_clip": 0.01132099, + "auxiliary_loss_mlp": 0.01044484, + "balance_loss_clip": 1.05459166, + "balance_loss_mlp": 1.02820039, + "epoch": 0.16084382798444663, + "flos": 13874482800000.0, + "grad_norm": 2.8915929772446, + "language_loss": 0.97698808, + "learning_rate": 3.823098817306453e-06, + "loss": 0.99875391, + "num_input_tokens_seen": 158376035, + "router_z_loss_clip": 0.77587891, + "router_z_loss_mlp": 0.1628418, + "step": 5543, + "time_per_iteration": 2.505518913269043 + }, + { + "auxiliary_loss_clip": 0.01040295, + "auxiliary_loss_mlp": 0.01007508, + "balance_loss_clip": 1.02068782, + "balance_loss_mlp": 1.00675738, + "epoch": 0.16087284545296268, + "flos": 67080308348160.0, + "grad_norm": 0.6435339141268893, + "language_loss": 0.50108105, + "learning_rate": 3.823021521159276e-06, + "loss": 0.52155912, + "num_input_tokens_seen": 158438465, + "router_z_loss_clip": 0.19628906, + "router_z_loss_mlp": 0.00750732, + "step": 5544, + "time_per_iteration": 3.1065120697021484 + }, + { + "auxiliary_loss_clip": 0.01040631, + "auxiliary_loss_mlp": 0.01005367, + "balance_loss_clip": 1.02122235, + "balance_loss_mlp": 1.00462806, + "epoch": 0.16090186292147873, + "flos": 69919620768000.0, + "grad_norm": 0.6577056375913635, + "language_loss": 0.46115091, + "learning_rate": 3.822944208910435e-06, + "loss": 0.48161089, + "num_input_tokens_seen": 158496845, + "router_z_loss_clip": 0.19433594, + "router_z_loss_mlp": 0.00738525, + "step": 5545, + "time_per_iteration": 3.11366605758667 + }, + { + "auxiliary_loss_clip": 0.0113439, + "auxiliary_loss_mlp": 0.01038387, + "balance_loss_clip": 1.05628812, + "balance_loss_mlp": 1.0230031, + "epoch": 0.16093088038999478, + "flos": 68165058147840.0, + "grad_norm": 2.206275169889389, + "language_loss": 0.68064344, + "learning_rate": 3.822866880560613e-06, + "loss": 0.70237124, + "num_input_tokens_seen": 158518430, + "router_z_loss_clip": 0.78125, + "router_z_loss_mlp": 0.15386963, + "step": 5546, + "time_per_iteration": 2.913856267929077 + }, + { + "auxiliary_loss_clip": 0.01123484, + "auxiliary_loss_mlp": 0.01032703, + "balance_loss_clip": 1.05267954, + "balance_loss_mlp": 1.01920283, + "epoch": 0.16095989785851084, + "flos": 16429598812800.0, + "grad_norm": 1.9552104977936264, + "language_loss": 0.70923257, + "learning_rate": 3.822789536110493e-06, + "loss": 0.73079443, + "num_input_tokens_seen": 158532480, + "router_z_loss_clip": 0.70849609, + "router_z_loss_mlp": 0.1350708, + "step": 5547, + "time_per_iteration": 4.84833288192749 + }, + { + "auxiliary_loss_clip": 0.01037028, + "auxiliary_loss_mlp": 0.01002524, + "balance_loss_clip": 1.01761639, + "balance_loss_mlp": 1.00173759, + "epoch": 0.16098891532702686, + "flos": 61393459294080.0, + "grad_norm": 0.6554833347989727, + "language_loss": 0.50039238, + "learning_rate": 3.822712175560759e-06, + "loss": 0.52078795, + "num_input_tokens_seen": 158597220, + "router_z_loss_clip": 0.19433594, + "router_z_loss_mlp": 0.00787354, + "step": 5548, + "time_per_iteration": 5.738279581069946 + }, + { + "auxiliary_loss_clip": 0.01137412, + "auxiliary_loss_mlp": 0.01039027, + "balance_loss_clip": 1.0553441, + "balance_loss_mlp": 1.02333927, + "epoch": 0.1610179327955429, + "flos": 37953699494400.0, + "grad_norm": 2.510992594334368, + "language_loss": 0.82887232, + "learning_rate": 3.8226347989120926e-06, + "loss": 0.85063672, + "num_input_tokens_seen": 158613170, + "router_z_loss_clip": 0.82080078, + "router_z_loss_mlp": 0.15686035, + "step": 5549, + "time_per_iteration": 5.115149021148682 + }, + { + "auxiliary_loss_clip": 0.01128422, + "auxiliary_loss_mlp": 0.01036735, + "balance_loss_clip": 1.0526197, + "balance_loss_mlp": 1.02234077, + "epoch": 0.16104695026405896, + "flos": 29233547642880.0, + "grad_norm": 2.3933311220812206, + "language_loss": 0.84713596, + "learning_rate": 3.822557406165178e-06, + "loss": 0.86878753, + "num_input_tokens_seen": 158627810, + "router_z_loss_clip": 0.75732422, + "router_z_loss_mlp": 0.14404297, + "step": 5550, + "time_per_iteration": 4.952232599258423 + }, + { + "auxiliary_loss_clip": 0.01134423, + "auxiliary_loss_mlp": 0.01046322, + "balance_loss_clip": 1.05383968, + "balance_loss_mlp": 1.02887011, + "epoch": 0.16107596773257501, + "flos": 35401995273600.0, + "grad_norm": 3.551345709044014, + "language_loss": 0.88492525, + "learning_rate": 3.822479997320699e-06, + "loss": 0.90673268, + "num_input_tokens_seen": 158641495, + "router_z_loss_clip": 0.80664062, + "router_z_loss_mlp": 0.17456055, + "step": 5551, + "time_per_iteration": 2.5501279830932617 + }, + { + "auxiliary_loss_clip": 0.0113194, + "auxiliary_loss_mlp": 0.01038618, + "balance_loss_clip": 1.05218267, + "balance_loss_mlp": 1.02385998, + "epoch": 0.16110498520109107, + "flos": 24675693943680.0, + "grad_norm": 2.0111593920406343, + "language_loss": 0.92290682, + "learning_rate": 3.82240257237934e-06, + "loss": 0.94461238, + "num_input_tokens_seen": 158657980, + "router_z_loss_clip": 0.796875, + "router_z_loss_mlp": 0.14764404, + "step": 5552, + "time_per_iteration": 2.5433053970336914 + }, + { + "auxiliary_loss_clip": 0.01130425, + "auxiliary_loss_mlp": 0.01051709, + "balance_loss_clip": 1.05014348, + "balance_loss_mlp": 1.0339644, + "epoch": 0.1611340026696071, + "flos": 11941914332160.0, + "grad_norm": 2.2358538327642234, + "language_loss": 0.84064907, + "learning_rate": 3.8223251313417825e-06, + "loss": 0.86247045, + "num_input_tokens_seen": 158670465, + "router_z_loss_clip": 0.80273438, + "router_z_loss_mlp": 0.1774292, + "step": 5553, + "time_per_iteration": 2.4518368244171143 + }, + { + "auxiliary_loss_clip": 0.01131784, + "auxiliary_loss_mlp": 0.01040008, + "balance_loss_clip": 1.05533552, + "balance_loss_mlp": 1.02454615, + "epoch": 0.16116302013812314, + "flos": 10518343539840.0, + "grad_norm": 2.5714589425880443, + "language_loss": 0.71964115, + "learning_rate": 3.8222476742087135e-06, + "loss": 0.74135911, + "num_input_tokens_seen": 158682600, + "router_z_loss_clip": 0.76416016, + "router_z_loss_mlp": 0.15447998, + "step": 5554, + "time_per_iteration": 2.4640142917633057 + }, + { + "auxiliary_loss_clip": 0.01035209, + "auxiliary_loss_mlp": 0.01014617, + "balance_loss_clip": 1.01592779, + "balance_loss_mlp": 1.01373506, + "epoch": 0.1611920376066392, + "flos": 62520694882560.0, + "grad_norm": 0.6717131202867388, + "language_loss": 0.5071348, + "learning_rate": 3.822170200980815e-06, + "loss": 0.52763307, + "num_input_tokens_seen": 158740720, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.0088501, + "step": 5555, + "time_per_iteration": 3.0446536540985107 + }, + { + "auxiliary_loss_clip": 0.01123052, + "auxiliary_loss_mlp": 0.0104248, + "balance_loss_clip": 1.05174935, + "balance_loss_mlp": 1.02782965, + "epoch": 0.16122105507515525, + "flos": 15261819747840.0, + "grad_norm": 3.4661076392921775, + "language_loss": 0.71846461, + "learning_rate": 3.822092711658772e-06, + "loss": 0.74011993, + "num_input_tokens_seen": 158753115, + "router_z_loss_clip": 0.71240234, + "router_z_loss_mlp": 0.14660645, + "step": 5556, + "time_per_iteration": 2.5249273777008057 + }, + { + "auxiliary_loss_clip": 0.01033349, + "auxiliary_loss_mlp": 0.01007458, + "balance_loss_clip": 1.0141511, + "balance_loss_mlp": 1.0065577, + "epoch": 0.1612500725436713, + "flos": 74775533523840.0, + "grad_norm": 0.6467286920713301, + "language_loss": 0.45614666, + "learning_rate": 3.822015206243269e-06, + "loss": 0.47655475, + "num_input_tokens_seen": 158816270, + "router_z_loss_clip": 0.19238281, + "router_z_loss_mlp": 0.00897217, + "step": 5557, + "time_per_iteration": 3.2399961948394775 + }, + { + "auxiliary_loss_clip": 0.01135094, + "auxiliary_loss_mlp": 0.010501, + "balance_loss_clip": 1.05420089, + "balance_loss_mlp": 1.03217721, + "epoch": 0.16127909001218735, + "flos": 28834257081600.0, + "grad_norm": 1.9942141026815925, + "language_loss": 0.77121091, + "learning_rate": 3.82193768473499e-06, + "loss": 0.79306281, + "num_input_tokens_seen": 158833175, + "router_z_loss_clip": 0.80908203, + "router_z_loss_mlp": 0.17926025, + "step": 5558, + "time_per_iteration": 2.7649617195129395 + }, + { + "auxiliary_loss_clip": 0.0114075, + "auxiliary_loss_mlp": 0.01043858, + "balance_loss_clip": 1.05576253, + "balance_loss_mlp": 1.02558351, + "epoch": 0.16130810748070337, + "flos": 18361413095040.0, + "grad_norm": 2.322615751952729, + "language_loss": 0.89837503, + "learning_rate": 3.82186014713462e-06, + "loss": 0.92022115, + "num_input_tokens_seen": 158848295, + "router_z_loss_clip": 0.85009766, + "router_z_loss_mlp": 0.18286133, + "step": 5559, + "time_per_iteration": 2.502542495727539 + }, + { + "auxiliary_loss_clip": 0.0103641, + "auxiliary_loss_mlp": 0.01004819, + "balance_loss_clip": 1.01730084, + "balance_loss_mlp": 1.00387728, + "epoch": 0.16133712494921942, + "flos": 62111204858880.0, + "grad_norm": 0.7641889403524551, + "language_loss": 0.48723868, + "learning_rate": 3.821782593442844e-06, + "loss": 0.50765097, + "num_input_tokens_seen": 158905200, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.00939941, + "step": 5560, + "time_per_iteration": 3.061027765274048 + }, + { + "auxiliary_loss_clip": 0.01122599, + "auxiliary_loss_mlp": 0.01041733, + "balance_loss_clip": 1.05056334, + "balance_loss_mlp": 1.02869761, + "epoch": 0.16136614241773548, + "flos": 24747622928640.0, + "grad_norm": 2.3189465527619704, + "language_loss": 0.82465744, + "learning_rate": 3.821705023660348e-06, + "loss": 0.84630072, + "num_input_tokens_seen": 158919015, + "router_z_loss_clip": 0.72021484, + "router_z_loss_mlp": 0.13043213, + "step": 5561, + "time_per_iteration": 2.5856614112854004 + }, + { + "auxiliary_loss_clip": 0.01147521, + "auxiliary_loss_mlp": 0.01054493, + "balance_loss_clip": 1.06225622, + "balance_loss_mlp": 1.03855515, + "epoch": 0.16139515988625153, + "flos": 28692230705280.0, + "grad_norm": 1.9527910195932316, + "language_loss": 0.85734677, + "learning_rate": 3.8216274377878155e-06, + "loss": 0.87936682, + "num_input_tokens_seen": 158936830, + "router_z_loss_clip": 0.85302734, + "router_z_loss_mlp": 0.15930176, + "step": 5562, + "time_per_iteration": 2.623365640640259 + }, + { + "auxiliary_loss_clip": 0.01036874, + "auxiliary_loss_mlp": 0.01013497, + "balance_loss_clip": 1.01769912, + "balance_loss_mlp": 1.01255512, + "epoch": 0.16142417735476758, + "flos": 61544428156800.0, + "grad_norm": 0.7166046651696053, + "language_loss": 0.47419187, + "learning_rate": 3.821549835825932e-06, + "loss": 0.49469563, + "num_input_tokens_seen": 159003245, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.00939941, + "step": 5563, + "time_per_iteration": 3.1641361713409424 + }, + { + "auxiliary_loss_clip": 0.01036341, + "auxiliary_loss_mlp": 0.01012035, + "balance_loss_clip": 1.01706755, + "balance_loss_mlp": 1.0110817, + "epoch": 0.16145319482328363, + "flos": 63358167456000.0, + "grad_norm": 0.6138347728843472, + "language_loss": 0.48106778, + "learning_rate": 3.821472217775383e-06, + "loss": 0.50155151, + "num_input_tokens_seen": 159066310, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.00952148, + "step": 5564, + "time_per_iteration": 3.0515716075897217 + }, + { + "auxiliary_loss_clip": 0.01130484, + "auxiliary_loss_mlp": 0.01039504, + "balance_loss_clip": 1.05398417, + "balance_loss_mlp": 1.02523494, + "epoch": 0.16148221229179965, + "flos": 21098847565440.0, + "grad_norm": 2.3023280196702762, + "language_loss": 0.76129174, + "learning_rate": 3.821394583636855e-06, + "loss": 0.78299165, + "num_input_tokens_seen": 159081575, + "router_z_loss_clip": 0.76513672, + "router_z_loss_mlp": 0.14276123, + "step": 5565, + "time_per_iteration": 2.529505491256714 + }, + { + "auxiliary_loss_clip": 0.01131358, + "auxiliary_loss_mlp": 0.01040455, + "balance_loss_clip": 1.05652452, + "balance_loss_mlp": 1.02521396, + "epoch": 0.1615112297603157, + "flos": 37735865464320.0, + "grad_norm": 2.470831805494957, + "language_loss": 0.8823933, + "learning_rate": 3.8213169334110325e-06, + "loss": 0.90411139, + "num_input_tokens_seen": 159099525, + "router_z_loss_clip": 0.74755859, + "router_z_loss_mlp": 0.15234375, + "step": 5566, + "time_per_iteration": 2.644620895385742 + }, + { + "auxiliary_loss_clip": 0.0113873, + "auxiliary_loss_mlp": 0.01046571, + "balance_loss_clip": 1.05475354, + "balance_loss_mlp": 1.02863038, + "epoch": 0.16154024722883176, + "flos": 32897514458880.0, + "grad_norm": 4.636617074913018, + "language_loss": 0.77619123, + "learning_rate": 3.821239267098602e-06, + "loss": 0.79804426, + "num_input_tokens_seen": 159118605, + "router_z_loss_clip": 0.83935547, + "router_z_loss_mlp": 0.17956543, + "step": 5567, + "time_per_iteration": 2.652318000793457 + }, + { + "auxiliary_loss_clip": 0.01131308, + "auxiliary_loss_mlp": 0.01038858, + "balance_loss_clip": 1.05561137, + "balance_loss_mlp": 1.02361679, + "epoch": 0.1615692646973478, + "flos": 31352816027520.0, + "grad_norm": 1.7841975868257551, + "language_loss": 0.77275372, + "learning_rate": 3.821161584700249e-06, + "loss": 0.79445541, + "num_input_tokens_seen": 159135420, + "router_z_loss_clip": 0.75537109, + "router_z_loss_mlp": 0.15234375, + "step": 5568, + "time_per_iteration": 2.582444906234741 + }, + { + "auxiliary_loss_clip": 0.01146806, + "auxiliary_loss_mlp": 0.01055981, + "balance_loss_clip": 1.06146955, + "balance_loss_mlp": 1.03767061, + "epoch": 0.16159828216586386, + "flos": 25148529601920.0, + "grad_norm": 2.2625036525448854, + "language_loss": 0.83244681, + "learning_rate": 3.821083886216661e-06, + "loss": 0.85447466, + "num_input_tokens_seen": 159150650, + "router_z_loss_clip": 0.85400391, + "router_z_loss_mlp": 0.18310547, + "step": 5569, + "time_per_iteration": 2.562990188598633 + }, + { + "auxiliary_loss_clip": 0.01036725, + "auxiliary_loss_mlp": 0.01008693, + "balance_loss_clip": 1.01749074, + "balance_loss_mlp": 1.00792968, + "epoch": 0.16162729963437988, + "flos": 74780489600640.0, + "grad_norm": 0.6221037488990485, + "language_loss": 0.47265363, + "learning_rate": 3.821006171648522e-06, + "loss": 0.49310789, + "num_input_tokens_seen": 159217835, + "router_z_loss_clip": 0.19238281, + "router_z_loss_mlp": 0.00762939, + "step": 5570, + "time_per_iteration": 3.1849143505096436 + }, + { + "auxiliary_loss_clip": 0.01033609, + "auxiliary_loss_mlp": 0.01008685, + "balance_loss_clip": 1.01435137, + "balance_loss_mlp": 1.00793362, + "epoch": 0.16165631710289594, + "flos": 74190476799360.0, + "grad_norm": 0.689637260041669, + "language_loss": 0.56032801, + "learning_rate": 3.820928440996521e-06, + "loss": 0.58075094, + "num_input_tokens_seen": 159283610, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.00750732, + "step": 5571, + "time_per_iteration": 3.1579113006591797 + }, + { + "auxiliary_loss_clip": 0.01131167, + "auxiliary_loss_mlp": 0.01036593, + "balance_loss_clip": 1.05358791, + "balance_loss_mlp": 1.02153718, + "epoch": 0.161685334571412, + "flos": 16975045814400.0, + "grad_norm": 2.2902054937719694, + "language_loss": 0.66960132, + "learning_rate": 3.820850694261342e-06, + "loss": 0.69127893, + "num_input_tokens_seen": 159296375, + "router_z_loss_clip": 0.77539062, + "router_z_loss_mlp": 0.15075684, + "step": 5572, + "time_per_iteration": 2.465695858001709 + }, + { + "auxiliary_loss_clip": 0.01032171, + "auxiliary_loss_mlp": 0.01008076, + "balance_loss_clip": 1.01285768, + "balance_loss_mlp": 1.00732231, + "epoch": 0.16171435203992804, + "flos": 67061094572160.0, + "grad_norm": 0.6663651918739254, + "language_loss": 0.48106289, + "learning_rate": 3.8207729314436725e-06, + "loss": 0.50146532, + "num_input_tokens_seen": 159352470, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.00753784, + "step": 5573, + "time_per_iteration": 2.9841020107269287 + }, + { + "auxiliary_loss_clip": 0.0112481, + "auxiliary_loss_mlp": 0.01039842, + "balance_loss_clip": 1.05382955, + "balance_loss_mlp": 1.02537012, + "epoch": 0.1617433695084441, + "flos": 16538048951040.0, + "grad_norm": 2.2957079504485747, + "language_loss": 0.70729196, + "learning_rate": 3.8206951525442e-06, + "loss": 0.72893852, + "num_input_tokens_seen": 159365130, + "router_z_loss_clip": 0.70947266, + "router_z_loss_mlp": 0.14465332, + "step": 5574, + "time_per_iteration": 2.4722366333007812 + }, + { + "auxiliary_loss_clip": 0.01032803, + "auxiliary_loss_mlp": 0.01004266, + "balance_loss_clip": 1.01365995, + "balance_loss_mlp": 1.003497, + "epoch": 0.16177238697696014, + "flos": 70793901803520.0, + "grad_norm": 0.6434762462284241, + "language_loss": 0.47438854, + "learning_rate": 3.820617357563612e-06, + "loss": 0.49475926, + "num_input_tokens_seen": 159425255, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.00765991, + "step": 5575, + "time_per_iteration": 3.073218822479248 + }, + { + "auxiliary_loss_clip": 0.01129271, + "auxiliary_loss_mlp": 0.01039752, + "balance_loss_clip": 1.05178595, + "balance_loss_mlp": 1.02508903, + "epoch": 0.16180140444547617, + "flos": 16863614847360.0, + "grad_norm": 2.8519309578605463, + "language_loss": 0.83587265, + "learning_rate": 3.820539546502594e-06, + "loss": 0.8575629, + "num_input_tokens_seen": 159439420, + "router_z_loss_clip": 0.77392578, + "router_z_loss_mlp": 0.14660645, + "step": 5576, + "time_per_iteration": 2.5321311950683594 + }, + { + "auxiliary_loss_clip": 0.01138896, + "auxiliary_loss_mlp": 0.01041366, + "balance_loss_clip": 1.05727875, + "balance_loss_mlp": 1.02507591, + "epoch": 0.16183042191399222, + "flos": 23253559695360.0, + "grad_norm": 2.347108512312909, + "language_loss": 0.93527395, + "learning_rate": 3.820461719361834e-06, + "loss": 0.95707655, + "num_input_tokens_seen": 159457200, + "router_z_loss_clip": 0.81665039, + "router_z_loss_mlp": 0.16296387, + "step": 5577, + "time_per_iteration": 2.516432762145996 + }, + { + "auxiliary_loss_clip": 0.01128183, + "auxiliary_loss_mlp": 0.01044048, + "balance_loss_clip": 1.05377388, + "balance_loss_mlp": 1.02907538, + "epoch": 0.16185943938250827, + "flos": 33320325450240.0, + "grad_norm": 2.248757697021121, + "language_loss": 0.83236611, + "learning_rate": 3.82038387614202e-06, + "loss": 0.85408843, + "num_input_tokens_seen": 159474880, + "router_z_loss_clip": 0.74414062, + "router_z_loss_mlp": 0.1496582, + "step": 5578, + "time_per_iteration": 2.5959548950195312 + }, + { + "auxiliary_loss_clip": 0.0112942, + "auxiliary_loss_mlp": 0.0103411, + "balance_loss_clip": 1.05658782, + "balance_loss_mlp": 1.02084768, + "epoch": 0.16188845685102432, + "flos": 26826455577600.0, + "grad_norm": 2.3408870559330075, + "language_loss": 0.86943948, + "learning_rate": 3.820306016843838e-06, + "loss": 0.89107478, + "num_input_tokens_seen": 159492310, + "router_z_loss_clip": 0.72802734, + "router_z_loss_mlp": 0.13256836, + "step": 5579, + "time_per_iteration": 2.564831495285034 + }, + { + "auxiliary_loss_clip": 0.01119713, + "auxiliary_loss_mlp": 0.01037427, + "balance_loss_clip": 1.05443764, + "balance_loss_mlp": 1.02576208, + "epoch": 0.16191747431954037, + "flos": 16391892510720.0, + "grad_norm": 3.7857327359992707, + "language_loss": 0.72189617, + "learning_rate": 3.820228141467978e-06, + "loss": 0.74346763, + "num_input_tokens_seen": 159504050, + "router_z_loss_clip": 0.65332031, + "router_z_loss_mlp": 0.11663818, + "step": 5580, + "time_per_iteration": 2.5736825466156006 + }, + { + "auxiliary_loss_clip": 0.01132667, + "auxiliary_loss_mlp": 0.01045359, + "balance_loss_clip": 1.05583084, + "balance_loss_mlp": 1.02887225, + "epoch": 0.16194649178805642, + "flos": 16901967594240.0, + "grad_norm": 2.858448289890264, + "language_loss": 0.76090097, + "learning_rate": 3.8201502500151255e-06, + "loss": 0.78268117, + "num_input_tokens_seen": 159516685, + "router_z_loss_clip": 0.76757812, + "router_z_loss_mlp": 0.16497803, + "step": 5581, + "time_per_iteration": 2.498579502105713 + }, + { + "auxiliary_loss_clip": 0.01139634, + "auxiliary_loss_mlp": 0.01046087, + "balance_loss_clip": 1.05863357, + "balance_loss_mlp": 1.02948141, + "epoch": 0.16197550925657245, + "flos": 12268629463680.0, + "grad_norm": 4.264394882796268, + "language_loss": 0.92013252, + "learning_rate": 3.82007234248597e-06, + "loss": 0.94198972, + "num_input_tokens_seen": 159527455, + "router_z_loss_clip": 0.81005859, + "router_z_loss_mlp": 0.16595459, + "step": 5582, + "time_per_iteration": 2.495628595352173 + }, + { + "auxiliary_loss_clip": 0.01141291, + "auxiliary_loss_mlp": 0.01050287, + "balance_loss_clip": 1.06042647, + "balance_loss_mlp": 1.03403258, + "epoch": 0.1620045267250885, + "flos": 16903296397440.0, + "grad_norm": 2.1883877185152687, + "language_loss": 0.82198155, + "learning_rate": 3.819994418881199e-06, + "loss": 0.84389734, + "num_input_tokens_seen": 159542235, + "router_z_loss_clip": 0.80908203, + "router_z_loss_mlp": 0.16235352, + "step": 5583, + "time_per_iteration": 2.4771957397460938 + }, + { + "auxiliary_loss_clip": 0.01140661, + "auxiliary_loss_mlp": 0.01042583, + "balance_loss_clip": 1.05903101, + "balance_loss_mlp": 1.02625728, + "epoch": 0.16203354419360455, + "flos": 33540314296320.0, + "grad_norm": 1.9601340694040437, + "language_loss": 0.81291705, + "learning_rate": 3.8199164792015e-06, + "loss": 0.83474952, + "num_input_tokens_seen": 159561265, + "router_z_loss_clip": 0.81542969, + "router_z_loss_mlp": 0.16326904, + "step": 5584, + "time_per_iteration": 2.6649551391601562 + }, + { + "auxiliary_loss_clip": 0.01138646, + "auxiliary_loss_mlp": 0.0103966, + "balance_loss_clip": 1.0603013, + "balance_loss_mlp": 1.0237397, + "epoch": 0.1620625616621206, + "flos": 29928742444800.0, + "grad_norm": 1.8540892469135668, + "language_loss": 0.82541627, + "learning_rate": 3.819838523447563e-06, + "loss": 0.84719932, + "num_input_tokens_seen": 159577985, + "router_z_loss_clip": 0.78222656, + "router_z_loss_mlp": 0.15936279, + "step": 5585, + "time_per_iteration": 2.6574742794036865 + }, + { + "auxiliary_loss_clip": 0.01043675, + "auxiliary_loss_mlp": 0.01015575, + "balance_loss_clip": 1.02380502, + "balance_loss_mlp": 1.01473451, + "epoch": 0.16209157913063665, + "flos": 67586648417280.0, + "grad_norm": 0.662514296144104, + "language_loss": 0.4634822, + "learning_rate": 3.8197605516200755e-06, + "loss": 0.48407471, + "num_input_tokens_seen": 159640755, + "router_z_loss_clip": 0.19921875, + "router_z_loss_mlp": 0.00842285, + "step": 5586, + "time_per_iteration": 3.094433069229126 + }, + { + "auxiliary_loss_clip": 0.01141582, + "auxiliary_loss_mlp": 0.0104086, + "balance_loss_clip": 1.06270218, + "balance_loss_mlp": 1.02542853, + "epoch": 0.16212059659915268, + "flos": 11028921413760.0, + "grad_norm": 3.1025919712613743, + "language_loss": 0.99565196, + "learning_rate": 3.819682563719727e-06, + "loss": 1.01747644, + "num_input_tokens_seen": 159651710, + "router_z_loss_clip": 0.7890625, + "router_z_loss_mlp": 0.15454102, + "step": 5587, + "time_per_iteration": 2.5102272033691406 + }, + { + "auxiliary_loss_clip": 0.01140822, + "auxiliary_loss_mlp": 0.01042708, + "balance_loss_clip": 1.05932426, + "balance_loss_mlp": 1.0273422, + "epoch": 0.16214961406766873, + "flos": 34526241780480.0, + "grad_norm": 1.6989110607117952, + "language_loss": 0.77172315, + "learning_rate": 3.819604559747205e-06, + "loss": 0.79355848, + "num_input_tokens_seen": 159674125, + "router_z_loss_clip": 0.81542969, + "router_z_loss_mlp": 0.15356445, + "step": 5588, + "time_per_iteration": 2.6691505908966064 + }, + { + "auxiliary_loss_clip": 0.01133532, + "auxiliary_loss_mlp": 0.01031827, + "balance_loss_clip": 1.0577898, + "balance_loss_mlp": 1.01864815, + "epoch": 0.16217863153618478, + "flos": 16063345785600.0, + "grad_norm": 2.8914897002113427, + "language_loss": 0.85804319, + "learning_rate": 3.819526539703199e-06, + "loss": 0.87969673, + "num_input_tokens_seen": 159686700, + "router_z_loss_clip": 0.7578125, + "router_z_loss_mlp": 0.1317749, + "step": 5589, + "time_per_iteration": 2.5038070678710938 + }, + { + "auxiliary_loss_clip": 0.01134071, + "auxiliary_loss_mlp": 0.01036067, + "balance_loss_clip": 1.05672944, + "balance_loss_mlp": 1.01974094, + "epoch": 0.16220764900470083, + "flos": 33248611946880.0, + "grad_norm": 2.2218926563852817, + "language_loss": 0.83378476, + "learning_rate": 3.819448503588399e-06, + "loss": 0.85548615, + "num_input_tokens_seen": 159707130, + "router_z_loss_clip": 0.77294922, + "router_z_loss_mlp": 0.16326904, + "step": 5590, + "time_per_iteration": 2.607043504714966 + }, + { + "auxiliary_loss_clip": 0.01137521, + "auxiliary_loss_mlp": 0.01042874, + "balance_loss_clip": 1.0593226, + "balance_loss_mlp": 1.02815211, + "epoch": 0.16223666647321688, + "flos": 12893224083840.0, + "grad_norm": 1.9777792810577794, + "language_loss": 0.84733683, + "learning_rate": 3.819370451403493e-06, + "loss": 0.8691408, + "num_input_tokens_seen": 159721455, + "router_z_loss_clip": 0.78173828, + "router_z_loss_mlp": 0.14715576, + "step": 5591, + "time_per_iteration": 2.500819444656372 + }, + { + "auxiliary_loss_clip": 0.01132097, + "auxiliary_loss_mlp": 0.01044833, + "balance_loss_clip": 1.05828881, + "balance_loss_mlp": 1.02987826, + "epoch": 0.16226568394173294, + "flos": 32556433887360.0, + "grad_norm": 2.583068885175035, + "language_loss": 0.89831829, + "learning_rate": 3.819292383149172e-06, + "loss": 0.92008764, + "num_input_tokens_seen": 159736490, + "router_z_loss_clip": 0.73876953, + "router_z_loss_mlp": 0.14953613, + "step": 5592, + "time_per_iteration": 2.601372003555298 + }, + { + "auxiliary_loss_clip": 0.01135959, + "auxiliary_loss_mlp": 0.01036905, + "balance_loss_clip": 1.06465006, + "balance_loss_mlp": 1.02489448, + "epoch": 0.16229470141024896, + "flos": 21320344782720.0, + "grad_norm": 1.6610297881623213, + "language_loss": 0.60086864, + "learning_rate": 3.819214298826124e-06, + "loss": 0.62259728, + "num_input_tokens_seen": 159750850, + "router_z_loss_clip": 0.71289062, + "router_z_loss_mlp": 0.12017822, + "step": 5593, + "time_per_iteration": 2.5412302017211914 + }, + { + "auxiliary_loss_clip": 0.01049019, + "auxiliary_loss_mlp": 0.0101224, + "balance_loss_clip": 1.02961266, + "balance_loss_mlp": 1.01143551, + "epoch": 0.162323718878765, + "flos": 64749383072640.0, + "grad_norm": 0.6416576596780261, + "language_loss": 0.4811703, + "learning_rate": 3.8191361984350385e-06, + "loss": 0.50178289, + "num_input_tokens_seen": 159816625, + "router_z_loss_clip": 0.19433594, + "router_z_loss_mlp": 0.00805664, + "step": 5594, + "time_per_iteration": 3.136432647705078 + }, + { + "auxiliary_loss_clip": 0.0112479, + "auxiliary_loss_mlp": 0.0103011, + "balance_loss_clip": 1.05650496, + "balance_loss_mlp": 1.01759267, + "epoch": 0.16235273634728106, + "flos": 16211764782720.0, + "grad_norm": 2.0605641748379444, + "language_loss": 0.81651509, + "learning_rate": 3.819058081976606e-06, + "loss": 0.83806407, + "num_input_tokens_seen": 159834430, + "router_z_loss_clip": 0.68310547, + "router_z_loss_mlp": 0.12512207, + "step": 5595, + "time_per_iteration": 2.5296990871429443 + }, + { + "auxiliary_loss_clip": 0.01045532, + "auxiliary_loss_mlp": 0.01002418, + "balance_loss_clip": 1.0262773, + "balance_loss_mlp": 1.00164342, + "epoch": 0.16238175381579711, + "flos": 74773773757440.0, + "grad_norm": 0.704325074057629, + "language_loss": 0.48881751, + "learning_rate": 3.818979949451517e-06, + "loss": 0.50929701, + "num_input_tokens_seen": 159891985, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.00775146, + "step": 5596, + "time_per_iteration": 3.0817763805389404 + }, + { + "auxiliary_loss_clip": 0.01141655, + "auxiliary_loss_mlp": 0.01050632, + "balance_loss_clip": 1.05962706, + "balance_loss_mlp": 1.03137922, + "epoch": 0.16241077128431317, + "flos": 17704894262400.0, + "grad_norm": 2.8523623100348425, + "language_loss": 0.89866424, + "learning_rate": 3.818901800860461e-06, + "loss": 0.92058712, + "num_input_tokens_seen": 159903915, + "router_z_loss_clip": 0.82080078, + "router_z_loss_mlp": 0.19226074, + "step": 5597, + "time_per_iteration": 2.5169949531555176 + }, + { + "auxiliary_loss_clip": 0.01147635, + "auxiliary_loss_mlp": 0.01043541, + "balance_loss_clip": 1.06580901, + "balance_loss_mlp": 1.0277102, + "epoch": 0.1624397887528292, + "flos": 31315001984640.0, + "grad_norm": 2.3047570123496195, + "language_loss": 0.88282692, + "learning_rate": 3.818823636204128e-06, + "loss": 0.90473878, + "num_input_tokens_seen": 159919525, + "router_z_loss_clip": 0.81787109, + "router_z_loss_mlp": 0.15844727, + "step": 5598, + "time_per_iteration": 2.5364904403686523 + }, + { + "auxiliary_loss_clip": 0.01143369, + "auxiliary_loss_mlp": 0.01042332, + "balance_loss_clip": 1.06174862, + "balance_loss_mlp": 1.02725172, + "epoch": 0.16246880622134524, + "flos": 16902542211840.0, + "grad_norm": 2.1745579853175747, + "language_loss": 0.71641421, + "learning_rate": 3.818745455483209e-06, + "loss": 0.73827118, + "num_input_tokens_seen": 159934550, + "router_z_loss_clip": 0.81640625, + "router_z_loss_mlp": 0.15081787, + "step": 5599, + "time_per_iteration": 2.5053446292877197 + }, + { + "auxiliary_loss_clip": 0.01139535, + "auxiliary_loss_mlp": 0.01034282, + "balance_loss_clip": 1.05962646, + "balance_loss_mlp": 1.01760447, + "epoch": 0.1624978236898613, + "flos": 18216513630720.0, + "grad_norm": 6.711004638828658, + "language_loss": 0.98296463, + "learning_rate": 3.818667258698394e-06, + "loss": 1.00470281, + "num_input_tokens_seen": 159947900, + "router_z_loss_clip": 0.79833984, + "router_z_loss_mlp": 0.16662598, + "step": 5600, + "time_per_iteration": 2.6228630542755127 + }, + { + "auxiliary_loss_clip": 0.0104254, + "auxiliary_loss_mlp": 0.0100951, + "balance_loss_clip": 1.02317166, + "balance_loss_mlp": 1.00863934, + "epoch": 0.16252684115837734, + "flos": 74791191853440.0, + "grad_norm": 0.6054098452121203, + "language_loss": 0.4593156, + "learning_rate": 3.818589045850373e-06, + "loss": 0.47983611, + "num_input_tokens_seen": 160015275, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.00872803, + "step": 5601, + "time_per_iteration": 3.2887094020843506 + }, + { + "auxiliary_loss_clip": 0.01131929, + "auxiliary_loss_mlp": 0.01040648, + "balance_loss_clip": 1.05501604, + "balance_loss_mlp": 1.02609241, + "epoch": 0.1625558586268934, + "flos": 21940126980480.0, + "grad_norm": 1.8361005119211926, + "language_loss": 0.81168973, + "learning_rate": 3.818510816939839e-06, + "loss": 0.83341551, + "num_input_tokens_seen": 160035810, + "router_z_loss_clip": 0.76904297, + "router_z_loss_mlp": 0.14550781, + "step": 5602, + "time_per_iteration": 2.7144558429718018 + }, + { + "auxiliary_loss_clip": 0.01134341, + "auxiliary_loss_mlp": 0.01039445, + "balance_loss_clip": 1.06226349, + "balance_loss_mlp": 1.02443612, + "epoch": 0.16258487609540945, + "flos": 29853832631040.0, + "grad_norm": 2.5581962575756996, + "language_loss": 0.74753368, + "learning_rate": 3.8184325719674804e-06, + "loss": 0.76927161, + "num_input_tokens_seen": 160048860, + "router_z_loss_clip": 0.72167969, + "router_z_loss_mlp": 0.15014648, + "step": 5603, + "time_per_iteration": 2.6150929927825928 + }, + { + "auxiliary_loss_clip": 0.01136571, + "auxiliary_loss_mlp": 0.01035984, + "balance_loss_clip": 1.05840302, + "balance_loss_mlp": 1.02093351, + "epoch": 0.16261389356392547, + "flos": 9164151866880.0, + "grad_norm": 3.327804457208737, + "language_loss": 0.7940042, + "learning_rate": 3.81835431093399e-06, + "loss": 0.81572986, + "num_input_tokens_seen": 160057555, + "router_z_loss_clip": 0.78125, + "router_z_loss_mlp": 0.1505127, + "step": 5604, + "time_per_iteration": 2.5221638679504395 + }, + { + "auxiliary_loss_clip": 0.01037735, + "auxiliary_loss_mlp": 0.01000849, + "balance_loss_clip": 1.01880813, + "balance_loss_mlp": 0.99999636, + "epoch": 0.16264291103244152, + "flos": 61824494499840.0, + "grad_norm": 0.6677066649406554, + "language_loss": 0.4796488, + "learning_rate": 3.818276033840059e-06, + "loss": 0.50003469, + "num_input_tokens_seen": 160116530, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.00854492, + "step": 5605, + "time_per_iteration": 3.0314950942993164 + }, + { + "auxiliary_loss_clip": 0.01036227, + "auxiliary_loss_mlp": 0.01001077, + "balance_loss_clip": 1.01728022, + "balance_loss_mlp": 1.00026023, + "epoch": 0.16267192850095757, + "flos": 67607729700480.0, + "grad_norm": 0.6401307606336013, + "language_loss": 0.45285559, + "learning_rate": 3.818197740686378e-06, + "loss": 0.47322863, + "num_input_tokens_seen": 160179475, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.00817871, + "step": 5606, + "time_per_iteration": 3.2163751125335693 + }, + { + "auxiliary_loss_clip": 0.01126974, + "auxiliary_loss_mlp": 0.01041938, + "balance_loss_clip": 1.05429125, + "balance_loss_mlp": 1.02815139, + "epoch": 0.16270094596947363, + "flos": 11828974993920.0, + "grad_norm": 2.48800948790209, + "language_loss": 0.78909355, + "learning_rate": 3.818119431473639e-06, + "loss": 0.81078261, + "num_input_tokens_seen": 160190225, + "router_z_loss_clip": 0.72705078, + "router_z_loss_mlp": 0.13763428, + "step": 5607, + "time_per_iteration": 2.5129570960998535 + }, + { + "auxiliary_loss_clip": 0.01035344, + "auxiliary_loss_mlp": 0.01003848, + "balance_loss_clip": 1.0162853, + "balance_loss_mlp": 1.00305521, + "epoch": 0.16272996343798968, + "flos": 60323033064960.0, + "grad_norm": 0.6661222518633827, + "language_loss": 0.48481095, + "learning_rate": 3.818041106202533e-06, + "loss": 0.50520289, + "num_input_tokens_seen": 160247810, + "router_z_loss_clip": 0.19042969, + "router_z_loss_mlp": 0.00793457, + "step": 5608, + "time_per_iteration": 2.9234778881073 + }, + { + "auxiliary_loss_clip": 0.01139716, + "auxiliary_loss_mlp": 0.01047217, + "balance_loss_clip": 1.05857539, + "balance_loss_mlp": 1.03112364, + "epoch": 0.16275898090650573, + "flos": 16101554878080.0, + "grad_norm": 2.6373155835901025, + "language_loss": 0.80786061, + "learning_rate": 3.817962764873752e-06, + "loss": 0.82972991, + "num_input_tokens_seen": 160261100, + "router_z_loss_clip": 0.81201172, + "router_z_loss_mlp": 0.16101074, + "step": 5609, + "time_per_iteration": 2.470688819885254 + }, + { + "auxiliary_loss_clip": 0.01133118, + "auxiliary_loss_mlp": 0.01052769, + "balance_loss_clip": 1.05422652, + "balance_loss_mlp": 1.03737903, + "epoch": 0.16278799837502175, + "flos": 25769353294080.0, + "grad_norm": 1.9144742386017377, + "language_loss": 0.94767982, + "learning_rate": 3.8178844074879894e-06, + "loss": 0.96953869, + "num_input_tokens_seen": 160277445, + "router_z_loss_clip": 0.78857422, + "router_z_loss_mlp": 0.15380859, + "step": 5610, + "time_per_iteration": 2.6030962467193604 + }, + { + "auxiliary_loss_clip": 0.01034599, + "auxiliary_loss_mlp": 0.01007397, + "balance_loss_clip": 1.0154736, + "balance_loss_mlp": 1.0066489, + "epoch": 0.1628170158435378, + "flos": 65940395258880.0, + "grad_norm": 0.6570434126629628, + "language_loss": 0.47102097, + "learning_rate": 3.817806034045935e-06, + "loss": 0.49144092, + "num_input_tokens_seen": 160339845, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.00747681, + "step": 5611, + "time_per_iteration": 3.1761441230773926 + }, + { + "auxiliary_loss_clip": 0.01033936, + "auxiliary_loss_mlp": 0.01006897, + "balance_loss_clip": 1.01484919, + "balance_loss_mlp": 1.00618124, + "epoch": 0.16284603331205386, + "flos": 74068271752320.0, + "grad_norm": 0.6827325652148613, + "language_loss": 0.45857167, + "learning_rate": 3.8177276445482825e-06, + "loss": 0.47898, + "num_input_tokens_seen": 160396030, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.00714111, + "step": 5612, + "time_per_iteration": 2.9693655967712402 + }, + { + "auxiliary_loss_clip": 0.0115023, + "auxiliary_loss_mlp": 0.01048273, + "balance_loss_clip": 1.06319129, + "balance_loss_mlp": 1.03163743, + "epoch": 0.1628750507805699, + "flos": 26312573652480.0, + "grad_norm": 4.141075252614059, + "language_loss": 0.82589769, + "learning_rate": 3.817649238995723e-06, + "loss": 0.84788269, + "num_input_tokens_seen": 160407955, + "router_z_loss_clip": 0.87011719, + "router_z_loss_mlp": 0.16619873, + "step": 5613, + "time_per_iteration": 2.5303871631622314 + }, + { + "auxiliary_loss_clip": 0.01031473, + "auxiliary_loss_mlp": 0.00999462, + "balance_loss_clip": 1.01234794, + "balance_loss_mlp": 0.99865782, + "epoch": 0.16290406824908596, + "flos": 74771223891840.0, + "grad_norm": 0.7174244075152693, + "language_loss": 0.48213658, + "learning_rate": 3.817570817388952e-06, + "loss": 0.50244594, + "num_input_tokens_seen": 160467250, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.00805664, + "step": 5614, + "time_per_iteration": 3.0799381732940674 + }, + { + "auxiliary_loss_clip": 0.01136027, + "auxiliary_loss_mlp": 0.01046806, + "balance_loss_clip": 1.06026483, + "balance_loss_mlp": 1.03157711, + "epoch": 0.16293308571760198, + "flos": 12706021376640.0, + "grad_norm": 2.2934903400904605, + "language_loss": 0.8170445, + "learning_rate": 3.817492379728657e-06, + "loss": 0.83887279, + "num_input_tokens_seen": 160479340, + "router_z_loss_clip": 0.75732422, + "router_z_loss_mlp": 0.15240479, + "step": 5615, + "time_per_iteration": 2.4614901542663574 + }, + { + "auxiliary_loss_clip": 0.0114329, + "auxiliary_loss_mlp": 0.01048232, + "balance_loss_clip": 1.05902481, + "balance_loss_mlp": 1.02996874, + "epoch": 0.16296210318611803, + "flos": 36278467038720.0, + "grad_norm": 1.6394535736951534, + "language_loss": 0.91251212, + "learning_rate": 3.817413926015537e-06, + "loss": 0.93442732, + "num_input_tokens_seen": 160504510, + "router_z_loss_clip": 0.84228516, + "router_z_loss_mlp": 0.18255615, + "step": 5616, + "time_per_iteration": 2.7779722213745117 + }, + { + "auxiliary_loss_clip": 0.01030392, + "auxiliary_loss_mlp": 0.01003591, + "balance_loss_clip": 1.01122344, + "balance_loss_mlp": 1.0027442, + "epoch": 0.1629911206546341, + "flos": 65874104709120.0, + "grad_norm": 0.7654764494589095, + "language_loss": 0.46885273, + "learning_rate": 3.81733545625028e-06, + "loss": 0.48919255, + "num_input_tokens_seen": 160561035, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.00848389, + "step": 5617, + "time_per_iteration": 3.031696081161499 + }, + { + "auxiliary_loss_clip": 0.01030526, + "auxiliary_loss_mlp": 0.01002705, + "balance_loss_clip": 1.01127958, + "balance_loss_mlp": 1.00194216, + "epoch": 0.16302013812315014, + "flos": 57190330356480.0, + "grad_norm": 0.6408295820769893, + "language_loss": 0.43823463, + "learning_rate": 3.817256970433581e-06, + "loss": 0.45856696, + "num_input_tokens_seen": 160621305, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.00762939, + "step": 5618, + "time_per_iteration": 5.287050485610962 + }, + { + "auxiliary_loss_clip": 0.01029813, + "auxiliary_loss_mlp": 0.01001499, + "balance_loss_clip": 1.01059318, + "balance_loss_mlp": 1.0007416, + "epoch": 0.1630491555916662, + "flos": 74779771328640.0, + "grad_norm": 0.6324137149909133, + "language_loss": 0.47280893, + "learning_rate": 3.817178468566134e-06, + "loss": 0.49312207, + "num_input_tokens_seen": 160686515, + "router_z_loss_clip": 0.19238281, + "router_z_loss_mlp": 0.00756836, + "step": 5619, + "time_per_iteration": 5.406815767288208 + }, + { + "auxiliary_loss_clip": 0.01029767, + "auxiliary_loss_mlp": 0.01002423, + "balance_loss_clip": 1.01059484, + "balance_loss_mlp": 1.00167549, + "epoch": 0.16307817306018224, + "flos": 73458258053760.0, + "grad_norm": 0.7079411250672262, + "language_loss": 0.50286674, + "learning_rate": 3.81709995064863e-06, + "loss": 0.52318865, + "num_input_tokens_seen": 160745365, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.00747681, + "step": 5620, + "time_per_iteration": 5.322272062301636 + }, + { + "auxiliary_loss_clip": 0.01110138, + "auxiliary_loss_mlp": 0.0103297, + "balance_loss_clip": 1.04512572, + "balance_loss_mlp": 1.02129948, + "epoch": 0.16310719052869826, + "flos": 23543502278400.0, + "grad_norm": 2.3943363619628677, + "language_loss": 0.79213017, + "learning_rate": 3.817021416681765e-06, + "loss": 0.8135612, + "num_input_tokens_seen": 160759550, + "router_z_loss_clip": 0.65087891, + "router_z_loss_mlp": 0.11676025, + "step": 5621, + "time_per_iteration": 4.965155839920044 + }, + { + "auxiliary_loss_clip": 0.01139605, + "auxiliary_loss_mlp": 0.01052111, + "balance_loss_clip": 1.05379128, + "balance_loss_mlp": 1.03473067, + "epoch": 0.16313620799721432, + "flos": 48426758962560.0, + "grad_norm": 2.6286062155705316, + "language_loss": 0.89219493, + "learning_rate": 3.816942866666231e-06, + "loss": 0.91411209, + "num_input_tokens_seen": 160779310, + "router_z_loss_clip": 0.859375, + "router_z_loss_mlp": 0.17382812, + "step": 5622, + "time_per_iteration": 2.690293073654175 + }, + { + "auxiliary_loss_clip": 0.01130477, + "auxiliary_loss_mlp": 0.01054242, + "balance_loss_clip": 1.05386913, + "balance_loss_mlp": 1.03781486, + "epoch": 0.16316522546573037, + "flos": 21790594661760.0, + "grad_norm": 2.857362343648802, + "language_loss": 0.68876243, + "learning_rate": 3.816864300602723e-06, + "loss": 0.71060961, + "num_input_tokens_seen": 160792455, + "router_z_loss_clip": 0.76513672, + "router_z_loss_mlp": 0.1640625, + "step": 5623, + "time_per_iteration": 2.4435369968414307 + }, + { + "auxiliary_loss_clip": 0.01030812, + "auxiliary_loss_mlp": 0.01021347, + "balance_loss_clip": 1.01152277, + "balance_loss_mlp": 1.02058721, + "epoch": 0.16319424293424642, + "flos": 63614246492160.0, + "grad_norm": 0.6278510129786723, + "language_loss": 0.46844453, + "learning_rate": 3.8167857184919335e-06, + "loss": 0.48896611, + "num_input_tokens_seen": 160855110, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.00759888, + "step": 5624, + "time_per_iteration": 3.0552685260772705 + }, + { + "auxiliary_loss_clip": 0.0114406, + "auxiliary_loss_mlp": 0.01052479, + "balance_loss_clip": 1.05964255, + "balance_loss_mlp": 1.03550911, + "epoch": 0.16322326040276247, + "flos": 46680208053120.0, + "grad_norm": 2.0854225489981335, + "language_loss": 0.93028009, + "learning_rate": 3.816707120334558e-06, + "loss": 0.95224547, + "num_input_tokens_seen": 160876605, + "router_z_loss_clip": 0.84375, + "router_z_loss_mlp": 0.16967773, + "step": 5625, + "time_per_iteration": 2.747439384460449 + }, + { + "auxiliary_loss_clip": 0.01124571, + "auxiliary_loss_mlp": 0.01048418, + "balance_loss_clip": 1.05039251, + "balance_loss_mlp": 1.0342319, + "epoch": 0.16325227787127852, + "flos": 15662295457920.0, + "grad_norm": 2.2346130051859565, + "language_loss": 0.70293128, + "learning_rate": 3.81662850613129e-06, + "loss": 0.72466123, + "num_input_tokens_seen": 160888710, + "router_z_loss_clip": 0.7421875, + "router_z_loss_mlp": 0.1418457, + "step": 5626, + "time_per_iteration": 2.518437385559082 + }, + { + "auxiliary_loss_clip": 0.01029676, + "auxiliary_loss_mlp": 0.01020658, + "balance_loss_clip": 1.01055896, + "balance_loss_mlp": 1.01999593, + "epoch": 0.16328129533979455, + "flos": 70580233751040.0, + "grad_norm": 0.6514501389945089, + "language_loss": 0.48513013, + "learning_rate": 3.816549875882824e-06, + "loss": 0.50563347, + "num_input_tokens_seen": 160950890, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.00662231, + "step": 5627, + "time_per_iteration": 3.082322120666504 + }, + { + "auxiliary_loss_clip": 0.01028729, + "auxiliary_loss_mlp": 0.01014833, + "balance_loss_clip": 1.00974667, + "balance_loss_mlp": 1.01420164, + "epoch": 0.1633103128083106, + "flos": 62508412431360.0, + "grad_norm": 0.6673742887437124, + "language_loss": 0.47776598, + "learning_rate": 3.816471229589854e-06, + "loss": 0.49820161, + "num_input_tokens_seen": 161006895, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.00631714, + "step": 5628, + "time_per_iteration": 2.995053768157959 + }, + { + "auxiliary_loss_clip": 0.01120355, + "auxiliary_loss_mlp": 0.01034648, + "balance_loss_clip": 1.0502528, + "balance_loss_mlp": 1.0212425, + "epoch": 0.16333933027682665, + "flos": 18144010028160.0, + "grad_norm": 2.2496244017205536, + "language_loss": 0.76832664, + "learning_rate": 3.816392567253075e-06, + "loss": 0.7898767, + "num_input_tokens_seen": 161021620, + "router_z_loss_clip": 0.70117188, + "router_z_loss_mlp": 0.13409424, + "step": 5629, + "time_per_iteration": 2.469223976135254 + }, + { + "auxiliary_loss_clip": 0.01028208, + "auxiliary_loss_mlp": 0.00999459, + "balance_loss_clip": 1.00897598, + "balance_loss_mlp": 0.99878234, + "epoch": 0.1633683477453427, + "flos": 63873198616320.0, + "grad_norm": 0.636045318918068, + "language_loss": 0.46110713, + "learning_rate": 3.816313888873182e-06, + "loss": 0.4813838, + "num_input_tokens_seen": 161083280, + "router_z_loss_clip": 0.19238281, + "router_z_loss_mlp": 0.0067749, + "step": 5630, + "time_per_iteration": 3.0573654174804688 + }, + { + "auxiliary_loss_clip": 0.01132948, + "auxiliary_loss_mlp": 0.01041702, + "balance_loss_clip": 1.05262971, + "balance_loss_mlp": 1.02606142, + "epoch": 0.16339736521385875, + "flos": 12124879234560.0, + "grad_norm": 2.526516357869479, + "language_loss": 0.76974171, + "learning_rate": 3.81623519445087e-06, + "loss": 0.79148817, + "num_input_tokens_seen": 161095980, + "router_z_loss_clip": 0.80224609, + "router_z_loss_mlp": 0.15649414, + "step": 5631, + "time_per_iteration": 2.503509283065796 + }, + { + "auxiliary_loss_clip": 0.01126256, + "auxiliary_loss_mlp": 0.01034511, + "balance_loss_clip": 1.05318451, + "balance_loss_mlp": 1.01939559, + "epoch": 0.16342638268237478, + "flos": 16243760822400.0, + "grad_norm": 2.499941475044382, + "language_loss": 0.61286843, + "learning_rate": 3.816156483986834e-06, + "loss": 0.63447613, + "num_input_tokens_seen": 161107775, + "router_z_loss_clip": 0.73095703, + "router_z_loss_mlp": 0.15106201, + "step": 5632, + "time_per_iteration": 2.4931018352508545 + }, + { + "auxiliary_loss_clip": 0.01126315, + "auxiliary_loss_mlp": 0.0104093, + "balance_loss_clip": 1.04989719, + "balance_loss_mlp": 1.02450264, + "epoch": 0.16345540015089083, + "flos": 14786506051200.0, + "grad_norm": 2.425135157490111, + "language_loss": 0.89277995, + "learning_rate": 3.816077757481768e-06, + "loss": 0.91445243, + "num_input_tokens_seen": 161120440, + "router_z_loss_clip": 0.76464844, + "router_z_loss_mlp": 0.16418457, + "step": 5633, + "time_per_iteration": 2.4884073734283447 + }, + { + "auxiliary_loss_clip": 0.01129214, + "auxiliary_loss_mlp": 0.0104583, + "balance_loss_clip": 1.05138052, + "balance_loss_mlp": 1.0290637, + "epoch": 0.16348441761940688, + "flos": 31604836826880.0, + "grad_norm": 2.2848561125361937, + "language_loss": 0.93230546, + "learning_rate": 3.815999014936369e-06, + "loss": 0.95405591, + "num_input_tokens_seen": 161134405, + "router_z_loss_clip": 0.77832031, + "router_z_loss_mlp": 0.16766357, + "step": 5634, + "time_per_iteration": 2.5700464248657227 + }, + { + "auxiliary_loss_clip": 0.0103095, + "auxiliary_loss_mlp": 0.01010709, + "balance_loss_clip": 1.01086283, + "balance_loss_mlp": 1.00996423, + "epoch": 0.16351343508792293, + "flos": 74782428935040.0, + "grad_norm": 0.593873188976517, + "language_loss": 0.52196914, + "learning_rate": 3.815920256351332e-06, + "loss": 0.54238576, + "num_input_tokens_seen": 161204830, + "router_z_loss_clip": 0.20117188, + "router_z_loss_mlp": 0.00744629, + "step": 5635, + "time_per_iteration": 3.263625383377075 + }, + { + "auxiliary_loss_clip": 0.01132967, + "auxiliary_loss_mlp": 0.01038421, + "balance_loss_clip": 1.05329204, + "balance_loss_mlp": 1.02210164, + "epoch": 0.16354245255643898, + "flos": 18946757128320.0, + "grad_norm": 2.048928589385822, + "language_loss": 0.84070736, + "learning_rate": 3.815841481727352e-06, + "loss": 0.86242127, + "num_input_tokens_seen": 161218870, + "router_z_loss_clip": 0.79589844, + "router_z_loss_mlp": 0.16296387, + "step": 5636, + "time_per_iteration": 2.482792377471924 + }, + { + "auxiliary_loss_clip": 0.01132288, + "auxiliary_loss_mlp": 0.01042033, + "balance_loss_clip": 1.05459881, + "balance_loss_mlp": 1.02530837, + "epoch": 0.16357147002495503, + "flos": 15952812658560.0, + "grad_norm": 2.800395862931091, + "language_loss": 0.93906879, + "learning_rate": 3.815762691065126e-06, + "loss": 0.96081191, + "num_input_tokens_seen": 161230220, + "router_z_loss_clip": 0.77685547, + "router_z_loss_mlp": 0.16729736, + "step": 5637, + "time_per_iteration": 2.581514835357666 + }, + { + "auxiliary_loss_clip": 0.01130804, + "auxiliary_loss_mlp": 0.01043963, + "balance_loss_clip": 1.05186319, + "balance_loss_mlp": 1.02781034, + "epoch": 0.16360048749347106, + "flos": 16541209347840.0, + "grad_norm": 2.095305272381245, + "language_loss": 0.75101691, + "learning_rate": 3.815683884365348e-06, + "loss": 0.77276456, + "num_input_tokens_seen": 161242745, + "router_z_loss_clip": 0.78759766, + "router_z_loss_mlp": 0.16156006, + "step": 5638, + "time_per_iteration": 2.512193441390991 + }, + { + "auxiliary_loss_clip": 0.01136618, + "auxiliary_loss_mlp": 0.01046803, + "balance_loss_clip": 1.05426764, + "balance_loss_mlp": 1.02989292, + "epoch": 0.1636295049619871, + "flos": 56633280284160.0, + "grad_norm": 2.549520297047926, + "language_loss": 0.86351442, + "learning_rate": 3.815605061628716e-06, + "loss": 0.88534862, + "num_input_tokens_seen": 161260630, + "router_z_loss_clip": 0.82373047, + "router_z_loss_mlp": 0.16931152, + "step": 5639, + "time_per_iteration": 2.7541913986206055 + }, + { + "auxiliary_loss_clip": 0.01030922, + "auxiliary_loss_mlp": 0.00998991, + "balance_loss_clip": 1.01187027, + "balance_loss_mlp": 0.99825162, + "epoch": 0.16365852243050316, + "flos": 74779412192640.0, + "grad_norm": 0.5919336711863002, + "language_loss": 0.44191971, + "learning_rate": 3.815526222855926e-06, + "loss": 0.46221882, + "num_input_tokens_seen": 161324025, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.00738525, + "step": 5640, + "time_per_iteration": 3.1844747066497803 + }, + { + "auxiliary_loss_clip": 0.01140902, + "auxiliary_loss_mlp": 0.01046512, + "balance_loss_clip": 1.05773759, + "balance_loss_mlp": 1.02907729, + "epoch": 0.1636875398990192, + "flos": 19608986223360.0, + "grad_norm": 2.3302780889511028, + "language_loss": 1.00241518, + "learning_rate": 3.8154473680476725e-06, + "loss": 1.02428937, + "num_input_tokens_seen": 161339465, + "router_z_loss_clip": 0.83154297, + "router_z_loss_mlp": 0.17425537, + "step": 5641, + "time_per_iteration": 2.5119075775146484 + }, + { + "auxiliary_loss_clip": 0.011298, + "auxiliary_loss_mlp": 0.01049144, + "balance_loss_clip": 1.05485988, + "balance_loss_mlp": 1.0328598, + "epoch": 0.16371655736753526, + "flos": 15698816611200.0, + "grad_norm": 1.9048701597875646, + "language_loss": 0.727364, + "learning_rate": 3.815368497204654e-06, + "loss": 0.74915349, + "num_input_tokens_seen": 161351985, + "router_z_loss_clip": 0.75048828, + "router_z_loss_mlp": 0.16265869, + "step": 5642, + "time_per_iteration": 2.5406646728515625 + }, + { + "auxiliary_loss_clip": 0.01127485, + "auxiliary_loss_mlp": 0.01037709, + "balance_loss_clip": 1.0525049, + "balance_loss_mlp": 1.02307618, + "epoch": 0.16374557483605132, + "flos": 31496889479040.0, + "grad_norm": 1.9531626913915696, + "language_loss": 0.65680766, + "learning_rate": 3.815289610327566e-06, + "loss": 0.67845958, + "num_input_tokens_seen": 161368360, + "router_z_loss_clip": 0.75, + "router_z_loss_mlp": 0.14624023, + "step": 5643, + "time_per_iteration": 2.6340434551239014 + }, + { + "auxiliary_loss_clip": 0.01136724, + "auxiliary_loss_mlp": 0.01048343, + "balance_loss_clip": 1.05874944, + "balance_loss_mlp": 1.0333699, + "epoch": 0.16377459230456734, + "flos": 74734735674240.0, + "grad_norm": 1.799110705056534, + "language_loss": 0.73672497, + "learning_rate": 3.815210707417106e-06, + "loss": 0.75857568, + "num_input_tokens_seen": 161395940, + "router_z_loss_clip": 0.78027344, + "router_z_loss_mlp": 0.14978027, + "step": 5644, + "time_per_iteration": 2.931936740875244 + }, + { + "auxiliary_loss_clip": 0.01032571, + "auxiliary_loss_mlp": 0.01016202, + "balance_loss_clip": 1.01373744, + "balance_loss_mlp": 1.01553488, + "epoch": 0.1638036097730834, + "flos": 64597121320320.0, + "grad_norm": 0.7390337384862689, + "language_loss": 0.50331807, + "learning_rate": 3.81513178847397e-06, + "loss": 0.5238058, + "num_input_tokens_seen": 161457300, + "router_z_loss_clip": 0.18847656, + "router_z_loss_mlp": 0.00668335, + "step": 5645, + "time_per_iteration": 3.049934148788452 + }, + { + "auxiliary_loss_clip": 0.01031548, + "auxiliary_loss_mlp": 0.01011338, + "balance_loss_clip": 1.01266396, + "balance_loss_mlp": 1.01066458, + "epoch": 0.16383262724159944, + "flos": 74781531095040.0, + "grad_norm": 0.6420097129225961, + "language_loss": 0.48767418, + "learning_rate": 3.815052853498855e-06, + "loss": 0.50810301, + "num_input_tokens_seen": 161526250, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.00674438, + "step": 5646, + "time_per_iteration": 3.2310397624969482 + }, + { + "auxiliary_loss_clip": 0.01138197, + "auxiliary_loss_mlp": 0.01040618, + "balance_loss_clip": 1.05706024, + "balance_loss_mlp": 1.02443576, + "epoch": 0.1638616447101155, + "flos": 32198979692160.0, + "grad_norm": 3.1796804348608774, + "language_loss": 0.86535656, + "learning_rate": 3.81497390249246e-06, + "loss": 0.88714468, + "num_input_tokens_seen": 161542895, + "router_z_loss_clip": 0.81152344, + "router_z_loss_mlp": 0.16174316, + "step": 5647, + "time_per_iteration": 2.6253767013549805 + }, + { + "auxiliary_loss_clip": 0.01129614, + "auxiliary_loss_mlp": 0.01041446, + "balance_loss_clip": 1.05475807, + "balance_loss_mlp": 1.02679563, + "epoch": 0.16389066217863155, + "flos": 17486018737920.0, + "grad_norm": 2.6033080478911117, + "language_loss": 0.80055445, + "learning_rate": 3.814894935455481e-06, + "loss": 0.82226503, + "num_input_tokens_seen": 161554750, + "router_z_loss_clip": 0.74902344, + "router_z_loss_mlp": 0.14648438, + "step": 5648, + "time_per_iteration": 2.584094762802124 + }, + { + "auxiliary_loss_clip": 0.01140987, + "auxiliary_loss_mlp": 0.0105392, + "balance_loss_clip": 1.05503201, + "balance_loss_mlp": 1.03631854, + "epoch": 0.16391967964714757, + "flos": 21353310489600.0, + "grad_norm": 2.3088957176234293, + "language_loss": 0.87001371, + "learning_rate": 3.814815952388614e-06, + "loss": 0.89196277, + "num_input_tokens_seen": 161568810, + "router_z_loss_clip": 0.85986328, + "router_z_loss_mlp": 0.17602539, + "step": 5649, + "time_per_iteration": 2.5650131702423096 + }, + { + "auxiliary_loss_clip": 0.01128111, + "auxiliary_loss_mlp": 0.01038735, + "balance_loss_clip": 1.05215669, + "balance_loss_mlp": 1.02376854, + "epoch": 0.16394869711566362, + "flos": 13033490693760.0, + "grad_norm": 2.783573687991608, + "language_loss": 1.0128113, + "learning_rate": 3.814736953292559e-06, + "loss": 1.03447974, + "num_input_tokens_seen": 161579495, + "router_z_loss_clip": 0.76025391, + "router_z_loss_mlp": 0.1496582, + "step": 5650, + "time_per_iteration": 2.5767855644226074 + }, + { + "auxiliary_loss_clip": 0.01133509, + "auxiliary_loss_mlp": 0.01041566, + "balance_loss_clip": 1.0579648, + "balance_loss_mlp": 1.02510333, + "epoch": 0.16397771458417967, + "flos": 32554171330560.0, + "grad_norm": 2.174845572675144, + "language_loss": 0.66559917, + "learning_rate": 3.8146579381680134e-06, + "loss": 0.68734992, + "num_input_tokens_seen": 161594395, + "router_z_loss_clip": 0.75537109, + "router_z_loss_mlp": 0.16479492, + "step": 5651, + "time_per_iteration": 2.602889060974121 + }, + { + "auxiliary_loss_clip": 0.01141658, + "auxiliary_loss_mlp": 0.0104833, + "balance_loss_clip": 1.05604768, + "balance_loss_mlp": 1.02923298, + "epoch": 0.16400673205269573, + "flos": 44121285198720.0, + "grad_norm": 9.344855697250756, + "language_loss": 1.0337075, + "learning_rate": 3.814578907015674e-06, + "loss": 1.05560756, + "num_input_tokens_seen": 161614170, + "router_z_loss_clip": 0.85693359, + "router_z_loss_mlp": 0.19085693, + "step": 5652, + "time_per_iteration": 2.737013101577759 + }, + { + "auxiliary_loss_clip": 0.01135946, + "auxiliary_loss_mlp": 0.01041413, + "balance_loss_clip": 1.05356264, + "balance_loss_mlp": 1.0258857, + "epoch": 0.16403574952121178, + "flos": 29672807063040.0, + "grad_norm": 2.5330850645160417, + "language_loss": 1.02728152, + "learning_rate": 3.8144998598362397e-06, + "loss": 1.0490551, + "num_input_tokens_seen": 161630635, + "router_z_loss_clip": 0.82470703, + "router_z_loss_mlp": 0.15527344, + "step": 5653, + "time_per_iteration": 2.5647919178009033 + }, + { + "auxiliary_loss_clip": 0.01032683, + "auxiliary_loss_mlp": 0.01001482, + "balance_loss_clip": 1.01352048, + "balance_loss_mlp": 1.00066495, + "epoch": 0.16406476698972783, + "flos": 72540919589760.0, + "grad_norm": 0.6478636712777854, + "language_loss": 0.4536041, + "learning_rate": 3.8144207966304084e-06, + "loss": 0.47394574, + "num_input_tokens_seen": 161691765, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.00817871, + "step": 5654, + "time_per_iteration": 3.1214113235473633 + }, + { + "auxiliary_loss_clip": 0.0113646, + "auxiliary_loss_mlp": 0.01045286, + "balance_loss_clip": 1.053617, + "balance_loss_mlp": 1.02925825, + "epoch": 0.16409378445824385, + "flos": 28327270567680.0, + "grad_norm": 2.1037506117534845, + "language_loss": 0.89861727, + "learning_rate": 3.814341717398878e-06, + "loss": 0.92043471, + "num_input_tokens_seen": 161708955, + "router_z_loss_clip": 0.82666016, + "router_z_loss_mlp": 0.16027832, + "step": 5655, + "time_per_iteration": 2.557050943374634 + }, + { + "auxiliary_loss_clip": 0.01033586, + "auxiliary_loss_mlp": 0.00999718, + "balance_loss_clip": 1.01446545, + "balance_loss_mlp": 0.99897307, + "epoch": 0.1641228019267599, + "flos": 64936621693440.0, + "grad_norm": 0.6581876778928859, + "language_loss": 0.50618201, + "learning_rate": 3.8142626221423475e-06, + "loss": 0.52651501, + "num_input_tokens_seen": 161771950, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.00744629, + "step": 5656, + "time_per_iteration": 3.1527748107910156 + }, + { + "auxiliary_loss_clip": 0.01034373, + "auxiliary_loss_mlp": 0.01002907, + "balance_loss_clip": 1.01530468, + "balance_loss_mlp": 1.00209033, + "epoch": 0.16415181939527596, + "flos": 74778837575040.0, + "grad_norm": 0.6671390849282658, + "language_loss": 0.53290617, + "learning_rate": 3.8141835108615155e-06, + "loss": 0.55327898, + "num_input_tokens_seen": 161839550, + "router_z_loss_clip": 0.19042969, + "router_z_loss_mlp": 0.00817871, + "step": 5657, + "time_per_iteration": 3.194150447845459 + }, + { + "auxiliary_loss_clip": 0.01034588, + "auxiliary_loss_mlp": 0.01000656, + "balance_loss_clip": 1.01562285, + "balance_loss_mlp": 0.99995273, + "epoch": 0.164180836863792, + "flos": 53902133671680.0, + "grad_norm": 0.7102476919695655, + "language_loss": 0.50967926, + "learning_rate": 3.8141043835570804e-06, + "loss": 0.53003168, + "num_input_tokens_seen": 161900760, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.00701904, + "step": 5658, + "time_per_iteration": 3.134477138519287 + }, + { + "auxiliary_loss_clip": 0.01134337, + "auxiliary_loss_mlp": 0.01047982, + "balance_loss_clip": 1.05927503, + "balance_loss_mlp": 1.03282464, + "epoch": 0.16420985433230806, + "flos": 31278768140160.0, + "grad_norm": 1.8487251085731415, + "language_loss": 0.67279923, + "learning_rate": 3.8140252402297415e-06, + "loss": 0.69462252, + "num_input_tokens_seen": 161916895, + "router_z_loss_clip": 0.75097656, + "router_z_loss_mlp": 0.15155029, + "step": 5659, + "time_per_iteration": 2.6249282360076904 + }, + { + "auxiliary_loss_clip": 0.01035396, + "auxiliary_loss_mlp": 0.01004032, + "balance_loss_clip": 1.0163784, + "balance_loss_mlp": 1.00326347, + "epoch": 0.1642388718008241, + "flos": 71824179605760.0, + "grad_norm": 0.6873364288013594, + "language_loss": 0.50023723, + "learning_rate": 3.813946080880198e-06, + "loss": 0.52063155, + "num_input_tokens_seen": 161981025, + "router_z_loss_clip": 0.19042969, + "router_z_loss_mlp": 0.00765991, + "step": 5660, + "time_per_iteration": 3.1398911476135254 + }, + { + "auxiliary_loss_clip": 0.01131509, + "auxiliary_loss_mlp": 0.01036585, + "balance_loss_clip": 1.05618405, + "balance_loss_mlp": 1.02129626, + "epoch": 0.16426788926934013, + "flos": 27629741381760.0, + "grad_norm": 3.227742290389448, + "language_loss": 0.97672808, + "learning_rate": 3.8138669055091483e-06, + "loss": 0.99840897, + "num_input_tokens_seen": 161995510, + "router_z_loss_clip": 0.75244141, + "router_z_loss_mlp": 0.15289307, + "step": 5661, + "time_per_iteration": 2.5874364376068115 + }, + { + "auxiliary_loss_clip": 0.01035868, + "auxiliary_loss_mlp": 0.00999919, + "balance_loss_clip": 1.01674128, + "balance_loss_mlp": 0.99918002, + "epoch": 0.16429690673785619, + "flos": 63830572151040.0, + "grad_norm": 0.6507657214816225, + "language_loss": 0.51581216, + "learning_rate": 3.813787714117292e-06, + "loss": 0.53617007, + "num_input_tokens_seen": 162058640, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.00738525, + "step": 5662, + "time_per_iteration": 3.02701473236084 + }, + { + "auxiliary_loss_clip": 0.01126211, + "auxiliary_loss_mlp": 0.01034964, + "balance_loss_clip": 1.0528779, + "balance_loss_mlp": 1.02040219, + "epoch": 0.16432592420637224, + "flos": 12923891320320.0, + "grad_norm": 2.580308987854389, + "language_loss": 0.85431242, + "learning_rate": 3.8137085067053287e-06, + "loss": 0.87592423, + "num_input_tokens_seen": 162069905, + "router_z_loss_clip": 0.73291016, + "router_z_loss_mlp": 0.14569092, + "step": 5663, + "time_per_iteration": 2.5221991539001465 + }, + { + "auxiliary_loss_clip": 0.01037356, + "auxiliary_loss_mlp": 0.0100352, + "balance_loss_clip": 1.01814163, + "balance_loss_mlp": 1.00272179, + "epoch": 0.1643549416748883, + "flos": 52156085552640.0, + "grad_norm": 0.6797364046254316, + "language_loss": 0.4830395, + "learning_rate": 3.8136292832739582e-06, + "loss": 0.50344825, + "num_input_tokens_seen": 162127965, + "router_z_loss_clip": 0.19238281, + "router_z_loss_mlp": 0.00799561, + "step": 5664, + "time_per_iteration": 2.968914270401001 + }, + { + "auxiliary_loss_clip": 0.01036213, + "auxiliary_loss_mlp": 0.01000468, + "balance_loss_clip": 1.01709831, + "balance_loss_mlp": 0.9997105, + "epoch": 0.16438395914340434, + "flos": 64594822849920.0, + "grad_norm": 0.839977545824509, + "language_loss": 0.51629889, + "learning_rate": 3.8135500438238797e-06, + "loss": 0.53666568, + "num_input_tokens_seen": 162185575, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.00756836, + "step": 5665, + "time_per_iteration": 3.045203924179077 + }, + { + "auxiliary_loss_clip": 0.01036404, + "auxiliary_loss_mlp": 0.0100709, + "balance_loss_clip": 1.01741803, + "balance_loss_mlp": 1.00624359, + "epoch": 0.16441297661192036, + "flos": 70793542667520.0, + "grad_norm": 0.6217555869976981, + "language_loss": 0.47767952, + "learning_rate": 3.8134707883557936e-06, + "loss": 0.49811447, + "num_input_tokens_seen": 162248505, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.00848389, + "step": 5666, + "time_per_iteration": 3.069720983505249 + }, + { + "auxiliary_loss_clip": 0.01133022, + "auxiliary_loss_mlp": 0.01045735, + "balance_loss_clip": 1.05752337, + "balance_loss_mlp": 1.02988648, + "epoch": 0.16444199408043642, + "flos": 26425477077120.0, + "grad_norm": 2.595872373828852, + "language_loss": 0.71295738, + "learning_rate": 3.813391516870399e-06, + "loss": 0.73474491, + "num_input_tokens_seen": 162265490, + "router_z_loss_clip": 0.75488281, + "router_z_loss_mlp": 0.1585083, + "step": 5667, + "time_per_iteration": 2.6884586811065674 + }, + { + "auxiliary_loss_clip": 0.01129838, + "auxiliary_loss_mlp": 0.01037968, + "balance_loss_clip": 1.0557698, + "balance_loss_mlp": 1.02210128, + "epoch": 0.16447101154895247, + "flos": 12343072400640.0, + "grad_norm": 2.640201376124729, + "language_loss": 0.79722977, + "learning_rate": 3.8133122293683977e-06, + "loss": 0.81890786, + "num_input_tokens_seen": 162277630, + "router_z_loss_clip": 0.74072266, + "router_z_loss_mlp": 0.15881348, + "step": 5668, + "time_per_iteration": 2.6434876918792725 + }, + { + "auxiliary_loss_clip": 0.01135374, + "auxiliary_loss_mlp": 0.01044588, + "balance_loss_clip": 1.05597234, + "balance_loss_mlp": 1.02838159, + "epoch": 0.16450002901746852, + "flos": 27628735800960.0, + "grad_norm": 1.733304314761794, + "language_loss": 0.81151855, + "learning_rate": 3.813232925850488e-06, + "loss": 0.83331811, + "num_input_tokens_seen": 162294125, + "router_z_loss_clip": 0.79492188, + "router_z_loss_mlp": 0.16204834, + "step": 5669, + "time_per_iteration": 2.6509718894958496 + }, + { + "auxiliary_loss_clip": 0.01127977, + "auxiliary_loss_mlp": 0.01035244, + "balance_loss_clip": 1.05737329, + "balance_loss_mlp": 1.02055097, + "epoch": 0.16452904648598457, + "flos": 20846359889280.0, + "grad_norm": 2.530532536268279, + "language_loss": 0.90530014, + "learning_rate": 3.813153606317372e-06, + "loss": 0.92693233, + "num_input_tokens_seen": 162307815, + "router_z_loss_clip": 0.70654297, + "router_z_loss_mlp": 0.14697266, + "step": 5670, + "time_per_iteration": 2.6242501735687256 + }, + { + "auxiliary_loss_clip": 0.01128228, + "auxiliary_loss_mlp": 0.01035459, + "balance_loss_clip": 1.05298042, + "balance_loss_mlp": 1.02033138, + "epoch": 0.16455806395450062, + "flos": 26132481838080.0, + "grad_norm": 3.3511910500444304, + "language_loss": 0.90969646, + "learning_rate": 3.8130742707697497e-06, + "loss": 0.9313333, + "num_input_tokens_seen": 162323345, + "router_z_loss_clip": 0.75244141, + "router_z_loss_mlp": 0.15118408, + "step": 5671, + "time_per_iteration": 2.6324377059936523 + }, + { + "auxiliary_loss_clip": 0.01132084, + "auxiliary_loss_mlp": 0.01035366, + "balance_loss_clip": 1.05898595, + "balance_loss_mlp": 1.02066731, + "epoch": 0.16458708142301665, + "flos": 30843279648000.0, + "grad_norm": 2.2519431823630365, + "language_loss": 0.65806419, + "learning_rate": 3.8129949192083215e-06, + "loss": 0.6797387, + "num_input_tokens_seen": 162339210, + "router_z_loss_clip": 0.73095703, + "router_z_loss_mlp": 0.14709473, + "step": 5672, + "time_per_iteration": 2.651137113571167 + }, + { + "auxiliary_loss_clip": 0.01131829, + "auxiliary_loss_mlp": 0.01040259, + "balance_loss_clip": 1.05829346, + "balance_loss_mlp": 1.02603781, + "epoch": 0.1646160988915327, + "flos": 37737194267520.0, + "grad_norm": 4.719816734241232, + "language_loss": 1.02851439, + "learning_rate": 3.8129155516337887e-06, + "loss": 1.05023527, + "num_input_tokens_seen": 162362150, + "router_z_loss_clip": 0.73535156, + "router_z_loss_mlp": 0.14221191, + "step": 5673, + "time_per_iteration": 2.7026383876800537 + }, + { + "auxiliary_loss_clip": 0.01046435, + "auxiliary_loss_mlp": 0.009974, + "balance_loss_clip": 1.02737808, + "balance_loss_mlp": 0.99660128, + "epoch": 0.16464511636004875, + "flos": 74780022723840.0, + "grad_norm": 0.6377580077619445, + "language_loss": 0.46081698, + "learning_rate": 3.8128361680468516e-06, + "loss": 0.48125535, + "num_input_tokens_seen": 162428305, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.00799561, + "step": 5674, + "time_per_iteration": 3.211299419403076 + }, + { + "auxiliary_loss_clip": 0.01141795, + "auxiliary_loss_mlp": 0.01036628, + "balance_loss_clip": 1.06012225, + "balance_loss_mlp": 1.01929474, + "epoch": 0.1646741338285648, + "flos": 21463987271040.0, + "grad_norm": 2.160038147020681, + "language_loss": 0.88739854, + "learning_rate": 3.8127567684482126e-06, + "loss": 0.90918279, + "num_input_tokens_seen": 162443055, + "router_z_loss_clip": 0.81542969, + "router_z_loss_mlp": 0.17327881, + "step": 5675, + "time_per_iteration": 2.5439655780792236 + }, + { + "auxiliary_loss_clip": 0.0113519, + "auxiliary_loss_mlp": 0.01037163, + "balance_loss_clip": 1.06217086, + "balance_loss_mlp": 1.02288175, + "epoch": 0.16470315129708085, + "flos": 74731647104640.0, + "grad_norm": 2.648504685438688, + "language_loss": 0.93506658, + "learning_rate": 3.8126773528385723e-06, + "loss": 0.95679009, + "num_input_tokens_seen": 162468485, + "router_z_loss_clip": 0.73046875, + "router_z_loss_mlp": 0.14282227, + "step": 5676, + "time_per_iteration": 2.965773105621338 + }, + { + "auxiliary_loss_clip": 0.01149178, + "auxiliary_loss_mlp": 0.01046704, + "balance_loss_clip": 1.06466722, + "balance_loss_mlp": 1.02780938, + "epoch": 0.16473216876559688, + "flos": 27777011143680.0, + "grad_norm": 2.269072499090365, + "language_loss": 0.87734741, + "learning_rate": 3.8125979212186316e-06, + "loss": 0.89930624, + "num_input_tokens_seen": 162485545, + "router_z_loss_clip": 0.84570312, + "router_z_loss_mlp": 0.18896484, + "step": 5677, + "time_per_iteration": 2.581794023513794 + }, + { + "auxiliary_loss_clip": 0.01140255, + "auxiliary_loss_mlp": 0.01041447, + "balance_loss_clip": 1.06351352, + "balance_loss_mlp": 1.02565229, + "epoch": 0.16476118623411293, + "flos": 30184246863360.0, + "grad_norm": 2.5532774091865327, + "language_loss": 0.76802588, + "learning_rate": 3.812518473589093e-06, + "loss": 0.78984296, + "num_input_tokens_seen": 162501620, + "router_z_loss_clip": 0.76757812, + "router_z_loss_mlp": 0.15808105, + "step": 5678, + "time_per_iteration": 2.6056952476501465 + }, + { + "auxiliary_loss_clip": 0.01049825, + "auxiliary_loss_mlp": 0.01003536, + "balance_loss_clip": 1.03091443, + "balance_loss_mlp": 1.00285912, + "epoch": 0.16479020370262898, + "flos": 74770182397440.0, + "grad_norm": 0.6576294419166444, + "language_loss": 0.45305657, + "learning_rate": 3.8124390099506573e-06, + "loss": 0.47359017, + "num_input_tokens_seen": 162560385, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.0067749, + "step": 5679, + "time_per_iteration": 3.0693371295928955 + }, + { + "auxiliary_loss_clip": 0.01140321, + "auxiliary_loss_mlp": 0.01042109, + "balance_loss_clip": 1.06549919, + "balance_loss_mlp": 1.02827442, + "epoch": 0.16481922117114503, + "flos": 31938231888000.0, + "grad_norm": 2.3520383365875457, + "language_loss": 0.94957137, + "learning_rate": 3.812359530304027e-06, + "loss": 0.97139573, + "num_input_tokens_seen": 162575985, + "router_z_loss_clip": 0.74804688, + "router_z_loss_mlp": 0.1383667, + "step": 5680, + "time_per_iteration": 2.651564598083496 + }, + { + "auxiliary_loss_clip": 0.01135043, + "auxiliary_loss_mlp": 0.01033999, + "balance_loss_clip": 1.06156027, + "balance_loss_mlp": 1.01985478, + "epoch": 0.16484823863966108, + "flos": 16683774428160.0, + "grad_norm": 3.100634182447165, + "language_loss": 0.55943191, + "learning_rate": 3.8122800346499044e-06, + "loss": 0.58112228, + "num_input_tokens_seen": 162590770, + "router_z_loss_clip": 0.73583984, + "router_z_loss_mlp": 0.14154053, + "step": 5681, + "time_per_iteration": 2.53782320022583 + }, + { + "auxiliary_loss_clip": 0.0104872, + "auxiliary_loss_mlp": 0.01003477, + "balance_loss_clip": 1.02983284, + "balance_loss_mlp": 1.00273538, + "epoch": 0.16487725610817713, + "flos": 63462631184640.0, + "grad_norm": 0.7106641395943095, + "language_loss": 0.53567731, + "learning_rate": 3.8122005229889907e-06, + "loss": 0.55619931, + "num_input_tokens_seen": 162650945, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.00741577, + "step": 5682, + "time_per_iteration": 3.0492026805877686 + }, + { + "auxiliary_loss_clip": 0.01143928, + "auxiliary_loss_mlp": 0.01041597, + "balance_loss_clip": 1.06238031, + "balance_loss_mlp": 1.02536678, + "epoch": 0.16490627357669316, + "flos": 32627393205120.0, + "grad_norm": 2.5118006372790895, + "language_loss": 0.85083854, + "learning_rate": 3.812120995321989e-06, + "loss": 0.87269378, + "num_input_tokens_seen": 162665420, + "router_z_loss_clip": 0.81494141, + "router_z_loss_mlp": 0.16235352, + "step": 5683, + "time_per_iteration": 2.6745293140411377 + }, + { + "auxiliary_loss_clip": 0.01050575, + "auxiliary_loss_mlp": 0.01001058, + "balance_loss_clip": 1.03163528, + "balance_loss_mlp": 1.00037003, + "epoch": 0.1649352910452092, + "flos": 61891646976000.0, + "grad_norm": 0.7069182275880898, + "language_loss": 0.42615426, + "learning_rate": 3.812041451649601e-06, + "loss": 0.44667059, + "num_input_tokens_seen": 162721645, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.00689697, + "step": 5684, + "time_per_iteration": 2.9786176681518555 + }, + { + "auxiliary_loss_clip": 0.01048029, + "auxiliary_loss_mlp": 0.01002202, + "balance_loss_clip": 1.02902269, + "balance_loss_mlp": 1.00145078, + "epoch": 0.16496430851372526, + "flos": 59388961841280.0, + "grad_norm": 0.6771466696102311, + "language_loss": 0.48162073, + "learning_rate": 3.8119618919725302e-06, + "loss": 0.50212306, + "num_input_tokens_seen": 162786680, + "router_z_loss_clip": 0.19042969, + "router_z_loss_mlp": 0.00750732, + "step": 5685, + "time_per_iteration": 3.175676107406616 + }, + { + "auxiliary_loss_clip": 0.01134704, + "auxiliary_loss_mlp": 0.01038998, + "balance_loss_clip": 1.06230414, + "balance_loss_mlp": 1.02393019, + "epoch": 0.1649933259822413, + "flos": 16758361019520.0, + "grad_norm": 2.023743335174309, + "language_loss": 0.80703306, + "learning_rate": 3.811882316291478e-06, + "loss": 0.82877016, + "num_input_tokens_seen": 162800670, + "router_z_loss_clip": 0.72509766, + "router_z_loss_mlp": 0.15063477, + "step": 5686, + "time_per_iteration": 2.5246870517730713 + }, + { + "auxiliary_loss_clip": 0.01142213, + "auxiliary_loss_mlp": 0.01039445, + "balance_loss_clip": 1.06383479, + "balance_loss_mlp": 1.02365017, + "epoch": 0.16502234345075736, + "flos": 18081454492800.0, + "grad_norm": 1.904295194204475, + "language_loss": 0.75096107, + "learning_rate": 3.8118027246071484e-06, + "loss": 0.77277768, + "num_input_tokens_seen": 162817905, + "router_z_loss_clip": 0.78320312, + "router_z_loss_mlp": 0.15795898, + "step": 5687, + "time_per_iteration": 2.5573995113372803 + }, + { + "auxiliary_loss_clip": 0.01140511, + "auxiliary_loss_mlp": 0.01037242, + "balance_loss_clip": 1.06275511, + "balance_loss_mlp": 1.02234089, + "epoch": 0.16505136091927342, + "flos": 28944143763840.0, + "grad_norm": 2.7298280117817852, + "language_loss": 0.78448313, + "learning_rate": 3.811723116920244e-06, + "loss": 0.8062607, + "num_input_tokens_seen": 162833955, + "router_z_loss_clip": 0.77783203, + "router_z_loss_mlp": 0.14886475, + "step": 5688, + "time_per_iteration": 2.628448486328125 + }, + { + "auxiliary_loss_clip": 0.01132605, + "auxiliary_loss_mlp": 0.01042686, + "balance_loss_clip": 1.0609951, + "balance_loss_mlp": 1.02885783, + "epoch": 0.16508037838778944, + "flos": 11977322163840.0, + "grad_norm": 3.1216869873027617, + "language_loss": 0.98652923, + "learning_rate": 3.811643493231468e-06, + "loss": 1.00828207, + "num_input_tokens_seen": 162846135, + "router_z_loss_clip": 0.71630859, + "router_z_loss_mlp": 0.13830566, + "step": 5689, + "time_per_iteration": 2.544774055480957 + }, + { + "auxiliary_loss_clip": 0.01146863, + "auxiliary_loss_mlp": 0.0105251, + "balance_loss_clip": 1.06345963, + "balance_loss_mlp": 1.03403878, + "epoch": 0.1651093958563055, + "flos": 30548021852160.0, + "grad_norm": 1.8798406829034158, + "language_loss": 0.95402771, + "learning_rate": 3.8115638535415235e-06, + "loss": 0.97602141, + "num_input_tokens_seen": 162866390, + "router_z_loss_clip": 0.83349609, + "router_z_loss_mlp": 0.18463135, + "step": 5690, + "time_per_iteration": 7.519484758377075 + }, + { + "auxiliary_loss_clip": 0.01048509, + "auxiliary_loss_mlp": 0.01008531, + "balance_loss_clip": 1.02951789, + "balance_loss_mlp": 1.00787532, + "epoch": 0.16513841332482154, + "flos": 72772149392640.0, + "grad_norm": 0.6597454613039335, + "language_loss": 0.50703961, + "learning_rate": 3.811484197851114e-06, + "loss": 0.52761, + "num_input_tokens_seen": 162929835, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.00656128, + "step": 5691, + "time_per_iteration": 5.691964626312256 + }, + { + "auxiliary_loss_clip": 0.01152861, + "auxiliary_loss_mlp": 0.01058613, + "balance_loss_clip": 1.06698012, + "balance_loss_mlp": 1.03920007, + "epoch": 0.1651674307933376, + "flos": 14604510816000.0, + "grad_norm": 2.5509788625338476, + "language_loss": 0.78579348, + "learning_rate": 3.8114045261609428e-06, + "loss": 0.80790818, + "num_input_tokens_seen": 162942410, + "router_z_loss_clip": 0.859375, + "router_z_loss_mlp": 0.19390869, + "step": 5692, + "time_per_iteration": 4.830523729324341 + }, + { + "auxiliary_loss_clip": 0.01048052, + "auxiliary_loss_mlp": 0.01003522, + "balance_loss_clip": 1.02897942, + "balance_loss_mlp": 1.00286961, + "epoch": 0.16519644826185365, + "flos": 74733373935360.0, + "grad_norm": 0.7738963412375884, + "language_loss": 0.47788274, + "learning_rate": 3.811324838471714e-06, + "loss": 0.49839851, + "num_input_tokens_seen": 162997310, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.00653076, + "step": 5693, + "time_per_iteration": 3.0797793865203857 + }, + { + "auxiliary_loss_clip": 0.01049098, + "auxiliary_loss_mlp": 0.01000895, + "balance_loss_clip": 1.03003109, + "balance_loss_mlp": 1.00026023, + "epoch": 0.16522546573036967, + "flos": 74780920563840.0, + "grad_norm": 0.6617781308948001, + "language_loss": 0.45871484, + "learning_rate": 3.811245134784131e-06, + "loss": 0.47921479, + "num_input_tokens_seen": 163058800, + "router_z_loss_clip": 0.19042969, + "router_z_loss_mlp": 0.00634766, + "step": 5694, + "time_per_iteration": 3.1791608333587646 + }, + { + "auxiliary_loss_clip": 0.01140893, + "auxiliary_loss_mlp": 0.01036601, + "balance_loss_clip": 1.06840205, + "balance_loss_mlp": 1.02120543, + "epoch": 0.16525448319888572, + "flos": 28761645738240.0, + "grad_norm": 2.2050863025727354, + "language_loss": 0.80848527, + "learning_rate": 3.8111654150988983e-06, + "loss": 0.83026022, + "num_input_tokens_seen": 163077395, + "router_z_loss_clip": 0.72509766, + "router_z_loss_mlp": 0.1539917, + "step": 5695, + "time_per_iteration": 2.6774814128875732 + }, + { + "auxiliary_loss_clip": 0.0115408, + "auxiliary_loss_mlp": 0.01043573, + "balance_loss_clip": 1.07098281, + "balance_loss_mlp": 1.02596545, + "epoch": 0.16528350066740177, + "flos": 74734017402240.0, + "grad_norm": 1.6746212658086905, + "language_loss": 0.77639371, + "learning_rate": 3.81108567941672e-06, + "loss": 0.79837018, + "num_input_tokens_seen": 163106130, + "router_z_loss_clip": 0.83203125, + "router_z_loss_mlp": 0.17602539, + "step": 5696, + "time_per_iteration": 2.9426991939544678 + }, + { + "auxiliary_loss_clip": 0.01144007, + "auxiliary_loss_mlp": 0.01040338, + "balance_loss_clip": 1.06435895, + "balance_loss_mlp": 1.02345788, + "epoch": 0.16531251813591782, + "flos": 12452635860480.0, + "grad_norm": 2.5615894986641, + "language_loss": 0.78377867, + "learning_rate": 3.8110059277382998e-06, + "loss": 0.8056221, + "num_input_tokens_seen": 163119230, + "router_z_loss_clip": 0.79638672, + "router_z_loss_mlp": 0.16882324, + "step": 5697, + "time_per_iteration": 2.5228278636932373 + }, + { + "auxiliary_loss_clip": 0.01147419, + "auxiliary_loss_mlp": 0.01046621, + "balance_loss_clip": 1.06452501, + "balance_loss_mlp": 1.02900147, + "epoch": 0.16534153560443388, + "flos": 38289141630720.0, + "grad_norm": 2.058304418800728, + "language_loss": 0.95173216, + "learning_rate": 3.810926160064342e-06, + "loss": 0.97367251, + "num_input_tokens_seen": 163141040, + "router_z_loss_clip": 0.83007812, + "router_z_loss_mlp": 0.1762085, + "step": 5698, + "time_per_iteration": 2.685396909713745 + }, + { + "auxiliary_loss_clip": 0.0114542, + "auxiliary_loss_mlp": 0.01041724, + "balance_loss_clip": 1.06584287, + "balance_loss_mlp": 1.02505875, + "epoch": 0.16537055307294993, + "flos": 15664127051520.0, + "grad_norm": 2.192229872166086, + "language_loss": 0.81542903, + "learning_rate": 3.8108463763955526e-06, + "loss": 0.83730042, + "num_input_tokens_seen": 163155495, + "router_z_loss_clip": 0.79541016, + "router_z_loss_mlp": 0.16650391, + "step": 5699, + "time_per_iteration": 2.5316293239593506 + }, + { + "auxiliary_loss_clip": 0.01046388, + "auxiliary_loss_mlp": 0.01006374, + "balance_loss_clip": 1.02725315, + "balance_loss_mlp": 1.00568008, + "epoch": 0.16539957054146595, + "flos": 50215256956800.0, + "grad_norm": 0.7131241841142753, + "language_loss": 0.44772083, + "learning_rate": 3.8107665767326343e-06, + "loss": 0.46824846, + "num_input_tokens_seen": 163212515, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.00695801, + "step": 5700, + "time_per_iteration": 2.991580009460449 + }, + { + "auxiliary_loss_clip": 0.01135563, + "auxiliary_loss_mlp": 0.01028288, + "balance_loss_clip": 1.06427598, + "balance_loss_mlp": 1.01467454, + "epoch": 0.165428588009982, + "flos": 30730699445760.0, + "grad_norm": 2.4523987025860134, + "language_loss": 0.9285031, + "learning_rate": 3.8106867610762935e-06, + "loss": 0.95014155, + "num_input_tokens_seen": 163227465, + "router_z_loss_clip": 0.71337891, + "router_z_loss_mlp": 0.13616943, + "step": 5701, + "time_per_iteration": 2.623849868774414 + }, + { + "auxiliary_loss_clip": 0.01146663, + "auxiliary_loss_mlp": 0.01055655, + "balance_loss_clip": 1.06160223, + "balance_loss_mlp": 1.03870916, + "epoch": 0.16545760547849805, + "flos": 15479869259520.0, + "grad_norm": 2.927439171238071, + "language_loss": 0.96824896, + "learning_rate": 3.810606929427234e-06, + "loss": 0.99027216, + "num_input_tokens_seen": 163240740, + "router_z_loss_clip": 0.85058594, + "router_z_loss_mlp": 0.16949463, + "step": 5702, + "time_per_iteration": 2.5091679096221924 + }, + { + "auxiliary_loss_clip": 0.01139892, + "auxiliary_loss_mlp": 0.01037663, + "balance_loss_clip": 1.06219256, + "balance_loss_mlp": 1.02262461, + "epoch": 0.1654866229470141, + "flos": 39676155356160.0, + "grad_norm": 3.4180808855339873, + "language_loss": 0.84323889, + "learning_rate": 3.810527081786162e-06, + "loss": 0.86501449, + "num_input_tokens_seen": 163262645, + "router_z_loss_clip": 0.77685547, + "router_z_loss_mlp": 0.1505127, + "step": 5703, + "time_per_iteration": 2.7370433807373047 + }, + { + "auxiliary_loss_clip": 0.01139192, + "auxiliary_loss_mlp": 0.01045122, + "balance_loss_clip": 1.05942404, + "balance_loss_mlp": 1.02771139, + "epoch": 0.16551564041553016, + "flos": 27919253001600.0, + "grad_norm": 2.683806637450351, + "language_loss": 0.97489166, + "learning_rate": 3.8104472181537813e-06, + "loss": 0.9967348, + "num_input_tokens_seen": 163275950, + "router_z_loss_clip": 0.79785156, + "router_z_loss_mlp": 0.17401123, + "step": 5704, + "time_per_iteration": 2.620088815689087 + }, + { + "auxiliary_loss_clip": 0.01138853, + "auxiliary_loss_mlp": 0.01041523, + "balance_loss_clip": 1.0648253, + "balance_loss_mlp": 1.02627027, + "epoch": 0.1655446578840462, + "flos": 24711209516160.0, + "grad_norm": 2.360659858322633, + "language_loss": 0.91447419, + "learning_rate": 3.810367338530799e-06, + "loss": 0.93627799, + "num_input_tokens_seen": 163292635, + "router_z_loss_clip": 0.73925781, + "router_z_loss_mlp": 0.15234375, + "step": 5705, + "time_per_iteration": 2.594574213027954 + }, + { + "auxiliary_loss_clip": 0.01139134, + "auxiliary_loss_mlp": 0.01041852, + "balance_loss_clip": 1.06392264, + "balance_loss_mlp": 1.02557945, + "epoch": 0.16557367535256223, + "flos": 12014597502720.0, + "grad_norm": 2.8994623975089944, + "language_loss": 1.01301599, + "learning_rate": 3.810287442917919e-06, + "loss": 1.0348258, + "num_input_tokens_seen": 163303900, + "router_z_loss_clip": 0.75244141, + "router_z_loss_mlp": 0.1628418, + "step": 5706, + "time_per_iteration": 2.517820119857788 + }, + { + "auxiliary_loss_clip": 0.01154312, + "auxiliary_loss_mlp": 0.01037245, + "balance_loss_clip": 1.07230449, + "balance_loss_mlp": 1.02053213, + "epoch": 0.16560269282107828, + "flos": 11211922229760.0, + "grad_norm": 3.0152695604817765, + "language_loss": 0.82194924, + "learning_rate": 3.8102075313158487e-06, + "loss": 0.8438648, + "num_input_tokens_seen": 163316165, + "router_z_loss_clip": 0.81982422, + "router_z_loss_mlp": 0.16723633, + "step": 5707, + "time_per_iteration": 2.509316921234131 + }, + { + "auxiliary_loss_clip": 0.01150716, + "auxiliary_loss_mlp": 0.01043422, + "balance_loss_clip": 1.06786442, + "balance_loss_mlp": 1.02532613, + "epoch": 0.16563171028959434, + "flos": 19237417983360.0, + "grad_norm": 2.8047623068475382, + "language_loss": 0.82320124, + "learning_rate": 3.8101276037252923e-06, + "loss": 0.8451426, + "num_input_tokens_seen": 163329610, + "router_z_loss_clip": 0.82861328, + "router_z_loss_mlp": 0.18103027, + "step": 5708, + "time_per_iteration": 2.5205774307250977 + }, + { + "auxiliary_loss_clip": 0.01148888, + "auxiliary_loss_mlp": 0.01039787, + "balance_loss_clip": 1.07030594, + "balance_loss_mlp": 1.02393174, + "epoch": 0.1656607277581104, + "flos": 56055154884480.0, + "grad_norm": 2.249865392414679, + "language_loss": 0.77961969, + "learning_rate": 3.8100476601469564e-06, + "loss": 0.8015064, + "num_input_tokens_seen": 163350825, + "router_z_loss_clip": 0.78613281, + "router_z_loss_mlp": 0.15844727, + "step": 5709, + "time_per_iteration": 2.8954529762268066 + }, + { + "auxiliary_loss_clip": 0.01152417, + "auxiliary_loss_mlp": 0.01043455, + "balance_loss_clip": 1.07311821, + "balance_loss_mlp": 1.02675438, + "epoch": 0.16568974522662644, + "flos": 35912178097920.0, + "grad_norm": 2.4120653524475744, + "language_loss": 0.83858317, + "learning_rate": 3.8099677005815475e-06, + "loss": 0.86054194, + "num_input_tokens_seen": 163367640, + "router_z_loss_clip": 0.79394531, + "router_z_loss_mlp": 0.16699219, + "step": 5710, + "time_per_iteration": 2.664179801940918 + }, + { + "auxiliary_loss_clip": 0.01156138, + "auxiliary_loss_mlp": 0.01047256, + "balance_loss_clip": 1.0753485, + "balance_loss_mlp": 1.03033447, + "epoch": 0.16571876269514246, + "flos": 11138233478400.0, + "grad_norm": 3.7772599731162484, + "language_loss": 0.84533894, + "learning_rate": 3.809887725029771e-06, + "loss": 0.86737287, + "num_input_tokens_seen": 163380420, + "router_z_loss_clip": 0.80712891, + "router_z_loss_mlp": 0.16918945, + "step": 5711, + "time_per_iteration": 2.655787944793701 + }, + { + "auxiliary_loss_clip": 0.01157816, + "auxiliary_loss_mlp": 0.01043194, + "balance_loss_clip": 1.07383394, + "balance_loss_mlp": 1.02470422, + "epoch": 0.16574778016365851, + "flos": 20917714256640.0, + "grad_norm": 1.9787846052976426, + "language_loss": 0.91789281, + "learning_rate": 3.8098077334923344e-06, + "loss": 0.9399029, + "num_input_tokens_seen": 163395670, + "router_z_loss_clip": 0.84130859, + "router_z_loss_mlp": 0.18493652, + "step": 5712, + "time_per_iteration": 2.633084774017334 + }, + { + "auxiliary_loss_clip": 0.01159301, + "auxiliary_loss_mlp": 0.01050638, + "balance_loss_clip": 1.07499313, + "balance_loss_mlp": 1.03257799, + "epoch": 0.16577679763217457, + "flos": 25807346904960.0, + "grad_norm": 2.7787286070145223, + "language_loss": 1.15489101, + "learning_rate": 3.809727725969943e-06, + "loss": 1.17699027, + "num_input_tokens_seen": 163415450, + "router_z_loss_clip": 0.84277344, + "router_z_loss_mlp": 0.18066406, + "step": 5713, + "time_per_iteration": 2.616039752960205 + }, + { + "auxiliary_loss_clip": 0.01062191, + "auxiliary_loss_mlp": 0.01000091, + "balance_loss_clip": 1.0430851, + "balance_loss_mlp": 0.99934274, + "epoch": 0.16580581510069062, + "flos": 69331583214720.0, + "grad_norm": 0.7913859326492467, + "language_loss": 0.48775244, + "learning_rate": 3.809647702463304e-06, + "loss": 0.50837529, + "num_input_tokens_seen": 163469760, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.00747681, + "step": 5714, + "time_per_iteration": 3.015605926513672 + }, + { + "auxiliary_loss_clip": 0.01158428, + "auxiliary_loss_mlp": 0.01044596, + "balance_loss_clip": 1.07965767, + "balance_loss_mlp": 1.02866375, + "epoch": 0.16583483256920667, + "flos": 32920316616960.0, + "grad_norm": 3.502100897866171, + "language_loss": 0.87116033, + "learning_rate": 3.8095676629731245e-06, + "loss": 0.89319062, + "num_input_tokens_seen": 163485215, + "router_z_loss_clip": 0.78759766, + "router_z_loss_mlp": 0.15930176, + "step": 5715, + "time_per_iteration": 2.6675398349761963 + }, + { + "auxiliary_loss_clip": 0.01161408, + "auxiliary_loss_mlp": 0.01050664, + "balance_loss_clip": 1.07541919, + "balance_loss_mlp": 1.03184712, + "epoch": 0.16586385003772272, + "flos": 37332013875840.0, + "grad_norm": 2.1571260546936744, + "language_loss": 1.00319362, + "learning_rate": 3.8094876075001113e-06, + "loss": 1.02531433, + "num_input_tokens_seen": 163503220, + "router_z_loss_clip": 0.86181641, + "router_z_loss_mlp": 0.18811035, + "step": 5716, + "time_per_iteration": 2.687002420425415 + }, + { + "auxiliary_loss_clip": 0.01151665, + "auxiliary_loss_mlp": 0.01046787, + "balance_loss_clip": 1.07567084, + "balance_loss_mlp": 1.0313201, + "epoch": 0.16589286750623874, + "flos": 30990262101120.0, + "grad_norm": 2.7156771755818125, + "language_loss": 0.60263097, + "learning_rate": 3.809407536044971e-06, + "loss": 0.62461549, + "num_input_tokens_seen": 163521080, + "router_z_loss_clip": 0.76025391, + "router_z_loss_mlp": 0.15460205, + "step": 5717, + "time_per_iteration": 2.5650124549865723 + }, + { + "auxiliary_loss_clip": 0.01148845, + "auxiliary_loss_mlp": 0.01035452, + "balance_loss_clip": 1.07351637, + "balance_loss_mlp": 1.02115297, + "epoch": 0.1659218849747548, + "flos": 25744109011200.0, + "grad_norm": 2.471954815676, + "language_loss": 0.74903393, + "learning_rate": 3.8093274486084108e-06, + "loss": 0.77087688, + "num_input_tokens_seen": 163543855, + "router_z_loss_clip": 0.75390625, + "router_z_loss_mlp": 0.14282227, + "step": 5718, + "time_per_iteration": 2.660449981689453 + }, + { + "auxiliary_loss_clip": 0.01156947, + "auxiliary_loss_mlp": 0.01041793, + "balance_loss_clip": 1.07511258, + "balance_loss_mlp": 1.02299416, + "epoch": 0.16595090244327085, + "flos": 29195194896000.0, + "grad_norm": 2.8045039554079403, + "language_loss": 0.9014101, + "learning_rate": 3.8092473451911385e-06, + "loss": 0.92339754, + "num_input_tokens_seen": 163556875, + "router_z_loss_clip": 0.81835938, + "router_z_loss_mlp": 0.18786621, + "step": 5719, + "time_per_iteration": 2.595879316329956 + }, + { + "auxiliary_loss_clip": 0.01158817, + "auxiliary_loss_mlp": 0.01039933, + "balance_loss_clip": 1.07576442, + "balance_loss_mlp": 1.02158689, + "epoch": 0.1659799199117869, + "flos": 45029501608320.0, + "grad_norm": 2.429033078274894, + "language_loss": 1.10272169, + "learning_rate": 3.809167225793862e-06, + "loss": 1.12470913, + "num_input_tokens_seen": 163570895, + "router_z_loss_clip": 0.83105469, + "router_z_loss_mlp": 0.18347168, + "step": 5720, + "time_per_iteration": 2.765749931335449 + }, + { + "auxiliary_loss_clip": 0.01159305, + "auxiliary_loss_mlp": 0.01047875, + "balance_loss_clip": 1.07511306, + "balance_loss_mlp": 1.02862298, + "epoch": 0.16600893738030295, + "flos": 27777406193280.0, + "grad_norm": 4.580299334499558, + "language_loss": 0.85899621, + "learning_rate": 3.8090870904172883e-06, + "loss": 0.88106799, + "num_input_tokens_seen": 163587150, + "router_z_loss_clip": 0.84228516, + "router_z_loss_mlp": 0.19238281, + "step": 5721, + "time_per_iteration": 2.645244836807251 + }, + { + "auxiliary_loss_clip": 0.01153301, + "auxiliary_loss_mlp": 0.01038355, + "balance_loss_clip": 1.07463384, + "balance_loss_mlp": 1.02175486, + "epoch": 0.166037954848819, + "flos": 31682511987840.0, + "grad_norm": 1.7679620142171024, + "language_loss": 0.87084097, + "learning_rate": 3.809006939062126e-06, + "loss": 0.89275753, + "num_input_tokens_seen": 163605820, + "router_z_loss_clip": 0.78759766, + "router_z_loss_mlp": 0.16589355, + "step": 5722, + "time_per_iteration": 2.661421775817871 + }, + { + "auxiliary_loss_clip": 0.01156949, + "auxiliary_loss_mlp": 0.01032313, + "balance_loss_clip": 1.07776403, + "balance_loss_mlp": 1.01720905, + "epoch": 0.16606697231733503, + "flos": 25879670939520.0, + "grad_norm": 1.94955304216184, + "language_loss": 0.75828505, + "learning_rate": 3.808926771729081e-06, + "loss": 0.78017765, + "num_input_tokens_seen": 163619870, + "router_z_loss_clip": 0.79248047, + "router_z_loss_mlp": 0.15118408, + "step": 5723, + "time_per_iteration": 2.6052370071411133 + }, + { + "auxiliary_loss_clip": 0.0106126, + "auxiliary_loss_mlp": 0.00998726, + "balance_loss_clip": 1.04189074, + "balance_loss_mlp": 0.99792719, + "epoch": 0.16609598978585108, + "flos": 73102886847360.0, + "grad_norm": 0.6313193212797676, + "language_loss": 0.47659227, + "learning_rate": 3.8088465884188636e-06, + "loss": 0.49719214, + "num_input_tokens_seen": 163685955, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.00799561, + "step": 5724, + "time_per_iteration": 3.271549940109253 + }, + { + "auxiliary_loss_clip": 0.01157771, + "auxiliary_loss_mlp": 0.01059654, + "balance_loss_clip": 1.07143712, + "balance_loss_mlp": 1.04012787, + "epoch": 0.16612500725436713, + "flos": 39524144999040.0, + "grad_norm": 2.1894443487760222, + "language_loss": 0.80919445, + "learning_rate": 3.808766389132181e-06, + "loss": 0.83136868, + "num_input_tokens_seen": 163706430, + "router_z_loss_clip": 0.86376953, + "router_z_loss_mlp": 0.19519043, + "step": 5725, + "time_per_iteration": 2.6885972023010254 + }, + { + "auxiliary_loss_clip": 0.01154575, + "auxiliary_loss_mlp": 0.01041629, + "balance_loss_clip": 1.07857108, + "balance_loss_mlp": 1.02585769, + "epoch": 0.16615402472288318, + "flos": 28032264167040.0, + "grad_norm": 2.0668028214910237, + "language_loss": 0.96203607, + "learning_rate": 3.808686173869742e-06, + "loss": 0.98399806, + "num_input_tokens_seen": 163724880, + "router_z_loss_clip": 0.76074219, + "router_z_loss_mlp": 0.15771484, + "step": 5726, + "time_per_iteration": 2.660820722579956 + }, + { + "auxiliary_loss_clip": 0.01152717, + "auxiliary_loss_mlp": 0.01039811, + "balance_loss_clip": 1.0738591, + "balance_loss_mlp": 1.02523172, + "epoch": 0.16618304219139923, + "flos": 30513691428480.0, + "grad_norm": 2.142289418210267, + "language_loss": 0.84307677, + "learning_rate": 3.8086059426322546e-06, + "loss": 0.8650021, + "num_input_tokens_seen": 163740370, + "router_z_loss_clip": 0.7890625, + "router_z_loss_mlp": 0.14581299, + "step": 5727, + "time_per_iteration": 2.6649394035339355 + }, + { + "auxiliary_loss_clip": 0.01151462, + "auxiliary_loss_mlp": 0.01050362, + "balance_loss_clip": 1.0729183, + "balance_loss_mlp": 1.03310072, + "epoch": 0.16621205965991526, + "flos": 16901069754240.0, + "grad_norm": 2.7549620159387294, + "language_loss": 0.91155773, + "learning_rate": 3.808525695420427e-06, + "loss": 0.93357599, + "num_input_tokens_seen": 163751635, + "router_z_loss_clip": 0.78613281, + "router_z_loss_mlp": 0.17248535, + "step": 5728, + "time_per_iteration": 2.513063430786133 + }, + { + "auxiliary_loss_clip": 0.01162209, + "auxiliary_loss_mlp": 0.01050838, + "balance_loss_clip": 1.07638597, + "balance_loss_mlp": 1.03330219, + "epoch": 0.1662410771284313, + "flos": 53899329432960.0, + "grad_norm": 2.2785305523789248, + "language_loss": 0.88376343, + "learning_rate": 3.8084454322349698e-06, + "loss": 0.90589392, + "num_input_tokens_seen": 163769320, + "router_z_loss_clip": 0.85791016, + "router_z_loss_mlp": 0.17541504, + "step": 5729, + "time_per_iteration": 2.795652389526367 + }, + { + "auxiliary_loss_clip": 0.01058836, + "auxiliary_loss_mlp": 0.01015088, + "balance_loss_clip": 1.0391916, + "balance_loss_mlp": 1.01428306, + "epoch": 0.16627009459694736, + "flos": 74776323623040.0, + "grad_norm": 0.6489323053686219, + "language_loss": 0.45549726, + "learning_rate": 3.80836515307659e-06, + "loss": 0.47623649, + "num_input_tokens_seen": 163833655, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.00805664, + "step": 5730, + "time_per_iteration": 3.174844264984131 + }, + { + "auxiliary_loss_clip": 0.01153496, + "auxiliary_loss_mlp": 0.01047773, + "balance_loss_clip": 1.07191932, + "balance_loss_mlp": 1.03030324, + "epoch": 0.1662991120654634, + "flos": 43538993821440.0, + "grad_norm": 1.7458472887194985, + "language_loss": 0.79034948, + "learning_rate": 3.808284857945998e-06, + "loss": 0.81236225, + "num_input_tokens_seen": 163855490, + "router_z_loss_clip": 0.81689453, + "router_z_loss_mlp": 0.17474365, + "step": 5731, + "time_per_iteration": 2.80802059173584 + }, + { + "auxiliary_loss_clip": 0.01157342, + "auxiliary_loss_mlp": 0.01052743, + "balance_loss_clip": 1.07367539, + "balance_loss_mlp": 1.03559434, + "epoch": 0.16632812953397946, + "flos": 11648057166720.0, + "grad_norm": 3.0651400029867144, + "language_loss": 0.86678672, + "learning_rate": 3.8082045468439015e-06, + "loss": 0.88888752, + "num_input_tokens_seen": 163866505, + "router_z_loss_clip": 0.8359375, + "router_z_loss_mlp": 0.17175293, + "step": 5732, + "time_per_iteration": 2.4967191219329834 + }, + { + "auxiliary_loss_clip": 0.01055254, + "auxiliary_loss_mlp": 0.01008851, + "balance_loss_clip": 1.03571439, + "balance_loss_mlp": 1.00798059, + "epoch": 0.16635714700249551, + "flos": 52502840472960.0, + "grad_norm": 0.6717634447543206, + "language_loss": 0.49566841, + "learning_rate": 3.808124219771011e-06, + "loss": 0.5163095, + "num_input_tokens_seen": 163926465, + "router_z_loss_clip": 0.1953125, + "router_z_loss_mlp": 0.00872803, + "step": 5733, + "time_per_iteration": 3.148301124572754 + }, + { + "auxiliary_loss_clip": 0.01152687, + "auxiliary_loss_mlp": 0.01039994, + "balance_loss_clip": 1.074682, + "balance_loss_mlp": 1.02529562, + "epoch": 0.16638616447101154, + "flos": 22998091190400.0, + "grad_norm": 1.8196422087542143, + "language_loss": 0.6960541, + "learning_rate": 3.8080438767280364e-06, + "loss": 0.71798092, + "num_input_tokens_seen": 163949090, + "router_z_loss_clip": 0.78027344, + "router_z_loss_mlp": 0.14703369, + "step": 5734, + "time_per_iteration": 2.773531436920166 + }, + { + "auxiliary_loss_clip": 0.01055121, + "auxiliary_loss_mlp": 0.01008018, + "balance_loss_clip": 1.03557849, + "balance_loss_mlp": 1.00718904, + "epoch": 0.1664151819395276, + "flos": 66719693756160.0, + "grad_norm": 0.6031528710129249, + "language_loss": 0.49125922, + "learning_rate": 3.807963517715686e-06, + "loss": 0.51189065, + "num_input_tokens_seen": 164017345, + "router_z_loss_clip": 0.1953125, + "router_z_loss_mlp": 0.00830078, + "step": 5735, + "time_per_iteration": 3.1755595207214355 + }, + { + "auxiliary_loss_clip": 0.01148351, + "auxiliary_loss_mlp": 0.01032986, + "balance_loss_clip": 1.06750607, + "balance_loss_mlp": 1.01689911, + "epoch": 0.16644419940804364, + "flos": 49010091834240.0, + "grad_norm": 3.1239639844968097, + "language_loss": 0.96164012, + "learning_rate": 3.8078831427346707e-06, + "loss": 0.98345345, + "num_input_tokens_seen": 164039625, + "router_z_loss_clip": 0.80810547, + "router_z_loss_mlp": 0.16082764, + "step": 5736, + "time_per_iteration": 2.7864255905151367 + }, + { + "auxiliary_loss_clip": 0.01148809, + "auxiliary_loss_mlp": 0.01038614, + "balance_loss_clip": 1.06624842, + "balance_loss_mlp": 1.02334368, + "epoch": 0.1664732168765597, + "flos": 36421822218240.0, + "grad_norm": 1.9761699921339173, + "language_loss": 0.83337319, + "learning_rate": 3.8078027517857e-06, + "loss": 0.85524738, + "num_input_tokens_seen": 164057405, + "router_z_loss_clip": 0.82568359, + "router_z_loss_mlp": 0.15258789, + "step": 5737, + "time_per_iteration": 2.6775712966918945 + }, + { + "auxiliary_loss_clip": 0.01146561, + "auxiliary_loss_mlp": 0.01046319, + "balance_loss_clip": 1.06659794, + "balance_loss_mlp": 1.02856326, + "epoch": 0.16650223434507574, + "flos": 20660162762880.0, + "grad_norm": 3.321219314897044, + "language_loss": 0.88985455, + "learning_rate": 3.8077223448694833e-06, + "loss": 0.91178334, + "num_input_tokens_seen": 164070905, + "router_z_loss_clip": 0.79980469, + "router_z_loss_mlp": 0.17730713, + "step": 5738, + "time_per_iteration": 2.551509380340576 + }, + { + "auxiliary_loss_clip": 0.01137677, + "auxiliary_loss_mlp": 0.01036363, + "balance_loss_clip": 1.0657413, + "balance_loss_mlp": 1.02350044, + "epoch": 0.16653125181359177, + "flos": 16538300346240.0, + "grad_norm": 2.38450419432797, + "language_loss": 0.62658668, + "learning_rate": 3.8076419219867317e-06, + "loss": 0.64832711, + "num_input_tokens_seen": 164085130, + "router_z_loss_clip": 0.71972656, + "router_z_loss_mlp": 0.12872314, + "step": 5739, + "time_per_iteration": 2.489520311355591 + }, + { + "auxiliary_loss_clip": 0.01051529, + "auxiliary_loss_mlp": 0.00999609, + "balance_loss_clip": 1.03175926, + "balance_loss_mlp": 0.99880439, + "epoch": 0.16656026928210782, + "flos": 74778873488640.0, + "grad_norm": 0.6948491571150405, + "language_loss": 0.49518475, + "learning_rate": 3.807561483138155e-06, + "loss": 0.51569617, + "num_input_tokens_seen": 164152530, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.00805664, + "step": 5740, + "time_per_iteration": 3.174288034439087 + }, + { + "auxiliary_loss_clip": 0.01143603, + "auxiliary_loss_mlp": 0.01042458, + "balance_loss_clip": 1.06675982, + "balance_loss_mlp": 1.02713919, + "epoch": 0.16658928675062387, + "flos": 10771298092800.0, + "grad_norm": 3.0287947769649297, + "language_loss": 0.79347765, + "learning_rate": 3.8074810283244638e-06, + "loss": 0.81533837, + "num_input_tokens_seen": 164163090, + "router_z_loss_clip": 0.76806641, + "router_z_loss_mlp": 0.15319824, + "step": 5741, + "time_per_iteration": 2.534113883972168 + }, + { + "auxiliary_loss_clip": 0.01136982, + "auxiliary_loss_mlp": 0.01044607, + "balance_loss_clip": 1.06211567, + "balance_loss_mlp": 1.02967572, + "epoch": 0.16661830421913992, + "flos": 59006975679360.0, + "grad_norm": 2.633835177068543, + "language_loss": 1.02268791, + "learning_rate": 3.8074005575463684e-06, + "loss": 1.04450381, + "num_input_tokens_seen": 164181850, + "router_z_loss_clip": 0.74853516, + "router_z_loss_mlp": 0.14929199, + "step": 5742, + "time_per_iteration": 2.8837552070617676 + }, + { + "auxiliary_loss_clip": 0.01148111, + "auxiliary_loss_mlp": 0.01038941, + "balance_loss_clip": 1.06836891, + "balance_loss_mlp": 1.02325332, + "epoch": 0.16664732168765597, + "flos": 28539286594560.0, + "grad_norm": 3.5258919125643544, + "language_loss": 0.96759748, + "learning_rate": 3.8073200708045806e-06, + "loss": 0.98946804, + "num_input_tokens_seen": 164196085, + "router_z_loss_clip": 0.79785156, + "router_z_loss_mlp": 0.15679932, + "step": 5743, + "time_per_iteration": 2.5963265895843506 + }, + { + "auxiliary_loss_clip": 0.01051515, + "auxiliary_loss_mlp": 0.01000635, + "balance_loss_clip": 1.03201962, + "balance_loss_mlp": 0.99972916, + "epoch": 0.16667633915617203, + "flos": 73925024313600.0, + "grad_norm": 0.6711763703947231, + "language_loss": 0.47944999, + "learning_rate": 3.80723956809981e-06, + "loss": 0.49997148, + "num_input_tokens_seen": 164251340, + "router_z_loss_clip": 0.1953125, + "router_z_loss_mlp": 0.0090332, + "step": 5744, + "time_per_iteration": 3.0810062885284424 + }, + { + "auxiliary_loss_clip": 0.01150743, + "auxiliary_loss_mlp": 0.01045018, + "balance_loss_clip": 1.06604195, + "balance_loss_mlp": 1.02670765, + "epoch": 0.16670535662468805, + "flos": 35804158922880.0, + "grad_norm": 2.449816566852161, + "language_loss": 1.02413571, + "learning_rate": 3.8071590494327683e-06, + "loss": 1.04609323, + "num_input_tokens_seen": 164267900, + "router_z_loss_clip": 0.84521484, + "router_z_loss_mlp": 0.1831665, + "step": 5745, + "time_per_iteration": 2.693096160888672 + }, + { + "auxiliary_loss_clip": 0.01152439, + "auxiliary_loss_mlp": 0.01049603, + "balance_loss_clip": 1.06848049, + "balance_loss_mlp": 1.03187013, + "epoch": 0.1667343740932041, + "flos": 25841390019840.0, + "grad_norm": 5.789738799040445, + "language_loss": 0.85002315, + "learning_rate": 3.807078514804167e-06, + "loss": 0.87204361, + "num_input_tokens_seen": 164283295, + "router_z_loss_clip": 0.83935547, + "router_z_loss_mlp": 0.17730713, + "step": 5746, + "time_per_iteration": 2.5974390506744385 + }, + { + "auxiliary_loss_clip": 0.01145289, + "auxiliary_loss_mlp": 0.01041991, + "balance_loss_clip": 1.06390941, + "balance_loss_mlp": 1.02598083, + "epoch": 0.16676339156172015, + "flos": 15991524541440.0, + "grad_norm": 3.384552609677209, + "language_loss": 0.97571701, + "learning_rate": 3.806997964214717e-06, + "loss": 0.99758983, + "num_input_tokens_seen": 164295815, + "router_z_loss_clip": 0.81396484, + "router_z_loss_mlp": 0.16009521, + "step": 5747, + "time_per_iteration": 2.512918472290039 + }, + { + "auxiliary_loss_clip": 0.01143663, + "auxiliary_loss_mlp": 0.01036972, + "balance_loss_clip": 1.06699145, + "balance_loss_mlp": 1.02137983, + "epoch": 0.1667924090302362, + "flos": 20005367783040.0, + "grad_norm": 1.8431189224372406, + "language_loss": 0.80030876, + "learning_rate": 3.8069173976651295e-06, + "loss": 0.82211512, + "num_input_tokens_seen": 164311750, + "router_z_loss_clip": 0.76708984, + "router_z_loss_mlp": 0.15588379, + "step": 5748, + "time_per_iteration": 2.5742909908294678 + }, + { + "auxiliary_loss_clip": 0.01145836, + "auxiliary_loss_mlp": 0.01045796, + "balance_loss_clip": 1.06927192, + "balance_loss_mlp": 1.02978075, + "epoch": 0.16682142649875226, + "flos": 24562610951040.0, + "grad_norm": 2.1872904912364666, + "language_loss": 0.87488091, + "learning_rate": 3.806836815156116e-06, + "loss": 0.89679736, + "num_input_tokens_seen": 164324380, + "router_z_loss_clip": 0.765625, + "router_z_loss_mlp": 0.16009521, + "step": 5749, + "time_per_iteration": 2.578540086746216 + }, + { + "auxiliary_loss_clip": 0.01138277, + "auxiliary_loss_mlp": 0.01036323, + "balance_loss_clip": 1.06064749, + "balance_loss_mlp": 1.020051, + "epoch": 0.1668504439672683, + "flos": 39861921519360.0, + "grad_norm": 1.7889369731583327, + "language_loss": 0.70067614, + "learning_rate": 3.806756216688389e-06, + "loss": 0.72242212, + "num_input_tokens_seen": 164344480, + "router_z_loss_clip": 0.77539062, + "router_z_loss_mlp": 0.16271973, + "step": 5750, + "time_per_iteration": 2.6074881553649902 + }, + { + "auxiliary_loss_clip": 0.01149215, + "auxiliary_loss_mlp": 0.01043669, + "balance_loss_clip": 1.0668757, + "balance_loss_mlp": 1.0266279, + "epoch": 0.16687946143578433, + "flos": 32343591847680.0, + "grad_norm": 1.6979254774178598, + "language_loss": 1.11049747, + "learning_rate": 3.8066756022626604e-06, + "loss": 1.13242638, + "num_input_tokens_seen": 164367315, + "router_z_loss_clip": 0.82470703, + "router_z_loss_mlp": 0.17053223, + "step": 5751, + "time_per_iteration": 2.6751368045806885 + }, + { + "auxiliary_loss_clip": 0.01140347, + "auxiliary_loss_mlp": 0.01038362, + "balance_loss_clip": 1.06305695, + "balance_loss_mlp": 1.02207232, + "epoch": 0.16690847890430038, + "flos": 11831417118720.0, + "grad_norm": 1.9879210691307614, + "language_loss": 0.7519303, + "learning_rate": 3.806594971879641e-06, + "loss": 0.7737174, + "num_input_tokens_seen": 164380495, + "router_z_loss_clip": 0.77197266, + "router_z_loss_mlp": 0.16308594, + "step": 5752, + "time_per_iteration": 2.50974440574646 + }, + { + "auxiliary_loss_clip": 0.0114516, + "auxiliary_loss_mlp": 0.01037149, + "balance_loss_clip": 1.06183863, + "balance_loss_mlp": 1.01974452, + "epoch": 0.16693749637281644, + "flos": 36248913123840.0, + "grad_norm": 1.9059602533768174, + "language_loss": 0.79605836, + "learning_rate": 3.806514325540044e-06, + "loss": 0.81788146, + "num_input_tokens_seen": 164402680, + "router_z_loss_clip": 0.83398438, + "router_z_loss_mlp": 0.17419434, + "step": 5753, + "time_per_iteration": 2.669724225997925 + }, + { + "auxiliary_loss_clip": 0.01049725, + "auxiliary_loss_mlp": 0.01010291, + "balance_loss_clip": 1.02957296, + "balance_loss_mlp": 1.00945044, + "epoch": 0.1669665138413325, + "flos": 60149262044160.0, + "grad_norm": 0.6456285665131631, + "language_loss": 0.46673244, + "learning_rate": 3.806433663244582e-06, + "loss": 0.48733264, + "num_input_tokens_seen": 164467445, + "router_z_loss_clip": 0.20214844, + "router_z_loss_mlp": 0.00842285, + "step": 5754, + "time_per_iteration": 3.2174453735351562 + }, + { + "auxiliary_loss_clip": 0.01146592, + "auxiliary_loss_mlp": 0.01039827, + "balance_loss_clip": 1.06365848, + "balance_loss_mlp": 1.0227325, + "epoch": 0.16699553130984854, + "flos": 36645977041920.0, + "grad_norm": 1.9873593209445028, + "language_loss": 0.79618317, + "learning_rate": 3.8063529849939663e-06, + "loss": 0.81804734, + "num_input_tokens_seen": 164488160, + "router_z_loss_clip": 0.82958984, + "router_z_loss_mlp": 0.17089844, + "step": 5755, + "time_per_iteration": 2.664226770401001 + }, + { + "auxiliary_loss_clip": 0.01140858, + "auxiliary_loss_mlp": 0.01043474, + "balance_loss_clip": 1.06382108, + "balance_loss_mlp": 1.02747011, + "epoch": 0.16702454877836456, + "flos": 16757570920320.0, + "grad_norm": 2.0892052076410828, + "language_loss": 0.82090628, + "learning_rate": 3.80627229078891e-06, + "loss": 0.8427496, + "num_input_tokens_seen": 164501670, + "router_z_loss_clip": 0.76953125, + "router_z_loss_mlp": 0.15985107, + "step": 5756, + "time_per_iteration": 2.5256247520446777 + }, + { + "auxiliary_loss_clip": 0.0115158, + "auxiliary_loss_mlp": 0.0104212, + "balance_loss_clip": 1.06661868, + "balance_loss_mlp": 1.02637255, + "epoch": 0.1670535662468806, + "flos": 47622683059200.0, + "grad_norm": 1.8530709201948108, + "language_loss": 0.89417434, + "learning_rate": 3.806191580630126e-06, + "loss": 0.91611135, + "num_input_tokens_seen": 164528155, + "router_z_loss_clip": 0.84912109, + "router_z_loss_mlp": 0.1574707, + "step": 5757, + "time_per_iteration": 2.8366281986236572 + }, + { + "auxiliary_loss_clip": 0.01155706, + "auxiliary_loss_mlp": 0.01050886, + "balance_loss_clip": 1.0710113, + "balance_loss_mlp": 1.031991, + "epoch": 0.16708258371539667, + "flos": 12342713264640.0, + "grad_norm": 2.376871262109412, + "language_loss": 0.86490405, + "learning_rate": 3.8061108545183275e-06, + "loss": 0.88696992, + "num_input_tokens_seen": 164541700, + "router_z_loss_clip": 0.84765625, + "router_z_loss_mlp": 0.18884277, + "step": 5758, + "time_per_iteration": 2.5213897228240967 + }, + { + "auxiliary_loss_clip": 0.01141854, + "auxiliary_loss_mlp": 0.01038759, + "balance_loss_clip": 1.06648982, + "balance_loss_mlp": 1.02431095, + "epoch": 0.16711160118391272, + "flos": 34203333490560.0, + "grad_norm": 1.7676893387858652, + "language_loss": 0.82046968, + "learning_rate": 3.806030112454227e-06, + "loss": 0.84227574, + "num_input_tokens_seen": 164560520, + "router_z_loss_clip": 0.75244141, + "router_z_loss_mlp": 0.14434814, + "step": 5759, + "time_per_iteration": 2.662755250930786 + }, + { + "auxiliary_loss_clip": 0.01054757, + "auxiliary_loss_mlp": 0.00999429, + "balance_loss_clip": 1.03450513, + "balance_loss_mlp": 0.99857062, + "epoch": 0.16714061865242877, + "flos": 59009528522880.0, + "grad_norm": 0.8279827965046812, + "language_loss": 0.48673779, + "learning_rate": 3.8059493544385373e-06, + "loss": 0.50727963, + "num_input_tokens_seen": 164624595, + "router_z_loss_clip": 0.203125, + "router_z_loss_mlp": 0.00860596, + "step": 5760, + "time_per_iteration": 5.528505563735962 + }, + { + "auxiliary_loss_clip": 0.01143097, + "auxiliary_loss_mlp": 0.01040315, + "balance_loss_clip": 1.0670675, + "balance_loss_mlp": 1.02534795, + "epoch": 0.16716963612094482, + "flos": 21573083854080.0, + "grad_norm": 1.9662931239937116, + "language_loss": 0.85435283, + "learning_rate": 3.805868580471973e-06, + "loss": 0.87618691, + "num_input_tokens_seen": 164638785, + "router_z_loss_clip": 0.76074219, + "router_z_loss_mlp": 0.14959717, + "step": 5761, + "time_per_iteration": 4.972748756408691 + }, + { + "auxiliary_loss_clip": 0.01140822, + "auxiliary_loss_mlp": 0.01043358, + "balance_loss_clip": 1.06514239, + "balance_loss_mlp": 1.02777159, + "epoch": 0.16719865358946084, + "flos": 32008329279360.0, + "grad_norm": 2.624840302643648, + "language_loss": 0.66685909, + "learning_rate": 3.8057877905552454e-06, + "loss": 0.68870091, + "num_input_tokens_seen": 164658040, + "router_z_loss_clip": 0.75732422, + "router_z_loss_mlp": 0.15582275, + "step": 5762, + "time_per_iteration": 5.116630792617798 + }, + { + "auxiliary_loss_clip": 0.01054355, + "auxiliary_loss_mlp": 0.01000531, + "balance_loss_clip": 1.03429174, + "balance_loss_mlp": 0.9995597, + "epoch": 0.1672276710579769, + "flos": 64809245088000.0, + "grad_norm": 0.6780294158909292, + "language_loss": 0.48502994, + "learning_rate": 3.8057069846890704e-06, + "loss": 0.50557882, + "num_input_tokens_seen": 164712740, + "router_z_loss_clip": 0.20117188, + "router_z_loss_mlp": 0.00970459, + "step": 5763, + "time_per_iteration": 5.352809429168701 + }, + { + "auxiliary_loss_clip": 0.0113654, + "auxiliary_loss_mlp": 0.01030736, + "balance_loss_clip": 1.06590235, + "balance_loss_mlp": 1.01923275, + "epoch": 0.16725668852649295, + "flos": 11649996501120.0, + "grad_norm": 2.7609414371402505, + "language_loss": 0.75315154, + "learning_rate": 3.8056261628741595e-06, + "loss": 0.77482432, + "num_input_tokens_seen": 164724520, + "router_z_loss_clip": 0.70654297, + "router_z_loss_mlp": 0.11505127, + "step": 5764, + "time_per_iteration": 2.5988006591796875 + }, + { + "auxiliary_loss_clip": 0.01145171, + "auxiliary_loss_mlp": 0.01036237, + "balance_loss_clip": 1.06790614, + "balance_loss_mlp": 1.01989341, + "epoch": 0.167285705995009, + "flos": 15369731182080.0, + "grad_norm": 3.8165010740004752, + "language_loss": 0.60313272, + "learning_rate": 3.8055453251112288e-06, + "loss": 0.62494683, + "num_input_tokens_seen": 164736580, + "router_z_loss_clip": 0.77294922, + "router_z_loss_mlp": 0.16333008, + "step": 5765, + "time_per_iteration": 2.4896230697631836 + }, + { + "auxiliary_loss_clip": 0.01052325, + "auxiliary_loss_mlp": 0.0100133, + "balance_loss_clip": 1.03248262, + "balance_loss_mlp": 1.00045383, + "epoch": 0.16731472346352505, + "flos": 74770936583040.0, + "grad_norm": 0.639646105008088, + "language_loss": 0.48570693, + "learning_rate": 3.8054644714009907e-06, + "loss": 0.50624347, + "num_input_tokens_seen": 164800840, + "router_z_loss_clip": 0.19921875, + "router_z_loss_mlp": 0.00878906, + "step": 5766, + "time_per_iteration": 3.1857569217681885 + }, + { + "auxiliary_loss_clip": 0.01150094, + "auxiliary_loss_mlp": 0.01046426, + "balance_loss_clip": 1.06816459, + "balance_loss_mlp": 1.03103626, + "epoch": 0.1673437409320411, + "flos": 44816910963840.0, + "grad_norm": 1.884619843563206, + "language_loss": 0.9754051, + "learning_rate": 3.8053836017441597e-06, + "loss": 0.99737036, + "num_input_tokens_seen": 164822490, + "router_z_loss_clip": 0.81884766, + "router_z_loss_mlp": 0.15393066, + "step": 5767, + "time_per_iteration": 2.786857843399048 + }, + { + "auxiliary_loss_clip": 0.01145188, + "auxiliary_loss_mlp": 0.01049007, + "balance_loss_clip": 1.06774771, + "balance_loss_mlp": 1.03397489, + "epoch": 0.16737275840055713, + "flos": 52510986904320.0, + "grad_norm": 3.191030294203872, + "language_loss": 0.98528779, + "learning_rate": 3.80530271614145e-06, + "loss": 1.0072298, + "num_input_tokens_seen": 164839300, + "router_z_loss_clip": 0.77441406, + "router_z_loss_mlp": 0.15014648, + "step": 5768, + "time_per_iteration": 2.893920421600342 + }, + { + "auxiliary_loss_clip": 0.01147758, + "auxiliary_loss_mlp": 0.01042031, + "balance_loss_clip": 1.06813216, + "balance_loss_mlp": 1.02719498, + "epoch": 0.16740177586907318, + "flos": 38177207873280.0, + "grad_norm": 1.7781949983402365, + "language_loss": 0.96332437, + "learning_rate": 3.8052218145935767e-06, + "loss": 0.98522222, + "num_input_tokens_seen": 164870590, + "router_z_loss_clip": 0.796875, + "router_z_loss_mlp": 0.14837646, + "step": 5769, + "time_per_iteration": 2.988825798034668 + }, + { + "auxiliary_loss_clip": 0.01046792, + "auxiliary_loss_mlp": 0.01000637, + "balance_loss_clip": 1.02717841, + "balance_loss_mlp": 0.99980277, + "epoch": 0.16743079333758923, + "flos": 67263883781760.0, + "grad_norm": 0.6999524881921253, + "language_loss": 0.49434161, + "learning_rate": 3.8051408971012533e-06, + "loss": 0.51481593, + "num_input_tokens_seen": 164935405, + "router_z_loss_clip": 0.19628906, + "router_z_loss_mlp": 0.00836182, + "step": 5770, + "time_per_iteration": 3.14874005317688 + }, + { + "auxiliary_loss_clip": 0.0113624, + "auxiliary_loss_mlp": 0.01044932, + "balance_loss_clip": 1.06186032, + "balance_loss_mlp": 1.02885103, + "epoch": 0.16745981080610528, + "flos": 14276933758080.0, + "grad_norm": 3.1145422185739102, + "language_loss": 0.7932297, + "learning_rate": 3.8050599636651952e-06, + "loss": 0.81504142, + "num_input_tokens_seen": 164949055, + "router_z_loss_clip": 0.74316406, + "router_z_loss_mlp": 0.16094971, + "step": 5771, + "time_per_iteration": 2.5217559337615967 + }, + { + "auxiliary_loss_clip": 0.01045537, + "auxiliary_loss_mlp": 0.00999183, + "balance_loss_clip": 1.02594721, + "balance_loss_mlp": 0.99834269, + "epoch": 0.16748882827462133, + "flos": 61084195194240.0, + "grad_norm": 0.6387886464083851, + "language_loss": 0.47732207, + "learning_rate": 3.8049790142861175e-06, + "loss": 0.49776927, + "num_input_tokens_seen": 165008915, + "router_z_loss_clip": 0.19628906, + "router_z_loss_mlp": 0.00842285, + "step": 5772, + "time_per_iteration": 3.0262503623962402 + }, + { + "auxiliary_loss_clip": 0.01139005, + "auxiliary_loss_mlp": 0.01039358, + "balance_loss_clip": 1.06394815, + "balance_loss_mlp": 1.0244627, + "epoch": 0.16751784574313736, + "flos": 11284533573120.0, + "grad_norm": 3.1720181694264813, + "language_loss": 0.93152022, + "learning_rate": 3.804898048964734e-06, + "loss": 0.95330387, + "num_input_tokens_seen": 165020825, + "router_z_loss_clip": 0.75, + "router_z_loss_mlp": 0.14892578, + "step": 5773, + "time_per_iteration": 2.5133702754974365 + }, + { + "auxiliary_loss_clip": 0.01131076, + "auxiliary_loss_mlp": 0.0103224, + "balance_loss_clip": 1.05933368, + "balance_loss_mlp": 1.01889467, + "epoch": 0.1675468632116534, + "flos": 10115605272960.0, + "grad_norm": 2.615657755136708, + "language_loss": 0.79143631, + "learning_rate": 3.8048170677017615e-06, + "loss": 0.81306946, + "num_input_tokens_seen": 165031875, + "router_z_loss_clip": 0.71582031, + "router_z_loss_mlp": 0.13354492, + "step": 5774, + "time_per_iteration": 2.502002716064453 + }, + { + "auxiliary_loss_clip": 0.0113507, + "auxiliary_loss_mlp": 0.01033446, + "balance_loss_clip": 1.06314635, + "balance_loss_mlp": 1.02008843, + "epoch": 0.16757588068016946, + "flos": 27667627251840.0, + "grad_norm": 1.9888601148260425, + "language_loss": 0.65084577, + "learning_rate": 3.8047360704979134e-06, + "loss": 0.67253089, + "num_input_tokens_seen": 165047685, + "router_z_loss_clip": 0.71875, + "router_z_loss_mlp": 0.13360596, + "step": 5775, + "time_per_iteration": 2.6463167667388916 + }, + { + "auxiliary_loss_clip": 0.01133928, + "auxiliary_loss_mlp": 0.01035293, + "balance_loss_clip": 1.05948257, + "balance_loss_mlp": 1.0208391, + "epoch": 0.1676048981486855, + "flos": 20923244951040.0, + "grad_norm": 2.211505871062322, + "language_loss": 0.95868236, + "learning_rate": 3.8046550573539066e-06, + "loss": 0.98037457, + "num_input_tokens_seen": 165064700, + "router_z_loss_clip": 0.74511719, + "router_z_loss_mlp": 0.14447021, + "step": 5776, + "time_per_iteration": 2.5337252616882324 + }, + { + "auxiliary_loss_clip": 0.01134903, + "auxiliary_loss_mlp": 0.01035456, + "balance_loss_clip": 1.06339955, + "balance_loss_mlp": 1.02087033, + "epoch": 0.16763391561720156, + "flos": 23178003436800.0, + "grad_norm": 1.7822490886598938, + "language_loss": 0.66657495, + "learning_rate": 3.8045740282704557e-06, + "loss": 0.68827856, + "num_input_tokens_seen": 165081715, + "router_z_loss_clip": 0.71435547, + "router_z_loss_mlp": 0.14581299, + "step": 5777, + "time_per_iteration": 2.5364906787872314 + }, + { + "auxiliary_loss_clip": 0.01136871, + "auxiliary_loss_mlp": 0.01038737, + "balance_loss_clip": 1.06254339, + "balance_loss_mlp": 1.02372873, + "epoch": 0.1676629330857176, + "flos": 17347009104000.0, + "grad_norm": 1.8883822582727725, + "language_loss": 0.69974196, + "learning_rate": 3.804492983248277e-06, + "loss": 0.72149801, + "num_input_tokens_seen": 165097375, + "router_z_loss_clip": 0.74365234, + "router_z_loss_mlp": 0.15008545, + "step": 5778, + "time_per_iteration": 2.5018513202667236 + }, + { + "auxiliary_loss_clip": 0.01134541, + "auxiliary_loss_mlp": 0.01039016, + "balance_loss_clip": 1.06263471, + "balance_loss_mlp": 1.02398992, + "epoch": 0.16769195055423364, + "flos": 10771585401600.0, + "grad_norm": 3.449753818444649, + "language_loss": 0.91402477, + "learning_rate": 3.804411922288086e-06, + "loss": 0.93576038, + "num_input_tokens_seen": 165107310, + "router_z_loss_clip": 0.71923828, + "router_z_loss_mlp": 0.15032959, + "step": 5779, + "time_per_iteration": 2.583808422088623 + }, + { + "auxiliary_loss_clip": 0.01137531, + "auxiliary_loss_mlp": 0.01041486, + "balance_loss_clip": 1.06202674, + "balance_loss_mlp": 1.0249989, + "epoch": 0.1677209680227497, + "flos": 33323198538240.0, + "grad_norm": 1.9615083619883915, + "language_loss": 0.8812176, + "learning_rate": 3.8043308453905984e-06, + "loss": 0.90300775, + "num_input_tokens_seen": 165126455, + "router_z_loss_clip": 0.75488281, + "router_z_loss_mlp": 0.16479492, + "step": 5780, + "time_per_iteration": 2.690080165863037 + }, + { + "auxiliary_loss_clip": 0.01048927, + "auxiliary_loss_mlp": 0.01008042, + "balance_loss_clip": 1.0293324, + "balance_loss_mlp": 1.00711238, + "epoch": 0.16774998549126574, + "flos": 60792097795200.0, + "grad_norm": 0.6624448060058713, + "language_loss": 0.46484578, + "learning_rate": 3.804249752556531e-06, + "loss": 0.48541546, + "num_input_tokens_seen": 165182925, + "router_z_loss_clip": 0.19628906, + "router_z_loss_mlp": 0.00927734, + "step": 5781, + "time_per_iteration": 2.9600636959075928 + }, + { + "auxiliary_loss_clip": 0.01139334, + "auxiliary_loss_mlp": 0.01032898, + "balance_loss_clip": 1.06161427, + "balance_loss_mlp": 1.01734114, + "epoch": 0.1677790029597818, + "flos": 44595701055360.0, + "grad_norm": 2.3823253532102466, + "language_loss": 0.74926567, + "learning_rate": 3.8041686437865995e-06, + "loss": 0.77098799, + "num_input_tokens_seen": 165200070, + "router_z_loss_clip": 0.77734375, + "router_z_loss_mlp": 0.15563965, + "step": 5782, + "time_per_iteration": 2.7569351196289062 + }, + { + "auxiliary_loss_clip": 0.01139599, + "auxiliary_loss_mlp": 0.01041025, + "balance_loss_clip": 1.06304169, + "balance_loss_mlp": 1.02520597, + "epoch": 0.16780802042829784, + "flos": 32372032440960.0, + "grad_norm": 2.276398140108763, + "language_loss": 0.79261136, + "learning_rate": 3.804087519081521e-06, + "loss": 0.8144176, + "num_input_tokens_seen": 165215085, + "router_z_loss_clip": 0.765625, + "router_z_loss_mlp": 0.15820312, + "step": 5783, + "time_per_iteration": 2.6318297386169434 + }, + { + "auxiliary_loss_clip": 0.01144984, + "auxiliary_loss_mlp": 0.01038196, + "balance_loss_clip": 1.06661308, + "balance_loss_mlp": 1.02206111, + "epoch": 0.1678370378968139, + "flos": 18194178349440.0, + "grad_norm": 4.058811116426, + "language_loss": 0.8292774, + "learning_rate": 3.804006378442011e-06, + "loss": 0.85110915, + "num_input_tokens_seen": 165228615, + "router_z_loss_clip": 0.78320312, + "router_z_loss_mlp": 0.16143799, + "step": 5784, + "time_per_iteration": 2.525939464569092 + }, + { + "auxiliary_loss_clip": 0.01137606, + "auxiliary_loss_mlp": 0.01038998, + "balance_loss_clip": 1.06189108, + "balance_loss_mlp": 1.02422857, + "epoch": 0.16786605536532992, + "flos": 74751291843840.0, + "grad_norm": 1.706340461486673, + "language_loss": 0.74257362, + "learning_rate": 3.803925221868787e-06, + "loss": 0.76433969, + "num_input_tokens_seen": 165254750, + "router_z_loss_clip": 0.7578125, + "router_z_loss_mlp": 0.14764404, + "step": 5785, + "time_per_iteration": 2.9914205074310303 + }, + { + "auxiliary_loss_clip": 0.01049186, + "auxiliary_loss_mlp": 0.01001667, + "balance_loss_clip": 1.02959907, + "balance_loss_mlp": 1.00074291, + "epoch": 0.16789507283384597, + "flos": 63361040544000.0, + "grad_norm": 0.6933673979729957, + "language_loss": 0.48774886, + "learning_rate": 3.803844049362565e-06, + "loss": 0.50825739, + "num_input_tokens_seen": 165320525, + "router_z_loss_clip": 0.19628906, + "router_z_loss_mlp": 0.00921631, + "step": 5786, + "time_per_iteration": 3.279507637023926 + }, + { + "auxiliary_loss_clip": 0.01148157, + "auxiliary_loss_mlp": 0.01044303, + "balance_loss_clip": 1.06502223, + "balance_loss_mlp": 1.02624249, + "epoch": 0.16792409030236202, + "flos": 29673273939840.0, + "grad_norm": 2.7050744535623292, + "language_loss": 0.95961761, + "learning_rate": 3.803762860924063e-06, + "loss": 0.98154211, + "num_input_tokens_seen": 165338780, + "router_z_loss_clip": 0.83105469, + "router_z_loss_mlp": 0.18060303, + "step": 5787, + "time_per_iteration": 2.61700439453125 + }, + { + "auxiliary_loss_clip": 0.01137528, + "auxiliary_loss_mlp": 0.01037865, + "balance_loss_clip": 1.06115937, + "balance_loss_mlp": 1.02132511, + "epoch": 0.16795310777087807, + "flos": 16282436791680.0, + "grad_norm": 2.0117220997553154, + "language_loss": 0.74937809, + "learning_rate": 3.803681656553997e-06, + "loss": 0.77113199, + "num_input_tokens_seen": 165354045, + "router_z_loss_clip": 0.76464844, + "router_z_loss_mlp": 0.16540527, + "step": 5788, + "time_per_iteration": 2.554205894470215 + }, + { + "auxiliary_loss_clip": 0.01044545, + "auxiliary_loss_mlp": 0.00999635, + "balance_loss_clip": 1.0248785, + "balance_loss_mlp": 0.99875832, + "epoch": 0.16798212523939413, + "flos": 68426922251520.0, + "grad_norm": 0.6303377382971683, + "language_loss": 0.48271191, + "learning_rate": 3.8036004362530847e-06, + "loss": 0.50315374, + "num_input_tokens_seen": 165418350, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.00878906, + "step": 5789, + "time_per_iteration": 3.168309211730957 + }, + { + "auxiliary_loss_clip": 0.01141058, + "auxiliary_loss_mlp": 0.01045222, + "balance_loss_clip": 1.06035447, + "balance_loss_mlp": 1.02712584, + "epoch": 0.16801114270791015, + "flos": 28176732668160.0, + "grad_norm": 2.343278525268062, + "language_loss": 0.81149226, + "learning_rate": 3.803519200022044e-06, + "loss": 0.83335507, + "num_input_tokens_seen": 165434475, + "router_z_loss_clip": 0.80664062, + "router_z_loss_mlp": 0.1809082, + "step": 5790, + "time_per_iteration": 2.614020347595215 + }, + { + "auxiliary_loss_clip": 0.01137817, + "auxiliary_loss_mlp": 0.01038287, + "balance_loss_clip": 1.06164408, + "balance_loss_mlp": 1.02337396, + "epoch": 0.1680401601764262, + "flos": 16355550925440.0, + "grad_norm": 2.9219145115050478, + "language_loss": 0.53173661, + "learning_rate": 3.8034379478615913e-06, + "loss": 0.55349767, + "num_input_tokens_seen": 165447440, + "router_z_loss_clip": 0.76074219, + "router_z_loss_mlp": 0.14923096, + "step": 5791, + "time_per_iteration": 2.5353822708129883 + }, + { + "auxiliary_loss_clip": 0.01128522, + "auxiliary_loss_mlp": 0.01042257, + "balance_loss_clip": 1.05659747, + "balance_loss_mlp": 1.02743363, + "epoch": 0.16806917764494225, + "flos": 26205416403840.0, + "grad_norm": 2.140277210963504, + "language_loss": 0.95363331, + "learning_rate": 3.8033566797724453e-06, + "loss": 0.97534108, + "num_input_tokens_seen": 165463330, + "router_z_loss_clip": 0.71875, + "router_z_loss_mlp": 0.14831543, + "step": 5792, + "time_per_iteration": 2.584050416946411 + }, + { + "auxiliary_loss_clip": 0.0113019, + "auxiliary_loss_mlp": 0.01042772, + "balance_loss_clip": 1.05715466, + "balance_loss_mlp": 1.02655935, + "epoch": 0.1680981951134583, + "flos": 30657657139200.0, + "grad_norm": 2.308870229476145, + "language_loss": 0.64712441, + "learning_rate": 3.8032753957553233e-06, + "loss": 0.668854, + "num_input_tokens_seen": 165482380, + "router_z_loss_clip": 0.73046875, + "router_z_loss_mlp": 0.1619873, + "step": 5793, + "time_per_iteration": 2.6341586112976074 + }, + { + "auxiliary_loss_clip": 0.01135261, + "auxiliary_loss_mlp": 0.01047524, + "balance_loss_clip": 1.05726802, + "balance_loss_mlp": 1.03017914, + "epoch": 0.16812721258197436, + "flos": 19345939948800.0, + "grad_norm": 2.950516323823487, + "language_loss": 0.90661711, + "learning_rate": 3.8031940958109436e-06, + "loss": 0.92844492, + "num_input_tokens_seen": 165493520, + "router_z_loss_clip": 0.77880859, + "router_z_loss_mlp": 0.17333984, + "step": 5794, + "time_per_iteration": 2.4984798431396484 + }, + { + "auxiliary_loss_clip": 0.01136397, + "auxiliary_loss_mlp": 0.01036146, + "balance_loss_clip": 1.06039453, + "balance_loss_mlp": 1.02079761, + "epoch": 0.1681562300504904, + "flos": 34269983176320.0, + "grad_norm": 2.1102579505773975, + "language_loss": 1.0439167, + "learning_rate": 3.803112779940023e-06, + "loss": 1.06564212, + "num_input_tokens_seen": 165510885, + "router_z_loss_clip": 0.75976562, + "router_z_loss_mlp": 0.15332031, + "step": 5795, + "time_per_iteration": 2.648344039916992 + }, + { + "auxiliary_loss_clip": 0.01142679, + "auxiliary_loss_mlp": 0.01037656, + "balance_loss_clip": 1.06183147, + "balance_loss_mlp": 1.02171791, + "epoch": 0.16818524751900643, + "flos": 37516415322240.0, + "grad_norm": 2.5442449577681585, + "language_loss": 0.93800676, + "learning_rate": 3.8030314481432815e-06, + "loss": 0.95981008, + "num_input_tokens_seen": 165527090, + "router_z_loss_clip": 0.80712891, + "router_z_loss_mlp": 0.15942383, + "step": 5796, + "time_per_iteration": 2.5906176567077637 + }, + { + "auxiliary_loss_clip": 0.01136051, + "auxiliary_loss_mlp": 0.01041751, + "balance_loss_clip": 1.06020844, + "balance_loss_mlp": 1.02741051, + "epoch": 0.16821426498752248, + "flos": 25631887944960.0, + "grad_norm": 1.9594272167218876, + "language_loss": 1.10189319, + "learning_rate": 3.8029501004214363e-06, + "loss": 1.12367117, + "num_input_tokens_seen": 165548625, + "router_z_loss_clip": 0.75927734, + "router_z_loss_mlp": 0.14343262, + "step": 5797, + "time_per_iteration": 2.6251943111419678 + }, + { + "auxiliary_loss_clip": 0.01133711, + "auxiliary_loss_mlp": 0.01042619, + "balance_loss_clip": 1.05822146, + "balance_loss_mlp": 1.02812362, + "epoch": 0.16824328245603853, + "flos": 16247424009600.0, + "grad_norm": 2.763149816393368, + "language_loss": 0.75039506, + "learning_rate": 3.8028687367752064e-06, + "loss": 0.77215838, + "num_input_tokens_seen": 165562355, + "router_z_loss_clip": 0.75634766, + "router_z_loss_mlp": 0.1451416, + "step": 5798, + "time_per_iteration": 2.543682813644409 + }, + { + "auxiliary_loss_clip": 0.01043428, + "auxiliary_loss_mlp": 0.01004939, + "balance_loss_clip": 1.02380681, + "balance_loss_mlp": 1.00394344, + "epoch": 0.16827229992455459, + "flos": 60441934060800.0, + "grad_norm": 0.6335231791874646, + "language_loss": 0.44897139, + "learning_rate": 3.8027873572053106e-06, + "loss": 0.46945506, + "num_input_tokens_seen": 165629105, + "router_z_loss_clip": 0.19628906, + "router_z_loss_mlp": 0.00994873, + "step": 5799, + "time_per_iteration": 3.2766025066375732 + }, + { + "auxiliary_loss_clip": 0.01042398, + "auxiliary_loss_mlp": 0.01008218, + "balance_loss_clip": 1.02283382, + "balance_loss_mlp": 1.00737178, + "epoch": 0.16830131739307064, + "flos": 63249322268160.0, + "grad_norm": 0.6117216742947745, + "language_loss": 0.47798747, + "learning_rate": 3.8027059617124673e-06, + "loss": 0.49849364, + "num_input_tokens_seen": 165692505, + "router_z_loss_clip": 0.1953125, + "router_z_loss_mlp": 0.00848389, + "step": 5800, + "time_per_iteration": 3.1601810455322266 + }, + { + "auxiliary_loss_clip": 0.01140608, + "auxiliary_loss_mlp": 0.01044265, + "balance_loss_clip": 1.06110418, + "balance_loss_mlp": 1.02694356, + "epoch": 0.1683303348615867, + "flos": 12200615061120.0, + "grad_norm": 2.4261048916273658, + "language_loss": 0.74068719, + "learning_rate": 3.8026245502973947e-06, + "loss": 0.76253593, + "num_input_tokens_seen": 165705290, + "router_z_loss_clip": 0.79589844, + "router_z_loss_mlp": 0.17315674, + "step": 5801, + "time_per_iteration": 2.4917023181915283 + }, + { + "auxiliary_loss_clip": 0.01043395, + "auxiliary_loss_mlp": 0.01002517, + "balance_loss_clip": 1.02375376, + "balance_loss_mlp": 1.00156307, + "epoch": 0.1683593523301027, + "flos": 74772875917440.0, + "grad_norm": 0.6417662403532448, + "language_loss": 0.44706362, + "learning_rate": 3.8025431229608127e-06, + "loss": 0.46752274, + "num_input_tokens_seen": 165771815, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.00952148, + "step": 5802, + "time_per_iteration": 3.184701442718506 + }, + { + "auxiliary_loss_clip": 0.01127285, + "auxiliary_loss_mlp": 0.01035103, + "balance_loss_clip": 1.05875468, + "balance_loss_mlp": 1.02123356, + "epoch": 0.16838836979861876, + "flos": 29892113550720.0, + "grad_norm": 2.659546566658851, + "language_loss": 0.79651141, + "learning_rate": 3.8024616797034414e-06, + "loss": 0.81813526, + "num_input_tokens_seen": 165785130, + "router_z_loss_clip": 0.68505859, + "router_z_loss_mlp": 0.13879395, + "step": 5803, + "time_per_iteration": 2.5933730602264404 + }, + { + "auxiliary_loss_clip": 0.01139009, + "auxiliary_loss_mlp": 0.01039254, + "balance_loss_clip": 1.06571114, + "balance_loss_mlp": 1.02401328, + "epoch": 0.16841738726713482, + "flos": 29856490237440.0, + "grad_norm": 1.79935489937145, + "language_loss": 0.75254202, + "learning_rate": 3.8023802205259986e-06, + "loss": 0.77432466, + "num_input_tokens_seen": 165802470, + "router_z_loss_clip": 0.73242188, + "router_z_loss_mlp": 0.15240479, + "step": 5804, + "time_per_iteration": 2.6079089641571045 + }, + { + "auxiliary_loss_clip": 0.01148936, + "auxiliary_loss_mlp": 0.0104636, + "balance_loss_clip": 1.06333399, + "balance_loss_mlp": 1.0266428, + "epoch": 0.16844640473565087, + "flos": 18221864757120.0, + "grad_norm": 2.494072008625288, + "language_loss": 0.82283819, + "learning_rate": 3.8022987454292043e-06, + "loss": 0.84479117, + "num_input_tokens_seen": 165821490, + "router_z_loss_clip": 0.85546875, + "router_z_loss_mlp": 0.19726562, + "step": 5805, + "time_per_iteration": 2.5485939979553223 + }, + { + "auxiliary_loss_clip": 0.01147462, + "auxiliary_loss_mlp": 0.01056656, + "balance_loss_clip": 1.06098819, + "balance_loss_mlp": 1.03705788, + "epoch": 0.16847542220416692, + "flos": 13106604827520.0, + "grad_norm": 3.0106411036326857, + "language_loss": 0.96952921, + "learning_rate": 3.8022172544137785e-06, + "loss": 0.99157035, + "num_input_tokens_seen": 165831950, + "router_z_loss_clip": 0.86474609, + "router_z_loss_mlp": 0.19580078, + "step": 5806, + "time_per_iteration": 2.524508476257324 + }, + { + "auxiliary_loss_clip": 0.01142963, + "auxiliary_loss_mlp": 0.01034056, + "balance_loss_clip": 1.05856419, + "balance_loss_mlp": 1.01597166, + "epoch": 0.16850443967268294, + "flos": 13582924104960.0, + "grad_norm": 1.940242929937938, + "language_loss": 0.71565819, + "learning_rate": 3.80213574748044e-06, + "loss": 0.73742843, + "num_input_tokens_seen": 165846945, + "router_z_loss_clip": 0.84472656, + "router_z_loss_mlp": 0.1807251, + "step": 5807, + "time_per_iteration": 2.527190923690796 + }, + { + "auxiliary_loss_clip": 0.01138937, + "auxiliary_loss_mlp": 0.01037833, + "balance_loss_clip": 1.06045008, + "balance_loss_mlp": 1.0217402, + "epoch": 0.168533457141199, + "flos": 11977968608640.0, + "grad_norm": 3.362920120575483, + "language_loss": 1.02001405, + "learning_rate": 3.8020542246299096e-06, + "loss": 1.04178166, + "num_input_tokens_seen": 165858120, + "router_z_loss_clip": 0.78613281, + "router_z_loss_mlp": 0.16113281, + "step": 5808, + "time_per_iteration": 2.4931752681732178 + }, + { + "auxiliary_loss_clip": 0.01040565, + "auxiliary_loss_mlp": 0.00999231, + "balance_loss_clip": 1.02128291, + "balance_loss_mlp": 0.99840277, + "epoch": 0.16856247460971505, + "flos": 69406026151680.0, + "grad_norm": 0.7373685702334885, + "language_loss": 0.49484035, + "learning_rate": 3.8019726858629073e-06, + "loss": 0.51523829, + "num_input_tokens_seen": 165915995, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.00830078, + "step": 5809, + "time_per_iteration": 3.087857484817505 + }, + { + "auxiliary_loss_clip": 0.01134004, + "auxiliary_loss_mlp": 0.01042539, + "balance_loss_clip": 1.05701017, + "balance_loss_mlp": 1.02691126, + "epoch": 0.1685914920782311, + "flos": 28907083906560.0, + "grad_norm": 3.417368359755431, + "language_loss": 0.95842302, + "learning_rate": 3.801891131180153e-06, + "loss": 0.98018837, + "num_input_tokens_seen": 165929190, + "router_z_loss_clip": 0.76953125, + "router_z_loss_mlp": 0.15625, + "step": 5810, + "time_per_iteration": 2.624286413192749 + }, + { + "auxiliary_loss_clip": 0.01134899, + "auxiliary_loss_mlp": 0.01039296, + "balance_loss_clip": 1.05749667, + "balance_loss_mlp": 1.02280307, + "epoch": 0.16862050954674715, + "flos": 26426410830720.0, + "grad_norm": 2.578876220560653, + "language_loss": 0.78125918, + "learning_rate": 3.8018095605823666e-06, + "loss": 0.80300117, + "num_input_tokens_seen": 165944625, + "router_z_loss_clip": 0.77441406, + "router_z_loss_mlp": 0.16485596, + "step": 5811, + "time_per_iteration": 2.574235439300537 + }, + { + "auxiliary_loss_clip": 0.01128944, + "auxiliary_loss_mlp": 0.01036463, + "balance_loss_clip": 1.05784035, + "balance_loss_mlp": 1.02238488, + "epoch": 0.1686495270152632, + "flos": 15453655309440.0, + "grad_norm": 2.8116050268869186, + "language_loss": 0.75481892, + "learning_rate": 3.80172797407027e-06, + "loss": 0.77647299, + "num_input_tokens_seen": 165959700, + "router_z_loss_clip": 0.7109375, + "router_z_loss_mlp": 0.14074707, + "step": 5812, + "time_per_iteration": 2.5084474086761475 + }, + { + "auxiliary_loss_clip": 0.01131172, + "auxiliary_loss_mlp": 0.01037081, + "balance_loss_clip": 1.05905271, + "balance_loss_mlp": 1.02172101, + "epoch": 0.16867854448377922, + "flos": 30512578106880.0, + "grad_norm": 1.843999413054696, + "language_loss": 0.85212559, + "learning_rate": 3.801646371644582e-06, + "loss": 0.87380809, + "num_input_tokens_seen": 165979685, + "router_z_loss_clip": 0.72167969, + "router_z_loss_mlp": 0.15362549, + "step": 5813, + "time_per_iteration": 2.607754945755005 + }, + { + "auxiliary_loss_clip": 0.01124276, + "auxiliary_loss_mlp": 0.01036466, + "balance_loss_clip": 1.05566978, + "balance_loss_mlp": 1.02279246, + "epoch": 0.16870756195229528, + "flos": 37773823161600.0, + "grad_norm": 1.8913638880140178, + "language_loss": 0.86490375, + "learning_rate": 3.8015647533060246e-06, + "loss": 0.88651121, + "num_input_tokens_seen": 166001715, + "router_z_loss_clip": 0.68652344, + "router_z_loss_mlp": 0.13665771, + "step": 5814, + "time_per_iteration": 2.681959867477417 + }, + { + "auxiliary_loss_clip": 0.01134398, + "auxiliary_loss_mlp": 0.01039346, + "balance_loss_clip": 1.05790496, + "balance_loss_mlp": 1.02319884, + "epoch": 0.16873657942081133, + "flos": 35255982487680.0, + "grad_norm": 1.7259892404490287, + "language_loss": 0.85244453, + "learning_rate": 3.8014831190553182e-06, + "loss": 0.87418199, + "num_input_tokens_seen": 166022175, + "router_z_loss_clip": 0.76464844, + "router_z_loss_mlp": 0.16137695, + "step": 5815, + "time_per_iteration": 2.658459186553955 + }, + { + "auxiliary_loss_clip": 0.01138716, + "auxiliary_loss_mlp": 0.01039551, + "balance_loss_clip": 1.05948758, + "balance_loss_mlp": 1.02238512, + "epoch": 0.16876559688932738, + "flos": 28546325660160.0, + "grad_norm": 2.011056412088316, + "language_loss": 0.84615433, + "learning_rate": 3.801401468893184e-06, + "loss": 0.86793709, + "num_input_tokens_seen": 166043360, + "router_z_loss_clip": 0.79150391, + "router_z_loss_mlp": 0.17163086, + "step": 5816, + "time_per_iteration": 2.656139373779297 + }, + { + "auxiliary_loss_clip": 0.01122816, + "auxiliary_loss_mlp": 0.01039601, + "balance_loss_clip": 1.0538156, + "balance_loss_mlp": 1.02579093, + "epoch": 0.16879461435784343, + "flos": 33432115553280.0, + "grad_norm": 2.1053636201661456, + "language_loss": 0.69162995, + "learning_rate": 3.801319802820343e-06, + "loss": 0.71325409, + "num_input_tokens_seen": 166066215, + "router_z_loss_clip": 0.69091797, + "router_z_loss_mlp": 0.13824463, + "step": 5817, + "time_per_iteration": 2.8063948154449463 + }, + { + "auxiliary_loss_clip": 0.01038722, + "auxiliary_loss_mlp": 0.01001647, + "balance_loss_clip": 1.01939225, + "balance_loss_mlp": 1.00081301, + "epoch": 0.16882363182635945, + "flos": 53609820791040.0, + "grad_norm": 0.6703809465242028, + "language_loss": 0.48477915, + "learning_rate": 3.8012381208375165e-06, + "loss": 0.50518286, + "num_input_tokens_seen": 166124090, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.00836182, + "step": 5818, + "time_per_iteration": 2.9703359603881836 + }, + { + "auxiliary_loss_clip": 0.01137905, + "auxiliary_loss_mlp": 0.01048263, + "balance_loss_clip": 1.05987513, + "balance_loss_mlp": 1.03169322, + "epoch": 0.1688526492948755, + "flos": 32482278259200.0, + "grad_norm": 1.609485890234291, + "language_loss": 0.76045346, + "learning_rate": 3.801156422945426e-06, + "loss": 0.78231514, + "num_input_tokens_seen": 166142885, + "router_z_loss_clip": 0.77880859, + "router_z_loss_mlp": 0.16577148, + "step": 5819, + "time_per_iteration": 2.6262903213500977 + }, + { + "auxiliary_loss_clip": 0.01134244, + "auxiliary_loss_mlp": 0.01048856, + "balance_loss_clip": 1.05698109, + "balance_loss_mlp": 1.03290594, + "epoch": 0.16888166676339156, + "flos": 28402324035840.0, + "grad_norm": 2.8534508491508985, + "language_loss": 1.07462192, + "learning_rate": 3.8010747091447926e-06, + "loss": 1.09645295, + "num_input_tokens_seen": 166160600, + "router_z_loss_clip": 0.77246094, + "router_z_loss_mlp": 0.15960693, + "step": 5820, + "time_per_iteration": 2.661543369293213 + }, + { + "auxiliary_loss_clip": 0.01126802, + "auxiliary_loss_mlp": 0.01035641, + "balance_loss_clip": 1.05082774, + "balance_loss_mlp": 1.01970315, + "epoch": 0.1689106842319076, + "flos": 21250498786560.0, + "grad_norm": 1.9203332386588037, + "language_loss": 0.79451454, + "learning_rate": 3.8009929794363394e-06, + "loss": 0.81613904, + "num_input_tokens_seen": 166177365, + "router_z_loss_clip": 0.75927734, + "router_z_loss_mlp": 0.15917969, + "step": 5821, + "time_per_iteration": 2.4814109802246094 + }, + { + "auxiliary_loss_clip": 0.01038345, + "auxiliary_loss_mlp": 0.01003737, + "balance_loss_clip": 1.01915431, + "balance_loss_mlp": 1.00283074, + "epoch": 0.16893970170042366, + "flos": 74780022723840.0, + "grad_norm": 0.6495131043662712, + "language_loss": 0.48476291, + "learning_rate": 3.800911233820787e-06, + "loss": 0.5051837, + "num_input_tokens_seen": 166247295, + "router_z_loss_clip": 0.19238281, + "router_z_loss_mlp": 0.0090332, + "step": 5822, + "time_per_iteration": 3.2607829570770264 + }, + { + "auxiliary_loss_clip": 0.01131035, + "auxiliary_loss_mlp": 0.01038548, + "balance_loss_clip": 1.05821657, + "balance_loss_mlp": 1.02381933, + "epoch": 0.1689687191689397, + "flos": 30446754433920.0, + "grad_norm": 1.636626323579499, + "language_loss": 0.83641654, + "learning_rate": 3.800829472298858e-06, + "loss": 0.85811234, + "num_input_tokens_seen": 166267035, + "router_z_loss_clip": 0.72851562, + "router_z_loss_mlp": 0.14733887, + "step": 5823, + "time_per_iteration": 2.638629674911499 + }, + { + "auxiliary_loss_clip": 0.0113646, + "auxiliary_loss_mlp": 0.01050155, + "balance_loss_clip": 1.05458713, + "balance_loss_mlp": 1.03182054, + "epoch": 0.16899773663745574, + "flos": 32627572773120.0, + "grad_norm": 2.2935237048297297, + "language_loss": 0.82529235, + "learning_rate": 3.8007476948712745e-06, + "loss": 0.84715855, + "num_input_tokens_seen": 166281890, + "router_z_loss_clip": 0.81835938, + "router_z_loss_mlp": 0.18334961, + "step": 5824, + "time_per_iteration": 2.659066915512085 + }, + { + "auxiliary_loss_clip": 0.0103892, + "auxiliary_loss_mlp": 0.01003281, + "balance_loss_clip": 1.01968002, + "balance_loss_mlp": 1.00232744, + "epoch": 0.1690267541059718, + "flos": 74736893468160.0, + "grad_norm": 0.6659549159468989, + "language_loss": 0.48498216, + "learning_rate": 3.8006659015387584e-06, + "loss": 0.50540423, + "num_input_tokens_seen": 166341265, + "router_z_loss_clip": 0.19238281, + "router_z_loss_mlp": 0.00952148, + "step": 5825, + "time_per_iteration": 3.1238691806793213 + }, + { + "auxiliary_loss_clip": 0.01039341, + "auxiliary_loss_mlp": 0.01003959, + "balance_loss_clip": 1.02002525, + "balance_loss_mlp": 1.00300491, + "epoch": 0.16905577157448784, + "flos": 70329649495680.0, + "grad_norm": 0.6215741040076016, + "language_loss": 0.46130699, + "learning_rate": 3.8005840923020324e-06, + "loss": 0.48174, + "num_input_tokens_seen": 166405385, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.00952148, + "step": 5826, + "time_per_iteration": 3.144329786300659 + }, + { + "auxiliary_loss_clip": 0.01139945, + "auxiliary_loss_mlp": 0.01044265, + "balance_loss_clip": 1.05871201, + "balance_loss_mlp": 1.02734351, + "epoch": 0.1690847890430039, + "flos": 38357012378880.0, + "grad_norm": 1.9714627508947828, + "language_loss": 0.83035493, + "learning_rate": 3.8005022671618194e-06, + "loss": 0.85219705, + "num_input_tokens_seen": 166422040, + "router_z_loss_clip": 0.81298828, + "router_z_loss_mlp": 0.16918945, + "step": 5827, + "time_per_iteration": 2.7174160480499268 + }, + { + "auxiliary_loss_clip": 0.01128739, + "auxiliary_loss_mlp": 0.01058814, + "balance_loss_clip": 1.05358386, + "balance_loss_mlp": 1.04253578, + "epoch": 0.16911380651151994, + "flos": 18031896702720.0, + "grad_norm": 3.177850138272301, + "language_loss": 0.99472111, + "learning_rate": 3.8004204261188415e-06, + "loss": 1.01659667, + "num_input_tokens_seen": 166432185, + "router_z_loss_clip": 0.75146484, + "router_z_loss_mlp": 0.1628418, + "step": 5828, + "time_per_iteration": 2.5247254371643066 + }, + { + "auxiliary_loss_clip": 0.01129215, + "auxiliary_loss_mlp": 0.01040179, + "balance_loss_clip": 1.05362666, + "balance_loss_mlp": 1.02500343, + "epoch": 0.169142823980036, + "flos": 19274082791040.0, + "grad_norm": 2.320767296142059, + "language_loss": 0.9779551, + "learning_rate": 3.800338569173822e-06, + "loss": 0.99964905, + "num_input_tokens_seen": 166445915, + "router_z_loss_clip": 0.75585938, + "router_z_loss_mlp": 0.15179443, + "step": 5829, + "time_per_iteration": 2.5427656173706055 + }, + { + "auxiliary_loss_clip": 0.01039744, + "auxiliary_loss_mlp": 0.00998807, + "balance_loss_clip": 1.0204072, + "balance_loss_mlp": 0.99782974, + "epoch": 0.16917184144855202, + "flos": 74772373127040.0, + "grad_norm": 0.7021320344670522, + "language_loss": 0.49939799, + "learning_rate": 3.8002566963274836e-06, + "loss": 0.5197835, + "num_input_tokens_seen": 166507545, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.00976562, + "step": 5830, + "time_per_iteration": 3.082277297973633 + }, + { + "auxiliary_loss_clip": 0.01136139, + "auxiliary_loss_mlp": 0.01048962, + "balance_loss_clip": 1.05402374, + "balance_loss_mlp": 1.03062177, + "epoch": 0.16920085891706807, + "flos": 16184006547840.0, + "grad_norm": 1.840201989363569, + "language_loss": 0.66893029, + "learning_rate": 3.80017480758055e-06, + "loss": 0.6907813, + "num_input_tokens_seen": 166524965, + "router_z_loss_clip": 0.82128906, + "router_z_loss_mlp": 0.18347168, + "step": 5831, + "time_per_iteration": 4.813066720962524 + }, + { + "auxiliary_loss_clip": 0.01129762, + "auxiliary_loss_mlp": 0.01040736, + "balance_loss_clip": 1.05432022, + "balance_loss_mlp": 1.02497017, + "epoch": 0.16922987638558412, + "flos": 16027327422720.0, + "grad_norm": 2.381423419259929, + "language_loss": 0.84597659, + "learning_rate": 3.800092902933744e-06, + "loss": 0.86768156, + "num_input_tokens_seen": 166537770, + "router_z_loss_clip": 0.75439453, + "router_z_loss_mlp": 0.15777588, + "step": 5832, + "time_per_iteration": 2.4867985248565674 + }, + { + "auxiliary_loss_clip": 0.01127087, + "auxiliary_loss_mlp": 0.01044582, + "balance_loss_clip": 1.05270863, + "balance_loss_mlp": 1.02897179, + "epoch": 0.16925889385410017, + "flos": 24056163141120.0, + "grad_norm": 2.246093454152997, + "language_loss": 0.91681546, + "learning_rate": 3.800010982387788e-06, + "loss": 0.93853211, + "num_input_tokens_seen": 166554020, + "router_z_loss_clip": 0.74365234, + "router_z_loss_mlp": 0.15618896, + "step": 5833, + "time_per_iteration": 7.403854846954346 + }, + { + "auxiliary_loss_clip": 0.01121668, + "auxiliary_loss_mlp": 0.01036496, + "balance_loss_clip": 1.05185103, + "balance_loss_mlp": 1.02294135, + "epoch": 0.16928791132261622, + "flos": 15586272322560.0, + "grad_norm": 3.006904949351681, + "language_loss": 0.62166619, + "learning_rate": 3.7999290459434076e-06, + "loss": 0.64324784, + "num_input_tokens_seen": 166564430, + "router_z_loss_clip": 0.69921875, + "router_z_loss_mlp": 0.13549805, + "step": 5834, + "time_per_iteration": 4.801820516586304 + }, + { + "auxiliary_loss_clip": 0.01127887, + "auxiliary_loss_mlp": 0.01047174, + "balance_loss_clip": 1.0545187, + "balance_loss_mlp": 1.03350115, + "epoch": 0.16931692879113225, + "flos": 17630918202240.0, + "grad_norm": 2.636340333631986, + "language_loss": 0.96626389, + "learning_rate": 3.7998470936013253e-06, + "loss": 0.98801446, + "num_input_tokens_seen": 166575925, + "router_z_loss_clip": 0.73388672, + "router_z_loss_mlp": 0.13671875, + "step": 5835, + "time_per_iteration": 2.4897167682647705 + }, + { + "auxiliary_loss_clip": 0.01130423, + "auxiliary_loss_mlp": 0.01034604, + "balance_loss_clip": 1.05716443, + "balance_loss_mlp": 1.01947057, + "epoch": 0.1693459462596483, + "flos": 74731287968640.0, + "grad_norm": 2.257160143263928, + "language_loss": 0.93893939, + "learning_rate": 3.799765125362265e-06, + "loss": 0.96058965, + "num_input_tokens_seen": 166600065, + "router_z_loss_clip": 0.73193359, + "router_z_loss_mlp": 0.15124512, + "step": 5836, + "time_per_iteration": 2.9187586307525635 + }, + { + "auxiliary_loss_clip": 0.01132879, + "auxiliary_loss_mlp": 0.01040171, + "balance_loss_clip": 1.05793309, + "balance_loss_mlp": 1.0257107, + "epoch": 0.16937496372816435, + "flos": 27739771718400.0, + "grad_norm": 1.9223545138630478, + "language_loss": 0.9458496, + "learning_rate": 3.7996831412269514e-06, + "loss": 0.96758014, + "num_input_tokens_seen": 166618370, + "router_z_loss_clip": 0.74951172, + "router_z_loss_mlp": 0.14465332, + "step": 5837, + "time_per_iteration": 2.570755958557129 + }, + { + "auxiliary_loss_clip": 0.01042469, + "auxiliary_loss_mlp": 0.00997631, + "balance_loss_clip": 1.02320695, + "balance_loss_mlp": 0.9967249, + "epoch": 0.1694039811966804, + "flos": 61312659649920.0, + "grad_norm": 0.6710962319413605, + "language_loss": 0.51909733, + "learning_rate": 3.799601141196107e-06, + "loss": 0.53949833, + "num_input_tokens_seen": 166679065, + "router_z_loss_clip": 0.19238281, + "router_z_loss_mlp": 0.0090332, + "step": 5838, + "time_per_iteration": 3.061084508895874 + }, + { + "auxiliary_loss_clip": 0.01137691, + "auxiliary_loss_mlp": 0.01042209, + "balance_loss_clip": 1.05973983, + "balance_loss_mlp": 1.02758765, + "epoch": 0.16943299866519645, + "flos": 26753161875840.0, + "grad_norm": 2.1743327294306765, + "language_loss": 0.78109789, + "learning_rate": 3.799519125270458e-06, + "loss": 0.80289686, + "num_input_tokens_seen": 166692725, + "router_z_loss_clip": 0.78027344, + "router_z_loss_mlp": 0.14624023, + "step": 5839, + "time_per_iteration": 2.6101319789886475 + }, + { + "auxiliary_loss_clip": 0.0112627, + "auxiliary_loss_mlp": 0.010319, + "balance_loss_clip": 1.05708766, + "balance_loss_mlp": 1.01792288, + "epoch": 0.1694620161337125, + "flos": 16133874140160.0, + "grad_norm": 4.07079770939145, + "language_loss": 0.97815603, + "learning_rate": 3.7994370934507276e-06, + "loss": 0.99973774, + "num_input_tokens_seen": 166702790, + "router_z_loss_clip": 0.69189453, + "router_z_loss_mlp": 0.13964844, + "step": 5840, + "time_per_iteration": 2.480983018875122 + }, + { + "auxiliary_loss_clip": 0.01122631, + "auxiliary_loss_mlp": 0.01045193, + "balance_loss_clip": 1.05348086, + "balance_loss_mlp": 1.03058398, + "epoch": 0.16949103360222853, + "flos": 17631780128640.0, + "grad_norm": 2.145617523395642, + "language_loss": 0.69444954, + "learning_rate": 3.7993550457376406e-06, + "loss": 0.71612775, + "num_input_tokens_seen": 166717190, + "router_z_loss_clip": 0.69091797, + "router_z_loss_mlp": 0.14605713, + "step": 5841, + "time_per_iteration": 2.5360794067382812 + }, + { + "auxiliary_loss_clip": 0.01041427, + "auxiliary_loss_mlp": 0.01003285, + "balance_loss_clip": 1.0221231, + "balance_loss_mlp": 1.00243819, + "epoch": 0.16952005107074458, + "flos": 70800617646720.0, + "grad_norm": 0.6260877693946378, + "language_loss": 0.49956071, + "learning_rate": 3.799272982131922e-06, + "loss": 0.52000785, + "num_input_tokens_seen": 166784350, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.00848389, + "step": 5842, + "time_per_iteration": 3.1793410778045654 + }, + { + "auxiliary_loss_clip": 0.01126205, + "auxiliary_loss_mlp": 0.01035037, + "balance_loss_clip": 1.05396068, + "balance_loss_mlp": 1.01977766, + "epoch": 0.16954906853926063, + "flos": 16177973063040.0, + "grad_norm": 2.2933831167385703, + "language_loss": 0.70145398, + "learning_rate": 3.799190902634296e-06, + "loss": 0.72306639, + "num_input_tokens_seen": 166799780, + "router_z_loss_clip": 0.72265625, + "router_z_loss_mlp": 0.15264893, + "step": 5843, + "time_per_iteration": 2.5291008949279785 + }, + { + "auxiliary_loss_clip": 0.01041802, + "auxiliary_loss_mlp": 0.010009, + "balance_loss_clip": 1.02246428, + "balance_loss_mlp": 0.99998802, + "epoch": 0.16957808600777668, + "flos": 60572899048320.0, + "grad_norm": 0.6926333094727932, + "language_loss": 0.49599007, + "learning_rate": 3.799108807245488e-06, + "loss": 0.51641709, + "num_input_tokens_seen": 166855080, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.00909424, + "step": 5844, + "time_per_iteration": 2.939159631729126 + }, + { + "auxiliary_loss_clip": 0.01040991, + "auxiliary_loss_mlp": 0.01000864, + "balance_loss_clip": 1.02155709, + "balance_loss_mlp": 1.00000572, + "epoch": 0.16960710347629274, + "flos": 55396268732160.0, + "grad_norm": 0.8760494737534793, + "language_loss": 0.51881897, + "learning_rate": 3.7990266959662227e-06, + "loss": 0.53923756, + "num_input_tokens_seen": 166915120, + "router_z_loss_clip": 0.19433594, + "router_z_loss_mlp": 0.00860596, + "step": 5845, + "time_per_iteration": 3.038628101348877 + }, + { + "auxiliary_loss_clip": 0.01119083, + "auxiliary_loss_mlp": 0.01046777, + "balance_loss_clip": 1.05216062, + "balance_loss_mlp": 1.0319891, + "epoch": 0.1696361209448088, + "flos": 15333425510400.0, + "grad_norm": 3.6208177173334897, + "language_loss": 0.95790774, + "learning_rate": 3.798944568797226e-06, + "loss": 0.97956634, + "num_input_tokens_seen": 166926430, + "router_z_loss_clip": 0.66943359, + "router_z_loss_mlp": 0.14788818, + "step": 5846, + "time_per_iteration": 2.491603374481201 + }, + { + "auxiliary_loss_clip": 0.01039423, + "auxiliary_loss_mlp": 0.01003771, + "balance_loss_clip": 1.02007771, + "balance_loss_mlp": 1.00284755, + "epoch": 0.1696651384133248, + "flos": 72300391142400.0, + "grad_norm": 0.5768938466419463, + "language_loss": 0.43568331, + "learning_rate": 3.798862425739223e-06, + "loss": 0.45611525, + "num_input_tokens_seen": 166997215, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.00921631, + "step": 5847, + "time_per_iteration": 3.3297178745269775 + }, + { + "auxiliary_loss_clip": 0.01129199, + "auxiliary_loss_mlp": 0.01035857, + "balance_loss_clip": 1.05614972, + "balance_loss_mlp": 1.02192092, + "epoch": 0.16969415588184086, + "flos": 12454251972480.0, + "grad_norm": 2.091568679233062, + "language_loss": 0.77645397, + "learning_rate": 3.798780266792939e-06, + "loss": 0.79810452, + "num_input_tokens_seen": 167009195, + "router_z_loss_clip": 0.72998047, + "router_z_loss_mlp": 0.13934326, + "step": 5848, + "time_per_iteration": 2.5248780250549316 + }, + { + "auxiliary_loss_clip": 0.01125305, + "auxiliary_loss_mlp": 0.01037155, + "balance_loss_clip": 1.05266762, + "balance_loss_mlp": 1.02230144, + "epoch": 0.16972317335035692, + "flos": 24418824808320.0, + "grad_norm": 2.4105251836863144, + "language_loss": 0.73223633, + "learning_rate": 3.7986980919590998e-06, + "loss": 0.75386095, + "num_input_tokens_seen": 167027385, + "router_z_loss_clip": 0.7265625, + "router_z_loss_mlp": 0.1484375, + "step": 5849, + "time_per_iteration": 2.607924222946167 + }, + { + "auxiliary_loss_clip": 0.01140072, + "auxiliary_loss_mlp": 0.01042663, + "balance_loss_clip": 1.06022453, + "balance_loss_mlp": 1.0260098, + "epoch": 0.16975219081887297, + "flos": 17633180759040.0, + "grad_norm": 2.8638686257346966, + "language_loss": 0.77660072, + "learning_rate": 3.7986159012384312e-06, + "loss": 0.79842812, + "num_input_tokens_seen": 167041060, + "router_z_loss_clip": 0.79833984, + "router_z_loss_mlp": 0.16650391, + "step": 5850, + "time_per_iteration": 2.527557849884033 + }, + { + "auxiliary_loss_clip": 0.01132656, + "auxiliary_loss_mlp": 0.01038475, + "balance_loss_clip": 1.05603051, + "balance_loss_mlp": 1.02322805, + "epoch": 0.16978120828738902, + "flos": 29308924333440.0, + "grad_norm": 1.8962166043119009, + "language_loss": 0.73135543, + "learning_rate": 3.7985336946316585e-06, + "loss": 0.75306678, + "num_input_tokens_seen": 167060555, + "router_z_loss_clip": 0.76660156, + "router_z_loss_mlp": 0.15234375, + "step": 5851, + "time_per_iteration": 2.6199002265930176 + }, + { + "auxiliary_loss_clip": 0.01130875, + "auxiliary_loss_mlp": 0.01043075, + "balance_loss_clip": 1.05877161, + "balance_loss_mlp": 1.0278163, + "epoch": 0.16981022575590504, + "flos": 26277273561600.0, + "grad_norm": 2.882040496333225, + "language_loss": 0.88260949, + "learning_rate": 3.7984514721395096e-06, + "loss": 0.90434891, + "num_input_tokens_seen": 167073395, + "router_z_loss_clip": 0.72021484, + "router_z_loss_mlp": 0.15258789, + "step": 5852, + "time_per_iteration": 2.576247215270996 + }, + { + "auxiliary_loss_clip": 0.01035554, + "auxiliary_loss_mlp": 0.01006096, + "balance_loss_clip": 1.01623797, + "balance_loss_mlp": 1.00523138, + "epoch": 0.1698392432244211, + "flos": 62590397224320.0, + "grad_norm": 0.6625453798407032, + "language_loss": 0.53652918, + "learning_rate": 3.7983692337627087e-06, + "loss": 0.55694568, + "num_input_tokens_seen": 167137485, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.00866699, + "step": 5853, + "time_per_iteration": 3.118860960006714 + }, + { + "auxiliary_loss_clip": 0.01132991, + "auxiliary_loss_mlp": 0.01046068, + "balance_loss_clip": 1.05620968, + "balance_loss_mlp": 1.02936101, + "epoch": 0.16986826069293715, + "flos": 45809877513600.0, + "grad_norm": 1.9059951336485716, + "language_loss": 0.92060971, + "learning_rate": 3.7982869795019835e-06, + "loss": 0.94240034, + "num_input_tokens_seen": 167159265, + "router_z_loss_clip": 0.76806641, + "router_z_loss_mlp": 0.16717529, + "step": 5854, + "time_per_iteration": 2.7817904949188232 + }, + { + "auxiliary_loss_clip": 0.01121924, + "auxiliary_loss_mlp": 0.01039667, + "balance_loss_clip": 1.05216932, + "balance_loss_mlp": 1.02482533, + "epoch": 0.1698972781614532, + "flos": 18802504108800.0, + "grad_norm": 2.4492530794510223, + "language_loss": 0.6119833, + "learning_rate": 3.7982047093580594e-06, + "loss": 0.63359916, + "num_input_tokens_seen": 167172395, + "router_z_loss_clip": 0.69726562, + "router_z_loss_mlp": 0.14849854, + "step": 5855, + "time_per_iteration": 2.5217108726501465 + }, + { + "auxiliary_loss_clip": 0.01132296, + "auxiliary_loss_mlp": 0.01039735, + "balance_loss_clip": 1.05624557, + "balance_loss_mlp": 1.02311683, + "epoch": 0.16992629562996925, + "flos": 39010047592320.0, + "grad_norm": 2.922981421464579, + "language_loss": 1.03962088, + "learning_rate": 3.798122423331664e-06, + "loss": 1.06134105, + "num_input_tokens_seen": 167186715, + "router_z_loss_clip": 0.76025391, + "router_z_loss_mlp": 0.16619873, + "step": 5856, + "time_per_iteration": 2.706166982650757 + }, + { + "auxiliary_loss_clip": 0.01037757, + "auxiliary_loss_mlp": 0.01005382, + "balance_loss_clip": 1.01823115, + "balance_loss_mlp": 1.00438678, + "epoch": 0.1699553130984853, + "flos": 74138587603200.0, + "grad_norm": 0.6343682514532625, + "language_loss": 0.46500921, + "learning_rate": 3.7980401214235237e-06, + "loss": 0.48544061, + "num_input_tokens_seen": 167254625, + "router_z_loss_clip": 0.1953125, + "router_z_loss_mlp": 0.00994873, + "step": 5857, + "time_per_iteration": 3.3125104904174805 + }, + { + "auxiliary_loss_clip": 0.01037303, + "auxiliary_loss_mlp": 0.01006822, + "balance_loss_clip": 1.01790404, + "balance_loss_mlp": 1.00580847, + "epoch": 0.16998433056700132, + "flos": 59472918904320.0, + "grad_norm": 0.6773060085195699, + "language_loss": 0.44664836, + "learning_rate": 3.7979578036343652e-06, + "loss": 0.46708962, + "num_input_tokens_seen": 167303015, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.01013184, + "step": 5858, + "time_per_iteration": 2.840139389038086 + }, + { + "auxiliary_loss_clip": 0.011393, + "auxiliary_loss_mlp": 0.01050106, + "balance_loss_clip": 1.058707, + "balance_loss_mlp": 1.03115165, + "epoch": 0.17001334803551738, + "flos": 17594037912960.0, + "grad_norm": 2.268905269309207, + "language_loss": 0.8425076, + "learning_rate": 3.7978754699649166e-06, + "loss": 0.86440158, + "num_input_tokens_seen": 167315840, + "router_z_loss_clip": 0.8046875, + "router_z_loss_mlp": 0.1895752, + "step": 5859, + "time_per_iteration": 2.496007204055786 + }, + { + "auxiliary_loss_clip": 0.01126037, + "auxiliary_loss_mlp": 0.01035837, + "balance_loss_clip": 1.0555675, + "balance_loss_mlp": 1.02208638, + "epoch": 0.17004236550403343, + "flos": 16209286744320.0, + "grad_norm": 2.357243949078669, + "language_loss": 0.76068151, + "learning_rate": 3.7977931204159037e-06, + "loss": 0.78230023, + "num_input_tokens_seen": 167327310, + "router_z_loss_clip": 0.70605469, + "router_z_loss_mlp": 0.13757324, + "step": 5860, + "time_per_iteration": 2.480562448501587 + }, + { + "auxiliary_loss_clip": 0.01124045, + "auxiliary_loss_mlp": 0.01031788, + "balance_loss_clip": 1.0553267, + "balance_loss_mlp": 1.01729178, + "epoch": 0.17007138297254948, + "flos": 11720237546880.0, + "grad_norm": 2.987361349843267, + "language_loss": 0.92425299, + "learning_rate": 3.7977107549880545e-06, + "loss": 0.94581133, + "num_input_tokens_seen": 167337135, + "router_z_loss_clip": 0.68701172, + "router_z_loss_mlp": 0.14501953, + "step": 5861, + "time_per_iteration": 2.4892067909240723 + }, + { + "auxiliary_loss_clip": 0.01128574, + "auxiliary_loss_mlp": 0.0104763, + "balance_loss_clip": 1.05673122, + "balance_loss_mlp": 1.03207898, + "epoch": 0.17010040044106553, + "flos": 10989419431680.0, + "grad_norm": 2.483374947719727, + "language_loss": 0.7327897, + "learning_rate": 3.7976283736820968e-06, + "loss": 0.75455177, + "num_input_tokens_seen": 167347905, + "router_z_loss_clip": 0.71923828, + "router_z_loss_mlp": 0.15557861, + "step": 5862, + "time_per_iteration": 2.54640793800354 + }, + { + "auxiliary_loss_clip": 0.01139902, + "auxiliary_loss_mlp": 0.01047846, + "balance_loss_clip": 1.06190825, + "balance_loss_mlp": 1.03143108, + "epoch": 0.17012941790958158, + "flos": 17158010716800.0, + "grad_norm": 3.1200980767013253, + "language_loss": 0.93615651, + "learning_rate": 3.7975459764987575e-06, + "loss": 0.95803398, + "num_input_tokens_seen": 167359915, + "router_z_loss_clip": 0.78027344, + "router_z_loss_mlp": 0.16412354, + "step": 5863, + "time_per_iteration": 2.4776430130004883 + }, + { + "auxiliary_loss_clip": 0.01138495, + "auxiliary_loss_mlp": 0.01043942, + "balance_loss_clip": 1.06171632, + "balance_loss_mlp": 1.02688861, + "epoch": 0.1701584353780976, + "flos": 36310822214400.0, + "grad_norm": 3.218570212546146, + "language_loss": 0.67576361, + "learning_rate": 3.797463563438765e-06, + "loss": 0.69758797, + "num_input_tokens_seen": 167375490, + "router_z_loss_clip": 0.76757812, + "router_z_loss_mlp": 0.1706543, + "step": 5864, + "time_per_iteration": 2.6393635272979736 + }, + { + "auxiliary_loss_clip": 0.0112376, + "auxiliary_loss_mlp": 0.0103649, + "balance_loss_clip": 1.05427122, + "balance_loss_mlp": 1.02221513, + "epoch": 0.17018745284661366, + "flos": 11134498464000.0, + "grad_norm": 2.448425033850232, + "language_loss": 0.84713024, + "learning_rate": 3.7973811345028464e-06, + "loss": 0.86873275, + "num_input_tokens_seen": 167385620, + "router_z_loss_clip": 0.6953125, + "router_z_loss_mlp": 0.14282227, + "step": 5865, + "time_per_iteration": 2.4928011894226074 + }, + { + "auxiliary_loss_clip": 0.01036302, + "auxiliary_loss_mlp": 0.01033, + "balance_loss_clip": 1.01690018, + "balance_loss_mlp": 1.03199899, + "epoch": 0.1702164703151297, + "flos": 69157309403520.0, + "grad_norm": 0.6863221552895621, + "language_loss": 0.49541786, + "learning_rate": 3.7972986896917306e-06, + "loss": 0.5161109, + "num_input_tokens_seen": 167449985, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.01000977, + "step": 5866, + "time_per_iteration": 3.2641122341156006 + }, + { + "auxiliary_loss_clip": 0.01036343, + "auxiliary_loss_mlp": 0.01022958, + "balance_loss_clip": 1.01697922, + "balance_loss_mlp": 1.02200472, + "epoch": 0.17024548778364576, + "flos": 57619533968640.0, + "grad_norm": 0.7186982929703696, + "language_loss": 0.54574776, + "learning_rate": 3.7972162290061462e-06, + "loss": 0.5663408, + "num_input_tokens_seen": 167507145, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.00952148, + "step": 5867, + "time_per_iteration": 3.0635135173797607 + }, + { + "auxiliary_loss_clip": 0.010355, + "auxiliary_loss_mlp": 0.01002092, + "balance_loss_clip": 1.01617813, + "balance_loss_mlp": 1.00116849, + "epoch": 0.1702745052521618, + "flos": 74784260528640.0, + "grad_norm": 0.630590242106845, + "language_loss": 0.51397979, + "learning_rate": 3.7971337524468197e-06, + "loss": 0.5343557, + "num_input_tokens_seen": 167573420, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.00921631, + "step": 5868, + "time_per_iteration": 3.1941394805908203 + }, + { + "auxiliary_loss_clip": 0.01035631, + "auxiliary_loss_mlp": 0.01001028, + "balance_loss_clip": 1.01626265, + "balance_loss_mlp": 1.00008643, + "epoch": 0.17030352272067784, + "flos": 74780525514240.0, + "grad_norm": 0.7118665849078888, + "language_loss": 0.44442731, + "learning_rate": 3.7970512600144816e-06, + "loss": 0.46479392, + "num_input_tokens_seen": 167634810, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.00939941, + "step": 5869, + "time_per_iteration": 3.1408824920654297 + }, + { + "auxiliary_loss_clip": 0.01130115, + "auxiliary_loss_mlp": 0.01042092, + "balance_loss_clip": 1.05772281, + "balance_loss_mlp": 1.02661836, + "epoch": 0.1703325401891939, + "flos": 26138766718080.0, + "grad_norm": 1.7486546305711643, + "language_loss": 0.72006863, + "learning_rate": 3.796968751709859e-06, + "loss": 0.74179065, + "num_input_tokens_seen": 167650695, + "router_z_loss_clip": 0.72363281, + "router_z_loss_mlp": 0.15466309, + "step": 5870, + "time_per_iteration": 2.661215305328369 + }, + { + "auxiliary_loss_clip": 0.01038692, + "auxiliary_loss_mlp": 0.01000691, + "balance_loss_clip": 1.01918435, + "balance_loss_mlp": 0.99975491, + "epoch": 0.17036155765770994, + "flos": 74784260528640.0, + "grad_norm": 0.5976450942024771, + "language_loss": 0.4604913, + "learning_rate": 3.7968862275336813e-06, + "loss": 0.48088515, + "num_input_tokens_seen": 167720695, + "router_z_loss_clip": 0.1953125, + "router_z_loss_mlp": 0.00933838, + "step": 5871, + "time_per_iteration": 3.2331535816192627 + }, + { + "auxiliary_loss_clip": 0.0113783, + "auxiliary_loss_mlp": 0.01045964, + "balance_loss_clip": 1.0587039, + "balance_loss_mlp": 1.03032374, + "epoch": 0.170390575126226, + "flos": 16211800696320.0, + "grad_norm": 1.9933911044231447, + "language_loss": 0.715756, + "learning_rate": 3.7968036874866778e-06, + "loss": 0.73759389, + "num_input_tokens_seen": 167737125, + "router_z_loss_clip": 0.79052734, + "router_z_loss_mlp": 0.15637207, + "step": 5872, + "time_per_iteration": 2.5804831981658936 + }, + { + "auxiliary_loss_clip": 0.01037042, + "auxiliary_loss_mlp": 0.01004782, + "balance_loss_clip": 1.01759982, + "balance_loss_mlp": 1.00382221, + "epoch": 0.17041959259474204, + "flos": 71402945834880.0, + "grad_norm": 0.7189489690247871, + "language_loss": 0.481408, + "learning_rate": 3.796721131569577e-06, + "loss": 0.50182629, + "num_input_tokens_seen": 167796325, + "router_z_loss_clip": 0.19433594, + "router_z_loss_mlp": 0.00958252, + "step": 5873, + "time_per_iteration": 3.0582196712493896 + }, + { + "auxiliary_loss_clip": 0.01143195, + "auxiliary_loss_mlp": 0.01056227, + "balance_loss_clip": 1.06181622, + "balance_loss_mlp": 1.03805935, + "epoch": 0.1704486100632581, + "flos": 26134169777280.0, + "grad_norm": 2.3844664809862817, + "language_loss": 0.87155128, + "learning_rate": 3.7966385597831074e-06, + "loss": 0.89354551, + "num_input_tokens_seen": 167814060, + "router_z_loss_clip": 0.81347656, + "router_z_loss_mlp": 0.1817627, + "step": 5874, + "time_per_iteration": 2.5804378986358643 + }, + { + "auxiliary_loss_clip": 0.01142263, + "auxiliary_loss_mlp": 0.01056105, + "balance_loss_clip": 1.06264162, + "balance_loss_mlp": 1.03986311, + "epoch": 0.17047762753177412, + "flos": 28688387950080.0, + "grad_norm": 2.145474980560695, + "language_loss": 0.88681245, + "learning_rate": 3.7965559721279995e-06, + "loss": 0.90879613, + "num_input_tokens_seen": 167829370, + "router_z_loss_clip": 0.79541016, + "router_z_loss_mlp": 0.16247559, + "step": 5875, + "time_per_iteration": 2.5817954540252686 + }, + { + "auxiliary_loss_clip": 0.01126658, + "auxiliary_loss_mlp": 0.01036456, + "balance_loss_clip": 1.05596209, + "balance_loss_mlp": 1.02244306, + "epoch": 0.17050664500029017, + "flos": 35036173209600.0, + "grad_norm": 4.415399529086282, + "language_loss": 0.7326318, + "learning_rate": 3.796473368604982e-06, + "loss": 0.75426292, + "num_input_tokens_seen": 167846005, + "router_z_loss_clip": 0.70800781, + "router_z_loss_mlp": 0.14001465, + "step": 5876, + "time_per_iteration": 2.5765645503997803 + }, + { + "auxiliary_loss_clip": 0.01133396, + "auxiliary_loss_mlp": 0.01048862, + "balance_loss_clip": 1.05640888, + "balance_loss_mlp": 1.03275633, + "epoch": 0.17053566246880622, + "flos": 12157880855040.0, + "grad_norm": 2.5338328419700527, + "language_loss": 0.78141814, + "learning_rate": 3.7963907492147847e-06, + "loss": 0.80324066, + "num_input_tokens_seen": 167856425, + "router_z_loss_clip": 0.77001953, + "router_z_loss_mlp": 0.16113281, + "step": 5877, + "time_per_iteration": 2.481065511703491 + }, + { + "auxiliary_loss_clip": 0.01033932, + "auxiliary_loss_mlp": 0.01035836, + "balance_loss_clip": 1.01455879, + "balance_loss_mlp": 1.03489399, + "epoch": 0.17056467993732227, + "flos": 63028327841280.0, + "grad_norm": 0.6931442366159942, + "language_loss": 0.49549738, + "learning_rate": 3.7963081139581375e-06, + "loss": 0.516195, + "num_input_tokens_seen": 167914435, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.00939941, + "step": 5878, + "time_per_iteration": 3.0561094284057617 + }, + { + "auxiliary_loss_clip": 0.01138395, + "auxiliary_loss_mlp": 0.01055095, + "balance_loss_clip": 1.06005704, + "balance_loss_mlp": 1.03778028, + "epoch": 0.17059369740583832, + "flos": 11098336446720.0, + "grad_norm": 3.09357290226073, + "language_loss": 0.82581651, + "learning_rate": 3.7962254628357704e-06, + "loss": 0.84775138, + "num_input_tokens_seen": 167924330, + "router_z_loss_clip": 0.78320312, + "router_z_loss_mlp": 0.17321777, + "step": 5879, + "time_per_iteration": 2.6618077754974365 + }, + { + "auxiliary_loss_clip": 0.01136625, + "auxiliary_loss_mlp": 0.01041513, + "balance_loss_clip": 1.05679786, + "balance_loss_mlp": 1.02532494, + "epoch": 0.17062271487435435, + "flos": 15626420749440.0, + "grad_norm": 3.1312108349891337, + "language_loss": 0.84908664, + "learning_rate": 3.7961427958484135e-06, + "loss": 0.87086809, + "num_input_tokens_seen": 167938535, + "router_z_loss_clip": 0.79736328, + "router_z_loss_mlp": 0.1618042, + "step": 5880, + "time_per_iteration": 2.5475454330444336 + }, + { + "auxiliary_loss_clip": 0.01032963, + "auxiliary_loss_mlp": 0.01028145, + "balance_loss_clip": 1.01361084, + "balance_loss_mlp": 1.0271492, + "epoch": 0.1706517323428704, + "flos": 74771331632640.0, + "grad_norm": 0.7311322299439214, + "language_loss": 0.47425604, + "learning_rate": 3.7960601129967957e-06, + "loss": 0.49486709, + "num_input_tokens_seen": 168003245, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.00994873, + "step": 5881, + "time_per_iteration": 3.1726975440979004 + }, + { + "auxiliary_loss_clip": 0.01127219, + "auxiliary_loss_mlp": 0.01046442, + "balance_loss_clip": 1.05560231, + "balance_loss_mlp": 1.02943039, + "epoch": 0.17068074981138645, + "flos": 27849083783040.0, + "grad_norm": 1.8486813016212678, + "language_loss": 0.93429649, + "learning_rate": 3.7959774142816484e-06, + "loss": 0.95603311, + "num_input_tokens_seen": 168019610, + "router_z_loss_clip": 0.71630859, + "router_z_loss_mlp": 0.17010498, + "step": 5882, + "time_per_iteration": 2.572629928588867 + }, + { + "auxiliary_loss_clip": 0.01033659, + "auxiliary_loss_mlp": 0.0101208, + "balance_loss_clip": 1.0142467, + "balance_loss_mlp": 1.01099539, + "epoch": 0.1707097672799025, + "flos": 57221644037760.0, + "grad_norm": 0.694039352909132, + "language_loss": 0.51764405, + "learning_rate": 3.7958946997037026e-06, + "loss": 0.53810143, + "num_input_tokens_seen": 168080495, + "router_z_loss_clip": 0.19433594, + "router_z_loss_mlp": 0.01086426, + "step": 5883, + "time_per_iteration": 3.0333540439605713 + }, + { + "auxiliary_loss_clip": 0.0113109, + "auxiliary_loss_mlp": 0.01036558, + "balance_loss_clip": 1.05368495, + "balance_loss_mlp": 1.01937366, + "epoch": 0.17073878474841855, + "flos": 17813919018240.0, + "grad_norm": 2.419424912643685, + "language_loss": 0.86183131, + "learning_rate": 3.795811969263687e-06, + "loss": 0.88350779, + "num_input_tokens_seen": 168092615, + "router_z_loss_clip": 0.77441406, + "router_z_loss_mlp": 0.171875, + "step": 5884, + "time_per_iteration": 2.512660026550293 + }, + { + "auxiliary_loss_clip": 0.01137835, + "auxiliary_loss_mlp": 0.01037521, + "balance_loss_clip": 1.05858278, + "balance_loss_mlp": 1.02129614, + "epoch": 0.1707678022169346, + "flos": 16247819059200.0, + "grad_norm": 2.896347000897968, + "language_loss": 0.87602794, + "learning_rate": 3.795729222962334e-06, + "loss": 0.89778149, + "num_input_tokens_seen": 168104960, + "router_z_loss_clip": 0.79248047, + "router_z_loss_mlp": 0.16210938, + "step": 5885, + "time_per_iteration": 2.5127718448638916 + }, + { + "auxiliary_loss_clip": 0.01035089, + "auxiliary_loss_mlp": 0.01001365, + "balance_loss_clip": 1.01572192, + "balance_loss_mlp": 1.00041723, + "epoch": 0.17079681968545063, + "flos": 64199734179840.0, + "grad_norm": 0.6562636610673568, + "language_loss": 0.4938671, + "learning_rate": 3.795646460800374e-06, + "loss": 0.51423162, + "num_input_tokens_seen": 168167295, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.00946045, + "step": 5886, + "time_per_iteration": 3.0569984912872314 + }, + { + "auxiliary_loss_clip": 0.01132405, + "auxiliary_loss_mlp": 0.0104381, + "balance_loss_clip": 1.05767179, + "balance_loss_mlp": 1.02848601, + "epoch": 0.17082583715396668, + "flos": 26498232074880.0, + "grad_norm": 2.1996576177533536, + "language_loss": 0.94847858, + "learning_rate": 3.795563682778537e-06, + "loss": 0.97024077, + "num_input_tokens_seen": 168184215, + "router_z_loss_clip": 0.74707031, + "router_z_loss_mlp": 0.15332031, + "step": 5887, + "time_per_iteration": 2.555767774581909 + }, + { + "auxiliary_loss_clip": 0.01134954, + "auxiliary_loss_mlp": 0.01046623, + "balance_loss_clip": 1.05984163, + "balance_loss_mlp": 1.0293076, + "epoch": 0.17085485462248273, + "flos": 25114235091840.0, + "grad_norm": 1.9558356988046426, + "language_loss": 0.91950035, + "learning_rate": 3.795480888897556e-06, + "loss": 0.94131607, + "num_input_tokens_seen": 168202200, + "router_z_loss_clip": 0.75097656, + "router_z_loss_mlp": 0.17321777, + "step": 5888, + "time_per_iteration": 2.5890867710113525 + }, + { + "auxiliary_loss_clip": 0.01033136, + "auxiliary_loss_mlp": 0.01004064, + "balance_loss_clip": 1.01412034, + "balance_loss_mlp": 1.00321198, + "epoch": 0.17088387209099878, + "flos": 59005542113280.0, + "grad_norm": 0.9442743597741875, + "language_loss": 0.53273225, + "learning_rate": 3.79539807915816e-06, + "loss": 0.55310428, + "num_input_tokens_seen": 168255920, + "router_z_loss_clip": 0.19042969, + "router_z_loss_mlp": 0.00854492, + "step": 5889, + "time_per_iteration": 2.8643290996551514 + }, + { + "auxiliary_loss_clip": 0.01032614, + "auxiliary_loss_mlp": 0.010091, + "balance_loss_clip": 1.01365077, + "balance_loss_mlp": 1.00823593, + "epoch": 0.17091288955951484, + "flos": 61470883059840.0, + "grad_norm": 0.6667441212419174, + "language_loss": 0.48816785, + "learning_rate": 3.795315253561083e-06, + "loss": 0.50858504, + "num_input_tokens_seen": 168319750, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.00866699, + "step": 5890, + "time_per_iteration": 3.109872341156006 + }, + { + "auxiliary_loss_clip": 0.01032224, + "auxiliary_loss_mlp": 0.01007836, + "balance_loss_clip": 1.01326096, + "balance_loss_mlp": 1.00691211, + "epoch": 0.1709419070280309, + "flos": 58719154976640.0, + "grad_norm": 0.6832936166901724, + "language_loss": 0.5211674, + "learning_rate": 3.7952324121070543e-06, + "loss": 0.54156804, + "num_input_tokens_seen": 168379485, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.00921631, + "step": 5891, + "time_per_iteration": 2.961212158203125 + }, + { + "auxiliary_loss_clip": 0.01114328, + "auxiliary_loss_mlp": 0.0103659, + "balance_loss_clip": 1.04970121, + "balance_loss_mlp": 1.02319121, + "epoch": 0.1709709244965469, + "flos": 11762756271360.0, + "grad_norm": 2.2545449532895976, + "language_loss": 0.6731447, + "learning_rate": 3.7951495547968067e-06, + "loss": 0.69465387, + "num_input_tokens_seen": 168391740, + "router_z_loss_clip": 0.64648438, + "router_z_loss_mlp": 0.1340332, + "step": 5892, + "time_per_iteration": 2.511726140975952 + }, + { + "auxiliary_loss_clip": 0.01030443, + "auxiliary_loss_mlp": 0.0099642, + "balance_loss_clip": 1.01155078, + "balance_loss_mlp": 0.9956274, + "epoch": 0.17099994196506296, + "flos": 74299250160000.0, + "grad_norm": 0.642815990458968, + "language_loss": 0.489923, + "learning_rate": 3.7950666816310726e-06, + "loss": 0.51019168, + "num_input_tokens_seen": 168455950, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.00793457, + "step": 5893, + "time_per_iteration": 3.1185929775238037 + }, + { + "auxiliary_loss_clip": 0.0114261, + "auxiliary_loss_mlp": 0.0104803, + "balance_loss_clip": 1.06128049, + "balance_loss_mlp": 1.02881384, + "epoch": 0.17102895943357901, + "flos": 39962865715200.0, + "grad_norm": 2.0639826039724363, + "language_loss": 1.02393174, + "learning_rate": 3.7949837926105826e-06, + "loss": 1.04583812, + "num_input_tokens_seen": 168477530, + "router_z_loss_clip": 0.81298828, + "router_z_loss_mlp": 0.1920166, + "step": 5894, + "time_per_iteration": 2.7332844734191895 + }, + { + "auxiliary_loss_clip": 0.01031913, + "auxiliary_loss_mlp": 0.01001438, + "balance_loss_clip": 1.01290536, + "balance_loss_mlp": 1.00059116, + "epoch": 0.17105797690209507, + "flos": 74776179968640.0, + "grad_norm": 0.6375905000879951, + "language_loss": 0.45025659, + "learning_rate": 3.794900887736069e-06, + "loss": 0.47059011, + "num_input_tokens_seen": 168538945, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.00848389, + "step": 5895, + "time_per_iteration": 3.1083691120147705 + }, + { + "auxiliary_loss_clip": 0.01033162, + "auxiliary_loss_mlp": 0.00999984, + "balance_loss_clip": 1.01404953, + "balance_loss_mlp": 0.99908412, + "epoch": 0.17108699437061112, + "flos": 64598198728320.0, + "grad_norm": 0.6503934527762635, + "language_loss": 0.46823835, + "learning_rate": 3.7948179670082646e-06, + "loss": 0.48856986, + "num_input_tokens_seen": 168601545, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.00897217, + "step": 5896, + "time_per_iteration": 3.0065174102783203 + }, + { + "auxiliary_loss_clip": 0.01132614, + "auxiliary_loss_mlp": 0.01049666, + "balance_loss_clip": 1.05799437, + "balance_loss_mlp": 1.03366244, + "epoch": 0.17111601183912714, + "flos": 30987532667520.0, + "grad_norm": 2.534893907093333, + "language_loss": 0.71496868, + "learning_rate": 3.794735030427902e-06, + "loss": 0.73679149, + "num_input_tokens_seen": 168621200, + "router_z_loss_clip": 0.74609375, + "router_z_loss_mlp": 0.16003418, + "step": 5897, + "time_per_iteration": 2.6475136280059814 + }, + { + "auxiliary_loss_clip": 0.01032898, + "auxiliary_loss_mlp": 0.01010783, + "balance_loss_clip": 1.01369345, + "balance_loss_mlp": 1.00993061, + "epoch": 0.1711450293076432, + "flos": 69233548020480.0, + "grad_norm": 0.6262667663221663, + "language_loss": 0.51371348, + "learning_rate": 3.794652077995713e-06, + "loss": 0.53415024, + "num_input_tokens_seen": 168686480, + "router_z_loss_clip": 0.19238281, + "router_z_loss_mlp": 0.00854492, + "step": 5898, + "time_per_iteration": 3.095874071121216 + }, + { + "auxiliary_loss_clip": 0.01139582, + "auxiliary_loss_mlp": 0.01057027, + "balance_loss_clip": 1.05863738, + "balance_loss_mlp": 1.03890681, + "epoch": 0.17117404677615924, + "flos": 32082520821120.0, + "grad_norm": 3.2727751982566233, + "language_loss": 0.85561371, + "learning_rate": 3.794569109712431e-06, + "loss": 0.87757981, + "num_input_tokens_seen": 168702015, + "router_z_loss_clip": 0.80957031, + "router_z_loss_mlp": 0.18139648, + "step": 5899, + "time_per_iteration": 2.610074281692505 + }, + { + "auxiliary_loss_clip": 0.01032889, + "auxiliary_loss_mlp": 0.01010205, + "balance_loss_clip": 1.01372623, + "balance_loss_mlp": 1.0093708, + "epoch": 0.1712030642446753, + "flos": 74772480867840.0, + "grad_norm": 0.672779399355133, + "language_loss": 0.48786587, + "learning_rate": 3.794486125578788e-06, + "loss": 0.50829685, + "num_input_tokens_seen": 168764760, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.00836182, + "step": 5900, + "time_per_iteration": 3.0762436389923096 + }, + { + "auxiliary_loss_clip": 0.011309, + "auxiliary_loss_mlp": 0.01039405, + "balance_loss_clip": 1.05711305, + "balance_loss_mlp": 1.02579761, + "epoch": 0.17123208171319135, + "flos": 34202076514560.0, + "grad_norm": 2.199523409081311, + "language_loss": 1.0032928, + "learning_rate": 3.7944031255955178e-06, + "loss": 1.02499604, + "num_input_tokens_seen": 168789380, + "router_z_loss_clip": 0.73779297, + "router_z_loss_mlp": 0.1361084, + "step": 5901, + "time_per_iteration": 2.7463886737823486 + }, + { + "auxiliary_loss_clip": 0.01031111, + "auxiliary_loss_mlp": 0.01014699, + "balance_loss_clip": 1.01212192, + "balance_loss_mlp": 1.01385903, + "epoch": 0.1712610991817074, + "flos": 72394296272640.0, + "grad_norm": 0.6720490422985607, + "language_loss": 0.46406189, + "learning_rate": 3.7943201097633527e-06, + "loss": 0.48451999, + "num_input_tokens_seen": 168843170, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.00842285, + "step": 5902, + "time_per_iteration": 3.025273084640503 + }, + { + "auxiliary_loss_clip": 0.01131418, + "auxiliary_loss_mlp": 0.01042576, + "balance_loss_clip": 1.05949986, + "balance_loss_mlp": 1.0277698, + "epoch": 0.17129011665022342, + "flos": 15222856469760.0, + "grad_norm": 2.4394260493197892, + "language_loss": 0.95481884, + "learning_rate": 3.794237078083026e-06, + "loss": 0.97655886, + "num_input_tokens_seen": 168853980, + "router_z_loss_clip": 0.71923828, + "router_z_loss_mlp": 0.14794922, + "step": 5903, + "time_per_iteration": 4.930679798126221 + }, + { + "auxiliary_loss_clip": 0.01030579, + "auxiliary_loss_mlp": 0.01009318, + "balance_loss_clip": 1.01173043, + "balance_loss_mlp": 1.00848949, + "epoch": 0.17131913411873947, + "flos": 61432458485760.0, + "grad_norm": 0.705013046413014, + "language_loss": 0.50528336, + "learning_rate": 3.7941540305552724e-06, + "loss": 0.52568233, + "num_input_tokens_seen": 168915210, + "router_z_loss_clip": 0.18847656, + "router_z_loss_mlp": 0.00830078, + "step": 5904, + "time_per_iteration": 7.838313341140747 + }, + { + "auxiliary_loss_clip": 0.01129548, + "auxiliary_loss_mlp": 0.0103677, + "balance_loss_clip": 1.05263412, + "balance_loss_mlp": 1.02139163, + "epoch": 0.17134815158725553, + "flos": 18799631020800.0, + "grad_norm": 3.1293744265551102, + "language_loss": 0.8302393, + "learning_rate": 3.794070967180824e-06, + "loss": 0.85190243, + "num_input_tokens_seen": 168927295, + "router_z_loss_clip": 0.76953125, + "router_z_loss_mlp": 0.15393066, + "step": 5905, + "time_per_iteration": 4.8297810554504395 + }, + { + "auxiliary_loss_clip": 0.01129808, + "auxiliary_loss_mlp": 0.01034605, + "balance_loss_clip": 1.05680192, + "balance_loss_mlp": 1.01966166, + "epoch": 0.17137716905577158, + "flos": 23946563767680.0, + "grad_norm": 2.8275933875294608, + "language_loss": 0.75179636, + "learning_rate": 3.793987887960414e-06, + "loss": 0.77344048, + "num_input_tokens_seen": 168944685, + "router_z_loss_clip": 0.72949219, + "router_z_loss_mlp": 0.14959717, + "step": 5906, + "time_per_iteration": 2.5945956707000732 + }, + { + "auxiliary_loss_clip": 0.01142715, + "auxiliary_loss_mlp": 0.01039208, + "balance_loss_clip": 1.06038451, + "balance_loss_mlp": 1.02083778, + "epoch": 0.17140618652428763, + "flos": 29861159005440.0, + "grad_norm": 2.2498535410411695, + "language_loss": 0.80544251, + "learning_rate": 3.7939047928947775e-06, + "loss": 0.82726175, + "num_input_tokens_seen": 168968445, + "router_z_loss_clip": 0.82421875, + "router_z_loss_mlp": 0.18365479, + "step": 5907, + "time_per_iteration": 2.7663967609405518 + }, + { + "auxiliary_loss_clip": 0.01133015, + "auxiliary_loss_mlp": 0.01039347, + "balance_loss_clip": 1.05922723, + "balance_loss_mlp": 1.02395105, + "epoch": 0.17143520399280368, + "flos": 17048698652160.0, + "grad_norm": 2.5901144315218403, + "language_loss": 0.70795459, + "learning_rate": 3.7938216819846485e-06, + "loss": 0.72967815, + "num_input_tokens_seen": 168982190, + "router_z_loss_clip": 0.73779297, + "router_z_loss_mlp": 0.15393066, + "step": 5908, + "time_per_iteration": 2.5018773078918457 + }, + { + "auxiliary_loss_clip": 0.01128213, + "auxiliary_loss_mlp": 0.0103521, + "balance_loss_clip": 1.05876088, + "balance_loss_mlp": 1.02109551, + "epoch": 0.1714642214613197, + "flos": 74744611914240.0, + "grad_norm": 1.5740141915491062, + "language_loss": 0.88670909, + "learning_rate": 3.79373855523076e-06, + "loss": 0.90834332, + "num_input_tokens_seen": 169011425, + "router_z_loss_clip": 0.6953125, + "router_z_loss_mlp": 0.14105225, + "step": 5909, + "time_per_iteration": 2.980968952178955 + }, + { + "auxiliary_loss_clip": 0.01135297, + "auxiliary_loss_mlp": 0.01040781, + "balance_loss_clip": 1.06204569, + "balance_loss_mlp": 1.02487886, + "epoch": 0.17149323892983576, + "flos": 31537576609920.0, + "grad_norm": 1.9023224302170156, + "language_loss": 0.74869478, + "learning_rate": 3.7936554126338473e-06, + "loss": 0.77045554, + "num_input_tokens_seen": 169028505, + "router_z_loss_clip": 0.73193359, + "router_z_loss_mlp": 0.15899658, + "step": 5910, + "time_per_iteration": 2.6332783699035645 + }, + { + "auxiliary_loss_clip": 0.01141913, + "auxiliary_loss_mlp": 0.01042696, + "balance_loss_clip": 1.06493032, + "balance_loss_mlp": 1.02613759, + "epoch": 0.1715222563983518, + "flos": 15223359260160.0, + "grad_norm": 2.6312299095956218, + "language_loss": 0.84331238, + "learning_rate": 3.793572254194643e-06, + "loss": 0.86515844, + "num_input_tokens_seen": 169041460, + "router_z_loss_clip": 0.77001953, + "router_z_loss_mlp": 0.16552734, + "step": 5911, + "time_per_iteration": 2.588310718536377 + }, + { + "auxiliary_loss_clip": 0.01144099, + "auxiliary_loss_mlp": 0.01043052, + "balance_loss_clip": 1.06379414, + "balance_loss_mlp": 1.02445495, + "epoch": 0.17155127386686786, + "flos": 74732149895040.0, + "grad_norm": 2.1263064901563777, + "language_loss": 0.84870028, + "learning_rate": 3.793489079913884e-06, + "loss": 0.87057185, + "num_input_tokens_seen": 169064675, + "router_z_loss_clip": 0.80273438, + "router_z_loss_mlp": 0.18603516, + "step": 5912, + "time_per_iteration": 2.9564995765686035 + }, + { + "auxiliary_loss_clip": 0.01146401, + "auxiliary_loss_mlp": 0.01040499, + "balance_loss_clip": 1.06823301, + "balance_loss_mlp": 1.02610469, + "epoch": 0.1715802913353839, + "flos": 65430137629440.0, + "grad_norm": 2.7376618323992834, + "language_loss": 0.92481226, + "learning_rate": 3.7934058897923032e-06, + "loss": 0.94668126, + "num_input_tokens_seen": 169091725, + "router_z_loss_clip": 0.78320312, + "router_z_loss_mlp": 0.14398193, + "step": 5913, + "time_per_iteration": 2.904019594192505 + }, + { + "auxiliary_loss_clip": 0.01142848, + "auxiliary_loss_mlp": 0.01036429, + "balance_loss_clip": 1.06756997, + "balance_loss_mlp": 1.0219686, + "epoch": 0.17160930880389993, + "flos": 47119898436480.0, + "grad_norm": 1.6390440506475332, + "language_loss": 0.77241415, + "learning_rate": 3.7933226838306356e-06, + "loss": 0.79420686, + "num_input_tokens_seen": 169116930, + "router_z_loss_clip": 0.75195312, + "router_z_loss_mlp": 0.14453125, + "step": 5914, + "time_per_iteration": 2.6947617530822754 + }, + { + "auxiliary_loss_clip": 0.01051727, + "auxiliary_loss_mlp": 0.0099766, + "balance_loss_clip": 1.03265834, + "balance_loss_mlp": 0.99672455, + "epoch": 0.171638326272416, + "flos": 60040129547520.0, + "grad_norm": 0.7095962772638718, + "language_loss": 0.50678283, + "learning_rate": 3.7932394620296167e-06, + "loss": 0.52727669, + "num_input_tokens_seen": 169174730, + "router_z_loss_clip": 0.19042969, + "router_z_loss_mlp": 0.00933838, + "step": 5915, + "time_per_iteration": 3.069044589996338 + }, + { + "auxiliary_loss_clip": 0.01151313, + "auxiliary_loss_mlp": 0.01055859, + "balance_loss_clip": 1.06567192, + "balance_loss_mlp": 1.03613544, + "epoch": 0.17166734374093204, + "flos": 49119475726080.0, + "grad_norm": 2.154780316915541, + "language_loss": 0.86653912, + "learning_rate": 3.7931562243899816e-06, + "loss": 0.88861084, + "num_input_tokens_seen": 169194995, + "router_z_loss_clip": 0.85693359, + "router_z_loss_mlp": 0.19726562, + "step": 5916, + "time_per_iteration": 2.783808946609497 + }, + { + "auxiliary_loss_clip": 0.01138984, + "auxiliary_loss_mlp": 0.01039987, + "balance_loss_clip": 1.06530297, + "balance_loss_mlp": 1.02564049, + "epoch": 0.1716963612094481, + "flos": 49703024079360.0, + "grad_norm": 1.9043920992572987, + "language_loss": 0.73549783, + "learning_rate": 3.7930729709124643e-06, + "loss": 0.75728756, + "num_input_tokens_seen": 169214065, + "router_z_loss_clip": 0.73681641, + "router_z_loss_mlp": 0.14349365, + "step": 5917, + "time_per_iteration": 2.8058407306671143 + }, + { + "auxiliary_loss_clip": 0.011388, + "auxiliary_loss_mlp": 0.01040555, + "balance_loss_clip": 1.06386161, + "balance_loss_mlp": 1.02555251, + "epoch": 0.17172537867796414, + "flos": 27337392587520.0, + "grad_norm": 2.8920507167828515, + "language_loss": 0.7155509, + "learning_rate": 3.7929897015978013e-06, + "loss": 0.73734438, + "num_input_tokens_seen": 169228850, + "router_z_loss_clip": 0.75048828, + "router_z_loss_mlp": 0.14990234, + "step": 5918, + "time_per_iteration": 2.5697643756866455 + }, + { + "auxiliary_loss_clip": 0.01049141, + "auxiliary_loss_mlp": 0.0100764, + "balance_loss_clip": 1.02992296, + "balance_loss_mlp": 1.00675762, + "epoch": 0.1717543961464802, + "flos": 74772480867840.0, + "grad_norm": 0.6366245202396597, + "language_loss": 0.49443933, + "learning_rate": 3.792906416446728e-06, + "loss": 0.51500714, + "num_input_tokens_seen": 169294270, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.0088501, + "step": 5919, + "time_per_iteration": 3.1627891063690186 + }, + { + "auxiliary_loss_clip": 0.01045787, + "auxiliary_loss_mlp": 0.01003416, + "balance_loss_clip": 1.02665555, + "balance_loss_mlp": 1.00255775, + "epoch": 0.17178341361499622, + "flos": 73023416006400.0, + "grad_norm": 0.6704342510256082, + "language_loss": 0.48604512, + "learning_rate": 3.7928231154599796e-06, + "loss": 0.50653714, + "num_input_tokens_seen": 169356315, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.00860596, + "step": 5920, + "time_per_iteration": 3.1498546600341797 + }, + { + "auxiliary_loss_clip": 0.01044753, + "auxiliary_loss_mlp": 0.01005981, + "balance_loss_clip": 1.0256902, + "balance_loss_mlp": 1.00505686, + "epoch": 0.17181243108351227, + "flos": 63214022177280.0, + "grad_norm": 0.6095136586658155, + "language_loss": 0.47051847, + "learning_rate": 3.7927397986382913e-06, + "loss": 0.49102587, + "num_input_tokens_seen": 169420745, + "router_z_loss_clip": 0.19042969, + "router_z_loss_mlp": 0.00921631, + "step": 5921, + "time_per_iteration": 3.1077370643615723 + }, + { + "auxiliary_loss_clip": 0.0114493, + "auxiliary_loss_mlp": 0.01036621, + "balance_loss_clip": 1.06563973, + "balance_loss_mlp": 1.02056384, + "epoch": 0.17184144855202832, + "flos": 74592709297920.0, + "grad_norm": 1.818549731193175, + "language_loss": 0.81545126, + "learning_rate": 3.7926564659824003e-06, + "loss": 0.8372668, + "num_input_tokens_seen": 169445415, + "router_z_loss_clip": 0.79296875, + "router_z_loss_mlp": 0.16064453, + "step": 5922, + "time_per_iteration": 2.9810738563537598 + }, + { + "auxiliary_loss_clip": 0.01042579, + "auxiliary_loss_mlp": 0.0100054, + "balance_loss_clip": 1.02348852, + "balance_loss_mlp": 0.99968797, + "epoch": 0.17187046602054437, + "flos": 72409775034240.0, + "grad_norm": 0.6641708088937154, + "language_loss": 0.48479921, + "learning_rate": 3.792573117493042e-06, + "loss": 0.50523043, + "num_input_tokens_seen": 169512865, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.00854492, + "step": 5923, + "time_per_iteration": 3.2504889965057373 + }, + { + "auxiliary_loss_clip": 0.01131133, + "auxiliary_loss_mlp": 0.0104277, + "balance_loss_clip": 1.05884027, + "balance_loss_mlp": 1.02651548, + "epoch": 0.17189948348906042, + "flos": 43975954771200.0, + "grad_norm": 1.936870251336733, + "language_loss": 0.8552928, + "learning_rate": 3.792489753170953e-06, + "loss": 0.8770318, + "num_input_tokens_seen": 169534260, + "router_z_loss_clip": 0.72265625, + "router_z_loss_mlp": 0.16241455, + "step": 5924, + "time_per_iteration": 2.812190294265747 + }, + { + "auxiliary_loss_clip": 0.01136984, + "auxiliary_loss_mlp": 0.01050134, + "balance_loss_clip": 1.06179011, + "balance_loss_mlp": 1.03239608, + "epoch": 0.17192850095757647, + "flos": 18432300585600.0, + "grad_norm": 3.0979488399980406, + "language_loss": 1.14144969, + "learning_rate": 3.792406373016868e-06, + "loss": 1.16332078, + "num_input_tokens_seen": 169546285, + "router_z_loss_clip": 0.75195312, + "router_z_loss_mlp": 0.17736816, + "step": 5925, + "time_per_iteration": 2.5391643047332764 + }, + { + "auxiliary_loss_clip": 0.01134029, + "auxiliary_loss_mlp": 0.01035015, + "balance_loss_clip": 1.06121361, + "balance_loss_mlp": 1.01963079, + "epoch": 0.1719575184260925, + "flos": 16357310691840.0, + "grad_norm": 2.3261353379341085, + "language_loss": 0.7380513, + "learning_rate": 3.792322977031525e-06, + "loss": 0.75974166, + "num_input_tokens_seen": 169563250, + "router_z_loss_clip": 0.72851562, + "router_z_loss_mlp": 0.15374756, + "step": 5926, + "time_per_iteration": 2.5504074096679688 + }, + { + "auxiliary_loss_clip": 0.0113809, + "auxiliary_loss_mlp": 0.01045866, + "balance_loss_clip": 1.06338179, + "balance_loss_mlp": 1.03139424, + "epoch": 0.17198653589460855, + "flos": 33395378918400.0, + "grad_norm": 3.26830525803389, + "language_loss": 0.90767908, + "learning_rate": 3.7922395652156607e-06, + "loss": 0.92951864, + "num_input_tokens_seen": 169581370, + "router_z_loss_clip": 0.74658203, + "router_z_loss_mlp": 0.14471436, + "step": 5927, + "time_per_iteration": 2.6772332191467285 + }, + { + "auxiliary_loss_clip": 0.01044667, + "auxiliary_loss_mlp": 0.00998456, + "balance_loss_clip": 1.02560282, + "balance_loss_mlp": 0.997693, + "epoch": 0.1720155533631246, + "flos": 67730973264000.0, + "grad_norm": 0.7044409143623319, + "language_loss": 0.46306145, + "learning_rate": 3.7921561375700107e-06, + "loss": 0.48349267, + "num_input_tokens_seen": 169641765, + "router_z_loss_clip": 0.19042969, + "router_z_loss_mlp": 0.00762939, + "step": 5928, + "time_per_iteration": 3.119088888168335 + }, + { + "auxiliary_loss_clip": 0.01134652, + "auxiliary_loss_mlp": 0.01036929, + "balance_loss_clip": 1.06128848, + "balance_loss_mlp": 1.02077007, + "epoch": 0.17204457083164065, + "flos": 35985112663680.0, + "grad_norm": 1.9880140643882172, + "language_loss": 0.87952721, + "learning_rate": 3.7920726940953127e-06, + "loss": 0.90124309, + "num_input_tokens_seen": 169661490, + "router_z_loss_clip": 0.734375, + "router_z_loss_mlp": 0.16168213, + "step": 5929, + "time_per_iteration": 2.68107271194458 + }, + { + "auxiliary_loss_clip": 0.0104372, + "auxiliary_loss_mlp": 0.00998566, + "balance_loss_clip": 1.02461481, + "balance_loss_mlp": 0.99774343, + "epoch": 0.1720735883001567, + "flos": 58717143815040.0, + "grad_norm": 0.7061489573973816, + "language_loss": 0.48504841, + "learning_rate": 3.7919892347923036e-06, + "loss": 0.50547123, + "num_input_tokens_seen": 169723845, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.00823975, + "step": 5930, + "time_per_iteration": 3.1114282608032227 + }, + { + "auxiliary_loss_clip": 0.01134379, + "auxiliary_loss_mlp": 0.01047428, + "balance_loss_clip": 1.06037724, + "balance_loss_mlp": 1.03195441, + "epoch": 0.17210260576867273, + "flos": 14312162021760.0, + "grad_norm": 2.439386174276212, + "language_loss": 0.80764514, + "learning_rate": 3.7919057596617207e-06, + "loss": 0.82946324, + "num_input_tokens_seen": 169736185, + "router_z_loss_clip": 0.73974609, + "router_z_loss_mlp": 0.15478516, + "step": 5931, + "time_per_iteration": 2.4976651668548584 + }, + { + "auxiliary_loss_clip": 0.011399, + "auxiliary_loss_mlp": 0.01043622, + "balance_loss_clip": 1.06630325, + "balance_loss_mlp": 1.028512, + "epoch": 0.17213162323718878, + "flos": 28980664917120.0, + "grad_norm": 2.5262444803024104, + "language_loss": 0.81629157, + "learning_rate": 3.791822268704301e-06, + "loss": 0.8381269, + "num_input_tokens_seen": 169755475, + "router_z_loss_clip": 0.73681641, + "router_z_loss_mlp": 0.15112305, + "step": 5932, + "time_per_iteration": 2.616837501525879 + }, + { + "auxiliary_loss_clip": 0.0104169, + "auxiliary_loss_mlp": 0.01003678, + "balance_loss_clip": 1.02252805, + "balance_loss_mlp": 1.0028733, + "epoch": 0.17216064070570483, + "flos": 65627470949760.0, + "grad_norm": 0.6658733539199099, + "language_loss": 0.47848186, + "learning_rate": 3.791738761920781e-06, + "loss": 0.49893552, + "num_input_tokens_seen": 169823805, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.00805664, + "step": 5933, + "time_per_iteration": 3.2530596256256104 + }, + { + "auxiliary_loss_clip": 0.01139375, + "auxiliary_loss_mlp": 0.01038292, + "balance_loss_clip": 1.06123328, + "balance_loss_mlp": 1.02215672, + "epoch": 0.17218965817422088, + "flos": 32299708406400.0, + "grad_norm": 2.103999020417484, + "language_loss": 0.80687761, + "learning_rate": 3.7916552393119004e-06, + "loss": 0.82865429, + "num_input_tokens_seen": 169840350, + "router_z_loss_clip": 0.78222656, + "router_z_loss_mlp": 0.16143799, + "step": 5934, + "time_per_iteration": 2.6463911533355713 + }, + { + "auxiliary_loss_clip": 0.01038882, + "auxiliary_loss_mlp": 0.01005558, + "balance_loss_clip": 1.01974607, + "balance_loss_mlp": 1.00472915, + "epoch": 0.17221867564273693, + "flos": 58683028872960.0, + "grad_norm": 0.6779197461369398, + "language_loss": 0.50768167, + "learning_rate": 3.791571700878395e-06, + "loss": 0.52812606, + "num_input_tokens_seen": 169902070, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.00830078, + "step": 5935, + "time_per_iteration": 3.1143436431884766 + }, + { + "auxiliary_loss_clip": 0.01127899, + "auxiliary_loss_mlp": 0.01048136, + "balance_loss_clip": 1.05408072, + "balance_loss_mlp": 1.03393817, + "epoch": 0.172247693111253, + "flos": 34598134851840.0, + "grad_norm": 2.1720216247679485, + "language_loss": 0.84415841, + "learning_rate": 3.7914881466210035e-06, + "loss": 0.8659187, + "num_input_tokens_seen": 169921545, + "router_z_loss_clip": 0.73828125, + "router_z_loss_mlp": 0.14202881, + "step": 5936, + "time_per_iteration": 2.682739019393921 + }, + { + "auxiliary_loss_clip": 0.01035864, + "auxiliary_loss_mlp": 0.01006967, + "balance_loss_clip": 1.01682138, + "balance_loss_mlp": 1.00616217, + "epoch": 0.172276710579769, + "flos": 62331014136960.0, + "grad_norm": 0.6765061976341811, + "language_loss": 0.46013215, + "learning_rate": 3.791404576540464e-06, + "loss": 0.48056045, + "num_input_tokens_seen": 169982565, + "router_z_loss_clip": 0.19042969, + "router_z_loss_mlp": 0.00805664, + "step": 5937, + "time_per_iteration": 3.0389723777770996 + }, + { + "auxiliary_loss_clip": 0.01034104, + "auxiliary_loss_mlp": 0.01003103, + "balance_loss_clip": 1.01519084, + "balance_loss_mlp": 1.00232792, + "epoch": 0.17230572804828506, + "flos": 63951484308480.0, + "grad_norm": 0.6369444321232449, + "language_loss": 0.49007809, + "learning_rate": 3.791320990637514e-06, + "loss": 0.51045012, + "num_input_tokens_seen": 170046125, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.00775146, + "step": 5938, + "time_per_iteration": 3.1232712268829346 + }, + { + "auxiliary_loss_clip": 0.01130446, + "auxiliary_loss_mlp": 0.01047448, + "balance_loss_clip": 1.05541015, + "balance_loss_mlp": 1.03094935, + "epoch": 0.1723347455168011, + "flos": 32008616588160.0, + "grad_norm": 2.114908651729791, + "language_loss": 0.80711401, + "learning_rate": 3.7912373889128926e-06, + "loss": 0.82889301, + "num_input_tokens_seen": 170062945, + "router_z_loss_clip": 0.75, + "router_z_loss_mlp": 0.16491699, + "step": 5939, + "time_per_iteration": 2.650965690612793 + }, + { + "auxiliary_loss_clip": 0.01132865, + "auxiliary_loss_mlp": 0.01038005, + "balance_loss_clip": 1.05526137, + "balance_loss_mlp": 1.02216804, + "epoch": 0.17236376298531716, + "flos": 28326552295680.0, + "grad_norm": 2.013532168057257, + "language_loss": 0.96551263, + "learning_rate": 3.7911537713673374e-06, + "loss": 0.98722136, + "num_input_tokens_seen": 170079885, + "router_z_loss_clip": 0.77636719, + "router_z_loss_mlp": 0.1583252, + "step": 5940, + "time_per_iteration": 2.615699052810669 + }, + { + "auxiliary_loss_clip": 0.01132804, + "auxiliary_loss_mlp": 0.01038079, + "balance_loss_clip": 1.058815, + "balance_loss_mlp": 1.02194405, + "epoch": 0.17239278045383322, + "flos": 18728599875840.0, + "grad_norm": 2.419263320820175, + "language_loss": 0.8005392, + "learning_rate": 3.7910701380015872e-06, + "loss": 0.82224798, + "num_input_tokens_seen": 170094190, + "router_z_loss_clip": 0.74023438, + "router_z_loss_mlp": 0.16125488, + "step": 5941, + "time_per_iteration": 2.5448696613311768 + }, + { + "auxiliary_loss_clip": 0.01134515, + "auxiliary_loss_mlp": 0.01038387, + "balance_loss_clip": 1.05882549, + "balance_loss_mlp": 1.02360511, + "epoch": 0.17242179792234927, + "flos": 11610386778240.0, + "grad_norm": 2.685451837254641, + "language_loss": 0.95134211, + "learning_rate": 3.790986488816381e-06, + "loss": 0.97307116, + "num_input_tokens_seen": 170104810, + "router_z_loss_clip": 0.75732422, + "router_z_loss_mlp": 0.14764404, + "step": 5942, + "time_per_iteration": 2.5076143741607666 + }, + { + "auxiliary_loss_clip": 0.01033727, + "auxiliary_loss_mlp": 0.0100632, + "balance_loss_clip": 1.01471567, + "balance_loss_mlp": 1.00543833, + "epoch": 0.1724508153908653, + "flos": 55550900782080.0, + "grad_norm": 0.6804806937887309, + "language_loss": 0.47917622, + "learning_rate": 3.7909028238124572e-06, + "loss": 0.49957669, + "num_input_tokens_seen": 170166425, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.0088501, + "step": 5943, + "time_per_iteration": 3.0833613872528076 + }, + { + "auxiliary_loss_clip": 0.01129686, + "auxiliary_loss_mlp": 0.01039516, + "balance_loss_clip": 1.0553205, + "balance_loss_mlp": 1.0241971, + "epoch": 0.17247983285938134, + "flos": 20988960883200.0, + "grad_norm": 2.3072752910096135, + "language_loss": 0.6601932, + "learning_rate": 3.790819142990555e-06, + "loss": 0.68188524, + "num_input_tokens_seen": 170178940, + "router_z_loss_clip": 0.74316406, + "router_z_loss_mlp": 0.15325928, + "step": 5944, + "time_per_iteration": 2.5479774475097656 + }, + { + "auxiliary_loss_clip": 0.01134992, + "auxiliary_loss_mlp": 0.01034517, + "balance_loss_clip": 1.05979085, + "balance_loss_mlp": 1.01938319, + "epoch": 0.1725088503278974, + "flos": 30331372970880.0, + "grad_norm": 1.5926230476410013, + "language_loss": 0.73055184, + "learning_rate": 3.7907354463514137e-06, + "loss": 0.75224698, + "num_input_tokens_seen": 170199570, + "router_z_loss_clip": 0.75146484, + "router_z_loss_mlp": 0.15142822, + "step": 5945, + "time_per_iteration": 2.6326425075531006 + }, + { + "auxiliary_loss_clip": 0.01121766, + "auxiliary_loss_mlp": 0.01035648, + "balance_loss_clip": 1.05169129, + "balance_loss_mlp": 1.01821971, + "epoch": 0.17253786779641345, + "flos": 15114514072320.0, + "grad_norm": 2.8863623349004253, + "language_loss": 0.75246787, + "learning_rate": 3.7906517338957718e-06, + "loss": 0.77404201, + "num_input_tokens_seen": 170212160, + "router_z_loss_clip": 0.70117188, + "router_z_loss_mlp": 0.17419434, + "step": 5946, + "time_per_iteration": 2.548004388809204 + }, + { + "auxiliary_loss_clip": 0.01131514, + "auxiliary_loss_mlp": 0.01038017, + "balance_loss_clip": 1.05984139, + "balance_loss_mlp": 1.02264476, + "epoch": 0.1725668852649295, + "flos": 25623304594560.0, + "grad_norm": 2.001810541278969, + "language_loss": 0.85824823, + "learning_rate": 3.7905680056243696e-06, + "loss": 0.87994361, + "num_input_tokens_seen": 170228050, + "router_z_loss_clip": 0.71728516, + "router_z_loss_mlp": 0.15368652, + "step": 5947, + "time_per_iteration": 2.594254493713379 + }, + { + "auxiliary_loss_clip": 0.01131104, + "auxiliary_loss_mlp": 0.01033457, + "balance_loss_clip": 1.05694902, + "balance_loss_mlp": 1.01738751, + "epoch": 0.17259590273344552, + "flos": 12560726862720.0, + "grad_norm": 2.263262029088512, + "language_loss": 0.57545048, + "learning_rate": 3.790484261537946e-06, + "loss": 0.59709609, + "num_input_tokens_seen": 170239915, + "router_z_loss_clip": 0.74121094, + "router_z_loss_mlp": 0.1605835, + "step": 5948, + "time_per_iteration": 2.5093517303466797 + }, + { + "auxiliary_loss_clip": 0.01130471, + "auxiliary_loss_mlp": 0.01038313, + "balance_loss_clip": 1.05407453, + "balance_loss_mlp": 1.02124262, + "epoch": 0.17262492020196157, + "flos": 41895649664640.0, + "grad_norm": 1.944643646681813, + "language_loss": 0.72432053, + "learning_rate": 3.7904005016372413e-06, + "loss": 0.7460084, + "num_input_tokens_seen": 170258735, + "router_z_loss_clip": 0.76416016, + "router_z_loss_mlp": 0.1706543, + "step": 5949, + "time_per_iteration": 2.776010751724243 + }, + { + "auxiliary_loss_clip": 0.01138468, + "auxiliary_loss_mlp": 0.01048897, + "balance_loss_clip": 1.05973911, + "balance_loss_mlp": 1.02994323, + "epoch": 0.17265393767047763, + "flos": 44928736980480.0, + "grad_norm": 1.990398440880716, + "language_loss": 0.93204546, + "learning_rate": 3.7903167259229944e-06, + "loss": 0.95391905, + "num_input_tokens_seen": 170279315, + "router_z_loss_clip": 0.78710938, + "router_z_loss_mlp": 0.1895752, + "step": 5950, + "time_per_iteration": 2.73532772064209 + }, + { + "auxiliary_loss_clip": 0.01128687, + "auxiliary_loss_mlp": 0.01042525, + "balance_loss_clip": 1.054456, + "balance_loss_mlp": 1.02630663, + "epoch": 0.17268295513899368, + "flos": 34092836277120.0, + "grad_norm": 2.4139439236279, + "language_loss": 0.78378606, + "learning_rate": 3.790232934395946e-06, + "loss": 0.80549812, + "num_input_tokens_seen": 170299645, + "router_z_loss_clip": 0.74267578, + "router_z_loss_mlp": 0.16217041, + "step": 5951, + "time_per_iteration": 2.677814483642578 + }, + { + "auxiliary_loss_clip": 0.01136866, + "auxiliary_loss_mlp": 0.01036178, + "balance_loss_clip": 1.06056094, + "balance_loss_mlp": 1.02103794, + "epoch": 0.17271197260750973, + "flos": 38514625257600.0, + "grad_norm": 2.4211725423617265, + "language_loss": 0.6740036, + "learning_rate": 3.7901491270568354e-06, + "loss": 0.69573402, + "num_input_tokens_seen": 170320760, + "router_z_loss_clip": 0.76269531, + "router_z_loss_mlp": 0.15136719, + "step": 5952, + "time_per_iteration": 2.7207186222076416 + }, + { + "auxiliary_loss_clip": 0.01127587, + "auxiliary_loss_mlp": 0.01037606, + "balance_loss_clip": 1.05658257, + "balance_loss_mlp": 1.02283561, + "epoch": 0.17274099007602578, + "flos": 27778950478080.0, + "grad_norm": 1.9906001940444205, + "language_loss": 0.75498307, + "learning_rate": 3.790065303906404e-06, + "loss": 0.77663505, + "num_input_tokens_seen": 170346050, + "router_z_loss_clip": 0.71044922, + "router_z_loss_mlp": 0.14764404, + "step": 5953, + "time_per_iteration": 2.7523231506347656 + }, + { + "auxiliary_loss_clip": 0.01035463, + "auxiliary_loss_mlp": 0.01004849, + "balance_loss_clip": 1.01661861, + "balance_loss_mlp": 1.00397897, + "epoch": 0.1727700075445418, + "flos": 74773342794240.0, + "grad_norm": 0.6382492676626761, + "language_loss": 0.46261209, + "learning_rate": 3.7899814649453915e-06, + "loss": 0.48301524, + "num_input_tokens_seen": 170412375, + "router_z_loss_clip": 0.18847656, + "router_z_loss_mlp": 0.00872803, + "step": 5954, + "time_per_iteration": 3.2005600929260254 + }, + { + "auxiliary_loss_clip": 0.01138547, + "auxiliary_loss_mlp": 0.01052725, + "balance_loss_clip": 1.06309938, + "balance_loss_mlp": 1.03687036, + "epoch": 0.17279902501305786, + "flos": 18144692386560.0, + "grad_norm": 2.046980718819428, + "language_loss": 0.94353878, + "learning_rate": 3.7898976101745383e-06, + "loss": 0.96545148, + "num_input_tokens_seen": 170428060, + "router_z_loss_clip": 0.75488281, + "router_z_loss_mlp": 0.1585083, + "step": 5955, + "time_per_iteration": 2.5469532012939453 + }, + { + "auxiliary_loss_clip": 0.01136839, + "auxiliary_loss_mlp": 0.01047208, + "balance_loss_clip": 1.05695391, + "balance_loss_mlp": 1.02930903, + "epoch": 0.1728280424815739, + "flos": 38432891859840.0, + "grad_norm": 2.8009416087952497, + "language_loss": 0.92317122, + "learning_rate": 3.789813739594584e-06, + "loss": 0.94501168, + "num_input_tokens_seen": 170446315, + "router_z_loss_clip": 0.79882812, + "router_z_loss_mlp": 0.17889404, + "step": 5956, + "time_per_iteration": 2.7246718406677246 + }, + { + "auxiliary_loss_clip": 0.01135471, + "auxiliary_loss_mlp": 0.01041983, + "balance_loss_clip": 1.05809224, + "balance_loss_mlp": 1.02468538, + "epoch": 0.17285705995008996, + "flos": 62328604947840.0, + "grad_norm": 1.9656473825156353, + "language_loss": 0.90360832, + "learning_rate": 3.789729853206272e-06, + "loss": 0.92538285, + "num_input_tokens_seen": 170469490, + "router_z_loss_clip": 0.77441406, + "router_z_loss_mlp": 0.17303467, + "step": 5957, + "time_per_iteration": 2.8641884326934814 + }, + { + "auxiliary_loss_clip": 0.01126754, + "auxiliary_loss_mlp": 0.01037761, + "balance_loss_clip": 1.05428398, + "balance_loss_mlp": 1.0219481, + "epoch": 0.172886077418606, + "flos": 31649151231360.0, + "grad_norm": 1.7968148086310614, + "language_loss": 0.77388006, + "learning_rate": 3.7896459510103406e-06, + "loss": 0.79552519, + "num_input_tokens_seen": 170491935, + "router_z_loss_clip": 0.72412109, + "router_z_loss_mlp": 0.15808105, + "step": 5958, + "time_per_iteration": 2.722830295562744 + }, + { + "auxiliary_loss_clip": 0.01130656, + "auxiliary_loss_mlp": 0.01043546, + "balance_loss_clip": 1.05561972, + "balance_loss_mlp": 1.02757204, + "epoch": 0.17291509488712203, + "flos": 25298959760640.0, + "grad_norm": 1.8717294644715754, + "language_loss": 0.79173541, + "learning_rate": 3.7895620330075326e-06, + "loss": 0.8134774, + "num_input_tokens_seen": 170509955, + "router_z_loss_clip": 0.75097656, + "router_z_loss_mlp": 0.159729, + "step": 5959, + "time_per_iteration": 2.6146962642669678 + }, + { + "auxiliary_loss_clip": 0.01123977, + "auxiliary_loss_mlp": 0.01042967, + "balance_loss_clip": 1.05415201, + "balance_loss_mlp": 1.02837563, + "epoch": 0.17294411235563809, + "flos": 18325933436160.0, + "grad_norm": 1.876997598299576, + "language_loss": 0.69729567, + "learning_rate": 3.7894780991985887e-06, + "loss": 0.71896505, + "num_input_tokens_seen": 170523850, + "router_z_loss_clip": 0.69921875, + "router_z_loss_mlp": 0.14599609, + "step": 5960, + "time_per_iteration": 2.5815629959106445 + }, + { + "auxiliary_loss_clip": 0.01036617, + "auxiliary_loss_mlp": 0.01000512, + "balance_loss_clip": 1.01762128, + "balance_loss_mlp": 0.99957657, + "epoch": 0.17297312982415414, + "flos": 74776359536640.0, + "grad_norm": 0.6298238769406204, + "language_loss": 0.42818069, + "learning_rate": 3.7893941495842494e-06, + "loss": 0.44855195, + "num_input_tokens_seen": 170592165, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.00933838, + "step": 5961, + "time_per_iteration": 3.2693865299224854 + }, + { + "auxiliary_loss_clip": 0.01035796, + "auxiliary_loss_mlp": 0.00998805, + "balance_loss_clip": 1.01692796, + "balance_loss_mlp": 0.99800634, + "epoch": 0.1730021472926702, + "flos": 63788628044160.0, + "grad_norm": 0.6598667492987448, + "language_loss": 0.48749718, + "learning_rate": 3.7893101841652574e-06, + "loss": 0.5078432, + "num_input_tokens_seen": 170655785, + "router_z_loss_clip": 0.18847656, + "router_z_loss_mlp": 0.00799561, + "step": 5962, + "time_per_iteration": 3.2341628074645996 + }, + { + "auxiliary_loss_clip": 0.01134672, + "auxiliary_loss_mlp": 0.01038665, + "balance_loss_clip": 1.06061649, + "balance_loss_mlp": 1.02273309, + "epoch": 0.17303116476118624, + "flos": 11211132130560.0, + "grad_norm": 2.174731728979186, + "language_loss": 0.73654777, + "learning_rate": 3.7892262029423534e-06, + "loss": 0.75828111, + "num_input_tokens_seen": 170668735, + "router_z_loss_clip": 0.73974609, + "router_z_loss_mlp": 0.15924072, + "step": 5963, + "time_per_iteration": 2.5606982707977295 + }, + { + "auxiliary_loss_clip": 0.0113081, + "auxiliary_loss_mlp": 0.01035436, + "balance_loss_clip": 1.0563097, + "balance_loss_mlp": 1.02004635, + "epoch": 0.1730601822297023, + "flos": 22414758318720.0, + "grad_norm": 2.011005233972091, + "language_loss": 0.88893121, + "learning_rate": 3.7891422059162804e-06, + "loss": 0.91059369, + "num_input_tokens_seen": 170684035, + "router_z_loss_clip": 0.74511719, + "router_z_loss_mlp": 0.15380859, + "step": 5964, + "time_per_iteration": 2.726465940475464 + }, + { + "auxiliary_loss_clip": 0.01138024, + "auxiliary_loss_mlp": 0.01039388, + "balance_loss_clip": 1.05897498, + "balance_loss_mlp": 1.02270484, + "epoch": 0.17308919969821832, + "flos": 74736567267840.0, + "grad_norm": 2.0208957819109665, + "language_loss": 0.81969059, + "learning_rate": 3.789058193087778e-06, + "loss": 0.84146464, + "num_input_tokens_seen": 170714490, + "router_z_loss_clip": 0.79101562, + "router_z_loss_mlp": 0.16693115, + "step": 5965, + "time_per_iteration": 3.164729356765747 + }, + { + "auxiliary_loss_clip": 0.01129228, + "auxiliary_loss_mlp": 0.01041304, + "balance_loss_clip": 1.06008339, + "balance_loss_mlp": 1.02599168, + "epoch": 0.17311821716673437, + "flos": 17666828824320.0, + "grad_norm": 2.176864683287672, + "language_loss": 0.72636008, + "learning_rate": 3.788974164457591e-06, + "loss": 0.74806535, + "num_input_tokens_seen": 170729325, + "router_z_loss_clip": 0.69189453, + "router_z_loss_mlp": 0.15307617, + "step": 5966, + "time_per_iteration": 2.618767738342285 + }, + { + "auxiliary_loss_clip": 0.01036115, + "auxiliary_loss_mlp": 0.0100477, + "balance_loss_clip": 1.01733494, + "balance_loss_mlp": 1.00392365, + "epoch": 0.17314723463525042, + "flos": 55498433990400.0, + "grad_norm": 0.7029267712711114, + "language_loss": 0.48364305, + "learning_rate": 3.78889012002646e-06, + "loss": 0.50405192, + "num_input_tokens_seen": 170782530, + "router_z_loss_clip": 0.1875, + "router_z_loss_mlp": 0.00848389, + "step": 5967, + "time_per_iteration": 2.912775754928589 + }, + { + "auxiliary_loss_clip": 0.01035459, + "auxiliary_loss_mlp": 0.01003209, + "balance_loss_clip": 1.01658416, + "balance_loss_mlp": 1.00234473, + "epoch": 0.17317625210376647, + "flos": 63135125953920.0, + "grad_norm": 0.6618432337083907, + "language_loss": 0.53768438, + "learning_rate": 3.788806059795127e-06, + "loss": 0.55807102, + "num_input_tokens_seen": 170841870, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.00866699, + "step": 5968, + "time_per_iteration": 3.0775890350341797 + }, + { + "auxiliary_loss_clip": 0.01034056, + "auxiliary_loss_mlp": 0.01002481, + "balance_loss_clip": 1.01521087, + "balance_loss_mlp": 1.00166464, + "epoch": 0.17320526957228252, + "flos": 69322535965440.0, + "grad_norm": 0.721408065876217, + "language_loss": 0.49578595, + "learning_rate": 3.7887219837643355e-06, + "loss": 0.51615131, + "num_input_tokens_seen": 170905640, + "router_z_loss_clip": 0.18847656, + "router_z_loss_mlp": 0.00817871, + "step": 5969, + "time_per_iteration": 3.2763991355895996 + }, + { + "auxiliary_loss_clip": 0.01132004, + "auxiliary_loss_mlp": 0.01046712, + "balance_loss_clip": 1.05666912, + "balance_loss_mlp": 1.0315485, + "epoch": 0.17323428704079857, + "flos": 17122064181120.0, + "grad_norm": 2.3913989387780727, + "language_loss": 0.7171706, + "learning_rate": 3.7886378919348274e-06, + "loss": 0.7389577, + "num_input_tokens_seen": 170920345, + "router_z_loss_clip": 0.75341797, + "router_z_loss_mlp": 0.15167236, + "step": 5970, + "time_per_iteration": 2.599247694015503 + }, + { + "auxiliary_loss_clip": 0.01032672, + "auxiliary_loss_mlp": 0.01001419, + "balance_loss_clip": 1.01389599, + "balance_loss_mlp": 1.00059605, + "epoch": 0.1732633045093146, + "flos": 74774456115840.0, + "grad_norm": 0.6715802803810482, + "language_loss": 0.48828954, + "learning_rate": 3.7885537843073464e-06, + "loss": 0.50863045, + "num_input_tokens_seen": 170984390, + "router_z_loss_clip": 0.1875, + "router_z_loss_mlp": 0.00823975, + "step": 5971, + "time_per_iteration": 3.174818515777588 + }, + { + "auxiliary_loss_clip": 0.01125174, + "auxiliary_loss_mlp": 0.0103223, + "balance_loss_clip": 1.05499053, + "balance_loss_mlp": 1.01871204, + "epoch": 0.17329232197783065, + "flos": 28615166075520.0, + "grad_norm": 1.8880823164971205, + "language_loss": 0.76548654, + "learning_rate": 3.788469660882634e-06, + "loss": 0.78706062, + "num_input_tokens_seen": 171001155, + "router_z_loss_clip": 0.70214844, + "router_z_loss_mlp": 0.13525391, + "step": 5972, + "time_per_iteration": 2.6780598163604736 + }, + { + "auxiliary_loss_clip": 0.01129547, + "auxiliary_loss_mlp": 0.0104042, + "balance_loss_clip": 1.05437517, + "balance_loss_mlp": 1.0242312, + "epoch": 0.1733213394463467, + "flos": 22027355159040.0, + "grad_norm": 2.372646674549798, + "language_loss": 0.83183587, + "learning_rate": 3.7883855216614335e-06, + "loss": 0.85353547, + "num_input_tokens_seen": 171017045, + "router_z_loss_clip": 0.75195312, + "router_z_loss_mlp": 0.16174316, + "step": 5973, + "time_per_iteration": 2.540937662124634 + }, + { + "auxiliary_loss_clip": 0.01032544, + "auxiliary_loss_mlp": 0.01010947, + "balance_loss_clip": 1.01369679, + "balance_loss_mlp": 1.01014829, + "epoch": 0.17335035691486275, + "flos": 56022910427520.0, + "grad_norm": 0.6702109626553158, + "language_loss": 0.49382615, + "learning_rate": 3.7883013666444886e-06, + "loss": 0.51426101, + "num_input_tokens_seen": 171073410, + "router_z_loss_clip": 0.18847656, + "router_z_loss_mlp": 0.00799561, + "step": 5974, + "time_per_iteration": 5.298434257507324 + }, + { + "auxiliary_loss_clip": 0.01127773, + "auxiliary_loss_mlp": 0.01039306, + "balance_loss_clip": 1.05265069, + "balance_loss_mlp": 1.02402377, + "epoch": 0.1733793743833788, + "flos": 15919559642880.0, + "grad_norm": 3.7413732246553186, + "language_loss": 0.70357269, + "learning_rate": 3.7882171958325426e-06, + "loss": 0.72524345, + "num_input_tokens_seen": 171088920, + "router_z_loss_clip": 0.75195312, + "router_z_loss_mlp": 0.152771, + "step": 5975, + "time_per_iteration": 5.0916924476623535 + }, + { + "auxiliary_loss_clip": 0.01031694, + "auxiliary_loss_mlp": 0.010017, + "balance_loss_clip": 1.01282763, + "balance_loss_mlp": 1.00088334, + "epoch": 0.17340839185189483, + "flos": 74775102560640.0, + "grad_norm": 0.6145757803515374, + "language_loss": 0.47599399, + "learning_rate": 3.7881330092263386e-06, + "loss": 0.49632791, + "num_input_tokens_seen": 171153495, + "router_z_loss_clip": 0.18847656, + "router_z_loss_mlp": 0.00817871, + "step": 5976, + "time_per_iteration": 5.526043653488159 + }, + { + "auxiliary_loss_clip": 0.01138274, + "auxiliary_loss_mlp": 0.01044335, + "balance_loss_clip": 1.05744493, + "balance_loss_mlp": 1.02710915, + "epoch": 0.17343740932041088, + "flos": 46311872037120.0, + "grad_norm": 2.5181064807228712, + "language_loss": 0.75631183, + "learning_rate": 3.7880488068266205e-06, + "loss": 0.77813792, + "num_input_tokens_seen": 171170025, + "router_z_loss_clip": 0.80908203, + "router_z_loss_mlp": 0.17236328, + "step": 5977, + "time_per_iteration": 2.665738582611084 + }, + { + "auxiliary_loss_clip": 0.01133015, + "auxiliary_loss_mlp": 0.01048562, + "balance_loss_clip": 1.05598044, + "balance_loss_mlp": 1.03253436, + "epoch": 0.17346642678892693, + "flos": 35328414263040.0, + "grad_norm": 3.1603650656373823, + "language_loss": 0.84233701, + "learning_rate": 3.787964588634131e-06, + "loss": 0.86415291, + "num_input_tokens_seen": 171186580, + "router_z_loss_clip": 0.77050781, + "router_z_loss_mlp": 0.16009521, + "step": 5978, + "time_per_iteration": 2.5949714183807373 + }, + { + "auxiliary_loss_clip": 0.0113383, + "auxiliary_loss_mlp": 0.01042894, + "balance_loss_clip": 1.05464613, + "balance_loss_mlp": 1.02676463, + "epoch": 0.17349544425744298, + "flos": 28943605059840.0, + "grad_norm": 3.4130082905484502, + "language_loss": 0.98557699, + "learning_rate": 3.7878803546496157e-06, + "loss": 1.00734425, + "num_input_tokens_seen": 171200445, + "router_z_loss_clip": 0.79199219, + "router_z_loss_mlp": 0.16125488, + "step": 5979, + "time_per_iteration": 2.590810537338257 + }, + { + "auxiliary_loss_clip": 0.01132227, + "auxiliary_loss_mlp": 0.01040573, + "balance_loss_clip": 1.05923879, + "balance_loss_mlp": 1.02289414, + "epoch": 0.17352446172595903, + "flos": 17121920526720.0, + "grad_norm": 2.804777534156646, + "language_loss": 0.74838996, + "learning_rate": 3.7877961048738172e-06, + "loss": 0.770118, + "num_input_tokens_seen": 171212690, + "router_z_loss_clip": 0.72998047, + "router_z_loss_mlp": 0.17663574, + "step": 5980, + "time_per_iteration": 2.519787549972534 + }, + { + "auxiliary_loss_clip": 0.01143575, + "auxiliary_loss_mlp": 0.01040026, + "balance_loss_clip": 1.06103373, + "balance_loss_mlp": 1.02331328, + "epoch": 0.17355347919447509, + "flos": 40983051795840.0, + "grad_norm": 2.24285169283302, + "language_loss": 0.88556707, + "learning_rate": 3.78771183930748e-06, + "loss": 0.90740299, + "num_input_tokens_seen": 171230875, + "router_z_loss_clip": 0.82568359, + "router_z_loss_mlp": 0.16729736, + "step": 5981, + "time_per_iteration": 2.6937038898468018 + }, + { + "auxiliary_loss_clip": 0.01134652, + "auxiliary_loss_mlp": 0.01044876, + "balance_loss_clip": 1.05807412, + "balance_loss_mlp": 1.02818632, + "epoch": 0.1735824966629911, + "flos": 10955843193600.0, + "grad_norm": 2.6406192714312224, + "language_loss": 0.86821365, + "learning_rate": 3.7876275579513487e-06, + "loss": 0.89000893, + "num_input_tokens_seen": 171241535, + "router_z_loss_clip": 0.765625, + "router_z_loss_mlp": 0.16699219, + "step": 5982, + "time_per_iteration": 2.535151481628418 + }, + { + "auxiliary_loss_clip": 0.01133068, + "auxiliary_loss_mlp": 0.0104321, + "balance_loss_clip": 1.05773211, + "balance_loss_mlp": 1.02700305, + "epoch": 0.17361151413150716, + "flos": 27046013460480.0, + "grad_norm": 2.111624028018974, + "language_loss": 0.8381241, + "learning_rate": 3.787543260806167e-06, + "loss": 0.85988688, + "num_input_tokens_seen": 171258625, + "router_z_loss_clip": 0.75341797, + "router_z_loss_mlp": 0.16204834, + "step": 5983, + "time_per_iteration": 2.605484962463379 + }, + { + "auxiliary_loss_clip": 0.01141084, + "auxiliary_loss_mlp": 0.01051268, + "balance_loss_clip": 1.05982459, + "balance_loss_mlp": 1.03200364, + "epoch": 0.1736405316000232, + "flos": 19637139507840.0, + "grad_norm": 2.3533390932762708, + "language_loss": 0.95580894, + "learning_rate": 3.7874589478726807e-06, + "loss": 0.97773242, + "num_input_tokens_seen": 171271675, + "router_z_loss_clip": 0.81201172, + "router_z_loss_mlp": 0.19274902, + "step": 5984, + "time_per_iteration": 2.537721633911133 + }, + { + "auxiliary_loss_clip": 0.01134021, + "auxiliary_loss_mlp": 0.01040954, + "balance_loss_clip": 1.05873644, + "balance_loss_mlp": 1.02540886, + "epoch": 0.17366954906853926, + "flos": 27850663981440.0, + "grad_norm": 4.734893165630021, + "language_loss": 0.96610022, + "learning_rate": 3.7873746191516328e-06, + "loss": 0.98784995, + "num_input_tokens_seen": 171285130, + "router_z_loss_clip": 0.75341797, + "router_z_loss_mlp": 0.15557861, + "step": 5985, + "time_per_iteration": 2.62599515914917 + }, + { + "auxiliary_loss_clip": 0.01131497, + "auxiliary_loss_mlp": 0.01041391, + "balance_loss_clip": 1.05554295, + "balance_loss_mlp": 1.02634716, + "epoch": 0.17369856653705532, + "flos": 19164375676800.0, + "grad_norm": 2.8714382107004583, + "language_loss": 0.8773095, + "learning_rate": 3.7872902746437694e-06, + "loss": 0.89903843, + "num_input_tokens_seen": 171297485, + "router_z_loss_clip": 0.76025391, + "router_z_loss_mlp": 0.15026855, + "step": 5986, + "time_per_iteration": 2.534806728363037 + }, + { + "auxiliary_loss_clip": 0.01136612, + "auxiliary_loss_mlp": 0.01044176, + "balance_loss_clip": 1.05808818, + "balance_loss_mlp": 1.02671766, + "epoch": 0.17372758400557137, + "flos": 16393041745920.0, + "grad_norm": 2.5690112073672453, + "language_loss": 0.83989644, + "learning_rate": 3.7872059143498348e-06, + "loss": 0.86170423, + "num_input_tokens_seen": 171311350, + "router_z_loss_clip": 0.78564453, + "router_z_loss_mlp": 0.17462158, + "step": 5987, + "time_per_iteration": 2.535144805908203 + }, + { + "auxiliary_loss_clip": 0.01130324, + "auxiliary_loss_mlp": 0.01034918, + "balance_loss_clip": 1.05820644, + "balance_loss_mlp": 1.01914668, + "epoch": 0.1737566014740874, + "flos": 20186537005440.0, + "grad_norm": 3.48804505927822, + "language_loss": 1.01020789, + "learning_rate": 3.7871215382705746e-06, + "loss": 1.03186035, + "num_input_tokens_seen": 171324615, + "router_z_loss_clip": 0.72070312, + "router_z_loss_mlp": 0.15771484, + "step": 5988, + "time_per_iteration": 2.5765299797058105 + }, + { + "auxiliary_loss_clip": 0.01131718, + "auxiliary_loss_mlp": 0.01040841, + "balance_loss_clip": 1.0599432, + "balance_loss_mlp": 1.02577257, + "epoch": 0.17378561894260344, + "flos": 10553715457920.0, + "grad_norm": 3.2950915740765376, + "language_loss": 0.92518342, + "learning_rate": 3.7870371464067336e-06, + "loss": 0.94690907, + "num_input_tokens_seen": 171336265, + "router_z_loss_clip": 0.71826172, + "router_z_loss_mlp": 0.1506958, + "step": 5989, + "time_per_iteration": 2.6251182556152344 + }, + { + "auxiliary_loss_clip": 0.01126621, + "auxiliary_loss_mlp": 0.01038758, + "balance_loss_clip": 1.05536747, + "balance_loss_mlp": 1.02512622, + "epoch": 0.1738146364111195, + "flos": 21865863611520.0, + "grad_norm": 3.0343022978026375, + "language_loss": 1.0504967, + "learning_rate": 3.7869527387590578e-06, + "loss": 1.07215059, + "num_input_tokens_seen": 171349790, + "router_z_loss_clip": 0.71191406, + "router_z_loss_mlp": 0.1362915, + "step": 5990, + "time_per_iteration": 2.564777135848999 + }, + { + "auxiliary_loss_clip": 0.01129597, + "auxiliary_loss_mlp": 0.01042687, + "balance_loss_clip": 1.05558944, + "balance_loss_mlp": 1.02652264, + "epoch": 0.17384365387963555, + "flos": 28688567518080.0, + "grad_norm": 2.506338993029538, + "language_loss": 0.80407965, + "learning_rate": 3.786868315328292e-06, + "loss": 0.82580245, + "num_input_tokens_seen": 171363555, + "router_z_loss_clip": 0.73974609, + "router_z_loss_mlp": 0.16168213, + "step": 5991, + "time_per_iteration": 2.592374801635742 + }, + { + "auxiliary_loss_clip": 0.01131303, + "auxiliary_loss_mlp": 0.01040076, + "balance_loss_clip": 1.0564549, + "balance_loss_mlp": 1.02367258, + "epoch": 0.1738726713481516, + "flos": 12488259173760.0, + "grad_norm": 2.51428566699517, + "language_loss": 0.81338346, + "learning_rate": 3.786783876115182e-06, + "loss": 0.83509719, + "num_input_tokens_seen": 171374770, + "router_z_loss_clip": 0.74755859, + "router_z_loss_mlp": 0.16400146, + "step": 5992, + "time_per_iteration": 2.5229737758636475 + }, + { + "auxiliary_loss_clip": 0.01042079, + "auxiliary_loss_mlp": 0.01000849, + "balance_loss_clip": 1.02298784, + "balance_loss_mlp": 0.99999028, + "epoch": 0.17390168881666762, + "flos": 63537038208000.0, + "grad_norm": 0.7701415678509251, + "language_loss": 0.54229963, + "learning_rate": 3.786699421120474e-06, + "loss": 0.56272888, + "num_input_tokens_seen": 171430065, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.00860596, + "step": 5993, + "time_per_iteration": 3.0233078002929688 + }, + { + "auxiliary_loss_clip": 0.01129612, + "auxiliary_loss_mlp": 0.01033304, + "balance_loss_clip": 1.05781269, + "balance_loss_mlp": 1.01946414, + "epoch": 0.17393070628518367, + "flos": 16829356250880.0, + "grad_norm": 3.49519098783704, + "language_loss": 0.86931169, + "learning_rate": 3.786614950344914e-06, + "loss": 0.89094085, + "num_input_tokens_seen": 171443460, + "router_z_loss_clip": 0.71875, + "router_z_loss_mlp": 0.13842773, + "step": 5994, + "time_per_iteration": 2.539450168609619 + }, + { + "auxiliary_loss_clip": 0.01135171, + "auxiliary_loss_mlp": 0.01044533, + "balance_loss_clip": 1.05849791, + "balance_loss_mlp": 1.02699733, + "epoch": 0.17395972375369972, + "flos": 17886314880000.0, + "grad_norm": 2.329896046208121, + "language_loss": 0.89725786, + "learning_rate": 3.786530463789247e-06, + "loss": 0.91905487, + "num_input_tokens_seen": 171456305, + "router_z_loss_clip": 0.76660156, + "router_z_loss_mlp": 0.17523193, + "step": 5995, + "time_per_iteration": 2.50179386138916 + }, + { + "auxiliary_loss_clip": 0.01143534, + "auxiliary_loss_mlp": 0.01041878, + "balance_loss_clip": 1.06291795, + "balance_loss_mlp": 1.02533162, + "epoch": 0.17398874122221578, + "flos": 27557561001600.0, + "grad_norm": 2.09902642700371, + "language_loss": 0.8928299, + "learning_rate": 3.7864459614542206e-06, + "loss": 0.91468406, + "num_input_tokens_seen": 171469815, + "router_z_loss_clip": 0.80566406, + "router_z_loss_mlp": 0.16540527, + "step": 5996, + "time_per_iteration": 2.668743133544922 + }, + { + "auxiliary_loss_clip": 0.01134761, + "auxiliary_loss_mlp": 0.01039461, + "balance_loss_clip": 1.06171703, + "balance_loss_mlp": 1.02432704, + "epoch": 0.17401775869073183, + "flos": 30949395402240.0, + "grad_norm": 3.3888217140399846, + "language_loss": 0.75277293, + "learning_rate": 3.7863614433405804e-06, + "loss": 0.77451509, + "num_input_tokens_seen": 171489380, + "router_z_loss_clip": 0.73046875, + "router_z_loss_mlp": 0.15130615, + "step": 5997, + "time_per_iteration": 2.731968402862549 + }, + { + "auxiliary_loss_clip": 0.01044878, + "auxiliary_loss_mlp": 0.01010138, + "balance_loss_clip": 1.02543616, + "balance_loss_mlp": 1.00930941, + "epoch": 0.17404677615924788, + "flos": 59052801432960.0, + "grad_norm": 0.6904311736957286, + "language_loss": 0.51096708, + "learning_rate": 3.786276909449073e-06, + "loss": 0.53151727, + "num_input_tokens_seen": 171548035, + "router_z_loss_clip": 0.1953125, + "router_z_loss_mlp": 0.00830078, + "step": 5998, + "time_per_iteration": 3.0257728099823 + }, + { + "auxiliary_loss_clip": 0.01133034, + "auxiliary_loss_mlp": 0.01037427, + "balance_loss_clip": 1.05887103, + "balance_loss_mlp": 1.02299678, + "epoch": 0.1740757936277639, + "flos": 30876496750080.0, + "grad_norm": 2.741023561088121, + "language_loss": 0.6882844, + "learning_rate": 3.786192359780445e-06, + "loss": 0.70998907, + "num_input_tokens_seen": 171562540, + "router_z_loss_clip": 0.74072266, + "router_z_loss_mlp": 0.14434814, + "step": 5999, + "time_per_iteration": 2.590240001678467 + }, + { + "auxiliary_loss_clip": 0.01132533, + "auxiliary_loss_mlp": 0.01046104, + "balance_loss_clip": 1.05893862, + "balance_loss_mlp": 1.03117847, + "epoch": 0.17410481109627995, + "flos": 16683774428160.0, + "grad_norm": 2.796870957344143, + "language_loss": 0.73430109, + "learning_rate": 3.786107794335444e-06, + "loss": 0.75608742, + "num_input_tokens_seen": 171574435, + "router_z_loss_clip": 0.73681641, + "router_z_loss_mlp": 0.14935303, + "step": 6000, + "time_per_iteration": 2.5099422931671143 + }, + { + "auxiliary_loss_clip": 0.01131329, + "auxiliary_loss_mlp": 0.01043649, + "balance_loss_clip": 1.06031442, + "balance_loss_mlp": 1.02950454, + "epoch": 0.174133828564796, + "flos": 11537416298880.0, + "grad_norm": 2.9277621168243457, + "language_loss": 0.80916679, + "learning_rate": 3.7860232131148154e-06, + "loss": 0.83091658, + "num_input_tokens_seen": 171583860, + "router_z_loss_clip": 0.71044922, + "router_z_loss_mlp": 0.14141846, + "step": 6001, + "time_per_iteration": 2.5349223613739014 + }, + { + "auxiliary_loss_clip": 0.01137822, + "auxiliary_loss_mlp": 0.01040353, + "balance_loss_clip": 1.06246948, + "balance_loss_mlp": 1.02473116, + "epoch": 0.17416284603331206, + "flos": 21618511580160.0, + "grad_norm": 2.0502874987337285, + "language_loss": 0.70855904, + "learning_rate": 3.785938616119307e-06, + "loss": 0.73034072, + "num_input_tokens_seen": 171602675, + "router_z_loss_clip": 0.75292969, + "router_z_loss_mlp": 0.15612793, + "step": 6002, + "time_per_iteration": 2.6742289066314697 + }, + { + "auxiliary_loss_clip": 0.01130999, + "auxiliary_loss_mlp": 0.0103551, + "balance_loss_clip": 1.05898619, + "balance_loss_mlp": 1.02127075, + "epoch": 0.1741918635018281, + "flos": 16464755249280.0, + "grad_norm": 2.5966193636677923, + "language_loss": 0.65064645, + "learning_rate": 3.785854003349667e-06, + "loss": 0.67231148, + "num_input_tokens_seen": 171615115, + "router_z_loss_clip": 0.72021484, + "router_z_loss_mlp": 0.14245605, + "step": 6003, + "time_per_iteration": 2.5150113105773926 + }, + { + "auxiliary_loss_clip": 0.01047137, + "auxiliary_loss_mlp": 0.00997411, + "balance_loss_clip": 1.02729821, + "balance_loss_mlp": 0.99650508, + "epoch": 0.17422088097034416, + "flos": 65099546807040.0, + "grad_norm": 0.6521719416894752, + "language_loss": 0.4646365, + "learning_rate": 3.785769374806641e-06, + "loss": 0.48508197, + "num_input_tokens_seen": 171678265, + "router_z_loss_clip": 0.19921875, + "router_z_loss_mlp": 0.0090332, + "step": 6004, + "time_per_iteration": 3.279487371444702 + }, + { + "auxiliary_loss_clip": 0.0104579, + "auxiliary_loss_mlp": 0.0100095, + "balance_loss_clip": 1.02598548, + "balance_loss_mlp": 1.00003815, + "epoch": 0.17424989843886018, + "flos": 55070056391040.0, + "grad_norm": 0.6273192707057292, + "language_loss": 0.48029581, + "learning_rate": 3.7856847304909775e-06, + "loss": 0.50076318, + "num_input_tokens_seen": 171736745, + "router_z_loss_clip": 0.19824219, + "router_z_loss_mlp": 0.00909424, + "step": 6005, + "time_per_iteration": 3.095426559448242 + }, + { + "auxiliary_loss_clip": 0.01125763, + "auxiliary_loss_mlp": 0.01043079, + "balance_loss_clip": 1.05553174, + "balance_loss_mlp": 1.02827907, + "epoch": 0.17427891590737624, + "flos": 15991165405440.0, + "grad_norm": 4.200144024328865, + "language_loss": 0.76832891, + "learning_rate": 3.785600070403424e-06, + "loss": 0.79001725, + "num_input_tokens_seen": 171750050, + "router_z_loss_clip": 0.70263672, + "router_z_loss_mlp": 0.14801025, + "step": 6006, + "time_per_iteration": 2.5488393306732178 + }, + { + "auxiliary_loss_clip": 0.01044731, + "auxiliary_loss_mlp": 0.00999654, + "balance_loss_clip": 1.0249579, + "balance_loss_mlp": 0.99875432, + "epoch": 0.1743079333758923, + "flos": 59452091994240.0, + "grad_norm": 0.7096052609215996, + "language_loss": 0.52138758, + "learning_rate": 3.7855153945447275e-06, + "loss": 0.54183143, + "num_input_tokens_seen": 171810775, + "router_z_loss_clip": 0.19824219, + "router_z_loss_mlp": 0.00897217, + "step": 6007, + "time_per_iteration": 3.0555810928344727 + }, + { + "auxiliary_loss_clip": 0.01128728, + "auxiliary_loss_mlp": 0.01043293, + "balance_loss_clip": 1.05985773, + "balance_loss_mlp": 1.02797449, + "epoch": 0.17433695084440834, + "flos": 12820648654080.0, + "grad_norm": 2.756738171585207, + "language_loss": 0.84965682, + "learning_rate": 3.7854307029156375e-06, + "loss": 0.87137705, + "num_input_tokens_seen": 171824120, + "router_z_loss_clip": 0.68847656, + "router_z_loss_mlp": 0.15319824, + "step": 6008, + "time_per_iteration": 2.54571270942688 + }, + { + "auxiliary_loss_clip": 0.01042248, + "auxiliary_loss_mlp": 0.01002195, + "balance_loss_clip": 1.0225265, + "balance_loss_mlp": 1.00141454, + "epoch": 0.1743659683129244, + "flos": 63609972773760.0, + "grad_norm": 0.6750500307502993, + "language_loss": 0.46322477, + "learning_rate": 3.7853459955169002e-06, + "loss": 0.48366916, + "num_input_tokens_seen": 171882995, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.0078125, + "step": 6009, + "time_per_iteration": 3.062046527862549 + }, + { + "auxiliary_loss_clip": 0.01131929, + "auxiliary_loss_mlp": 0.01038117, + "balance_loss_clip": 1.05912435, + "balance_loss_mlp": 1.02384198, + "epoch": 0.17439498578144041, + "flos": 31066788026880.0, + "grad_norm": 1.7560287049473935, + "language_loss": 0.86578083, + "learning_rate": 3.785261272349265e-06, + "loss": 0.88748127, + "num_input_tokens_seen": 171903255, + "router_z_loss_clip": 0.72802734, + "router_z_loss_mlp": 0.1427002, + "step": 6010, + "time_per_iteration": 2.713862180709839 + }, + { + "auxiliary_loss_clip": 0.0112386, + "auxiliary_loss_mlp": 0.01041245, + "balance_loss_clip": 1.05469799, + "balance_loss_mlp": 1.0262723, + "epoch": 0.17442400324995647, + "flos": 20191241687040.0, + "grad_norm": 2.629642435378846, + "language_loss": 0.80332232, + "learning_rate": 3.7851765334134792e-06, + "loss": 0.8249734, + "num_input_tokens_seen": 171917135, + "router_z_loss_clip": 0.69140625, + "router_z_loss_mlp": 0.14990234, + "step": 6011, + "time_per_iteration": 2.547847270965576 + }, + { + "auxiliary_loss_clip": 0.01040344, + "auxiliary_loss_mlp": 0.01005781, + "balance_loss_clip": 1.02050686, + "balance_loss_mlp": 1.005, + "epoch": 0.17445302071847252, + "flos": 74784906973440.0, + "grad_norm": 0.6329157804544849, + "language_loss": 0.48242411, + "learning_rate": 3.785091778710293e-06, + "loss": 0.50288534, + "num_input_tokens_seen": 171984555, + "router_z_loss_clip": 0.19921875, + "router_z_loss_mlp": 0.0078125, + "step": 6012, + "time_per_iteration": 3.2386982440948486 + }, + { + "auxiliary_loss_clip": 0.011294, + "auxiliary_loss_mlp": 0.01036396, + "balance_loss_clip": 1.0564723, + "balance_loss_mlp": 1.02152467, + "epoch": 0.17448203818698857, + "flos": 18187282938240.0, + "grad_norm": 2.061841061107741, + "language_loss": 0.82762021, + "learning_rate": 3.785007008240453e-06, + "loss": 0.84927809, + "num_input_tokens_seen": 172001960, + "router_z_loss_clip": 0.72900391, + "router_z_loss_mlp": 0.14880371, + "step": 6013, + "time_per_iteration": 2.5320005416870117 + }, + { + "auxiliary_loss_clip": 0.01136858, + "auxiliary_loss_mlp": 0.01046136, + "balance_loss_clip": 1.05909193, + "balance_loss_mlp": 1.02926826, + "epoch": 0.17451105565550462, + "flos": 35074597783680.0, + "grad_norm": 2.2811070232995236, + "language_loss": 0.71328259, + "learning_rate": 3.784922222004709e-06, + "loss": 0.73511255, + "num_input_tokens_seen": 172018890, + "router_z_loss_clip": 0.77832031, + "router_z_loss_mlp": 0.16894531, + "step": 6014, + "time_per_iteration": 2.7336697578430176 + }, + { + "auxiliary_loss_clip": 0.01132844, + "auxiliary_loss_mlp": 0.01042449, + "balance_loss_clip": 1.05833983, + "balance_loss_mlp": 1.02670789, + "epoch": 0.17454007312402067, + "flos": 27957713489280.0, + "grad_norm": 2.1920041016712153, + "language_loss": 0.79435104, + "learning_rate": 3.78483742000381e-06, + "loss": 0.81610394, + "num_input_tokens_seen": 172033995, + "router_z_loss_clip": 0.74414062, + "router_z_loss_mlp": 0.15722656, + "step": 6015, + "time_per_iteration": 2.524869441986084 + }, + { + "auxiliary_loss_clip": 0.01040758, + "auxiliary_loss_mlp": 0.0100743, + "balance_loss_clip": 1.02091408, + "balance_loss_mlp": 1.00662565, + "epoch": 0.1745690905925367, + "flos": 60764698696320.0, + "grad_norm": 0.6309171202583513, + "language_loss": 0.4722833, + "learning_rate": 3.7847526022385045e-06, + "loss": 0.49276519, + "num_input_tokens_seen": 172104910, + "router_z_loss_clip": 0.19921875, + "router_z_loss_mlp": 0.00805664, + "step": 6016, + "time_per_iteration": 3.339719533920288 + }, + { + "auxiliary_loss_clip": 0.01039352, + "auxiliary_loss_mlp": 0.01004574, + "balance_loss_clip": 1.01970577, + "balance_loss_mlp": 1.00375772, + "epoch": 0.17459810806105275, + "flos": 74775677178240.0, + "grad_norm": 0.6171452486368376, + "language_loss": 0.47717711, + "learning_rate": 3.7846677687095408e-06, + "loss": 0.49761635, + "num_input_tokens_seen": 172172240, + "router_z_loss_clip": 0.19628906, + "router_z_loss_mlp": 0.00817871, + "step": 6017, + "time_per_iteration": 3.2630457878112793 + }, + { + "auxiliary_loss_clip": 0.01128734, + "auxiliary_loss_mlp": 0.01032447, + "balance_loss_clip": 1.05697417, + "balance_loss_mlp": 1.01851761, + "epoch": 0.1746271255295688, + "flos": 30155554874880.0, + "grad_norm": 1.7244341536232175, + "language_loss": 0.73650199, + "learning_rate": 3.784582919417671e-06, + "loss": 0.75811386, + "num_input_tokens_seen": 172191295, + "router_z_loss_clip": 0.71777344, + "router_z_loss_mlp": 0.13934326, + "step": 6018, + "time_per_iteration": 2.6107728481292725 + }, + { + "auxiliary_loss_clip": 0.01121796, + "auxiliary_loss_mlp": 0.01033458, + "balance_loss_clip": 1.05581748, + "balance_loss_mlp": 1.02075052, + "epoch": 0.17465614299808485, + "flos": 15774696092160.0, + "grad_norm": 2.0976049902914125, + "language_loss": 0.67406869, + "learning_rate": 3.7844980543636417e-06, + "loss": 0.69562119, + "num_input_tokens_seen": 172205730, + "router_z_loss_clip": 0.65966797, + "router_z_loss_mlp": 0.12713623, + "step": 6019, + "time_per_iteration": 2.5105533599853516 + }, + { + "auxiliary_loss_clip": 0.01126767, + "auxiliary_loss_mlp": 0.01039722, + "balance_loss_clip": 1.05474246, + "balance_loss_mlp": 1.02458203, + "epoch": 0.1746851604666009, + "flos": 29053527655680.0, + "grad_norm": 2.3173210887059006, + "language_loss": 0.65517378, + "learning_rate": 3.784413173548203e-06, + "loss": 0.67683864, + "num_input_tokens_seen": 172223885, + "router_z_loss_clip": 0.72070312, + "router_z_loss_mlp": 0.15142822, + "step": 6020, + "time_per_iteration": 2.6111574172973633 + }, + { + "auxiliary_loss_clip": 0.01134494, + "auxiliary_loss_mlp": 0.01048677, + "balance_loss_clip": 1.05946732, + "balance_loss_mlp": 1.03310823, + "epoch": 0.17471417793511693, + "flos": 12741501035520.0, + "grad_norm": 3.3166164567106136, + "language_loss": 0.85121888, + "learning_rate": 3.784328276972106e-06, + "loss": 0.87305063, + "num_input_tokens_seen": 172234455, + "router_z_loss_clip": 0.75097656, + "router_z_loss_mlp": 0.15570068, + "step": 6021, + "time_per_iteration": 2.56715726852417 + }, + { + "auxiliary_loss_clip": 0.01134905, + "auxiliary_loss_mlp": 0.01039415, + "balance_loss_clip": 1.06134987, + "balance_loss_mlp": 1.02395368, + "epoch": 0.17474319540363298, + "flos": 28249918629120.0, + "grad_norm": 2.3120986358033013, + "language_loss": 0.88893569, + "learning_rate": 3.7842433646360988e-06, + "loss": 0.91067886, + "num_input_tokens_seen": 172249335, + "router_z_loss_clip": 0.73583984, + "router_z_loss_mlp": 0.15466309, + "step": 6022, + "time_per_iteration": 2.63765549659729 + }, + { + "auxiliary_loss_clip": 0.01134444, + "auxiliary_loss_mlp": 0.01042524, + "balance_loss_clip": 1.05671644, + "balance_loss_mlp": 1.02585852, + "epoch": 0.17477221287214903, + "flos": 15556000135680.0, + "grad_norm": 3.1392564820411453, + "language_loss": 0.94395161, + "learning_rate": 3.7841584365409327e-06, + "loss": 0.96572125, + "num_input_tokens_seen": 172261790, + "router_z_loss_clip": 0.77734375, + "router_z_loss_mlp": 0.16656494, + "step": 6023, + "time_per_iteration": 2.5600409507751465 + }, + { + "auxiliary_loss_clip": 0.01035181, + "auxiliary_loss_mlp": 0.01000408, + "balance_loss_clip": 1.01559186, + "balance_loss_mlp": 0.99952537, + "epoch": 0.17480123034066508, + "flos": 66606000232320.0, + "grad_norm": 0.6639540952792905, + "language_loss": 0.48308444, + "learning_rate": 3.7840734926873574e-06, + "loss": 0.50344032, + "num_input_tokens_seen": 172323435, + "router_z_loss_clip": 0.1953125, + "router_z_loss_mlp": 0.0088501, + "step": 6024, + "time_per_iteration": 3.1607885360717773 + }, + { + "auxiliary_loss_clip": 0.01033726, + "auxiliary_loss_mlp": 0.00998567, + "balance_loss_clip": 1.0141573, + "balance_loss_mlp": 0.9977625, + "epoch": 0.17483024780918113, + "flos": 74776646845440.0, + "grad_norm": 0.6652771724393736, + "language_loss": 0.50670934, + "learning_rate": 3.7839885330761223e-06, + "loss": 0.52703226, + "num_input_tokens_seen": 172384080, + "router_z_loss_clip": 0.1953125, + "router_z_loss_mlp": 0.00805664, + "step": 6025, + "time_per_iteration": 3.191404342651367 + }, + { + "auxiliary_loss_clip": 0.01130621, + "auxiliary_loss_mlp": 0.0104418, + "balance_loss_clip": 1.05522418, + "balance_loss_mlp": 1.02915978, + "epoch": 0.17485926527769718, + "flos": 35150944141440.0, + "grad_norm": 2.2036306274290243, + "language_loss": 0.89843494, + "learning_rate": 3.7839035577079798e-06, + "loss": 0.92018294, + "num_input_tokens_seen": 172404595, + "router_z_loss_clip": 0.75390625, + "router_z_loss_mlp": 0.15014648, + "step": 6026, + "time_per_iteration": 2.7286150455474854 + }, + { + "auxiliary_loss_clip": 0.01032896, + "auxiliary_loss_mlp": 0.01004046, + "balance_loss_clip": 1.01322114, + "balance_loss_mlp": 1.00320578, + "epoch": 0.1748882827462132, + "flos": 56345890544640.0, + "grad_norm": 0.6503377130888086, + "language_loss": 0.47628725, + "learning_rate": 3.7838185665836784e-06, + "loss": 0.49665666, + "num_input_tokens_seen": 172462340, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.00842285, + "step": 6027, + "time_per_iteration": 2.977234363555908 + }, + { + "auxiliary_loss_clip": 0.01032898, + "auxiliary_loss_mlp": 0.01012485, + "balance_loss_clip": 1.01329708, + "balance_loss_mlp": 1.01169205, + "epoch": 0.17491730021472926, + "flos": 54782053142400.0, + "grad_norm": 0.6342707008034173, + "language_loss": 0.44857669, + "learning_rate": 3.78373355970397e-06, + "loss": 0.46903053, + "num_input_tokens_seen": 172519500, + "router_z_loss_clip": 0.19628906, + "router_z_loss_mlp": 0.00793457, + "step": 6028, + "time_per_iteration": 2.9662084579467773 + }, + { + "auxiliary_loss_clip": 0.01127749, + "auxiliary_loss_mlp": 0.01039256, + "balance_loss_clip": 1.05694544, + "balance_loss_mlp": 1.02445006, + "epoch": 0.1749463176832453, + "flos": 35003207502720.0, + "grad_norm": 1.867105619508051, + "language_loss": 0.98133147, + "learning_rate": 3.7836485370696044e-06, + "loss": 1.00300145, + "num_input_tokens_seen": 172540120, + "router_z_loss_clip": 0.70825195, + "router_z_loss_mlp": 0.14801025, + "step": 6029, + "time_per_iteration": 2.661341905593872 + }, + { + "auxiliary_loss_clip": 0.01032037, + "auxiliary_loss_mlp": 0.01017079, + "balance_loss_clip": 1.01242089, + "balance_loss_mlp": 1.01633728, + "epoch": 0.17497533515176136, + "flos": 58865311416960.0, + "grad_norm": 0.656308755795322, + "language_loss": 0.46445924, + "learning_rate": 3.783563498681334e-06, + "loss": 0.48495042, + "num_input_tokens_seen": 172599840, + "router_z_loss_clip": 0.19628906, + "router_z_loss_mlp": 0.00741577, + "step": 6030, + "time_per_iteration": 3.0000083446502686 + }, + { + "auxiliary_loss_clip": 0.01031613, + "auxiliary_loss_mlp": 0.01007833, + "balance_loss_clip": 1.01219988, + "balance_loss_mlp": 1.00706434, + "epoch": 0.17500435262027741, + "flos": 74770433792640.0, + "grad_norm": 0.6582853402243409, + "language_loss": 0.50107557, + "learning_rate": 3.7834784445399086e-06, + "loss": 0.52147001, + "num_input_tokens_seen": 172658185, + "router_z_loss_clip": 0.19433594, + "router_z_loss_mlp": 0.00769043, + "step": 6031, + "time_per_iteration": 3.072539806365967 + }, + { + "auxiliary_loss_clip": 0.01030994, + "auxiliary_loss_mlp": 0.01002862, + "balance_loss_clip": 1.01171434, + "balance_loss_mlp": 1.00203919, + "epoch": 0.17503337008879347, + "flos": 74770649274240.0, + "grad_norm": 0.6789325906341478, + "language_loss": 0.46846476, + "learning_rate": 3.7833933746460794e-06, + "loss": 0.48880336, + "num_input_tokens_seen": 172715750, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.00823975, + "step": 6032, + "time_per_iteration": 3.06565260887146 + }, + { + "auxiliary_loss_clip": 0.01030691, + "auxiliary_loss_mlp": 0.00998807, + "balance_loss_clip": 1.0113523, + "balance_loss_mlp": 0.99798459, + "epoch": 0.1750623875573095, + "flos": 63765825886080.0, + "grad_norm": 0.6562348064446664, + "language_loss": 0.4866901, + "learning_rate": 3.783308289000599e-06, + "loss": 0.50698507, + "num_input_tokens_seen": 172779315, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.00823975, + "step": 6033, + "time_per_iteration": 3.183562755584717 + }, + { + "auxiliary_loss_clip": 0.01128197, + "auxiliary_loss_mlp": 0.01042409, + "balance_loss_clip": 1.05646014, + "balance_loss_mlp": 1.02657175, + "epoch": 0.17509140502582554, + "flos": 34166309546880.0, + "grad_norm": 1.7179950940864486, + "language_loss": 0.83700752, + "learning_rate": 3.783223187604218e-06, + "loss": 0.85871363, + "num_input_tokens_seen": 172802010, + "router_z_loss_clip": 0.71777344, + "router_z_loss_mlp": 0.1585083, + "step": 6034, + "time_per_iteration": 2.7743074893951416 + }, + { + "auxiliary_loss_clip": 0.01142121, + "auxiliary_loss_mlp": 0.01033657, + "balance_loss_clip": 1.06206858, + "balance_loss_mlp": 1.01732492, + "epoch": 0.1751204224943416, + "flos": 33288329410560.0, + "grad_norm": 1.9992763341255975, + "language_loss": 0.87981296, + "learning_rate": 3.7831380704576875e-06, + "loss": 0.90157068, + "num_input_tokens_seen": 172820570, + "router_z_loss_clip": 0.80126953, + "router_z_loss_mlp": 0.16320801, + "step": 6035, + "time_per_iteration": 2.6663591861724854 + }, + { + "auxiliary_loss_clip": 0.01117134, + "auxiliary_loss_mlp": 0.01038372, + "balance_loss_clip": 1.04978704, + "balance_loss_mlp": 1.02452576, + "epoch": 0.17514943996285764, + "flos": 11466241499520.0, + "grad_norm": 2.132180431762711, + "language_loss": 0.75894421, + "learning_rate": 3.783052937561761e-06, + "loss": 0.78049928, + "num_input_tokens_seen": 172833630, + "router_z_loss_clip": 0.67382812, + "router_z_loss_mlp": 0.13861084, + "step": 6036, + "time_per_iteration": 2.500889778137207 + }, + { + "auxiliary_loss_clip": 0.01130628, + "auxiliary_loss_mlp": 0.01038438, + "balance_loss_clip": 1.0552206, + "balance_loss_mlp": 1.02255309, + "epoch": 0.1751784574313737, + "flos": 11208474524160.0, + "grad_norm": 3.020322293387472, + "language_loss": 0.97371113, + "learning_rate": 3.782967788917189e-06, + "loss": 0.99540174, + "num_input_tokens_seen": 172843570, + "router_z_loss_clip": 0.75439453, + "router_z_loss_mlp": 0.15875244, + "step": 6037, + "time_per_iteration": 2.5059680938720703 + }, + { + "auxiliary_loss_clip": 0.01132792, + "auxiliary_loss_mlp": 0.0103749, + "balance_loss_clip": 1.05733907, + "balance_loss_mlp": 1.02223694, + "epoch": 0.17520747489988972, + "flos": 23114155011840.0, + "grad_norm": 2.2858843356985106, + "language_loss": 0.87744194, + "learning_rate": 3.782882624524724e-06, + "loss": 0.89914477, + "num_input_tokens_seen": 172860850, + "router_z_loss_clip": 0.75439453, + "router_z_loss_mlp": 0.15246582, + "step": 6038, + "time_per_iteration": 2.61926531791687 + }, + { + "auxiliary_loss_clip": 0.01133558, + "auxiliary_loss_mlp": 0.01041315, + "balance_loss_clip": 1.05951798, + "balance_loss_mlp": 1.02683735, + "epoch": 0.17523649236840577, + "flos": 17125583713920.0, + "grad_norm": 2.1177622051092113, + "language_loss": 0.67384422, + "learning_rate": 3.7827974443851184e-06, + "loss": 0.695593, + "num_input_tokens_seen": 172875855, + "router_z_loss_clip": 0.73974609, + "router_z_loss_mlp": 0.14477539, + "step": 6039, + "time_per_iteration": 2.582275867462158 + }, + { + "auxiliary_loss_clip": 0.01139271, + "auxiliary_loss_mlp": 0.01043366, + "balance_loss_clip": 1.05852771, + "balance_loss_mlp": 1.02622354, + "epoch": 0.17526550983692182, + "flos": 19426631852160.0, + "grad_norm": 1.942568199421801, + "language_loss": 0.8045733, + "learning_rate": 3.7827122484991237e-06, + "loss": 0.82639968, + "num_input_tokens_seen": 172893015, + "router_z_loss_clip": 0.80810547, + "router_z_loss_mlp": 0.17132568, + "step": 6040, + "time_per_iteration": 2.5682284832000732 + }, + { + "auxiliary_loss_clip": 0.01034674, + "auxiliary_loss_mlp": 0.0100685, + "balance_loss_clip": 1.01531434, + "balance_loss_mlp": 1.00600326, + "epoch": 0.17529452730543787, + "flos": 66004351424640.0, + "grad_norm": 0.7032625877796841, + "language_loss": 0.50261253, + "learning_rate": 3.7826270368674937e-06, + "loss": 0.52302778, + "num_input_tokens_seen": 172945500, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.00848389, + "step": 6041, + "time_per_iteration": 3.005396842956543 + }, + { + "auxiliary_loss_clip": 0.01135599, + "auxiliary_loss_mlp": 0.01038643, + "balance_loss_clip": 1.05832946, + "balance_loss_mlp": 1.02256119, + "epoch": 0.17532354477395393, + "flos": 13948135637760.0, + "grad_norm": 2.7862318584046624, + "language_loss": 0.76784456, + "learning_rate": 3.78254180949098e-06, + "loss": 0.78958702, + "num_input_tokens_seen": 172959760, + "router_z_loss_clip": 0.7734375, + "router_z_loss_mlp": 0.16064453, + "step": 6042, + "time_per_iteration": 2.5215327739715576 + }, + { + "auxiliary_loss_clip": 0.01141066, + "auxiliary_loss_mlp": 0.01040305, + "balance_loss_clip": 1.05853188, + "balance_loss_mlp": 1.02303171, + "epoch": 0.17535256224246998, + "flos": 23324734494720.0, + "grad_norm": 2.771114759118208, + "language_loss": 0.98341531, + "learning_rate": 3.782456566370336e-06, + "loss": 1.005229, + "num_input_tokens_seen": 172971920, + "router_z_loss_clip": 0.82519531, + "router_z_loss_mlp": 0.17260742, + "step": 6043, + "time_per_iteration": 2.553314447402954 + }, + { + "auxiliary_loss_clip": 0.01141254, + "auxiliary_loss_mlp": 0.01055315, + "balance_loss_clip": 1.0649116, + "balance_loss_mlp": 1.03771973, + "epoch": 0.175381579710986, + "flos": 35332795722240.0, + "grad_norm": 2.1649178677201797, + "language_loss": 0.90268058, + "learning_rate": 3.782371307506314e-06, + "loss": 0.92464632, + "num_input_tokens_seen": 172995480, + "router_z_loss_clip": 0.76367188, + "router_z_loss_mlp": 0.17596436, + "step": 6044, + "time_per_iteration": 2.6814866065979004 + }, + { + "auxiliary_loss_clip": 0.0114244, + "auxiliary_loss_mlp": 0.01047535, + "balance_loss_clip": 1.05799699, + "balance_loss_mlp": 1.02927208, + "epoch": 0.17541059717950205, + "flos": 24749418608640.0, + "grad_norm": 1.9606097937644007, + "language_loss": 0.96287519, + "learning_rate": 3.782286032899668e-06, + "loss": 0.98477495, + "num_input_tokens_seen": 173016530, + "router_z_loss_clip": 0.84521484, + "router_z_loss_mlp": 0.18273926, + "step": 6045, + "time_per_iteration": 7.4904491901397705 + }, + { + "auxiliary_loss_clip": 0.01130252, + "auxiliary_loss_mlp": 0.01037151, + "balance_loss_clip": 1.05880761, + "balance_loss_mlp": 1.02286375, + "epoch": 0.1754396146480181, + "flos": 37845536664960.0, + "grad_norm": 1.8732211977161357, + "language_loss": 0.589459, + "learning_rate": 3.78220074255115e-06, + "loss": 0.61113304, + "num_input_tokens_seen": 173032935, + "router_z_loss_clip": 0.71484375, + "router_z_loss_mlp": 0.1428833, + "step": 6046, + "time_per_iteration": 5.041693449020386 + }, + { + "auxiliary_loss_clip": 0.01135704, + "auxiliary_loss_mlp": 0.01043979, + "balance_loss_clip": 1.06221759, + "balance_loss_mlp": 1.02842784, + "epoch": 0.17546863211653416, + "flos": 24999643728000.0, + "grad_norm": 2.9079038359696145, + "language_loss": 0.55231303, + "learning_rate": 3.782115436461514e-06, + "loss": 0.57410991, + "num_input_tokens_seen": 173045545, + "router_z_loss_clip": 0.73535156, + "router_z_loss_mlp": 0.15545654, + "step": 6047, + "time_per_iteration": 4.941274881362915 + }, + { + "auxiliary_loss_clip": 0.01137014, + "auxiliary_loss_mlp": 0.0104429, + "balance_loss_clip": 1.0614301, + "balance_loss_mlp": 1.02863801, + "epoch": 0.1754976495850502, + "flos": 12816482676480.0, + "grad_norm": 2.4385580399863636, + "language_loss": 0.84669739, + "learning_rate": 3.782030114631513e-06, + "loss": 0.86851048, + "num_input_tokens_seen": 173056890, + "router_z_loss_clip": 0.75537109, + "router_z_loss_mlp": 0.15643311, + "step": 6048, + "time_per_iteration": 2.5144383907318115 + }, + { + "auxiliary_loss_clip": 0.01042764, + "auxiliary_loss_mlp": 0.01013309, + "balance_loss_clip": 1.02373409, + "balance_loss_mlp": 1.01242733, + "epoch": 0.17552666705356626, + "flos": 70395904131840.0, + "grad_norm": 0.6538481913550376, + "language_loss": 0.50081038, + "learning_rate": 3.781944777061901e-06, + "loss": 0.52137119, + "num_input_tokens_seen": 173115155, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.0088501, + "step": 6049, + "time_per_iteration": 3.1015100479125977 + }, + { + "auxiliary_loss_clip": 0.01131907, + "auxiliary_loss_mlp": 0.01035137, + "balance_loss_clip": 1.06323254, + "balance_loss_mlp": 1.02250659, + "epoch": 0.17555568452208228, + "flos": 32589830557440.0, + "grad_norm": 2.2834463935731444, + "language_loss": 0.78404665, + "learning_rate": 3.781859423753432e-06, + "loss": 0.80571711, + "num_input_tokens_seen": 173128535, + "router_z_loss_clip": 0.68652344, + "router_z_loss_mlp": 0.12634277, + "step": 6050, + "time_per_iteration": 2.6579558849334717 + }, + { + "auxiliary_loss_clip": 0.01143759, + "auxiliary_loss_mlp": 0.01048676, + "balance_loss_clip": 1.06450582, + "balance_loss_mlp": 1.03201652, + "epoch": 0.17558470199059834, + "flos": 14352597757440.0, + "grad_norm": 2.5027184980486434, + "language_loss": 0.90296447, + "learning_rate": 3.7817740547068596e-06, + "loss": 0.92488885, + "num_input_tokens_seen": 173142920, + "router_z_loss_clip": 0.79394531, + "router_z_loss_mlp": 0.16662598, + "step": 6051, + "time_per_iteration": 2.566380023956299 + }, + { + "auxiliary_loss_clip": 0.01128834, + "auxiliary_loss_mlp": 0.01031521, + "balance_loss_clip": 1.05902779, + "balance_loss_mlp": 1.01831865, + "epoch": 0.1756137194591144, + "flos": 13691877033600.0, + "grad_norm": 2.193344232387236, + "language_loss": 0.71268356, + "learning_rate": 3.7816886699229373e-06, + "loss": 0.73428702, + "num_input_tokens_seen": 173156005, + "router_z_loss_clip": 0.69848633, + "router_z_loss_mlp": 0.13208008, + "step": 6052, + "time_per_iteration": 2.5246057510375977 + }, + { + "auxiliary_loss_clip": 0.01046581, + "auxiliary_loss_mlp": 0.01004293, + "balance_loss_clip": 1.02721405, + "balance_loss_mlp": 1.00338697, + "epoch": 0.17564273692763044, + "flos": 72436348120320.0, + "grad_norm": 0.6977719849687657, + "language_loss": 0.49064356, + "learning_rate": 3.7816032694024197e-06, + "loss": 0.51115233, + "num_input_tokens_seen": 173214675, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.0090332, + "step": 6053, + "time_per_iteration": 3.077208995819092 + }, + { + "auxiliary_loss_clip": 0.01134415, + "auxiliary_loss_mlp": 0.01032207, + "balance_loss_clip": 1.06097198, + "balance_loss_mlp": 1.01712704, + "epoch": 0.1756717543961465, + "flos": 16722127175040.0, + "grad_norm": 4.833870205872264, + "language_loss": 0.81351441, + "learning_rate": 3.7815178531460615e-06, + "loss": 0.83518058, + "num_input_tokens_seen": 173230215, + "router_z_loss_clip": 0.73535156, + "router_z_loss_mlp": 0.15081787, + "step": 6054, + "time_per_iteration": 2.476663827896118 + }, + { + "auxiliary_loss_clip": 0.01049177, + "auxiliary_loss_mlp": 0.01004717, + "balance_loss_clip": 1.02968967, + "balance_loss_mlp": 1.00374556, + "epoch": 0.1757007718646625, + "flos": 66957133633920.0, + "grad_norm": 0.6976064236327202, + "language_loss": 0.49648976, + "learning_rate": 3.7814324211546166e-06, + "loss": 0.51702875, + "num_input_tokens_seen": 173286990, + "router_z_loss_clip": 0.1953125, + "router_z_loss_mlp": 0.00970459, + "step": 6055, + "time_per_iteration": 3.0509676933288574 + }, + { + "auxiliary_loss_clip": 0.01133698, + "auxiliary_loss_mlp": 0.0103654, + "balance_loss_clip": 1.06457543, + "balance_loss_mlp": 1.02302217, + "epoch": 0.17572978933317857, + "flos": 13472067755520.0, + "grad_norm": 2.7757581505477744, + "language_loss": 0.80249929, + "learning_rate": 3.78134697342884e-06, + "loss": 0.8242017, + "num_input_tokens_seen": 173298645, + "router_z_loss_clip": 0.69140625, + "router_z_loss_mlp": 0.13531494, + "step": 6056, + "time_per_iteration": 2.488375425338745 + }, + { + "auxiliary_loss_clip": 0.01135773, + "auxiliary_loss_mlp": 0.01033417, + "balance_loss_clip": 1.06626618, + "balance_loss_mlp": 1.02048862, + "epoch": 0.17575880680169462, + "flos": 11940908751360.0, + "grad_norm": 2.3586003750326174, + "language_loss": 0.7400676, + "learning_rate": 3.7812615099694853e-06, + "loss": 0.76175946, + "num_input_tokens_seen": 173310600, + "router_z_loss_clip": 0.69433594, + "router_z_loss_mlp": 0.1293335, + "step": 6057, + "time_per_iteration": 2.509315252304077 + }, + { + "auxiliary_loss_clip": 0.01139491, + "auxiliary_loss_mlp": 0.010408, + "balance_loss_clip": 1.06667173, + "balance_loss_mlp": 1.02590537, + "epoch": 0.17578782427021067, + "flos": 16792655529600.0, + "grad_norm": 3.1993713411467217, + "language_loss": 0.675946, + "learning_rate": 3.781176030777309e-06, + "loss": 0.6977489, + "num_input_tokens_seen": 173323215, + "router_z_loss_clip": 0.72998047, + "router_z_loss_mlp": 0.14904785, + "step": 6058, + "time_per_iteration": 2.5410327911376953 + }, + { + "auxiliary_loss_clip": 0.01141349, + "auxiliary_loss_mlp": 0.01043885, + "balance_loss_clip": 1.06676173, + "balance_loss_mlp": 1.02874553, + "epoch": 0.17581684173872672, + "flos": 28725447807360.0, + "grad_norm": 2.226553487714164, + "language_loss": 0.79079878, + "learning_rate": 3.781090535853065e-06, + "loss": 0.81265116, + "num_input_tokens_seen": 173339625, + "router_z_loss_clip": 0.74658203, + "router_z_loss_mlp": 0.15136719, + "step": 6059, + "time_per_iteration": 2.6014926433563232 + }, + { + "auxiliary_loss_clip": 0.01140102, + "auxiliary_loss_mlp": 0.01043331, + "balance_loss_clip": 1.06652915, + "balance_loss_mlp": 1.02774405, + "epoch": 0.17584585920724277, + "flos": 18580791409920.0, + "grad_norm": 2.096154129340645, + "language_loss": 0.80999684, + "learning_rate": 3.781005025197508e-06, + "loss": 0.83183122, + "num_input_tokens_seen": 173356015, + "router_z_loss_clip": 0.73730469, + "router_z_loss_mlp": 0.15588379, + "step": 6060, + "time_per_iteration": 2.576718807220459 + }, + { + "auxiliary_loss_clip": 0.01049824, + "auxiliary_loss_mlp": 0.01007583, + "balance_loss_clip": 1.03053141, + "balance_loss_mlp": 1.00671887, + "epoch": 0.1758748766757588, + "flos": 58752407992320.0, + "grad_norm": 0.6896590927850704, + "language_loss": 0.5027017, + "learning_rate": 3.7809194988113943e-06, + "loss": 0.52327573, + "num_input_tokens_seen": 173414275, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.00866699, + "step": 6061, + "time_per_iteration": 3.0727121829986572 + }, + { + "auxiliary_loss_clip": 0.01136311, + "auxiliary_loss_mlp": 0.01042087, + "balance_loss_clip": 1.06371033, + "balance_loss_mlp": 1.0264945, + "epoch": 0.17590389414427485, + "flos": 50943809537280.0, + "grad_norm": 2.228304114538222, + "language_loss": 0.89746344, + "learning_rate": 3.7808339566954786e-06, + "loss": 0.91924739, + "num_input_tokens_seen": 173432185, + "router_z_loss_clip": 0.72607422, + "router_z_loss_mlp": 0.15600586, + "step": 6062, + "time_per_iteration": 2.80539870262146 + }, + { + "auxiliary_loss_clip": 0.01136234, + "auxiliary_loss_mlp": 0.01035713, + "balance_loss_clip": 1.0645256, + "balance_loss_mlp": 1.02147365, + "epoch": 0.1759329116127909, + "flos": 36313012944000.0, + "grad_norm": 2.3946608786799928, + "language_loss": 0.81035769, + "learning_rate": 3.7807483988505173e-06, + "loss": 0.83207715, + "num_input_tokens_seen": 173449820, + "router_z_loss_clip": 0.71704102, + "router_z_loss_mlp": 0.14227295, + "step": 6063, + "time_per_iteration": 2.6561808586120605 + }, + { + "auxiliary_loss_clip": 0.01044799, + "auxiliary_loss_mlp": 0.01003991, + "balance_loss_clip": 1.02551222, + "balance_loss_mlp": 1.00306082, + "epoch": 0.17596192908130695, + "flos": 73314938787840.0, + "grad_norm": 0.7012353420550718, + "language_loss": 0.5016585, + "learning_rate": 3.7806628252772654e-06, + "loss": 0.5221464, + "num_input_tokens_seen": 173514635, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.00927734, + "step": 6064, + "time_per_iteration": 3.128474712371826 + }, + { + "auxiliary_loss_clip": 0.01137511, + "auxiliary_loss_mlp": 0.01036659, + "balance_loss_clip": 1.06574583, + "balance_loss_mlp": 1.02286005, + "epoch": 0.175990946549823, + "flos": 33908219349120.0, + "grad_norm": 1.8859840774777106, + "language_loss": 0.76394081, + "learning_rate": 3.780577235976479e-06, + "loss": 0.7856825, + "num_input_tokens_seen": 173535110, + "router_z_loss_clip": 0.71777344, + "router_z_loss_mlp": 0.13787842, + "step": 6065, + "time_per_iteration": 2.645515203475952 + }, + { + "auxiliary_loss_clip": 0.01042793, + "auxiliary_loss_mlp": 0.01000909, + "balance_loss_clip": 1.02340388, + "balance_loss_mlp": 0.99999064, + "epoch": 0.17601996401833905, + "flos": 65941257185280.0, + "grad_norm": 0.6940721944908985, + "language_loss": 0.46215349, + "learning_rate": 3.780491630948914e-06, + "loss": 0.48259056, + "num_input_tokens_seen": 173595105, + "router_z_loss_clip": 0.19433594, + "router_z_loss_mlp": 0.00915527, + "step": 6066, + "time_per_iteration": 3.0078439712524414 + }, + { + "auxiliary_loss_clip": 0.01147319, + "auxiliary_loss_mlp": 0.01041086, + "balance_loss_clip": 1.06775653, + "balance_loss_mlp": 1.02436662, + "epoch": 0.17604898148685508, + "flos": 14127796488960.0, + "grad_norm": 3.186108234153674, + "language_loss": 0.89259017, + "learning_rate": 3.780406010195326e-06, + "loss": 0.91447425, + "num_input_tokens_seen": 173605935, + "router_z_loss_clip": 0.79541016, + "router_z_loss_mlp": 0.16711426, + "step": 6067, + "time_per_iteration": 2.497995615005493 + }, + { + "auxiliary_loss_clip": 0.0114028, + "auxiliary_loss_mlp": 0.01045511, + "balance_loss_clip": 1.06654537, + "balance_loss_mlp": 1.02993655, + "epoch": 0.17607799895537113, + "flos": 10661267756160.0, + "grad_norm": 2.7552257022950624, + "language_loss": 0.81355381, + "learning_rate": 3.7803203737164714e-06, + "loss": 0.83541173, + "num_input_tokens_seen": 173617090, + "router_z_loss_clip": 0.73828125, + "router_z_loss_mlp": 0.15594482, + "step": 6068, + "time_per_iteration": 2.533186435699463 + }, + { + "auxiliary_loss_clip": 0.01132049, + "auxiliary_loss_mlp": 0.01035634, + "balance_loss_clip": 1.06516671, + "balance_loss_mlp": 1.02159762, + "epoch": 0.17610701642388718, + "flos": 27013406889600.0, + "grad_norm": 1.8419705392037815, + "language_loss": 0.89438045, + "learning_rate": 3.780234721513108e-06, + "loss": 0.91605735, + "num_input_tokens_seen": 173635555, + "router_z_loss_clip": 0.66845703, + "router_z_loss_mlp": 0.14031982, + "step": 6069, + "time_per_iteration": 2.6086881160736084 + }, + { + "auxiliary_loss_clip": 0.01138662, + "auxiliary_loss_mlp": 0.01035483, + "balance_loss_clip": 1.06437588, + "balance_loss_mlp": 1.0197413, + "epoch": 0.17613603389240323, + "flos": 20845533876480.0, + "grad_norm": 1.9210607099033532, + "language_loss": 0.89534855, + "learning_rate": 3.7801490535859905e-06, + "loss": 0.91709006, + "num_input_tokens_seen": 173652970, + "router_z_loss_clip": 0.74365234, + "router_z_loss_mlp": 0.15734863, + "step": 6070, + "time_per_iteration": 2.5527853965759277 + }, + { + "auxiliary_loss_clip": 0.01049601, + "auxiliary_loss_mlp": 0.01004665, + "balance_loss_clip": 1.03029323, + "balance_loss_mlp": 1.00378251, + "epoch": 0.17616505136091928, + "flos": 64377778919040.0, + "grad_norm": 0.6634611692468938, + "language_loss": 0.54349017, + "learning_rate": 3.7800633699358757e-06, + "loss": 0.56403291, + "num_input_tokens_seen": 173714530, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.0088501, + "step": 6071, + "time_per_iteration": 3.084388017654419 + }, + { + "auxiliary_loss_clip": 0.01137336, + "auxiliary_loss_mlp": 0.01038925, + "balance_loss_clip": 1.06826591, + "balance_loss_mlp": 1.02553749, + "epoch": 0.1761940688294353, + "flos": 12048784272000.0, + "grad_norm": 2.5208249179884348, + "language_loss": 0.79482466, + "learning_rate": 3.7799776705635216e-06, + "loss": 0.81658721, + "num_input_tokens_seen": 173726120, + "router_z_loss_clip": 0.69042969, + "router_z_loss_mlp": 0.13391113, + "step": 6072, + "time_per_iteration": 2.52146577835083 + }, + { + "auxiliary_loss_clip": 0.01131359, + "auxiliary_loss_mlp": 0.01033422, + "balance_loss_clip": 1.06405663, + "balance_loss_mlp": 1.01978457, + "epoch": 0.17622308629795136, + "flos": 16683092069760.0, + "grad_norm": 2.310388487079457, + "language_loss": 0.74821943, + "learning_rate": 3.779891955469684e-06, + "loss": 0.7698673, + "num_input_tokens_seen": 173739255, + "router_z_loss_clip": 0.67382812, + "router_z_loss_mlp": 0.13635254, + "step": 6073, + "time_per_iteration": 2.4619498252868652 + }, + { + "auxiliary_loss_clip": 0.01135119, + "auxiliary_loss_mlp": 0.01033122, + "balance_loss_clip": 1.06515884, + "balance_loss_mlp": 1.01876926, + "epoch": 0.1762521037664674, + "flos": 35729033627520.0, + "grad_norm": 2.1059166990937124, + "language_loss": 0.75683796, + "learning_rate": 3.7798062246551206e-06, + "loss": 0.77852035, + "num_input_tokens_seen": 173756220, + "router_z_loss_clip": 0.69921875, + "router_z_loss_mlp": 0.14355469, + "step": 6074, + "time_per_iteration": 2.741229772567749 + }, + { + "auxiliary_loss_clip": 0.0114549, + "auxiliary_loss_mlp": 0.01045119, + "balance_loss_clip": 1.06674552, + "balance_loss_mlp": 1.02858496, + "epoch": 0.17628112123498346, + "flos": 36442077488640.0, + "grad_norm": 2.5002484214013188, + "language_loss": 0.74232876, + "learning_rate": 3.7797204781205886e-06, + "loss": 0.76423484, + "num_input_tokens_seen": 173776285, + "router_z_loss_clip": 0.78710938, + "router_z_loss_mlp": 0.16516113, + "step": 6075, + "time_per_iteration": 2.6412248611450195 + }, + { + "auxiliary_loss_clip": 0.01052648, + "auxiliary_loss_mlp": 0.01002815, + "balance_loss_clip": 1.033306, + "balance_loss_mlp": 1.00198102, + "epoch": 0.1763101387034995, + "flos": 52345909952640.0, + "grad_norm": 0.6624875930023051, + "language_loss": 0.48258325, + "learning_rate": 3.7796347158668455e-06, + "loss": 0.50313795, + "num_input_tokens_seen": 173835275, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.00836182, + "step": 6076, + "time_per_iteration": 3.0983736515045166 + }, + { + "auxiliary_loss_clip": 0.01141767, + "auxiliary_loss_mlp": 0.01039363, + "balance_loss_clip": 1.06616926, + "balance_loss_mlp": 1.02385426, + "epoch": 0.17633915617201557, + "flos": 21029001569280.0, + "grad_norm": 1.8553163286426964, + "language_loss": 0.74146801, + "learning_rate": 3.779548937894647e-06, + "loss": 0.76327932, + "num_input_tokens_seen": 173853540, + "router_z_loss_clip": 0.75537109, + "router_z_loss_mlp": 0.15509033, + "step": 6077, + "time_per_iteration": 2.5421524047851562 + }, + { + "auxiliary_loss_clip": 0.01047358, + "auxiliary_loss_mlp": 0.01002913, + "balance_loss_clip": 1.02792513, + "balance_loss_mlp": 1.00205421, + "epoch": 0.1763681736405316, + "flos": 71163853931520.0, + "grad_norm": 0.6420037038617274, + "language_loss": 0.46557987, + "learning_rate": 3.7794631442047534e-06, + "loss": 0.48608258, + "num_input_tokens_seen": 173918690, + "router_z_loss_clip": 0.1953125, + "router_z_loss_mlp": 0.00860596, + "step": 6078, + "time_per_iteration": 3.134603977203369 + }, + { + "auxiliary_loss_clip": 0.01138753, + "auxiliary_loss_mlp": 0.01043692, + "balance_loss_clip": 1.0648936, + "balance_loss_mlp": 1.0289402, + "epoch": 0.17639719110904764, + "flos": 29673453507840.0, + "grad_norm": 2.243878111236265, + "language_loss": 0.83515918, + "learning_rate": 3.779377334797921e-06, + "loss": 0.85698366, + "num_input_tokens_seen": 173937460, + "router_z_loss_clip": 0.73876953, + "router_z_loss_mlp": 0.14758301, + "step": 6079, + "time_per_iteration": 2.586841106414795 + }, + { + "auxiliary_loss_clip": 0.01043685, + "auxiliary_loss_mlp": 0.00999555, + "balance_loss_clip": 1.02440572, + "balance_loss_mlp": 0.99870265, + "epoch": 0.1764262085775637, + "flos": 66419156661120.0, + "grad_norm": 0.7716385115809283, + "language_loss": 0.51789552, + "learning_rate": 3.779291509674908e-06, + "loss": 0.53832787, + "num_input_tokens_seen": 173992715, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.00854492, + "step": 6080, + "time_per_iteration": 3.030250310897827 + }, + { + "auxiliary_loss_clip": 0.01136497, + "auxiliary_loss_mlp": 0.01039592, + "balance_loss_clip": 1.06379032, + "balance_loss_mlp": 1.02500057, + "epoch": 0.17645522604607974, + "flos": 31132288477440.0, + "grad_norm": 2.1299919496205835, + "language_loss": 0.72738868, + "learning_rate": 3.7792056688364725e-06, + "loss": 0.74914956, + "num_input_tokens_seen": 174009065, + "router_z_loss_clip": 0.72705078, + "router_z_loss_mlp": 0.14599609, + "step": 6081, + "time_per_iteration": 2.647091865539551 + }, + { + "auxiliary_loss_clip": 0.01137879, + "auxiliary_loss_mlp": 0.01038364, + "balance_loss_clip": 1.06379962, + "balance_loss_mlp": 1.02074504, + "epoch": 0.1764842435145958, + "flos": 41310054236160.0, + "grad_norm": 3.4852762643334447, + "language_loss": 0.83506846, + "learning_rate": 3.779119812283372e-06, + "loss": 0.85683084, + "num_input_tokens_seen": 174024975, + "router_z_loss_clip": 0.74072266, + "router_z_loss_mlp": 0.17626953, + "step": 6082, + "time_per_iteration": 2.692610025405884 + }, + { + "auxiliary_loss_clip": 0.01141575, + "auxiliary_loss_mlp": 0.01059568, + "balance_loss_clip": 1.06278765, + "balance_loss_mlp": 1.03929055, + "epoch": 0.17651326098311185, + "flos": 28689501271680.0, + "grad_norm": 1.9607177562864444, + "language_loss": 1.01352894, + "learning_rate": 3.779033940016366e-06, + "loss": 1.03554034, + "num_input_tokens_seen": 174044125, + "router_z_loss_clip": 0.78759766, + "router_z_loss_mlp": 0.20281982, + "step": 6083, + "time_per_iteration": 2.6261022090911865 + }, + { + "auxiliary_loss_clip": 0.01135852, + "auxiliary_loss_mlp": 0.01035696, + "balance_loss_clip": 1.06393075, + "balance_loss_mlp": 1.02052093, + "epoch": 0.17654227845162787, + "flos": 22886265173760.0, + "grad_norm": 2.4033207263548366, + "language_loss": 0.74289691, + "learning_rate": 3.7789480520362117e-06, + "loss": 0.76461232, + "num_input_tokens_seen": 174056960, + "router_z_loss_clip": 0.71875, + "router_z_loss_mlp": 0.15155029, + "step": 6084, + "time_per_iteration": 2.547776222229004 + }, + { + "auxiliary_loss_clip": 0.01143302, + "auxiliary_loss_mlp": 0.01042074, + "balance_loss_clip": 1.06565845, + "balance_loss_mlp": 1.0260644, + "epoch": 0.17657129592014392, + "flos": 31789381927680.0, + "grad_norm": 2.4590726184612053, + "language_loss": 0.81680632, + "learning_rate": 3.778862148343669e-06, + "loss": 0.83866012, + "num_input_tokens_seen": 174072830, + "router_z_loss_clip": 0.77685547, + "router_z_loss_mlp": 0.15997314, + "step": 6085, + "time_per_iteration": 2.631640672683716 + }, + { + "auxiliary_loss_clip": 0.01134591, + "auxiliary_loss_mlp": 0.01041306, + "balance_loss_clip": 1.06209207, + "balance_loss_mlp": 1.02548671, + "epoch": 0.17660031338865997, + "flos": 29378016144000.0, + "grad_norm": 2.653155855491774, + "language_loss": 0.98376459, + "learning_rate": 3.7787762289394954e-06, + "loss": 1.00552356, + "num_input_tokens_seen": 174087100, + "router_z_loss_clip": 0.72558594, + "router_z_loss_mlp": 0.1583252, + "step": 6086, + "time_per_iteration": 2.5959253311157227 + }, + { + "auxiliary_loss_clip": 0.01139504, + "auxiliary_loss_mlp": 0.01054291, + "balance_loss_clip": 1.06776702, + "balance_loss_mlp": 1.03778028, + "epoch": 0.17662933085717603, + "flos": 21537747849600.0, + "grad_norm": 2.2024126107735085, + "language_loss": 0.83860987, + "learning_rate": 3.778690293824451e-06, + "loss": 0.8605479, + "num_input_tokens_seen": 174103210, + "router_z_loss_clip": 0.71777344, + "router_z_loss_mlp": 0.1651001, + "step": 6087, + "time_per_iteration": 2.599961042404175 + }, + { + "auxiliary_loss_clip": 0.01131881, + "auxiliary_loss_mlp": 0.01032795, + "balance_loss_clip": 1.06217718, + "balance_loss_mlp": 1.01724458, + "epoch": 0.17665834832569208, + "flos": 19929057338880.0, + "grad_norm": 3.1349978931869202, + "language_loss": 1.03753424, + "learning_rate": 3.7786043429992935e-06, + "loss": 1.05918109, + "num_input_tokens_seen": 174115255, + "router_z_loss_clip": 0.69677734, + "router_z_loss_mlp": 0.15539551, + "step": 6088, + "time_per_iteration": 2.5310897827148438 + }, + { + "auxiliary_loss_clip": 0.01142937, + "auxiliary_loss_mlp": 0.01045684, + "balance_loss_clip": 1.06431639, + "balance_loss_mlp": 1.02950704, + "epoch": 0.1766873657942081, + "flos": 33212449929600.0, + "grad_norm": 2.0204676376163206, + "language_loss": 0.8552385, + "learning_rate": 3.7785183764647827e-06, + "loss": 0.87712467, + "num_input_tokens_seen": 174135190, + "router_z_loss_clip": 0.78515625, + "router_z_loss_mlp": 0.16186523, + "step": 6089, + "time_per_iteration": 2.669650077819824 + }, + { + "auxiliary_loss_clip": 0.01132885, + "auxiliary_loss_mlp": 0.0103665, + "balance_loss_clip": 1.06250858, + "balance_loss_mlp": 1.02140355, + "epoch": 0.17671638326272415, + "flos": 12266761956480.0, + "grad_norm": 3.7617600776420685, + "language_loss": 0.93404448, + "learning_rate": 3.7784323942216788e-06, + "loss": 0.95573986, + "num_input_tokens_seen": 174143995, + "router_z_loss_clip": 0.70410156, + "router_z_loss_mlp": 0.15246582, + "step": 6090, + "time_per_iteration": 2.481773853302002 + }, + { + "auxiliary_loss_clip": 0.01048638, + "auxiliary_loss_mlp": 0.01005043, + "balance_loss_clip": 1.02943563, + "balance_loss_mlp": 1.00426233, + "epoch": 0.1767454007312402, + "flos": 57214497231360.0, + "grad_norm": 0.7677823359898223, + "language_loss": 0.44741929, + "learning_rate": 3.7783463962707397e-06, + "loss": 0.46795607, + "num_input_tokens_seen": 174198785, + "router_z_loss_clip": 0.19238281, + "router_z_loss_mlp": 0.0078125, + "step": 6091, + "time_per_iteration": 2.9739315509796143 + }, + { + "auxiliary_loss_clip": 0.01049356, + "auxiliary_loss_mlp": 0.0100258, + "balance_loss_clip": 1.03005457, + "balance_loss_mlp": 1.00181997, + "epoch": 0.17677441819975626, + "flos": 61049864770560.0, + "grad_norm": 0.7501055502961772, + "language_loss": 0.4884108, + "learning_rate": 3.778260382612726e-06, + "loss": 0.50893021, + "num_input_tokens_seen": 174258665, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.00759888, + "step": 6092, + "time_per_iteration": 3.0796337127685547 + }, + { + "auxiliary_loss_clip": 0.01044644, + "auxiliary_loss_mlp": 0.00999426, + "balance_loss_clip": 1.02536893, + "balance_loss_mlp": 0.9986214, + "epoch": 0.1768034356682723, + "flos": 65876115870720.0, + "grad_norm": 2.052261507411921, + "language_loss": 0.55134845, + "learning_rate": 3.778174353248396e-06, + "loss": 0.57178915, + "num_input_tokens_seen": 174317125, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.00805664, + "step": 6093, + "time_per_iteration": 3.055837392807007 + }, + { + "auxiliary_loss_clip": 0.01140179, + "auxiliary_loss_mlp": 0.01044959, + "balance_loss_clip": 1.06445611, + "balance_loss_mlp": 1.02861547, + "epoch": 0.17683245313678836, + "flos": 20552574551040.0, + "grad_norm": 2.0334935829028313, + "language_loss": 0.74706459, + "learning_rate": 3.778088308178511e-06, + "loss": 0.76891595, + "num_input_tokens_seen": 174334160, + "router_z_loss_clip": 0.75634766, + "router_z_loss_mlp": 0.16345215, + "step": 6094, + "time_per_iteration": 2.5816378593444824 + }, + { + "auxiliary_loss_clip": 0.01140484, + "auxiliary_loss_mlp": 0.01036346, + "balance_loss_clip": 1.06818604, + "balance_loss_mlp": 1.02156973, + "epoch": 0.17686147060530438, + "flos": 32080042782720.0, + "grad_norm": 2.2925890589418905, + "language_loss": 1.00626981, + "learning_rate": 3.7780022474038313e-06, + "loss": 1.02803814, + "num_input_tokens_seen": 174350485, + "router_z_loss_clip": 0.72265625, + "router_z_loss_mlp": 0.14764404, + "step": 6095, + "time_per_iteration": 2.6170918941497803 + }, + { + "auxiliary_loss_clip": 0.01139655, + "auxiliary_loss_mlp": 0.01042114, + "balance_loss_clip": 1.0628103, + "balance_loss_mlp": 1.02659297, + "epoch": 0.17689048807382043, + "flos": 27154930475520.0, + "grad_norm": 3.6557222320661054, + "language_loss": 0.82964551, + "learning_rate": 3.7779161709251157e-06, + "loss": 0.8514632, + "num_input_tokens_seen": 174365630, + "router_z_loss_clip": 0.76806641, + "router_z_loss_mlp": 0.15530396, + "step": 6096, + "time_per_iteration": 2.6816277503967285 + }, + { + "auxiliary_loss_clip": 0.01134611, + "auxiliary_loss_mlp": 0.01033997, + "balance_loss_clip": 1.06127894, + "balance_loss_mlp": 1.01900053, + "epoch": 0.17691950554233649, + "flos": 32154521633280.0, + "grad_norm": 2.4730565815878847, + "language_loss": 0.69270378, + "learning_rate": 3.777830078743125e-06, + "loss": 0.71438992, + "num_input_tokens_seen": 174382355, + "router_z_loss_clip": 0.73339844, + "router_z_loss_mlp": 0.14996338, + "step": 6097, + "time_per_iteration": 2.606032133102417 + }, + { + "auxiliary_loss_clip": 0.01139529, + "auxiliary_loss_mlp": 0.01044525, + "balance_loss_clip": 1.06439316, + "balance_loss_mlp": 1.02848577, + "epoch": 0.17694852301085254, + "flos": 11536769854080.0, + "grad_norm": 3.3627518050667704, + "language_loss": 0.83645624, + "learning_rate": 3.77774397085862e-06, + "loss": 0.85829675, + "num_input_tokens_seen": 174391735, + "router_z_loss_clip": 0.75097656, + "router_z_loss_mlp": 0.16033936, + "step": 6098, + "time_per_iteration": 2.4905662536621094 + }, + { + "auxiliary_loss_clip": 0.01132417, + "auxiliary_loss_mlp": 0.01040423, + "balance_loss_clip": 1.06050825, + "balance_loss_mlp": 1.02651763, + "epoch": 0.1769775404793686, + "flos": 31173298830720.0, + "grad_norm": 2.444651600704913, + "language_loss": 0.86915731, + "learning_rate": 3.77765784727236e-06, + "loss": 0.89088577, + "num_input_tokens_seen": 174408855, + "router_z_loss_clip": 0.71923828, + "router_z_loss_mlp": 0.13909912, + "step": 6099, + "time_per_iteration": 2.5540175437927246 + }, + { + "auxiliary_loss_clip": 0.01127349, + "auxiliary_loss_mlp": 0.01031849, + "balance_loss_clip": 1.06113267, + "balance_loss_mlp": 1.01871228, + "epoch": 0.1770065579478846, + "flos": 22423485323520.0, + "grad_norm": 2.063475014175759, + "language_loss": 0.82897204, + "learning_rate": 3.777571707985108e-06, + "loss": 0.850564, + "num_input_tokens_seen": 174424265, + "router_z_loss_clip": 0.66210938, + "router_z_loss_mlp": 0.13140869, + "step": 6100, + "time_per_iteration": 2.636167287826538 + }, + { + "auxiliary_loss_clip": 0.01131621, + "auxiliary_loss_mlp": 0.01048015, + "balance_loss_clip": 1.05807734, + "balance_loss_mlp": 1.03350115, + "epoch": 0.17703557541640066, + "flos": 25546670928000.0, + "grad_norm": 3.224661001563884, + "language_loss": 0.878878, + "learning_rate": 3.7774855529976222e-06, + "loss": 0.90067434, + "num_input_tokens_seen": 174435695, + "router_z_loss_clip": 0.73632812, + "router_z_loss_mlp": 0.14520264, + "step": 6101, + "time_per_iteration": 2.581498384475708 + }, + { + "auxiliary_loss_clip": 0.01134747, + "auxiliary_loss_mlp": 0.0104142, + "balance_loss_clip": 1.06257939, + "balance_loss_mlp": 1.02524292, + "epoch": 0.17706459288491672, + "flos": 44520827155200.0, + "grad_norm": 2.302794443605033, + "language_loss": 0.89503813, + "learning_rate": 3.7773993823106647e-06, + "loss": 0.91679972, + "num_input_tokens_seen": 174451675, + "router_z_loss_clip": 0.72167969, + "router_z_loss_mlp": 0.1618042, + "step": 6102, + "time_per_iteration": 2.7290706634521484 + }, + { + "auxiliary_loss_clip": 0.01040923, + "auxiliary_loss_mlp": 0.01014182, + "balance_loss_clip": 1.02142358, + "balance_loss_mlp": 1.01331758, + "epoch": 0.17709361035343277, + "flos": 63576791585280.0, + "grad_norm": 0.8292509765271455, + "language_loss": 0.47920835, + "learning_rate": 3.777313195924997e-06, + "loss": 0.49975944, + "num_input_tokens_seen": 174515320, + "router_z_loss_clip": 0.1953125, + "router_z_loss_mlp": 0.00866699, + "step": 6103, + "time_per_iteration": 3.1029999256134033 + }, + { + "auxiliary_loss_clip": 0.01039931, + "auxiliary_loss_mlp": 0.01013488, + "balance_loss_clip": 1.02037048, + "balance_loss_mlp": 1.01262999, + "epoch": 0.17712262782194882, + "flos": 61711303766400.0, + "grad_norm": 0.7385948225915242, + "language_loss": 0.44390804, + "learning_rate": 3.77722699384138e-06, + "loss": 0.46444219, + "num_input_tokens_seen": 174579105, + "router_z_loss_clip": 0.1953125, + "router_z_loss_mlp": 0.00860596, + "step": 6104, + "time_per_iteration": 3.272993326187134 + }, + { + "auxiliary_loss_clip": 0.01133893, + "auxiliary_loss_mlp": 0.01040451, + "balance_loss_clip": 1.06194115, + "balance_loss_mlp": 1.02705765, + "epoch": 0.17715164529046487, + "flos": 37302352220160.0, + "grad_norm": 1.6347963618657253, + "language_loss": 0.76814157, + "learning_rate": 3.777140776060575e-06, + "loss": 0.78988504, + "num_input_tokens_seen": 174607770, + "router_z_loss_clip": 0.71972656, + "router_z_loss_mlp": 0.13391113, + "step": 6105, + "time_per_iteration": 3.0099661350250244 + }, + { + "auxiliary_loss_clip": 0.0113708, + "auxiliary_loss_mlp": 0.010448, + "balance_loss_clip": 1.06645465, + "balance_loss_mlp": 1.03085256, + "epoch": 0.1771806627589809, + "flos": 26937096445440.0, + "grad_norm": 2.3170543804832398, + "language_loss": 0.95413518, + "learning_rate": 3.777054542583343e-06, + "loss": 0.97595394, + "num_input_tokens_seen": 174623240, + "router_z_loss_clip": 0.70654297, + "router_z_loss_mlp": 0.13952637, + "step": 6106, + "time_per_iteration": 2.68371319770813 + }, + { + "auxiliary_loss_clip": 0.01138742, + "auxiliary_loss_mlp": 0.01046376, + "balance_loss_clip": 1.06425798, + "balance_loss_mlp": 1.03163302, + "epoch": 0.17720968022749695, + "flos": 74730677437440.0, + "grad_norm": 2.243057069123723, + "language_loss": 0.86517298, + "learning_rate": 3.776968293410447e-06, + "loss": 0.88702416, + "num_input_tokens_seen": 174643855, + "router_z_loss_clip": 0.74560547, + "router_z_loss_mlp": 0.14743042, + "step": 6107, + "time_per_iteration": 2.9831221103668213 + }, + { + "auxiliary_loss_clip": 0.01041396, + "auxiliary_loss_mlp": 0.01001101, + "balance_loss_clip": 1.02150786, + "balance_loss_mlp": 1.00027835, + "epoch": 0.177238697696013, + "flos": 74777005981440.0, + "grad_norm": 0.6742115548856774, + "language_loss": 0.46238551, + "learning_rate": 3.7768820285426477e-06, + "loss": 0.48281047, + "num_input_tokens_seen": 174709680, + "router_z_loss_clip": 0.19921875, + "router_z_loss_mlp": 0.00823975, + "step": 6108, + "time_per_iteration": 3.157836437225342 + }, + { + "auxiliary_loss_clip": 0.01140748, + "auxiliary_loss_mlp": 0.01043307, + "balance_loss_clip": 1.06282508, + "balance_loss_mlp": 1.02740431, + "epoch": 0.17726771516452905, + "flos": 31318485603840.0, + "grad_norm": 2.313406911703752, + "language_loss": 0.96935916, + "learning_rate": 3.7767957479807074e-06, + "loss": 0.99119973, + "num_input_tokens_seen": 174727330, + "router_z_loss_clip": 0.78027344, + "router_z_loss_mlp": 0.15887451, + "step": 6109, + "time_per_iteration": 2.64334774017334 + }, + { + "auxiliary_loss_clip": 0.01132641, + "auxiliary_loss_mlp": 0.01043592, + "balance_loss_clip": 1.06229413, + "balance_loss_mlp": 1.02887547, + "epoch": 0.1772967326330451, + "flos": 14838254570880.0, + "grad_norm": 2.1483165707837184, + "language_loss": 0.70330012, + "learning_rate": 3.7767094517253874e-06, + "loss": 0.72506249, + "num_input_tokens_seen": 174742645, + "router_z_loss_clip": 0.703125, + "router_z_loss_mlp": 0.1472168, + "step": 6110, + "time_per_iteration": 2.5376040935516357 + }, + { + "auxiliary_loss_clip": 0.01139855, + "auxiliary_loss_mlp": 0.01055495, + "balance_loss_clip": 1.06670284, + "balance_loss_mlp": 1.03924131, + "epoch": 0.17732575010156115, + "flos": 22342542024960.0, + "grad_norm": 4.4729057906754015, + "language_loss": 0.73861164, + "learning_rate": 3.776623139777451e-06, + "loss": 0.76056516, + "num_input_tokens_seen": 174758795, + "router_z_loss_clip": 0.73144531, + "router_z_loss_mlp": 0.16278076, + "step": 6111, + "time_per_iteration": 2.5503616333007812 + }, + { + "auxiliary_loss_clip": 0.01138548, + "auxiliary_loss_mlp": 0.01039115, + "balance_loss_clip": 1.06196761, + "balance_loss_mlp": 1.02439833, + "epoch": 0.17735476757007718, + "flos": 26063785077120.0, + "grad_norm": 1.578194320671514, + "language_loss": 0.69740307, + "learning_rate": 3.77653681213766e-06, + "loss": 0.71917963, + "num_input_tokens_seen": 174777715, + "router_z_loss_clip": 0.765625, + "router_z_loss_mlp": 0.14727783, + "step": 6112, + "time_per_iteration": 2.5228848457336426 + }, + { + "auxiliary_loss_clip": 0.01128411, + "auxiliary_loss_mlp": 0.01039141, + "balance_loss_clip": 1.06013656, + "balance_loss_mlp": 1.02577138, + "epoch": 0.17738378503859323, + "flos": 13947417365760.0, + "grad_norm": 2.499101495188165, + "language_loss": 0.71225798, + "learning_rate": 3.7764504688067774e-06, + "loss": 0.73393357, + "num_input_tokens_seen": 174791500, + "router_z_loss_clip": 0.68310547, + "router_z_loss_mlp": 0.1338501, + "step": 6113, + "time_per_iteration": 2.558253526687622 + }, + { + "auxiliary_loss_clip": 0.01039345, + "auxiliary_loss_mlp": 0.01005934, + "balance_loss_clip": 1.01963162, + "balance_loss_mlp": 1.00499272, + "epoch": 0.17741280250710928, + "flos": 67489618803840.0, + "grad_norm": 0.6480103211255528, + "language_loss": 0.47544605, + "learning_rate": 3.776364109785565e-06, + "loss": 0.49589884, + "num_input_tokens_seen": 174859110, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.00939941, + "step": 6114, + "time_per_iteration": 3.2287259101867676 + }, + { + "auxiliary_loss_clip": 0.01128714, + "auxiliary_loss_mlp": 0.0104096, + "balance_loss_clip": 1.05997515, + "balance_loss_mlp": 1.02839494, + "epoch": 0.17744181997562533, + "flos": 16793661110400.0, + "grad_norm": 2.711392471253473, + "language_loss": 0.71236598, + "learning_rate": 3.776277735074786e-06, + "loss": 0.73406273, + "num_input_tokens_seen": 174874420, + "router_z_loss_clip": 0.6875, + "router_z_loss_mlp": 0.12567139, + "step": 6115, + "time_per_iteration": 2.4705047607421875 + }, + { + "auxiliary_loss_clip": 0.0113153, + "auxiliary_loss_mlp": 0.01034522, + "balance_loss_clip": 1.06187177, + "balance_loss_mlp": 1.0211823, + "epoch": 0.17747083744414138, + "flos": 50149502133120.0, + "grad_norm": 2.3670527169314806, + "language_loss": 0.66732335, + "learning_rate": 3.7761913446752037e-06, + "loss": 0.68898386, + "num_input_tokens_seen": 174893120, + "router_z_loss_clip": 0.69677734, + "router_z_loss_mlp": 0.13348389, + "step": 6116, + "time_per_iteration": 4.95050573348999 + }, + { + "auxiliary_loss_clip": 0.0113201, + "auxiliary_loss_mlp": 0.01042789, + "balance_loss_clip": 1.06081462, + "balance_loss_mlp": 1.02717817, + "epoch": 0.1774998549126574, + "flos": 14792252227200.0, + "grad_norm": 3.420192005812148, + "language_loss": 0.9303028, + "learning_rate": 3.77610493858758e-06, + "loss": 0.95205075, + "num_input_tokens_seen": 174905470, + "router_z_loss_clip": 0.71289062, + "router_z_loss_mlp": 0.15600586, + "step": 6117, + "time_per_iteration": 7.24863338470459 + }, + { + "auxiliary_loss_clip": 0.01036669, + "auxiliary_loss_mlp": 0.00999051, + "balance_loss_clip": 1.01701355, + "balance_loss_mlp": 0.99810362, + "epoch": 0.17752887238117346, + "flos": 72549754335360.0, + "grad_norm": 0.646800376799609, + "language_loss": 0.47237349, + "learning_rate": 3.776018516812679e-06, + "loss": 0.49273068, + "num_input_tokens_seen": 174968590, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.00946045, + "step": 6118, + "time_per_iteration": 5.402390718460083 + }, + { + "auxiliary_loss_clip": 0.01038014, + "auxiliary_loss_mlp": 0.01003047, + "balance_loss_clip": 1.01842523, + "balance_loss_mlp": 1.0022006, + "epoch": 0.1775578898496895, + "flos": 68475043497600.0, + "grad_norm": 0.7568726207112998, + "language_loss": 0.47423369, + "learning_rate": 3.7759320793512643e-06, + "loss": 0.49464428, + "num_input_tokens_seen": 175034465, + "router_z_loss_clip": 0.19628906, + "router_z_loss_mlp": 0.00848389, + "step": 6119, + "time_per_iteration": 3.2212369441986084 + }, + { + "auxiliary_loss_clip": 0.01130625, + "auxiliary_loss_mlp": 0.01039464, + "balance_loss_clip": 1.06108332, + "balance_loss_mlp": 1.02500379, + "epoch": 0.17758690731820556, + "flos": 14895315325440.0, + "grad_norm": 2.745518118074428, + "language_loss": 0.87323344, + "learning_rate": 3.7758456262040986e-06, + "loss": 0.8949343, + "num_input_tokens_seen": 175045945, + "router_z_loss_clip": 0.69580078, + "router_z_loss_mlp": 0.14465332, + "step": 6120, + "time_per_iteration": 2.5366547107696533 + }, + { + "auxiliary_loss_clip": 0.01136779, + "auxiliary_loss_mlp": 0.0103561, + "balance_loss_clip": 1.05948305, + "balance_loss_mlp": 1.020262, + "epoch": 0.1776159247867216, + "flos": 47416233640320.0, + "grad_norm": 1.7969299395978557, + "language_loss": 0.75813001, + "learning_rate": 3.7757591573719456e-06, + "loss": 0.77985388, + "num_input_tokens_seen": 175068035, + "router_z_loss_clip": 0.77392578, + "router_z_loss_mlp": 0.15350342, + "step": 6121, + "time_per_iteration": 2.778834819793701 + }, + { + "auxiliary_loss_clip": 0.01132401, + "auxiliary_loss_mlp": 0.01033196, + "balance_loss_clip": 1.05804324, + "balance_loss_mlp": 1.01748991, + "epoch": 0.17764494225523766, + "flos": 32665566384000.0, + "grad_norm": 1.8603989396924294, + "language_loss": 0.98381281, + "learning_rate": 3.7756726728555686e-06, + "loss": 1.00546873, + "num_input_tokens_seen": 175086630, + "router_z_loss_clip": 0.74365234, + "router_z_loss_mlp": 0.15704346, + "step": 6122, + "time_per_iteration": 2.6617233753204346 + }, + { + "auxiliary_loss_clip": 0.01131193, + "auxiliary_loss_mlp": 0.01037872, + "balance_loss_clip": 1.06117439, + "balance_loss_mlp": 1.02351892, + "epoch": 0.1776739597237537, + "flos": 26976993477120.0, + "grad_norm": 1.9696469022705247, + "language_loss": 1.08391225, + "learning_rate": 3.775586172655733e-06, + "loss": 1.10560298, + "num_input_tokens_seen": 175105735, + "router_z_loss_clip": 0.70019531, + "router_z_loss_mlp": 0.14349365, + "step": 6123, + "time_per_iteration": 2.6437644958496094 + }, + { + "auxiliary_loss_clip": 0.01035909, + "auxiliary_loss_mlp": 0.01006042, + "balance_loss_clip": 1.01651454, + "balance_loss_mlp": 1.0051893, + "epoch": 0.17770297719226974, + "flos": 74777688339840.0, + "grad_norm": 0.643227394189484, + "language_loss": 0.4191255, + "learning_rate": 3.775499656773201e-06, + "loss": 0.43954504, + "num_input_tokens_seen": 175169920, + "router_z_loss_clip": 0.19433594, + "router_z_loss_mlp": 0.00854492, + "step": 6124, + "time_per_iteration": 3.1423227787017822 + }, + { + "auxiliary_loss_clip": 0.01132126, + "auxiliary_loss_mlp": 0.01032184, + "balance_loss_clip": 1.05845618, + "balance_loss_mlp": 1.01754475, + "epoch": 0.1777319946607858, + "flos": 14494480479360.0, + "grad_norm": 2.1818725917970605, + "language_loss": 0.7897436, + "learning_rate": 3.7754131252087377e-06, + "loss": 0.8113867, + "num_input_tokens_seen": 175183315, + "router_z_loss_clip": 0.73583984, + "router_z_loss_mlp": 0.1463623, + "step": 6125, + "time_per_iteration": 2.552809476852417 + }, + { + "auxiliary_loss_clip": 0.0112868, + "auxiliary_loss_mlp": 0.01033984, + "balance_loss_clip": 1.05916417, + "balance_loss_mlp": 1.01892757, + "epoch": 0.17776101212930184, + "flos": 30296934806400.0, + "grad_norm": 2.1084282494717153, + "language_loss": 0.82501006, + "learning_rate": 3.7753265779631076e-06, + "loss": 0.84663671, + "num_input_tokens_seen": 175200455, + "router_z_loss_clip": 0.69482422, + "router_z_loss_mlp": 0.15057373, + "step": 6126, + "time_per_iteration": 2.626863956451416 + }, + { + "auxiliary_loss_clip": 0.01138998, + "auxiliary_loss_mlp": 0.01032454, + "balance_loss_clip": 1.0640564, + "balance_loss_mlp": 1.01699281, + "epoch": 0.1777900295978179, + "flos": 35988057578880.0, + "grad_norm": 2.126877227175171, + "language_loss": 0.81593347, + "learning_rate": 3.7752400150370745e-06, + "loss": 0.83764797, + "num_input_tokens_seen": 175224690, + "router_z_loss_clip": 0.74951172, + "router_z_loss_mlp": 0.15472412, + "step": 6127, + "time_per_iteration": 2.7445404529571533 + }, + { + "auxiliary_loss_clip": 0.01136264, + "auxiliary_loss_mlp": 0.01039075, + "balance_loss_clip": 1.06010604, + "balance_loss_mlp": 1.02335131, + "epoch": 0.17781904706633395, + "flos": 16429347417600.0, + "grad_norm": 2.4767227203277615, + "language_loss": 0.78695792, + "learning_rate": 3.775153436431403e-06, + "loss": 0.80871129, + "num_input_tokens_seen": 175238140, + "router_z_loss_clip": 0.76123047, + "router_z_loss_mlp": 0.15722656, + "step": 6128, + "time_per_iteration": 2.4862661361694336 + }, + { + "auxiliary_loss_clip": 0.01034539, + "auxiliary_loss_mlp": 0.01000019, + "balance_loss_clip": 1.01540542, + "balance_loss_mlp": 0.99922037, + "epoch": 0.17784806453484997, + "flos": 56093726090880.0, + "grad_norm": 0.7079821679653335, + "language_loss": 0.50789207, + "learning_rate": 3.7750668421468584e-06, + "loss": 0.5282377, + "num_input_tokens_seen": 175293005, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.00799561, + "step": 6129, + "time_per_iteration": 2.9859273433685303 + }, + { + "auxiliary_loss_clip": 0.01133318, + "auxiliary_loss_mlp": 0.01045429, + "balance_loss_clip": 1.06163466, + "balance_loss_mlp": 1.02948523, + "epoch": 0.17787708200336602, + "flos": 19202189719680.0, + "grad_norm": 2.6484348350157987, + "language_loss": 0.98547554, + "learning_rate": 3.7749802321842052e-06, + "loss": 1.00726306, + "num_input_tokens_seen": 175306490, + "router_z_loss_clip": 0.71582031, + "router_z_loss_mlp": 0.15942383, + "step": 6130, + "time_per_iteration": 2.5285420417785645 + }, + { + "auxiliary_loss_clip": 0.01138588, + "auxiliary_loss_mlp": 0.01051728, + "balance_loss_clip": 1.06236148, + "balance_loss_mlp": 1.03631437, + "epoch": 0.17790609947188207, + "flos": 24966785761920.0, + "grad_norm": 1.6685458975482212, + "language_loss": 0.80455244, + "learning_rate": 3.7748936065442085e-06, + "loss": 0.82645565, + "num_input_tokens_seen": 175325270, + "router_z_loss_clip": 0.76171875, + "router_z_loss_mlp": 0.15411377, + "step": 6131, + "time_per_iteration": 2.5924744606018066 + }, + { + "auxiliary_loss_clip": 0.01135765, + "auxiliary_loss_mlp": 0.01038478, + "balance_loss_clip": 1.06136334, + "balance_loss_mlp": 1.02348137, + "epoch": 0.17793511694039812, + "flos": 42041734277760.0, + "grad_norm": 2.3946708123176035, + "language_loss": 0.79373085, + "learning_rate": 3.7748069652276325e-06, + "loss": 0.81547338, + "num_input_tokens_seen": 175347245, + "router_z_loss_clip": 0.74414062, + "router_z_loss_mlp": 0.14987183, + "step": 6132, + "time_per_iteration": 2.721566677093506 + }, + { + "auxiliary_loss_clip": 0.01035009, + "auxiliary_loss_mlp": 0.00997457, + "balance_loss_clip": 1.01607299, + "balance_loss_mlp": 0.99667019, + "epoch": 0.17796413440891418, + "flos": 54928676459520.0, + "grad_norm": 0.6903965106833847, + "language_loss": 0.52509868, + "learning_rate": 3.774720308235244e-06, + "loss": 0.54542339, + "num_input_tokens_seen": 175411825, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.00787354, + "step": 6133, + "time_per_iteration": 3.1286399364471436 + }, + { + "auxiliary_loss_clip": 0.01033527, + "auxiliary_loss_mlp": 0.00998943, + "balance_loss_clip": 1.01453066, + "balance_loss_mlp": 0.99820405, + "epoch": 0.1779931518774302, + "flos": 74768422631040.0, + "grad_norm": 0.6585864299714288, + "language_loss": 0.43457252, + "learning_rate": 3.7746336355678072e-06, + "loss": 0.45489722, + "num_input_tokens_seen": 175475600, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.00738525, + "step": 6134, + "time_per_iteration": 3.207908868789673 + }, + { + "auxiliary_loss_clip": 0.01032765, + "auxiliary_loss_mlp": 0.01001373, + "balance_loss_clip": 1.01378822, + "balance_loss_mlp": 1.0006398, + "epoch": 0.17802216934594625, + "flos": 60507937301760.0, + "grad_norm": 0.6626625242538734, + "language_loss": 0.50672865, + "learning_rate": 3.7745469472260885e-06, + "loss": 0.52707005, + "num_input_tokens_seen": 175534310, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.00732422, + "step": 6135, + "time_per_iteration": 3.054107904434204 + }, + { + "auxiliary_loss_clip": 0.0113992, + "auxiliary_loss_mlp": 0.01039053, + "balance_loss_clip": 1.06267059, + "balance_loss_mlp": 1.02351379, + "epoch": 0.1780511868144623, + "flos": 15080255475840.0, + "grad_norm": 2.542345219685283, + "language_loss": 0.91462028, + "learning_rate": 3.7744602432108527e-06, + "loss": 0.93641001, + "num_input_tokens_seen": 175552130, + "router_z_loss_clip": 0.77197266, + "router_z_loss_mlp": 0.15533447, + "step": 6136, + "time_per_iteration": 2.54594087600708 + }, + { + "auxiliary_loss_clip": 0.01141344, + "auxiliary_loss_mlp": 0.01046835, + "balance_loss_clip": 1.06236696, + "balance_loss_mlp": 1.02949059, + "epoch": 0.17808020428297835, + "flos": 44703863884800.0, + "grad_norm": 2.197814132452931, + "language_loss": 0.86087948, + "learning_rate": 3.7743735235228654e-06, + "loss": 0.8827613, + "num_input_tokens_seen": 175568115, + "router_z_loss_clip": 0.78955078, + "router_z_loss_mlp": 0.17346191, + "step": 6137, + "time_per_iteration": 2.7458572387695312 + }, + { + "auxiliary_loss_clip": 0.01134918, + "auxiliary_loss_mlp": 0.01043299, + "balance_loss_clip": 1.05801988, + "balance_loss_mlp": 1.02841532, + "epoch": 0.1781092217514944, + "flos": 17960865557760.0, + "grad_norm": 2.3743425907864415, + "language_loss": 0.83057034, + "learning_rate": 3.774286788162894e-06, + "loss": 0.8523525, + "num_input_tokens_seen": 175581160, + "router_z_loss_clip": 0.76904297, + "router_z_loss_mlp": 0.14886475, + "step": 6138, + "time_per_iteration": 2.502467155456543 + }, + { + "auxiliary_loss_clip": 0.01033495, + "auxiliary_loss_mlp": 0.01019753, + "balance_loss_clip": 1.01459539, + "balance_loss_mlp": 1.01904964, + "epoch": 0.17813823922001046, + "flos": 74776646845440.0, + "grad_norm": 0.6484873309652087, + "language_loss": 0.47373107, + "learning_rate": 3.7742000371317033e-06, + "loss": 0.49426353, + "num_input_tokens_seen": 175646350, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.00701904, + "step": 6139, + "time_per_iteration": 3.1543779373168945 + }, + { + "auxiliary_loss_clip": 0.01130121, + "auxiliary_loss_mlp": 0.01039812, + "balance_loss_clip": 1.05834842, + "balance_loss_mlp": 1.02531016, + "epoch": 0.17816725668852648, + "flos": 35770582684800.0, + "grad_norm": 3.2825268615084253, + "language_loss": 0.76380968, + "learning_rate": 3.77411327043006e-06, + "loss": 0.78550905, + "num_input_tokens_seen": 175662105, + "router_z_loss_clip": 0.71728516, + "router_z_loss_mlp": 0.14508057, + "step": 6140, + "time_per_iteration": 2.6382579803466797 + }, + { + "auxiliary_loss_clip": 0.0111807, + "auxiliary_loss_mlp": 0.01029852, + "balance_loss_clip": 1.05626941, + "balance_loss_mlp": 1.01801479, + "epoch": 0.17819627415704253, + "flos": 15408048015360.0, + "grad_norm": 1.8854130680803394, + "language_loss": 0.50772119, + "learning_rate": 3.7740264880587305e-06, + "loss": 0.52920038, + "num_input_tokens_seen": 175674670, + "router_z_loss_clip": 0.61816406, + "router_z_loss_mlp": 0.11834717, + "step": 6141, + "time_per_iteration": 2.479367733001709 + }, + { + "auxiliary_loss_clip": 0.01141856, + "auxiliary_loss_mlp": 0.01048599, + "balance_loss_clip": 1.06447721, + "balance_loss_mlp": 1.03169489, + "epoch": 0.17822529162555859, + "flos": 26058936741120.0, + "grad_norm": 2.0538823582651284, + "language_loss": 0.82414234, + "learning_rate": 3.7739396900184807e-06, + "loss": 0.84604681, + "num_input_tokens_seen": 175689255, + "router_z_loss_clip": 0.77539062, + "router_z_loss_mlp": 0.16900635, + "step": 6142, + "time_per_iteration": 2.5968167781829834 + }, + { + "auxiliary_loss_clip": 0.01135767, + "auxiliary_loss_mlp": 0.01036025, + "balance_loss_clip": 1.06078911, + "balance_loss_mlp": 1.02101016, + "epoch": 0.17825430909407464, + "flos": 26025396416640.0, + "grad_norm": 2.6717649839139583, + "language_loss": 0.71492022, + "learning_rate": 3.773852876310078e-06, + "loss": 0.73663807, + "num_input_tokens_seen": 175703565, + "router_z_loss_clip": 0.74951172, + "router_z_loss_mlp": 0.15020752, + "step": 6143, + "time_per_iteration": 2.6202900409698486 + }, + { + "auxiliary_loss_clip": 0.01128549, + "auxiliary_loss_mlp": 0.01039461, + "balance_loss_clip": 1.06065357, + "balance_loss_mlp": 1.02354097, + "epoch": 0.1782833265625907, + "flos": 15881925168000.0, + "grad_norm": 2.18846433064218, + "language_loss": 0.80537117, + "learning_rate": 3.7737660469342893e-06, + "loss": 0.82705134, + "num_input_tokens_seen": 175717800, + "router_z_loss_clip": 0.67871094, + "router_z_loss_mlp": 0.15917969, + "step": 6144, + "time_per_iteration": 2.539454698562622 + }, + { + "auxiliary_loss_clip": 0.01130377, + "auxiliary_loss_mlp": 0.01028632, + "balance_loss_clip": 1.06075609, + "balance_loss_mlp": 1.01459551, + "epoch": 0.17831234403110674, + "flos": 13874590540800.0, + "grad_norm": 2.056934115132262, + "language_loss": 0.66150367, + "learning_rate": 3.77367920189188e-06, + "loss": 0.68309379, + "num_input_tokens_seen": 175730040, + "router_z_loss_clip": 0.69628906, + "router_z_loss_mlp": 0.14044189, + "step": 6145, + "time_per_iteration": 2.5149614810943604 + }, + { + "auxiliary_loss_clip": 0.01143438, + "auxiliary_loss_mlp": 0.01042445, + "balance_loss_clip": 1.06753254, + "balance_loss_mlp": 1.02566016, + "epoch": 0.17834136149962276, + "flos": 26352327029760.0, + "grad_norm": 2.4862045654750378, + "language_loss": 0.76938856, + "learning_rate": 3.7735923411836196e-06, + "loss": 0.79124737, + "num_input_tokens_seen": 175745125, + "router_z_loss_clip": 0.75830078, + "router_z_loss_mlp": 0.16802979, + "step": 6146, + "time_per_iteration": 2.5956106185913086 + }, + { + "auxiliary_loss_clip": 0.01127161, + "auxiliary_loss_mlp": 0.01030224, + "balance_loss_clip": 1.05980015, + "balance_loss_mlp": 1.01614499, + "epoch": 0.17837037896813882, + "flos": 22303147783680.0, + "grad_norm": 11.823180375645725, + "language_loss": 0.72676253, + "learning_rate": 3.7735054648102733e-06, + "loss": 0.74833637, + "num_input_tokens_seen": 175762355, + "router_z_loss_clip": 0.67431641, + "router_z_loss_mlp": 0.14086914, + "step": 6147, + "time_per_iteration": 2.57775616645813 + }, + { + "auxiliary_loss_clip": 0.01129891, + "auxiliary_loss_mlp": 0.01032867, + "balance_loss_clip": 1.06061113, + "balance_loss_mlp": 1.01962328, + "epoch": 0.17839939643665487, + "flos": 19748642302080.0, + "grad_norm": 2.2342159711220138, + "language_loss": 0.78666687, + "learning_rate": 3.773418572772609e-06, + "loss": 0.80829448, + "num_input_tokens_seen": 175774915, + "router_z_loss_clip": 0.69238281, + "router_z_loss_mlp": 0.13238525, + "step": 6148, + "time_per_iteration": 2.5632171630859375 + }, + { + "auxiliary_loss_clip": 0.01131189, + "auxiliary_loss_mlp": 0.01028902, + "balance_loss_clip": 1.06189942, + "balance_loss_mlp": 1.01549077, + "epoch": 0.17842841390517092, + "flos": 48060865071360.0, + "grad_norm": 2.7393106448449855, + "language_loss": 0.93020129, + "learning_rate": 3.773331665071395e-06, + "loss": 0.95180213, + "num_input_tokens_seen": 175793720, + "router_z_loss_clip": 0.69262695, + "router_z_loss_mlp": 0.13421631, + "step": 6149, + "time_per_iteration": 2.82759952545166 + }, + { + "auxiliary_loss_clip": 0.0113254, + "auxiliary_loss_mlp": 0.01031726, + "balance_loss_clip": 1.06342793, + "balance_loss_mlp": 1.01873779, + "epoch": 0.17845743137368697, + "flos": 29642642616960.0, + "grad_norm": 2.3460050434142743, + "language_loss": 0.76695991, + "learning_rate": 3.773244741707397e-06, + "loss": 0.78860259, + "num_input_tokens_seen": 175813690, + "router_z_loss_clip": 0.69140625, + "router_z_loss_mlp": 0.13000488, + "step": 6150, + "time_per_iteration": 2.6270196437835693 + }, + { + "auxiliary_loss_clip": 0.0112488, + "auxiliary_loss_mlp": 0.01034172, + "balance_loss_clip": 1.0600872, + "balance_loss_mlp": 1.02198839, + "epoch": 0.178486448842203, + "flos": 30001748837760.0, + "grad_norm": 2.093173642310594, + "language_loss": 0.72381812, + "learning_rate": 3.773157802681385e-06, + "loss": 0.74540859, + "num_input_tokens_seen": 175829075, + "router_z_loss_clip": 0.6484375, + "router_z_loss_mlp": 0.12188721, + "step": 6151, + "time_per_iteration": 2.6312131881713867 + }, + { + "auxiliary_loss_clip": 0.01137201, + "auxiliary_loss_mlp": 0.01036463, + "balance_loss_clip": 1.06488013, + "balance_loss_mlp": 1.02084017, + "epoch": 0.17851546631071905, + "flos": 29892077637120.0, + "grad_norm": 2.015357282584174, + "language_loss": 0.96636474, + "learning_rate": 3.7730708479941246e-06, + "loss": 0.98810136, + "num_input_tokens_seen": 175846885, + "router_z_loss_clip": 0.72363281, + "router_z_loss_mlp": 0.15618896, + "step": 6152, + "time_per_iteration": 2.6258838176727295 + }, + { + "auxiliary_loss_clip": 0.01133027, + "auxiliary_loss_mlp": 0.01030876, + "balance_loss_clip": 1.06259274, + "balance_loss_mlp": 1.01629734, + "epoch": 0.1785444837792351, + "flos": 20625149980800.0, + "grad_norm": 2.341553603021499, + "language_loss": 0.98052227, + "learning_rate": 3.7729838776463856e-06, + "loss": 1.00216126, + "num_input_tokens_seen": 175861245, + "router_z_loss_clip": 0.70458984, + "router_z_loss_mlp": 0.14562988, + "step": 6153, + "time_per_iteration": 2.5319461822509766 + }, + { + "auxiliary_loss_clip": 0.01051476, + "auxiliary_loss_mlp": 0.01006293, + "balance_loss_clip": 1.03252363, + "balance_loss_mlp": 1.0055306, + "epoch": 0.17857350124775115, + "flos": 56085286394880.0, + "grad_norm": 0.7195309124097569, + "language_loss": 0.47123569, + "learning_rate": 3.7728968916389358e-06, + "loss": 0.49181336, + "num_input_tokens_seen": 175913410, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.00762939, + "step": 6154, + "time_per_iteration": 2.9396514892578125 + }, + { + "auxiliary_loss_clip": 0.01052196, + "auxiliary_loss_mlp": 0.01009858, + "balance_loss_clip": 1.03307891, + "balance_loss_mlp": 1.00901747, + "epoch": 0.1786025187162672, + "flos": 63059138732160.0, + "grad_norm": 0.7131389901254038, + "language_loss": 0.50674224, + "learning_rate": 3.7728098899725428e-06, + "loss": 0.52736276, + "num_input_tokens_seen": 175969820, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.00842285, + "step": 6155, + "time_per_iteration": 2.992950677871704 + }, + { + "auxiliary_loss_clip": 0.01130024, + "auxiliary_loss_mlp": 0.01054406, + "balance_loss_clip": 1.06273139, + "balance_loss_mlp": 1.03924298, + "epoch": 0.17863153618478325, + "flos": 36531421591680.0, + "grad_norm": 2.224637218607755, + "language_loss": 0.9287504, + "learning_rate": 3.772722872647976e-06, + "loss": 0.95059466, + "num_input_tokens_seen": 175985715, + "router_z_loss_clip": 0.67333984, + "router_z_loss_mlp": 0.15185547, + "step": 6156, + "time_per_iteration": 2.66074275970459 + }, + { + "auxiliary_loss_clip": 0.01046909, + "auxiliary_loss_mlp": 0.01007034, + "balance_loss_clip": 1.02791464, + "balance_loss_mlp": 1.00618207, + "epoch": 0.17866055365329928, + "flos": 55291840917120.0, + "grad_norm": 0.7014892199960823, + "language_loss": 0.50845301, + "learning_rate": 3.7726358396660027e-06, + "loss": 0.52899241, + "num_input_tokens_seen": 176046290, + "router_z_loss_clip": 0.19042969, + "router_z_loss_mlp": 0.00854492, + "step": 6157, + "time_per_iteration": 2.9857771396636963 + }, + { + "auxiliary_loss_clip": 0.01130429, + "auxiliary_loss_mlp": 0.01040052, + "balance_loss_clip": 1.06213832, + "balance_loss_mlp": 1.0265342, + "epoch": 0.17868957112181533, + "flos": 23146222878720.0, + "grad_norm": 1.8740842390739094, + "language_loss": 0.77301133, + "learning_rate": 3.7725487910273926e-06, + "loss": 0.79471612, + "num_input_tokens_seen": 176063220, + "router_z_loss_clip": 0.68310547, + "router_z_loss_mlp": 0.13531494, + "step": 6158, + "time_per_iteration": 2.579047918319702 + }, + { + "auxiliary_loss_clip": 0.0104143, + "auxiliary_loss_mlp": 0.00999597, + "balance_loss_clip": 1.02255177, + "balance_loss_mlp": 0.99876863, + "epoch": 0.17871858859033138, + "flos": 63092391747840.0, + "grad_norm": 0.6986677976561454, + "language_loss": 0.52167624, + "learning_rate": 3.7724617267329145e-06, + "loss": 0.54208654, + "num_input_tokens_seen": 176122185, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.00830078, + "step": 6159, + "time_per_iteration": 3.209693193435669 + }, + { + "auxiliary_loss_clip": 0.01136649, + "auxiliary_loss_mlp": 0.01045422, + "balance_loss_clip": 1.06328762, + "balance_loss_mlp": 1.02984166, + "epoch": 0.17874760605884743, + "flos": 11250274976640.0, + "grad_norm": 2.423960246574863, + "language_loss": 0.68448782, + "learning_rate": 3.772374646783337e-06, + "loss": 0.70630854, + "num_input_tokens_seen": 176135070, + "router_z_loss_clip": 0.734375, + "router_z_loss_mlp": 0.15576172, + "step": 6160, + "time_per_iteration": 2.521364688873291 + }, + { + "auxiliary_loss_clip": 0.01037329, + "auxiliary_loss_mlp": 0.01001179, + "balance_loss_clip": 1.01849842, + "balance_loss_mlp": 1.00039792, + "epoch": 0.17877662352736348, + "flos": 74787851888640.0, + "grad_norm": 0.7489001891648415, + "language_loss": 0.45020869, + "learning_rate": 3.7722875511794292e-06, + "loss": 0.47059375, + "num_input_tokens_seen": 176204470, + "router_z_loss_clip": 0.18847656, + "router_z_loss_mlp": 0.0078125, + "step": 6161, + "time_per_iteration": 3.236163377761841 + }, + { + "auxiliary_loss_clip": 0.01128707, + "auxiliary_loss_mlp": 0.0103984, + "balance_loss_clip": 1.06093061, + "balance_loss_mlp": 1.02603519, + "epoch": 0.1788056409958795, + "flos": 14496383900160.0, + "grad_norm": 2.308217376344025, + "language_loss": 0.68422049, + "learning_rate": 3.7722004399219616e-06, + "loss": 0.70590597, + "num_input_tokens_seen": 176220130, + "router_z_loss_clip": 0.67724609, + "router_z_loss_mlp": 0.13800049, + "step": 6162, + "time_per_iteration": 2.5426065921783447 + }, + { + "auxiliary_loss_clip": 0.01127275, + "auxiliary_loss_mlp": 0.01040687, + "balance_loss_clip": 1.06126964, + "balance_loss_mlp": 1.02629256, + "epoch": 0.17883465846439556, + "flos": 31789130532480.0, + "grad_norm": 2.1940153683559576, + "language_loss": 0.71011508, + "learning_rate": 3.772113313011702e-06, + "loss": 0.73179471, + "num_input_tokens_seen": 176236370, + "router_z_loss_clip": 0.65966797, + "router_z_loss_mlp": 0.14398193, + "step": 6163, + "time_per_iteration": 2.6577975749969482 + }, + { + "auxiliary_loss_clip": 0.01035286, + "auxiliary_loss_mlp": 0.01001862, + "balance_loss_clip": 1.01643372, + "balance_loss_mlp": 1.00109279, + "epoch": 0.1788636759329116, + "flos": 74771403459840.0, + "grad_norm": 0.7209752949112918, + "language_loss": 0.49304348, + "learning_rate": 3.77202617044942e-06, + "loss": 0.51341492, + "num_input_tokens_seen": 176294155, + "router_z_loss_clip": 0.18847656, + "router_z_loss_mlp": 0.00769043, + "step": 6164, + "time_per_iteration": 3.1019861698150635 + }, + { + "auxiliary_loss_clip": 0.01035134, + "auxiliary_loss_mlp": 0.01004394, + "balance_loss_clip": 1.01625919, + "balance_loss_mlp": 1.00358951, + "epoch": 0.17889269340142766, + "flos": 66382132717440.0, + "grad_norm": 0.697896295620558, + "language_loss": 0.5244109, + "learning_rate": 3.7719390122358867e-06, + "loss": 0.54480618, + "num_input_tokens_seen": 176351850, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.00805664, + "step": 6165, + "time_per_iteration": 3.041929006576538 + }, + { + "auxiliary_loss_clip": 0.01127938, + "auxiliary_loss_mlp": 0.01037468, + "balance_loss_clip": 1.05999541, + "balance_loss_mlp": 1.02476633, + "epoch": 0.1789217108699437, + "flos": 25987726028160.0, + "grad_norm": 1.7042873923213646, + "language_loss": 0.6381411, + "learning_rate": 3.77185183837187e-06, + "loss": 0.65979517, + "num_input_tokens_seen": 176370315, + "router_z_loss_clip": 0.67871094, + "router_z_loss_mlp": 0.12713623, + "step": 6166, + "time_per_iteration": 2.5522398948669434 + }, + { + "auxiliary_loss_clip": 0.01135127, + "auxiliary_loss_mlp": 0.01038288, + "balance_loss_clip": 1.05677009, + "balance_loss_mlp": 1.02120519, + "epoch": 0.17895072833845976, + "flos": 42478731141120.0, + "grad_norm": 3.2319196142274045, + "language_loss": 0.85792404, + "learning_rate": 3.7717646488581415e-06, + "loss": 0.87965822, + "num_input_tokens_seen": 176389365, + "router_z_loss_clip": 0.78369141, + "router_z_loss_mlp": 0.17089844, + "step": 6167, + "time_per_iteration": 2.6440019607543945 + }, + { + "auxiliary_loss_clip": 0.01120217, + "auxiliary_loss_mlp": 0.01039408, + "balance_loss_clip": 1.05420268, + "balance_loss_mlp": 1.02621746, + "epoch": 0.1789797458069758, + "flos": 32410780237440.0, + "grad_norm": 1.669045834108176, + "language_loss": 0.78202248, + "learning_rate": 3.7716774436954706e-06, + "loss": 0.80361867, + "num_input_tokens_seen": 176408085, + "router_z_loss_clip": 0.65991211, + "router_z_loss_mlp": 0.13195801, + "step": 6168, + "time_per_iteration": 2.651609182357788 + }, + { + "auxiliary_loss_clip": 0.01121619, + "auxiliary_loss_mlp": 0.01031405, + "balance_loss_clip": 1.05699122, + "balance_loss_mlp": 1.01876307, + "epoch": 0.17900876327549184, + "flos": 61486643174400.0, + "grad_norm": 1.8881469986537818, + "language_loss": 0.65566123, + "learning_rate": 3.7715902228846276e-06, + "loss": 0.6771915, + "num_input_tokens_seen": 176428395, + "router_z_loss_clip": 0.64697266, + "router_z_loss_mlp": 0.12658691, + "step": 6169, + "time_per_iteration": 2.816131353378296 + }, + { + "auxiliary_loss_clip": 0.01124651, + "auxiliary_loss_mlp": 0.01039032, + "balance_loss_clip": 1.05527723, + "balance_loss_mlp": 1.02641988, + "epoch": 0.1790377807440079, + "flos": 41275364676480.0, + "grad_norm": 2.5553863807155333, + "language_loss": 0.89350861, + "learning_rate": 3.7715029864263827e-06, + "loss": 0.91514546, + "num_input_tokens_seen": 176446355, + "router_z_loss_clip": 0.69433594, + "router_z_loss_mlp": 0.12612915, + "step": 6170, + "time_per_iteration": 2.7176897525787354 + }, + { + "auxiliary_loss_clip": 0.01120872, + "auxiliary_loss_mlp": 0.01038233, + "balance_loss_clip": 1.05745757, + "balance_loss_mlp": 1.02535856, + "epoch": 0.17906679821252394, + "flos": 26003563925760.0, + "grad_norm": 1.97185925942759, + "language_loss": 0.74951851, + "learning_rate": 3.7714157343215067e-06, + "loss": 0.77110952, + "num_input_tokens_seen": 176466480, + "router_z_loss_clip": 0.63378906, + "router_z_loss_mlp": 0.12866211, + "step": 6171, + "time_per_iteration": 2.715559959411621 + }, + { + "auxiliary_loss_clip": 0.01133397, + "auxiliary_loss_mlp": 0.01041292, + "balance_loss_clip": 1.05806255, + "balance_loss_mlp": 1.02537751, + "epoch": 0.17909581568104, + "flos": 19202333374080.0, + "grad_norm": 2.495457843781951, + "language_loss": 0.93870425, + "learning_rate": 3.7713284665707697e-06, + "loss": 0.96045113, + "num_input_tokens_seen": 176481065, + "router_z_loss_clip": 0.75390625, + "router_z_loss_mlp": 0.15917969, + "step": 6172, + "time_per_iteration": 2.552934408187866 + }, + { + "auxiliary_loss_clip": 0.01131952, + "auxiliary_loss_mlp": 0.01034526, + "balance_loss_clip": 1.06090641, + "balance_loss_mlp": 1.02091837, + "epoch": 0.17912483314955605, + "flos": 19493640673920.0, + "grad_norm": 2.2403668070920673, + "language_loss": 1.07509863, + "learning_rate": 3.771241183174943e-06, + "loss": 1.09676337, + "num_input_tokens_seen": 176493880, + "router_z_loss_clip": 0.71044922, + "router_z_loss_mlp": 0.13598633, + "step": 6173, + "time_per_iteration": 2.5189881324768066 + }, + { + "auxiliary_loss_clip": 0.01037772, + "auxiliary_loss_mlp": 0.01010738, + "balance_loss_clip": 1.01872802, + "balance_loss_mlp": 1.00995147, + "epoch": 0.17915385061807207, + "flos": 66354159000960.0, + "grad_norm": 0.6368877831128272, + "language_loss": 0.51995325, + "learning_rate": 3.7711538841347985e-06, + "loss": 0.54043835, + "num_input_tokens_seen": 176561780, + "router_z_loss_clip": 0.19042969, + "router_z_loss_mlp": 0.00787354, + "step": 6174, + "time_per_iteration": 3.20615291595459 + }, + { + "auxiliary_loss_clip": 0.01128858, + "auxiliary_loss_mlp": 0.01045117, + "balance_loss_clip": 1.05799985, + "balance_loss_mlp": 1.03094935, + "epoch": 0.17918286808658812, + "flos": 35985004922880.0, + "grad_norm": 1.9532164292838798, + "language_loss": 0.69033444, + "learning_rate": 3.771066569451105e-06, + "loss": 0.71207428, + "num_input_tokens_seen": 176578135, + "router_z_loss_clip": 0.70849609, + "router_z_loss_mlp": 0.14172363, + "step": 6175, + "time_per_iteration": 2.6441667079925537 + }, + { + "auxiliary_loss_clip": 0.01126409, + "auxiliary_loss_mlp": 0.01037827, + "balance_loss_clip": 1.05609393, + "balance_loss_mlp": 1.02302706, + "epoch": 0.17921188555510417, + "flos": 35436433438080.0, + "grad_norm": 1.9950910106305166, + "language_loss": 0.65375459, + "learning_rate": 3.7709792391246356e-06, + "loss": 0.67539692, + "num_input_tokens_seen": 176594965, + "router_z_loss_clip": 0.70361328, + "router_z_loss_mlp": 0.14807129, + "step": 6176, + "time_per_iteration": 2.5640628337860107 + }, + { + "auxiliary_loss_clip": 0.01128851, + "auxiliary_loss_mlp": 0.01042855, + "balance_loss_clip": 1.0572511, + "balance_loss_mlp": 1.02898455, + "epoch": 0.17924090302362022, + "flos": 15736594740480.0, + "grad_norm": 2.5661386752645834, + "language_loss": 0.73572767, + "learning_rate": 3.7708918931561606e-06, + "loss": 0.75744474, + "num_input_tokens_seen": 176608305, + "router_z_loss_clip": 0.71728516, + "router_z_loss_mlp": 0.13867188, + "step": 6177, + "time_per_iteration": 2.4638872146606445 + }, + { + "auxiliary_loss_clip": 0.0113246, + "auxiliary_loss_mlp": 0.0104444, + "balance_loss_clip": 1.05824077, + "balance_loss_mlp": 1.02885973, + "epoch": 0.17926992049213628, + "flos": 16501025007360.0, + "grad_norm": 2.8622269902947135, + "language_loss": 0.6814456, + "learning_rate": 3.770804531546452e-06, + "loss": 0.70321459, + "num_input_tokens_seen": 176623185, + "router_z_loss_clip": 0.74169922, + "router_z_loss_mlp": 0.15582275, + "step": 6178, + "time_per_iteration": 2.5005624294281006 + }, + { + "auxiliary_loss_clip": 0.01128304, + "auxiliary_loss_mlp": 0.01035835, + "balance_loss_clip": 1.0590682, + "balance_loss_mlp": 1.02284098, + "epoch": 0.1792989379606523, + "flos": 23762341889280.0, + "grad_norm": 2.1841947647082014, + "language_loss": 0.8513993, + "learning_rate": 3.7707171542962806e-06, + "loss": 0.87304068, + "num_input_tokens_seen": 176636225, + "router_z_loss_clip": 0.69189453, + "router_z_loss_mlp": 0.13000488, + "step": 6179, + "time_per_iteration": 2.5557498931884766 + }, + { + "auxiliary_loss_clip": 0.01036159, + "auxiliary_loss_mlp": 0.01000467, + "balance_loss_clip": 1.01699543, + "balance_loss_mlp": 0.99969816, + "epoch": 0.17932795542916835, + "flos": 59523697756800.0, + "grad_norm": 1.296529460662657, + "language_loss": 0.4768669, + "learning_rate": 3.7706297614064193e-06, + "loss": 0.49723321, + "num_input_tokens_seen": 176696010, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.00769043, + "step": 6180, + "time_per_iteration": 3.128244161605835 + }, + { + "auxiliary_loss_clip": 0.01135581, + "auxiliary_loss_mlp": 0.01040072, + "balance_loss_clip": 1.06247842, + "balance_loss_mlp": 1.02477753, + "epoch": 0.1793569728976844, + "flos": 17633324413440.0, + "grad_norm": 2.234238001532258, + "language_loss": 0.80081719, + "learning_rate": 3.7705423528776397e-06, + "loss": 0.82257378, + "num_input_tokens_seen": 176710660, + "router_z_loss_clip": 0.73095703, + "router_z_loss_mlp": 0.15289307, + "step": 6181, + "time_per_iteration": 2.461256742477417 + }, + { + "auxiliary_loss_clip": 0.01033924, + "auxiliary_loss_mlp": 0.01003255, + "balance_loss_clip": 1.01486301, + "balance_loss_mlp": 1.00242031, + "epoch": 0.17938599036620045, + "flos": 60185890938240.0, + "grad_norm": 0.6554476231739634, + "language_loss": 0.53258526, + "learning_rate": 3.770454928710713e-06, + "loss": 0.55295706, + "num_input_tokens_seen": 176776175, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.00836182, + "step": 6182, + "time_per_iteration": 3.095257043838501 + }, + { + "auxiliary_loss_clip": 0.0112837, + "auxiliary_loss_mlp": 0.01047925, + "balance_loss_clip": 1.05554211, + "balance_loss_mlp": 1.0333755, + "epoch": 0.1794150078347165, + "flos": 18434491315200.0, + "grad_norm": 3.6807235698900316, + "language_loss": 0.80329514, + "learning_rate": 3.7703674889064122e-06, + "loss": 0.8250581, + "num_input_tokens_seen": 176788445, + "router_z_loss_clip": 0.72851562, + "router_z_loss_mlp": 0.14556885, + "step": 6183, + "time_per_iteration": 2.494370222091675 + }, + { + "auxiliary_loss_clip": 0.01121796, + "auxiliary_loss_mlp": 0.01042269, + "balance_loss_clip": 1.0555346, + "balance_loss_mlp": 1.02885747, + "epoch": 0.17944402530323256, + "flos": 10333798439040.0, + "grad_norm": 2.5721177528717143, + "language_loss": 0.84318924, + "learning_rate": 3.770280033465509e-06, + "loss": 0.86482984, + "num_input_tokens_seen": 176798475, + "router_z_loss_clip": 0.66308594, + "router_z_loss_mlp": 0.1340332, + "step": 6184, + "time_per_iteration": 2.5144641399383545 + }, + { + "auxiliary_loss_clip": 0.01033001, + "auxiliary_loss_mlp": 0.01007805, + "balance_loss_clip": 1.01398921, + "balance_loss_mlp": 1.00691116, + "epoch": 0.17947304277174858, + "flos": 58357929853440.0, + "grad_norm": 0.6668643602623942, + "language_loss": 0.51795727, + "learning_rate": 3.770192562388777e-06, + "loss": 0.5383653, + "num_input_tokens_seen": 176861175, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.00891113, + "step": 6185, + "time_per_iteration": 3.04773211479187 + }, + { + "auxiliary_loss_clip": 0.01031198, + "auxiliary_loss_mlp": 0.01007155, + "balance_loss_clip": 1.01215172, + "balance_loss_mlp": 1.00628471, + "epoch": 0.17950206024026463, + "flos": 54011373909120.0, + "grad_norm": 0.6493949452943996, + "language_loss": 0.47417316, + "learning_rate": 3.7701050756769873e-06, + "loss": 0.4945567, + "num_input_tokens_seen": 176920120, + "router_z_loss_clip": 0.19042969, + "router_z_loss_mlp": 0.00872803, + "step": 6186, + "time_per_iteration": 3.0043957233428955 + }, + { + "auxiliary_loss_clip": 0.01029667, + "auxiliary_loss_mlp": 0.01001222, + "balance_loss_clip": 1.01070547, + "balance_loss_mlp": 1.00036418, + "epoch": 0.17953107770878068, + "flos": 68900440268160.0, + "grad_norm": 0.648264119353648, + "language_loss": 0.44966882, + "learning_rate": 3.7700175733309133e-06, + "loss": 0.46997768, + "num_input_tokens_seen": 176983185, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.00860596, + "step": 6187, + "time_per_iteration": 5.338307857513428 + }, + { + "auxiliary_loss_clip": 0.01122574, + "auxiliary_loss_mlp": 0.010388, + "balance_loss_clip": 1.05375576, + "balance_loss_mlp": 1.02527606, + "epoch": 0.17956009517729674, + "flos": 16610265244800.0, + "grad_norm": 2.4611219165677904, + "language_loss": 0.91147321, + "learning_rate": 3.7699300553513276e-06, + "loss": 0.93308699, + "num_input_tokens_seen": 176999020, + "router_z_loss_clip": 0.68945312, + "router_z_loss_mlp": 0.13537598, + "step": 6188, + "time_per_iteration": 7.422165870666504 + }, + { + "auxiliary_loss_clip": 0.01027368, + "auxiliary_loss_mlp": 0.01001007, + "balance_loss_clip": 1.00827944, + "balance_loss_mlp": 1.00010741, + "epoch": 0.1795891126458128, + "flos": 74790976371840.0, + "grad_norm": 0.6174746818876156, + "language_loss": 0.49060661, + "learning_rate": 3.7698425217390044e-06, + "loss": 0.51089036, + "num_input_tokens_seen": 177075540, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.00897217, + "step": 6189, + "time_per_iteration": 5.662339210510254 + }, + { + "auxiliary_loss_clip": 0.01027938, + "auxiliary_loss_mlp": 0.0100013, + "balance_loss_clip": 1.0088799, + "balance_loss_mlp": 0.99923003, + "epoch": 0.17961813011432884, + "flos": 74775856746240.0, + "grad_norm": 0.6085831322322328, + "language_loss": 0.47404864, + "learning_rate": 3.769754972494715e-06, + "loss": 0.4943293, + "num_input_tokens_seen": 177142345, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.00897217, + "step": 6190, + "time_per_iteration": 3.1993072032928467 + }, + { + "auxiliary_loss_clip": 0.01027123, + "auxiliary_loss_mlp": 0.00998862, + "balance_loss_clip": 1.00793636, + "balance_loss_mlp": 0.9979561, + "epoch": 0.17964714758284486, + "flos": 74777041895040.0, + "grad_norm": 0.6731531267224, + "language_loss": 0.49699485, + "learning_rate": 3.7696674076192337e-06, + "loss": 0.51725471, + "num_input_tokens_seen": 177205015, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.0090332, + "step": 6191, + "time_per_iteration": 3.171485185623169 + }, + { + "auxiliary_loss_clip": 0.01136194, + "auxiliary_loss_mlp": 0.01043406, + "balance_loss_clip": 1.05957401, + "balance_loss_mlp": 1.02715206, + "epoch": 0.17967616505136091, + "flos": 36676967500800.0, + "grad_norm": 2.5038471023202855, + "language_loss": 0.98372954, + "learning_rate": 3.7695798271133343e-06, + "loss": 1.00552547, + "num_input_tokens_seen": 177222030, + "router_z_loss_clip": 0.76660156, + "router_z_loss_mlp": 0.16253662, + "step": 6192, + "time_per_iteration": 2.6708922386169434 + }, + { + "auxiliary_loss_clip": 0.01138637, + "auxiliary_loss_mlp": 0.01047858, + "balance_loss_clip": 1.0611639, + "balance_loss_mlp": 1.03134787, + "epoch": 0.17970518251987697, + "flos": 18287472948480.0, + "grad_norm": 2.3430894473792168, + "language_loss": 0.9491142, + "learning_rate": 3.769492230977789e-06, + "loss": 0.97097921, + "num_input_tokens_seen": 177236170, + "router_z_loss_clip": 0.77539062, + "router_z_loss_mlp": 0.16503906, + "step": 6193, + "time_per_iteration": 2.4944746494293213 + }, + { + "auxiliary_loss_clip": 0.01133813, + "auxiliary_loss_mlp": 0.01031644, + "balance_loss_clip": 1.05672216, + "balance_loss_mlp": 1.01650429, + "epoch": 0.17973419998839302, + "flos": 52728856848000.0, + "grad_norm": 2.7229911663268114, + "language_loss": 0.91528511, + "learning_rate": 3.7694046192133725e-06, + "loss": 0.9369396, + "num_input_tokens_seen": 177253105, + "router_z_loss_clip": 0.77099609, + "router_z_loss_mlp": 0.15130615, + "step": 6194, + "time_per_iteration": 2.8214516639709473 + }, + { + "auxiliary_loss_clip": 0.01133207, + "auxiliary_loss_mlp": 0.01047324, + "balance_loss_clip": 1.05588269, + "balance_loss_mlp": 1.03064108, + "epoch": 0.17976321745690907, + "flos": 74732688599040.0, + "grad_norm": 2.4525782068923525, + "language_loss": 0.87159479, + "learning_rate": 3.7693169918208588e-06, + "loss": 0.89340007, + "num_input_tokens_seen": 177277155, + "router_z_loss_clip": 0.7734375, + "router_z_loss_mlp": 0.16680908, + "step": 6195, + "time_per_iteration": 2.936786651611328 + }, + { + "auxiliary_loss_clip": 0.01129839, + "auxiliary_loss_mlp": 0.01044091, + "balance_loss_clip": 1.05519521, + "balance_loss_mlp": 1.02840269, + "epoch": 0.1797922349254251, + "flos": 59007478469760.0, + "grad_norm": 3.095849892552639, + "language_loss": 0.82116085, + "learning_rate": 3.769229348801021e-06, + "loss": 0.84290028, + "num_input_tokens_seen": 177297170, + "router_z_loss_clip": 0.74658203, + "router_z_loss_mlp": 0.15704346, + "step": 6196, + "time_per_iteration": 2.897655487060547 + }, + { + "auxiliary_loss_clip": 0.01144163, + "auxiliary_loss_mlp": 0.01050777, + "balance_loss_clip": 1.06081736, + "balance_loss_mlp": 1.0330267, + "epoch": 0.17982125239394114, + "flos": 16503215736960.0, + "grad_norm": 2.442452110066776, + "language_loss": 0.95864117, + "learning_rate": 3.769141690154634e-06, + "loss": 0.98059058, + "num_input_tokens_seen": 177312830, + "router_z_loss_clip": 0.83447266, + "router_z_loss_mlp": 0.17736816, + "step": 6197, + "time_per_iteration": 2.4995479583740234 + }, + { + "auxiliary_loss_clip": 0.01130645, + "auxiliary_loss_mlp": 0.01036648, + "balance_loss_clip": 1.05795336, + "balance_loss_mlp": 1.02221727, + "epoch": 0.1798502698624572, + "flos": 16538264432640.0, + "grad_norm": 2.5409411471390406, + "language_loss": 0.9155997, + "learning_rate": 3.7690540158824717e-06, + "loss": 0.93727267, + "num_input_tokens_seen": 177325760, + "router_z_loss_clip": 0.72705078, + "router_z_loss_mlp": 0.14416504, + "step": 6198, + "time_per_iteration": 2.513803720474243 + }, + { + "auxiliary_loss_clip": 0.0112991, + "auxiliary_loss_mlp": 0.01039067, + "balance_loss_clip": 1.05673707, + "balance_loss_mlp": 1.02329016, + "epoch": 0.17987928733097325, + "flos": 21537496454400.0, + "grad_norm": 2.128520999099143, + "language_loss": 0.88636446, + "learning_rate": 3.768966325985308e-06, + "loss": 0.90805429, + "num_input_tokens_seen": 177340490, + "router_z_loss_clip": 0.73242188, + "router_z_loss_mlp": 0.15771484, + "step": 6199, + "time_per_iteration": 2.51246976852417 + }, + { + "auxiliary_loss_clip": 0.01136521, + "auxiliary_loss_mlp": 0.01041833, + "balance_loss_clip": 1.06150401, + "balance_loss_mlp": 1.0266341, + "epoch": 0.1799083047994893, + "flos": 16684277218560.0, + "grad_norm": 2.627560773768236, + "language_loss": 0.77469444, + "learning_rate": 3.7688786204639182e-06, + "loss": 0.79647803, + "num_input_tokens_seen": 177354790, + "router_z_loss_clip": 0.75, + "router_z_loss_mlp": 0.15185547, + "step": 6200, + "time_per_iteration": 2.4963035583496094 + }, + { + "auxiliary_loss_clip": 0.01136468, + "auxiliary_loss_mlp": 0.01038313, + "balance_loss_clip": 1.06274629, + "balance_loss_mlp": 1.02429366, + "epoch": 0.17993732226800535, + "flos": 27264422108160.0, + "grad_norm": 1.8969376305211085, + "language_loss": 0.79757595, + "learning_rate": 3.768790899319077e-06, + "loss": 0.81932378, + "num_input_tokens_seen": 177373085, + "router_z_loss_clip": 0.73730469, + "router_z_loss_mlp": 0.14019775, + "step": 6201, + "time_per_iteration": 2.62019419670105 + }, + { + "auxiliary_loss_clip": 0.01138459, + "auxiliary_loss_mlp": 0.01046454, + "balance_loss_clip": 1.06129956, + "balance_loss_mlp": 1.02936554, + "epoch": 0.17996633973652137, + "flos": 11320982899200.0, + "grad_norm": 2.397418773749441, + "language_loss": 0.8063758, + "learning_rate": 3.768703162551558e-06, + "loss": 0.8282249, + "num_input_tokens_seen": 177384055, + "router_z_loss_clip": 0.77197266, + "router_z_loss_mlp": 0.17089844, + "step": 6202, + "time_per_iteration": 2.5257275104522705 + }, + { + "auxiliary_loss_clip": 0.01031326, + "auxiliary_loss_mlp": 0.01012219, + "balance_loss_clip": 1.01250315, + "balance_loss_mlp": 1.01132476, + "epoch": 0.17999535720503743, + "flos": 60905180787840.0, + "grad_norm": 0.7009511462246246, + "language_loss": 0.5016464, + "learning_rate": 3.7686154101621374e-06, + "loss": 0.52208185, + "num_input_tokens_seen": 177444265, + "router_z_loss_clip": 0.18847656, + "router_z_loss_mlp": 0.00891113, + "step": 6203, + "time_per_iteration": 3.098519802093506 + }, + { + "auxiliary_loss_clip": 0.0113077, + "auxiliary_loss_mlp": 0.01042291, + "balance_loss_clip": 1.05782938, + "balance_loss_mlp": 1.02637076, + "epoch": 0.18002437467355348, + "flos": 38794296551040.0, + "grad_norm": 2.8385301814228154, + "language_loss": 0.9515605, + "learning_rate": 3.76852764215159e-06, + "loss": 0.9732911, + "num_input_tokens_seen": 177460245, + "router_z_loss_clip": 0.72851562, + "router_z_loss_mlp": 0.15911865, + "step": 6204, + "time_per_iteration": 2.6711325645446777 + }, + { + "auxiliary_loss_clip": 0.01134027, + "auxiliary_loss_mlp": 0.01044287, + "balance_loss_clip": 1.05902767, + "balance_loss_mlp": 1.02788985, + "epoch": 0.18005339214206953, + "flos": 11684506492800.0, + "grad_norm": 2.237437187381709, + "language_loss": 0.89207703, + "learning_rate": 3.768439858520691e-06, + "loss": 0.9138602, + "num_input_tokens_seen": 177471920, + "router_z_loss_clip": 0.75048828, + "router_z_loss_mlp": 0.16400146, + "step": 6205, + "time_per_iteration": 2.5145838260650635 + }, + { + "auxiliary_loss_clip": 0.01132441, + "auxiliary_loss_mlp": 0.01043064, + "balance_loss_clip": 1.05831671, + "balance_loss_mlp": 1.02649999, + "epoch": 0.18008240961058558, + "flos": 26973940821120.0, + "grad_norm": 2.878316490585907, + "language_loss": 0.88893181, + "learning_rate": 3.768352059270215e-06, + "loss": 0.91068685, + "num_input_tokens_seen": 177487840, + "router_z_loss_clip": 0.74121094, + "router_z_loss_mlp": 0.16558838, + "step": 6206, + "time_per_iteration": 2.5919172763824463 + }, + { + "auxiliary_loss_clip": 0.01134718, + "auxiliary_loss_mlp": 0.01044315, + "balance_loss_clip": 1.06197047, + "balance_loss_mlp": 1.02893138, + "epoch": 0.18011142707910163, + "flos": 41128346309760.0, + "grad_norm": 2.3768828969519356, + "language_loss": 0.96728444, + "learning_rate": 3.7682642444009383e-06, + "loss": 0.98907483, + "num_input_tokens_seen": 177504210, + "router_z_loss_clip": 0.72753906, + "router_z_loss_mlp": 0.15393066, + "step": 6207, + "time_per_iteration": 2.713256359100342 + }, + { + "auxiliary_loss_clip": 0.01031805, + "auxiliary_loss_mlp": 0.01003208, + "balance_loss_clip": 1.01322448, + "balance_loss_mlp": 1.00237989, + "epoch": 0.18014044454761766, + "flos": 63174663849600.0, + "grad_norm": 0.6719838788921375, + "language_loss": 0.48323208, + "learning_rate": 3.768176413913636e-06, + "loss": 0.50358218, + "num_input_tokens_seen": 177565315, + "router_z_loss_clip": 0.18554688, + "router_z_loss_mlp": 0.00830078, + "step": 6208, + "time_per_iteration": 3.022933006286621 + }, + { + "auxiliary_loss_clip": 0.01143655, + "auxiliary_loss_mlp": 0.01049535, + "balance_loss_clip": 1.06616509, + "balance_loss_mlp": 1.03210104, + "epoch": 0.1801694620161337, + "flos": 31896072299520.0, + "grad_norm": 2.596851997189921, + "language_loss": 0.9335748, + "learning_rate": 3.7680885678090847e-06, + "loss": 0.95550668, + "num_input_tokens_seen": 177578495, + "router_z_loss_clip": 0.77539062, + "router_z_loss_mlp": 0.17456055, + "step": 6209, + "time_per_iteration": 2.6435258388519287 + }, + { + "auxiliary_loss_clip": 0.01033443, + "auxiliary_loss_mlp": 0.00997972, + "balance_loss_clip": 1.01498032, + "balance_loss_mlp": 0.99715537, + "epoch": 0.18019847948464976, + "flos": 74771187978240.0, + "grad_norm": 0.6534924370675417, + "language_loss": 0.45276886, + "learning_rate": 3.768000706088059e-06, + "loss": 0.47308302, + "num_input_tokens_seen": 177646060, + "router_z_loss_clip": 0.18457031, + "router_z_loss_mlp": 0.00817871, + "step": 6210, + "time_per_iteration": 3.176239252090454 + }, + { + "auxiliary_loss_clip": 0.01032652, + "auxiliary_loss_mlp": 0.00999853, + "balance_loss_clip": 1.01414275, + "balance_loss_mlp": 0.99903029, + "epoch": 0.1802274969531658, + "flos": 65872991387520.0, + "grad_norm": 0.8313069993270643, + "language_loss": 0.4779374, + "learning_rate": 3.767912828751336e-06, + "loss": 0.49826241, + "num_input_tokens_seen": 177706775, + "router_z_loss_clip": 0.18554688, + "router_z_loss_mlp": 0.00823975, + "step": 6211, + "time_per_iteration": 3.194725275039673 + }, + { + "auxiliary_loss_clip": 0.01134753, + "auxiliary_loss_mlp": 0.01039224, + "balance_loss_clip": 1.05873907, + "balance_loss_mlp": 1.02321982, + "epoch": 0.18025651442168186, + "flos": 16537653901440.0, + "grad_norm": 2.9442522566739395, + "language_loss": 0.73007524, + "learning_rate": 3.7678249357996915e-06, + "loss": 0.75181508, + "num_input_tokens_seen": 177717830, + "router_z_loss_clip": 0.75927734, + "router_z_loss_mlp": 0.16009521, + "step": 6212, + "time_per_iteration": 2.5706443786621094 + }, + { + "auxiliary_loss_clip": 0.0113864, + "auxiliary_loss_mlp": 0.01044417, + "balance_loss_clip": 1.06157708, + "balance_loss_mlp": 1.02787042, + "epoch": 0.1802855318901979, + "flos": 18470868814080.0, + "grad_norm": 2.4707929602015035, + "language_loss": 0.81135648, + "learning_rate": 3.7677370272339015e-06, + "loss": 0.83318704, + "num_input_tokens_seen": 177730415, + "router_z_loss_clip": 0.77050781, + "router_z_loss_mlp": 0.16540527, + "step": 6213, + "time_per_iteration": 2.5118560791015625 + }, + { + "auxiliary_loss_clip": 0.01131683, + "auxiliary_loss_mlp": 0.01038925, + "balance_loss_clip": 1.05783081, + "balance_loss_mlp": 1.02276587, + "epoch": 0.18031454935871394, + "flos": 23725677081600.0, + "grad_norm": 2.563701689278854, + "language_loss": 0.82102561, + "learning_rate": 3.767649103054743e-06, + "loss": 0.84273171, + "num_input_tokens_seen": 177743490, + "router_z_loss_clip": 0.73730469, + "router_z_loss_mlp": 0.16174316, + "step": 6214, + "time_per_iteration": 2.5557844638824463 + }, + { + "auxiliary_loss_clip": 0.01031881, + "auxiliary_loss_mlp": 0.00999241, + "balance_loss_clip": 1.01315498, + "balance_loss_mlp": 0.99836463, + "epoch": 0.18034356682723, + "flos": 65068125384960.0, + "grad_norm": 0.6576741841478496, + "language_loss": 0.46632349, + "learning_rate": 3.7675611632629923e-06, + "loss": 0.48663473, + "num_input_tokens_seen": 177803610, + "router_z_loss_clip": 0.1875, + "router_z_loss_mlp": 0.00878906, + "step": 6215, + "time_per_iteration": 3.008373498916626 + }, + { + "auxiliary_loss_clip": 0.01141752, + "auxiliary_loss_mlp": 0.01044343, + "balance_loss_clip": 1.06477499, + "balance_loss_mlp": 1.02701592, + "epoch": 0.18037258429574604, + "flos": 28176912236160.0, + "grad_norm": 2.1171142177936684, + "language_loss": 0.89424336, + "learning_rate": 3.767473207859426e-06, + "loss": 0.91610432, + "num_input_tokens_seen": 177820825, + "router_z_loss_clip": 0.77001953, + "router_z_loss_mlp": 0.17321777, + "step": 6216, + "time_per_iteration": 2.627312660217285 + }, + { + "auxiliary_loss_clip": 0.01137855, + "auxiliary_loss_mlp": 0.01038389, + "balance_loss_clip": 1.06035268, + "balance_loss_mlp": 1.0221405, + "epoch": 0.1804016017642621, + "flos": 19870021336320.0, + "grad_norm": 2.326507384826652, + "language_loss": 0.88025093, + "learning_rate": 3.7673852368448217e-06, + "loss": 0.90201342, + "num_input_tokens_seen": 177837370, + "router_z_loss_clip": 0.77441406, + "router_z_loss_mlp": 0.16271973, + "step": 6217, + "time_per_iteration": 2.587313175201416 + }, + { + "auxiliary_loss_clip": 0.01147371, + "auxiliary_loss_mlp": 0.01047993, + "balance_loss_clip": 1.0638597, + "balance_loss_mlp": 1.02878869, + "epoch": 0.18043061923277814, + "flos": 24964810513920.0, + "grad_norm": 2.611989800120777, + "language_loss": 0.93504137, + "learning_rate": 3.767297250219955e-06, + "loss": 0.95699501, + "num_input_tokens_seen": 177851580, + "router_z_loss_clip": 0.83496094, + "router_z_loss_mlp": 0.19213867, + "step": 6218, + "time_per_iteration": 2.5899572372436523 + }, + { + "auxiliary_loss_clip": 0.01031636, + "auxiliary_loss_mlp": 0.01003913, + "balance_loss_clip": 1.0131824, + "balance_loss_mlp": 1.00301254, + "epoch": 0.18045963670129417, + "flos": 59460136640640.0, + "grad_norm": 0.6758147085068168, + "language_loss": 0.49870145, + "learning_rate": 3.7672092479856045e-06, + "loss": 0.51905692, + "num_input_tokens_seen": 177907410, + "router_z_loss_clip": 0.18457031, + "router_z_loss_mlp": 0.00897217, + "step": 6219, + "time_per_iteration": 3.052781581878662 + }, + { + "auxiliary_loss_clip": 0.01129145, + "auxiliary_loss_mlp": 0.01038913, + "balance_loss_clip": 1.06084406, + "balance_loss_mlp": 1.02499545, + "epoch": 0.18048865416981022, + "flos": 32484612643200.0, + "grad_norm": 1.994520927288974, + "language_loss": 0.66860569, + "learning_rate": 3.767121230142546e-06, + "loss": 0.69028628, + "num_input_tokens_seen": 177924770, + "router_z_loss_clip": 0.68310547, + "router_z_loss_mlp": 0.13909912, + "step": 6220, + "time_per_iteration": 2.6443567276000977 + }, + { + "auxiliary_loss_clip": 0.01125906, + "auxiliary_loss_mlp": 0.0103879, + "balance_loss_clip": 1.0585283, + "balance_loss_mlp": 1.02469373, + "epoch": 0.18051767163832627, + "flos": 19457622311040.0, + "grad_norm": 3.9400948830286358, + "language_loss": 0.78222537, + "learning_rate": 3.7670331966915586e-06, + "loss": 0.80387235, + "num_input_tokens_seen": 177937550, + "router_z_loss_clip": 0.67382812, + "router_z_loss_mlp": 0.14099121, + "step": 6221, + "time_per_iteration": 2.501567840576172 + }, + { + "auxiliary_loss_clip": 0.01138343, + "auxiliary_loss_mlp": 0.01045314, + "balance_loss_clip": 1.06037974, + "balance_loss_mlp": 1.02742088, + "epoch": 0.18054668910684232, + "flos": 13727320778880.0, + "grad_norm": 2.3339342182989515, + "language_loss": 0.86743784, + "learning_rate": 3.7669451476334187e-06, + "loss": 0.88927442, + "num_input_tokens_seen": 177949185, + "router_z_loss_clip": 0.77929688, + "router_z_loss_mlp": 0.17889404, + "step": 6222, + "time_per_iteration": 2.5583033561706543 + }, + { + "auxiliary_loss_clip": 0.01144079, + "auxiliary_loss_mlp": 0.01046985, + "balance_loss_clip": 1.06143594, + "balance_loss_mlp": 1.02883554, + "epoch": 0.18057570657535837, + "flos": 29636465477760.0, + "grad_norm": 2.527960865645036, + "language_loss": 0.98682296, + "learning_rate": 3.7668570829689043e-06, + "loss": 1.00873363, + "num_input_tokens_seen": 177965635, + "router_z_loss_clip": 0.82617188, + "router_z_loss_mlp": 0.18164062, + "step": 6223, + "time_per_iteration": 2.593003988265991 + }, + { + "auxiliary_loss_clip": 0.01129365, + "auxiliary_loss_mlp": 0.01038921, + "balance_loss_clip": 1.05893755, + "balance_loss_mlp": 1.02416301, + "epoch": 0.18060472404387443, + "flos": 15152794992000.0, + "grad_norm": 2.0590244755751335, + "language_loss": 0.70139825, + "learning_rate": 3.766769002698793e-06, + "loss": 0.72308111, + "num_input_tokens_seen": 177981025, + "router_z_loss_clip": 0.70458984, + "router_z_loss_mlp": 0.14752197, + "step": 6224, + "time_per_iteration": 2.5889103412628174 + }, + { + "auxiliary_loss_clip": 0.0111935, + "auxiliary_loss_mlp": 0.0103727, + "balance_loss_clip": 1.05438673, + "balance_loss_mlp": 1.02422297, + "epoch": 0.18063374151239045, + "flos": 34834392558720.0, + "grad_norm": 4.891585218420838, + "language_loss": 0.62958133, + "learning_rate": 3.766680906823863e-06, + "loss": 0.65114748, + "num_input_tokens_seen": 177997365, + "router_z_loss_clip": 0.64990234, + "router_z_loss_mlp": 0.13037109, + "step": 6225, + "time_per_iteration": 2.6533758640289307 + }, + { + "auxiliary_loss_clip": 0.01031442, + "auxiliary_loss_mlp": 0.01001944, + "balance_loss_clip": 1.01296115, + "balance_loss_mlp": 1.00102639, + "epoch": 0.1806627589809065, + "flos": 58729174871040.0, + "grad_norm": 0.6670552126275561, + "language_loss": 0.51191509, + "learning_rate": 3.766592795344892e-06, + "loss": 0.53224891, + "num_input_tokens_seen": 178057810, + "router_z_loss_clip": 0.18457031, + "router_z_loss_mlp": 0.00915527, + "step": 6226, + "time_per_iteration": 3.1419126987457275 + }, + { + "auxiliary_loss_clip": 0.01133762, + "auxiliary_loss_mlp": 0.01039856, + "balance_loss_clip": 1.05931532, + "balance_loss_mlp": 1.02256465, + "epoch": 0.18069177644942255, + "flos": 28211601795840.0, + "grad_norm": 2.2293345986984128, + "language_loss": 0.9966749, + "learning_rate": 3.766504668262659e-06, + "loss": 1.01841116, + "num_input_tokens_seen": 178077585, + "router_z_loss_clip": 0.74414062, + "router_z_loss_mlp": 0.17279053, + "step": 6227, + "time_per_iteration": 2.67069935798645 + }, + { + "auxiliary_loss_clip": 0.01125489, + "auxiliary_loss_mlp": 0.01032835, + "balance_loss_clip": 1.05710125, + "balance_loss_mlp": 1.01858902, + "epoch": 0.1807207939179386, + "flos": 38100071416320.0, + "grad_norm": 2.211709305985636, + "language_loss": 0.7596432, + "learning_rate": 3.7664165255779413e-06, + "loss": 0.7812264, + "num_input_tokens_seen": 178093170, + "router_z_loss_clip": 0.68408203, + "router_z_loss_mlp": 0.14239502, + "step": 6228, + "time_per_iteration": 2.5963048934936523 + }, + { + "auxiliary_loss_clip": 0.01033384, + "auxiliary_loss_mlp": 0.01000895, + "balance_loss_clip": 1.01478338, + "balance_loss_mlp": 0.99997157, + "epoch": 0.18074981138645466, + "flos": 60793247030400.0, + "grad_norm": 0.7162056147122439, + "language_loss": 0.52674115, + "learning_rate": 3.766328367291519e-06, + "loss": 0.54708397, + "num_input_tokens_seen": 178151655, + "router_z_loss_clip": 0.18554688, + "router_z_loss_mlp": 0.00921631, + "step": 6229, + "time_per_iteration": 3.0593223571777344 + }, + { + "auxiliary_loss_clip": 0.010317, + "auxiliary_loss_mlp": 0.0100232, + "balance_loss_clip": 1.01310611, + "balance_loss_mlp": 1.00128925, + "epoch": 0.18077882885497068, + "flos": 64516608984960.0, + "grad_norm": 0.7159979680826828, + "language_loss": 0.46202376, + "learning_rate": 3.766240193404169e-06, + "loss": 0.48236397, + "num_input_tokens_seen": 178203670, + "router_z_loss_clip": 0.18554688, + "router_z_loss_mlp": 0.01031494, + "step": 6230, + "time_per_iteration": 2.955040693283081 + }, + { + "auxiliary_loss_clip": 0.0114173, + "auxiliary_loss_mlp": 0.01050465, + "balance_loss_clip": 1.06294012, + "balance_loss_mlp": 1.03236866, + "epoch": 0.18080784632348673, + "flos": 28473319267200.0, + "grad_norm": 4.731706253575552, + "language_loss": 0.92197955, + "learning_rate": 3.766152003916671e-06, + "loss": 0.94390148, + "num_input_tokens_seen": 178225155, + "router_z_loss_clip": 0.78808594, + "router_z_loss_mlp": 0.18084717, + "step": 6231, + "time_per_iteration": 2.7046852111816406 + }, + { + "auxiliary_loss_clip": 0.01134895, + "auxiliary_loss_mlp": 0.01040268, + "balance_loss_clip": 1.06109166, + "balance_loss_mlp": 1.02483058, + "epoch": 0.18083686379200278, + "flos": 29929640284800.0, + "grad_norm": 1.6322419505654118, + "language_loss": 0.90232909, + "learning_rate": 3.7660637988298047e-06, + "loss": 0.92408073, + "num_input_tokens_seen": 178244820, + "router_z_loss_clip": 0.73779297, + "router_z_loss_mlp": 0.15429688, + "step": 6232, + "time_per_iteration": 2.5862412452697754 + }, + { + "auxiliary_loss_clip": 0.01136945, + "auxiliary_loss_mlp": 0.01038232, + "balance_loss_clip": 1.06196046, + "balance_loss_mlp": 1.02262187, + "epoch": 0.18086588126051883, + "flos": 21974960194560.0, + "grad_norm": 1.9306234149651589, + "language_loss": 0.7452749, + "learning_rate": 3.7659755781443473e-06, + "loss": 0.76702666, + "num_input_tokens_seen": 178257730, + "router_z_loss_clip": 0.74951172, + "router_z_loss_mlp": 0.15594482, + "step": 6233, + "time_per_iteration": 2.5244736671447754 + }, + { + "auxiliary_loss_clip": 0.01031964, + "auxiliary_loss_mlp": 0.01002028, + "balance_loss_clip": 1.01333499, + "balance_loss_mlp": 1.0010922, + "epoch": 0.1808948987290349, + "flos": 74782931725440.0, + "grad_norm": 0.6365891182470685, + "language_loss": 0.48280513, + "learning_rate": 3.7658873418610797e-06, + "loss": 0.5031451, + "num_input_tokens_seen": 178326940, + "router_z_loss_clip": 0.18652344, + "router_z_loss_mlp": 0.00933838, + "step": 6234, + "time_per_iteration": 3.2424044609069824 + }, + { + "auxiliary_loss_clip": 0.01137911, + "auxiliary_loss_mlp": 0.0104275, + "balance_loss_clip": 1.05985594, + "balance_loss_mlp": 1.02609611, + "epoch": 0.18092391619755094, + "flos": 16317880536960.0, + "grad_norm": 2.3103960157542507, + "language_loss": 0.84856373, + "learning_rate": 3.76579908998078e-06, + "loss": 0.87037027, + "num_input_tokens_seen": 178339300, + "router_z_loss_clip": 0.78076172, + "router_z_loss_mlp": 0.16650391, + "step": 6235, + "time_per_iteration": 2.629584789276123 + }, + { + "auxiliary_loss_clip": 0.01029964, + "auxiliary_loss_mlp": 0.01003387, + "balance_loss_clip": 1.01131082, + "balance_loss_mlp": 1.00235546, + "epoch": 0.18095293366606696, + "flos": 57358247460480.0, + "grad_norm": 0.6896245772801797, + "language_loss": 0.4503065, + "learning_rate": 3.7657108225042284e-06, + "loss": 0.47064003, + "num_input_tokens_seen": 178387785, + "router_z_loss_clip": 0.18652344, + "router_z_loss_mlp": 0.01031494, + "step": 6236, + "time_per_iteration": 2.8501482009887695 + }, + { + "auxiliary_loss_clip": 0.01131305, + "auxiliary_loss_mlp": 0.0104105, + "balance_loss_clip": 1.05841208, + "balance_loss_mlp": 1.0252192, + "epoch": 0.180981951134583, + "flos": 39450994951680.0, + "grad_norm": 2.340618769852879, + "language_loss": 0.7387172, + "learning_rate": 3.765622539432204e-06, + "loss": 0.76044071, + "num_input_tokens_seen": 178403970, + "router_z_loss_clip": 0.72802734, + "router_z_loss_mlp": 0.15838623, + "step": 6237, + "time_per_iteration": 2.6556167602539062 + }, + { + "auxiliary_loss_clip": 0.01029467, + "auxiliary_loss_mlp": 0.01003295, + "balance_loss_clip": 1.010957, + "balance_loss_mlp": 1.00234747, + "epoch": 0.18101096860309906, + "flos": 60811060176000.0, + "grad_norm": 0.6715438877357616, + "language_loss": 0.505256, + "learning_rate": 3.7655342407654873e-06, + "loss": 0.52558362, + "num_input_tokens_seen": 178460070, + "router_z_loss_clip": 0.18554688, + "router_z_loss_mlp": 0.00946045, + "step": 6238, + "time_per_iteration": 3.076474189758301 + }, + { + "auxiliary_loss_clip": 0.0112545, + "auxiliary_loss_mlp": 0.01036944, + "balance_loss_clip": 1.05812228, + "balance_loss_mlp": 1.02422392, + "epoch": 0.18103998607161512, + "flos": 23289075267840.0, + "grad_norm": 1.745857404392243, + "language_loss": 0.83740044, + "learning_rate": 3.7654459265048574e-06, + "loss": 0.85902435, + "num_input_tokens_seen": 178481805, + "router_z_loss_clip": 0.67285156, + "router_z_loss_mlp": 0.12719727, + "step": 6239, + "time_per_iteration": 2.564929962158203 + }, + { + "auxiliary_loss_clip": 0.01140361, + "auxiliary_loss_mlp": 0.01040336, + "balance_loss_clip": 1.06156278, + "balance_loss_mlp": 1.02367079, + "epoch": 0.18106900354013117, + "flos": 45983864016000.0, + "grad_norm": 2.1882895011497947, + "language_loss": 0.94699442, + "learning_rate": 3.7653575966510942e-06, + "loss": 0.96880138, + "num_input_tokens_seen": 178501045, + "router_z_loss_clip": 0.78808594, + "router_z_loss_mlp": 0.16674805, + "step": 6240, + "time_per_iteration": 2.743134021759033 + }, + { + "auxiliary_loss_clip": 0.01028965, + "auxiliary_loss_mlp": 0.01002095, + "balance_loss_clip": 1.01027489, + "balance_loss_mlp": 1.00117683, + "epoch": 0.1810980210086472, + "flos": 56050417267200.0, + "grad_norm": 0.7331592802862269, + "language_loss": 0.52226949, + "learning_rate": 3.765269251204979e-06, + "loss": 0.54258013, + "num_input_tokens_seen": 178551160, + "router_z_loss_clip": 0.1875, + "router_z_loss_mlp": 0.00915527, + "step": 6241, + "time_per_iteration": 2.843376874923706 + }, + { + "auxiliary_loss_clip": 0.01028686, + "auxiliary_loss_mlp": 0.00999546, + "balance_loss_clip": 1.01000929, + "balance_loss_mlp": 0.99869919, + "epoch": 0.18112703847716324, + "flos": 67799634111360.0, + "grad_norm": 0.6796192884571883, + "language_loss": 0.53926301, + "learning_rate": 3.765180890167292e-06, + "loss": 0.55954534, + "num_input_tokens_seen": 178612890, + "router_z_loss_clip": 0.1875, + "router_z_loss_mlp": 0.00848389, + "step": 6242, + "time_per_iteration": 3.054624557495117 + }, + { + "auxiliary_loss_clip": 0.01140798, + "auxiliary_loss_mlp": 0.01042731, + "balance_loss_clip": 1.062953, + "balance_loss_mlp": 1.02629185, + "epoch": 0.1811560559456793, + "flos": 32232232707840.0, + "grad_norm": 2.094319265430559, + "language_loss": 0.93355906, + "learning_rate": 3.7650925135388125e-06, + "loss": 0.95539439, + "num_input_tokens_seen": 178629255, + "router_z_loss_clip": 0.77832031, + "router_z_loss_mlp": 0.16442871, + "step": 6243, + "time_per_iteration": 2.5923991203308105 + }, + { + "auxiliary_loss_clip": 0.01028177, + "auxiliary_loss_mlp": 0.00998871, + "balance_loss_clip": 1.00968528, + "balance_loss_mlp": 0.99800032, + "epoch": 0.18118507341419535, + "flos": 58196043256320.0, + "grad_norm": 0.7535661404481059, + "language_loss": 0.48836994, + "learning_rate": 3.7650041213203216e-06, + "loss": 0.50864041, + "num_input_tokens_seen": 178683205, + "router_z_loss_clip": 0.18554688, + "router_z_loss_mlp": 0.00872803, + "step": 6244, + "time_per_iteration": 2.9875409603118896 + }, + { + "auxiliary_loss_clip": 0.01130384, + "auxiliary_loss_mlp": 0.01030489, + "balance_loss_clip": 1.05650663, + "balance_loss_mlp": 1.01533794, + "epoch": 0.1812140908827114, + "flos": 37451561316480.0, + "grad_norm": 2.0129093286726913, + "language_loss": 0.95165253, + "learning_rate": 3.7649157135126e-06, + "loss": 0.97326124, + "num_input_tokens_seen": 178708350, + "router_z_loss_clip": 0.73876953, + "router_z_loss_mlp": 0.15136719, + "step": 6245, + "time_per_iteration": 2.813817262649536 + }, + { + "auxiliary_loss_clip": 0.01027339, + "auxiliary_loss_mlp": 0.01000268, + "balance_loss_clip": 1.00894785, + "balance_loss_mlp": 0.99935001, + "epoch": 0.18124310835122745, + "flos": 55911048497280.0, + "grad_norm": 0.6613000669864462, + "language_loss": 0.43661612, + "learning_rate": 3.764827290116429e-06, + "loss": 0.45689219, + "num_input_tokens_seen": 178765950, + "router_z_loss_clip": 0.18359375, + "router_z_loss_mlp": 0.00915527, + "step": 6246, + "time_per_iteration": 3.0308678150177 + }, + { + "auxiliary_loss_clip": 0.0102655, + "auxiliary_loss_mlp": 0.00999604, + "balance_loss_clip": 1.00824237, + "balance_loss_mlp": 0.99865037, + "epoch": 0.18127212581974347, + "flos": 61770091351680.0, + "grad_norm": 0.6787785569316445, + "language_loss": 0.50227839, + "learning_rate": 3.7647388511325888e-06, + "loss": 0.52253991, + "num_input_tokens_seen": 178829750, + "router_z_loss_clip": 0.18359375, + "router_z_loss_mlp": 0.00952148, + "step": 6247, + "time_per_iteration": 3.1815950870513916 + }, + { + "auxiliary_loss_clip": 0.0102573, + "auxiliary_loss_mlp": 0.01002396, + "balance_loss_clip": 1.00748515, + "balance_loss_mlp": 1.00141859, + "epoch": 0.18130114328825953, + "flos": 57848429387520.0, + "grad_norm": 0.6705918659182412, + "language_loss": 0.49966851, + "learning_rate": 3.764650396561861e-06, + "loss": 0.51994973, + "num_input_tokens_seen": 178890870, + "router_z_loss_clip": 0.18261719, + "router_z_loss_mlp": 0.00976562, + "step": 6248, + "time_per_iteration": 3.1355392932891846 + }, + { + "auxiliary_loss_clip": 0.010252, + "auxiliary_loss_mlp": 0.0100321, + "balance_loss_clip": 1.00704789, + "balance_loss_mlp": 1.0023098, + "epoch": 0.18133016075677558, + "flos": 73644491093760.0, + "grad_norm": 0.655992347188059, + "language_loss": 0.47550255, + "learning_rate": 3.7645619264050267e-06, + "loss": 0.49578667, + "num_input_tokens_seen": 178952635, + "router_z_loss_clip": 0.18164062, + "router_z_loss_mlp": 0.00897217, + "step": 6249, + "time_per_iteration": 3.1043171882629395 + }, + { + "auxiliary_loss_clip": 0.01023803, + "auxiliary_loss_mlp": 0.01006402, + "balance_loss_clip": 1.00575757, + "balance_loss_mlp": 1.00546646, + "epoch": 0.18135917822529163, + "flos": 62552798663040.0, + "grad_norm": 0.6581019686590595, + "language_loss": 0.46503186, + "learning_rate": 3.764473440662868e-06, + "loss": 0.48533392, + "num_input_tokens_seen": 179014065, + "router_z_loss_clip": 0.1796875, + "router_z_loss_mlp": 0.00933838, + "step": 6250, + "time_per_iteration": 3.04498291015625 + }, + { + "auxiliary_loss_clip": 0.01136056, + "auxiliary_loss_mlp": 0.01040169, + "balance_loss_clip": 1.05819929, + "balance_loss_mlp": 1.02280045, + "epoch": 0.18138819569380768, + "flos": 54411487505280.0, + "grad_norm": 2.0332588015766584, + "language_loss": 0.69686133, + "learning_rate": 3.7643849393361654e-06, + "loss": 0.71862358, + "num_input_tokens_seen": 179034755, + "router_z_loss_clip": 0.77880859, + "router_z_loss_mlp": 0.17370605, + "step": 6251, + "time_per_iteration": 2.7227330207824707 + }, + { + "auxiliary_loss_clip": 0.01126904, + "auxiliary_loss_mlp": 0.01043288, + "balance_loss_clip": 1.05379534, + "balance_loss_mlp": 1.02509701, + "epoch": 0.18141721316232373, + "flos": 18837480977280.0, + "grad_norm": 1.987850462779664, + "language_loss": 0.6723578, + "learning_rate": 3.764296422425701e-06, + "loss": 0.69405967, + "num_input_tokens_seen": 179053040, + "router_z_loss_clip": 0.72998047, + "router_z_loss_mlp": 0.18200684, + "step": 6252, + "time_per_iteration": 2.640368938446045 + }, + { + "auxiliary_loss_clip": 0.01137378, + "auxiliary_loss_mlp": 0.01047742, + "balance_loss_clip": 1.06135774, + "balance_loss_mlp": 1.03144574, + "epoch": 0.18144623063083976, + "flos": 12452097156480.0, + "grad_norm": 2.119201759124329, + "language_loss": 0.70704067, + "learning_rate": 3.7642078899322568e-06, + "loss": 0.72889185, + "num_input_tokens_seen": 179067685, + "router_z_loss_clip": 0.76025391, + "router_z_loss_mlp": 0.16278076, + "step": 6253, + "time_per_iteration": 2.484581470489502 + }, + { + "auxiliary_loss_clip": 0.01023808, + "auxiliary_loss_mlp": 0.00998887, + "balance_loss_clip": 1.00555468, + "balance_loss_mlp": 0.99805874, + "epoch": 0.1814752480993558, + "flos": 74783470429440.0, + "grad_norm": 0.6438228049982117, + "language_loss": 0.47297755, + "learning_rate": 3.764119341856615e-06, + "loss": 0.4932045, + "num_input_tokens_seen": 179133970, + "router_z_loss_clip": 0.18261719, + "router_z_loss_mlp": 0.00830078, + "step": 6254, + "time_per_iteration": 3.1850709915161133 + }, + { + "auxiliary_loss_clip": 0.01139602, + "auxiliary_loss_mlp": 0.01043239, + "balance_loss_clip": 1.05889988, + "balance_loss_mlp": 1.02511883, + "epoch": 0.18150426556787186, + "flos": 31350912606720.0, + "grad_norm": 2.210111405809756, + "language_loss": 0.85748291, + "learning_rate": 3.764030778199557e-06, + "loss": 0.87931132, + "num_input_tokens_seen": 179151270, + "router_z_loss_clip": 0.80712891, + "router_z_loss_mlp": 0.18139648, + "step": 6255, + "time_per_iteration": 2.636690855026245 + }, + { + "auxiliary_loss_clip": 0.01123212, + "auxiliary_loss_mlp": 0.01035552, + "balance_loss_clip": 1.05492365, + "balance_loss_mlp": 1.02063894, + "epoch": 0.1815332830363879, + "flos": 12815153873280.0, + "grad_norm": 2.9721903943685692, + "language_loss": 0.83433437, + "learning_rate": 3.7639421989618653e-06, + "loss": 0.85592198, + "num_input_tokens_seen": 179161915, + "router_z_loss_clip": 0.68359375, + "router_z_loss_mlp": 0.14910889, + "step": 6256, + "time_per_iteration": 2.4687247276306152 + }, + { + "auxiliary_loss_clip": 0.01024817, + "auxiliary_loss_mlp": 0.00999312, + "balance_loss_clip": 1.00641084, + "balance_loss_mlp": 0.99845982, + "epoch": 0.18156230050490396, + "flos": 74776862327040.0, + "grad_norm": 0.7749857062577944, + "language_loss": 0.43793142, + "learning_rate": 3.763853604144322e-06, + "loss": 0.45817274, + "num_input_tokens_seen": 179226700, + "router_z_loss_clip": 0.18359375, + "router_z_loss_mlp": 0.00854492, + "step": 6257, + "time_per_iteration": 3.1153171062469482 + }, + { + "auxiliary_loss_clip": 0.01132523, + "auxiliary_loss_mlp": 0.01045959, + "balance_loss_clip": 1.05700111, + "balance_loss_mlp": 1.02903724, + "epoch": 0.18159131797341999, + "flos": 34453450869120.0, + "grad_norm": 3.548710919303992, + "language_loss": 1.00790501, + "learning_rate": 3.76376499374771e-06, + "loss": 1.02968979, + "num_input_tokens_seen": 179245835, + "router_z_loss_clip": 0.75488281, + "router_z_loss_mlp": 0.16931152, + "step": 6258, + "time_per_iteration": 2.5591440200805664 + }, + { + "auxiliary_loss_clip": 0.01132878, + "auxiliary_loss_mlp": 0.01038295, + "balance_loss_clip": 1.0557909, + "balance_loss_mlp": 1.02137947, + "epoch": 0.18162033544193604, + "flos": 32883795463680.0, + "grad_norm": 1.8768826361484439, + "language_loss": 0.95829183, + "learning_rate": 3.763676367772812e-06, + "loss": 0.9800036, + "num_input_tokens_seen": 179264610, + "router_z_loss_clip": 0.77148438, + "router_z_loss_mlp": 0.16912842, + "step": 6259, + "time_per_iteration": 9.805782556533813 + }, + { + "auxiliary_loss_clip": 0.01130363, + "auxiliary_loss_mlp": 0.01034453, + "balance_loss_clip": 1.05614281, + "balance_loss_mlp": 1.01892042, + "epoch": 0.1816493529104521, + "flos": 33726367768320.0, + "grad_norm": 2.712908401402195, + "language_loss": 0.69910026, + "learning_rate": 3.763587726220411e-06, + "loss": 0.72074842, + "num_input_tokens_seen": 179281400, + "router_z_loss_clip": 0.74267578, + "router_z_loss_mlp": 0.15533447, + "step": 6260, + "time_per_iteration": 4.960230350494385 + }, + { + "auxiliary_loss_clip": 0.0102646, + "auxiliary_loss_mlp": 0.01003601, + "balance_loss_clip": 1.00820005, + "balance_loss_mlp": 1.0028199, + "epoch": 0.18167837037896814, + "flos": 71085316844160.0, + "grad_norm": 0.7314984343919383, + "language_loss": 0.45437413, + "learning_rate": 3.7634990690912894e-06, + "loss": 0.47467473, + "num_input_tokens_seen": 179336110, + "router_z_loss_clip": 0.18359375, + "router_z_loss_mlp": 0.00778198, + "step": 6261, + "time_per_iteration": 3.0172650814056396 + }, + { + "auxiliary_loss_clip": 0.01135864, + "auxiliary_loss_mlp": 0.01048184, + "balance_loss_clip": 1.05825973, + "balance_loss_mlp": 1.03171492, + "epoch": 0.1817073878474842, + "flos": 33758722944000.0, + "grad_norm": 2.309584330873718, + "language_loss": 1.07818699, + "learning_rate": 3.7634103963862304e-06, + "loss": 1.10002744, + "num_input_tokens_seen": 179356520, + "router_z_loss_clip": 0.77539062, + "router_z_loss_mlp": 0.16461182, + "step": 6262, + "time_per_iteration": 2.6560583114624023 + }, + { + "auxiliary_loss_clip": 0.0112795, + "auxiliary_loss_mlp": 0.01044874, + "balance_loss_clip": 1.05537784, + "balance_loss_mlp": 1.02955568, + "epoch": 0.18173640531600024, + "flos": 13072346231040.0, + "grad_norm": 4.935291022361984, + "language_loss": 0.82241631, + "learning_rate": 3.7633217081060168e-06, + "loss": 0.84414446, + "num_input_tokens_seen": 179369545, + "router_z_loss_clip": 0.72558594, + "router_z_loss_mlp": 0.15319824, + "step": 6263, + "time_per_iteration": 2.5466387271881104 + }, + { + "auxiliary_loss_clip": 0.01026176, + "auxiliary_loss_mlp": 0.0100006, + "balance_loss_clip": 1.00775981, + "balance_loss_mlp": 0.99932367, + "epoch": 0.18176542278451627, + "flos": 72734012127360.0, + "grad_norm": 0.6908644269457984, + "language_loss": 0.49908078, + "learning_rate": 3.7632330042514325e-06, + "loss": 0.51934314, + "num_input_tokens_seen": 179430290, + "router_z_loss_clip": 0.18359375, + "router_z_loss_mlp": 0.00735474, + "step": 6264, + "time_per_iteration": 3.1084675788879395 + }, + { + "auxiliary_loss_clip": 0.01134716, + "auxiliary_loss_mlp": 0.01036196, + "balance_loss_clip": 1.05951273, + "balance_loss_mlp": 1.02152145, + "epoch": 0.18179444025303232, + "flos": 39887093975040.0, + "grad_norm": 2.1322182684168323, + "language_loss": 1.01913154, + "learning_rate": 3.763144284823261e-06, + "loss": 1.04084063, + "num_input_tokens_seen": 179447695, + "router_z_loss_clip": 0.75244141, + "router_z_loss_mlp": 0.14678955, + "step": 6265, + "time_per_iteration": 2.6949453353881836 + }, + { + "auxiliary_loss_clip": 0.01024064, + "auxiliary_loss_mlp": 0.01003171, + "balance_loss_clip": 1.00575137, + "balance_loss_mlp": 1.00240791, + "epoch": 0.18182345772154837, + "flos": 66147922085760.0, + "grad_norm": 0.7224839617124285, + "language_loss": 0.47088814, + "learning_rate": 3.7630555498222856e-06, + "loss": 0.49116042, + "num_input_tokens_seen": 179497500, + "router_z_loss_clip": 0.18359375, + "router_z_loss_mlp": 0.00762939, + "step": 6266, + "time_per_iteration": 2.9283735752105713 + }, + { + "auxiliary_loss_clip": 0.01126555, + "auxiliary_loss_mlp": 0.01039322, + "balance_loss_clip": 1.05588841, + "balance_loss_mlp": 1.02529144, + "epoch": 0.18185247519006442, + "flos": 23981289240960.0, + "grad_norm": 2.866267521029868, + "language_loss": 0.87959486, + "learning_rate": 3.76296679924929e-06, + "loss": 0.90125358, + "num_input_tokens_seen": 179511440, + "router_z_loss_clip": 0.70703125, + "router_z_loss_mlp": 0.14031982, + "step": 6267, + "time_per_iteration": 2.5662221908569336 + }, + { + "auxiliary_loss_clip": 0.01125173, + "auxiliary_loss_mlp": 0.01034819, + "balance_loss_clip": 1.05341816, + "balance_loss_mlp": 1.01969719, + "epoch": 0.18188149265858047, + "flos": 22958984257920.0, + "grad_norm": 3.0847845758149264, + "language_loss": 0.78791231, + "learning_rate": 3.7628780331050586e-06, + "loss": 0.80951226, + "num_input_tokens_seen": 179524475, + "router_z_loss_clip": 0.71728516, + "router_z_loss_mlp": 0.15130615, + "step": 6268, + "time_per_iteration": 2.540923595428467 + }, + { + "auxiliary_loss_clip": 0.01139177, + "auxiliary_loss_mlp": 0.01048764, + "balance_loss_clip": 1.06119561, + "balance_loss_mlp": 1.03166926, + "epoch": 0.18191051012709653, + "flos": 35881726343040.0, + "grad_norm": 2.010260523726314, + "language_loss": 0.8910566, + "learning_rate": 3.762789251390375e-06, + "loss": 0.91293597, + "num_input_tokens_seen": 179548470, + "router_z_loss_clip": 0.78027344, + "router_z_loss_mlp": 0.17108154, + "step": 6269, + "time_per_iteration": 2.6597707271575928 + }, + { + "auxiliary_loss_clip": 0.01131243, + "auxiliary_loss_mlp": 0.01040728, + "balance_loss_clip": 1.05693078, + "balance_loss_mlp": 1.02495062, + "epoch": 0.18193952759561255, + "flos": 34123898563200.0, + "grad_norm": 2.209292500154974, + "language_loss": 0.87597227, + "learning_rate": 3.7627004541060233e-06, + "loss": 0.89769197, + "num_input_tokens_seen": 179567995, + "router_z_loss_clip": 0.74316406, + "router_z_loss_mlp": 0.15789795, + "step": 6270, + "time_per_iteration": 2.6791648864746094 + }, + { + "auxiliary_loss_clip": 0.01132686, + "auxiliary_loss_mlp": 0.01037649, + "balance_loss_clip": 1.05831766, + "balance_loss_mlp": 1.02236652, + "epoch": 0.1819685450641286, + "flos": 33796213764480.0, + "grad_norm": 2.266046189548211, + "language_loss": 0.79973453, + "learning_rate": 3.7626116412527876e-06, + "loss": 0.82143784, + "num_input_tokens_seen": 179583595, + "router_z_loss_clip": 0.74365234, + "router_z_loss_mlp": 0.152771, + "step": 6271, + "time_per_iteration": 2.556579351425171 + }, + { + "auxiliary_loss_clip": 0.01137681, + "auxiliary_loss_mlp": 0.01037653, + "balance_loss_clip": 1.06079912, + "balance_loss_mlp": 1.02082062, + "epoch": 0.18199756253264465, + "flos": 43464191748480.0, + "grad_norm": 2.0487715264234843, + "language_loss": 0.73808098, + "learning_rate": 3.762522812831453e-06, + "loss": 0.75983429, + "num_input_tokens_seen": 179600390, + "router_z_loss_clip": 0.76757812, + "router_z_loss_mlp": 0.168396, + "step": 6272, + "time_per_iteration": 2.723914861679077 + }, + { + "auxiliary_loss_clip": 0.01127836, + "auxiliary_loss_mlp": 0.01035365, + "balance_loss_clip": 1.05875397, + "balance_loss_mlp": 1.02134633, + "epoch": 0.1820265800011607, + "flos": 30111420038400.0, + "grad_norm": 2.152099550615902, + "language_loss": 0.79845798, + "learning_rate": 3.762433968842804e-06, + "loss": 0.82009, + "num_input_tokens_seen": 179614200, + "router_z_loss_clip": 0.68994141, + "router_z_loss_mlp": 0.14019775, + "step": 6273, + "time_per_iteration": 2.6374194622039795 + }, + { + "auxiliary_loss_clip": 0.01133513, + "auxiliary_loss_mlp": 0.01041678, + "balance_loss_clip": 1.05864739, + "balance_loss_mlp": 1.02537012, + "epoch": 0.18205559746967676, + "flos": 19966368591360.0, + "grad_norm": 2.3397824290044356, + "language_loss": 0.72440207, + "learning_rate": 3.762345109287624e-06, + "loss": 0.74615395, + "num_input_tokens_seen": 179627600, + "router_z_loss_clip": 0.74853516, + "router_z_loss_mlp": 0.16290283, + "step": 6274, + "time_per_iteration": 2.5280184745788574 + }, + { + "auxiliary_loss_clip": 0.01128636, + "auxiliary_loss_mlp": 0.01042602, + "balance_loss_clip": 1.05631804, + "balance_loss_mlp": 1.02764702, + "epoch": 0.18208461493819278, + "flos": 34742100562560.0, + "grad_norm": 2.3136168092931735, + "language_loss": 0.87897247, + "learning_rate": 3.7622562341666997e-06, + "loss": 0.90068483, + "num_input_tokens_seen": 179645390, + "router_z_loss_clip": 0.72363281, + "router_z_loss_mlp": 0.1496582, + "step": 6275, + "time_per_iteration": 2.63919734954834 + }, + { + "auxiliary_loss_clip": 0.01128184, + "auxiliary_loss_mlp": 0.01036435, + "balance_loss_clip": 1.05809498, + "balance_loss_mlp": 1.02148604, + "epoch": 0.18211363240670883, + "flos": 16684851836160.0, + "grad_norm": 2.5498954930531994, + "language_loss": 0.64385343, + "learning_rate": 3.762167343480815e-06, + "loss": 0.66549969, + "num_input_tokens_seen": 179658815, + "router_z_loss_clip": 0.70117188, + "router_z_loss_mlp": 0.14959717, + "step": 6276, + "time_per_iteration": 2.5401079654693604 + }, + { + "auxiliary_loss_clip": 0.01029072, + "auxiliary_loss_mlp": 0.01019026, + "balance_loss_clip": 1.01077437, + "balance_loss_mlp": 1.01820326, + "epoch": 0.18214264987522488, + "flos": 74773917411840.0, + "grad_norm": 0.6775487271470946, + "language_loss": 0.47187302, + "learning_rate": 3.762078437230755e-06, + "loss": 0.49235395, + "num_input_tokens_seen": 179721375, + "router_z_loss_clip": 0.18359375, + "router_z_loss_mlp": 0.00823975, + "step": 6277, + "time_per_iteration": 3.131080150604248 + }, + { + "auxiliary_loss_clip": 0.01131919, + "auxiliary_loss_mlp": 0.01041411, + "balance_loss_clip": 1.057441, + "balance_loss_mlp": 1.02587843, + "epoch": 0.18217166734374093, + "flos": 47890290360960.0, + "grad_norm": 2.4703622446859237, + "language_loss": 1.09258127, + "learning_rate": 3.761989515417306e-06, + "loss": 1.11431468, + "num_input_tokens_seen": 179744325, + "router_z_loss_clip": 0.74389648, + "router_z_loss_mlp": 0.15533447, + "step": 6278, + "time_per_iteration": 2.8373219966888428 + }, + { + "auxiliary_loss_clip": 0.01131214, + "auxiliary_loss_mlp": 0.01042917, + "balance_loss_clip": 1.06093431, + "balance_loss_mlp": 1.02730632, + "epoch": 0.18220068481225699, + "flos": 12348243959040.0, + "grad_norm": 2.6154702707041606, + "language_loss": 0.75448173, + "learning_rate": 3.761900578041252e-06, + "loss": 0.776223, + "num_input_tokens_seen": 179756965, + "router_z_loss_clip": 0.70263672, + "router_z_loss_mlp": 0.15625, + "step": 6279, + "time_per_iteration": 2.5126760005950928 + }, + { + "auxiliary_loss_clip": 0.01130258, + "auxiliary_loss_mlp": 0.01046405, + "balance_loss_clip": 1.05697, + "balance_loss_mlp": 1.02981162, + "epoch": 0.18222970228077304, + "flos": 33467271989760.0, + "grad_norm": 2.2745635282188204, + "language_loss": 0.82801759, + "learning_rate": 3.7618116251033785e-06, + "loss": 0.84978431, + "num_input_tokens_seen": 179772335, + "router_z_loss_clip": 0.73291016, + "router_z_loss_mlp": 0.16601562, + "step": 6280, + "time_per_iteration": 2.6258578300476074 + }, + { + "auxiliary_loss_clip": 0.0103106, + "auxiliary_loss_mlp": 0.01005472, + "balance_loss_clip": 1.0130018, + "balance_loss_mlp": 1.00464332, + "epoch": 0.18225871974928906, + "flos": 62695794706560.0, + "grad_norm": 0.6144213418917721, + "language_loss": 0.49202448, + "learning_rate": 3.7617226566044727e-06, + "loss": 0.51238978, + "num_input_tokens_seen": 179833975, + "router_z_loss_clip": 0.18066406, + "router_z_loss_mlp": 0.00830078, + "step": 6281, + "time_per_iteration": 3.0614230632781982 + }, + { + "auxiliary_loss_clip": 0.01124995, + "auxiliary_loss_mlp": 0.01032292, + "balance_loss_clip": 1.05504215, + "balance_loss_mlp": 1.01807046, + "epoch": 0.1822877372178051, + "flos": 35839889976960.0, + "grad_norm": 2.165760028497278, + "language_loss": 0.71898091, + "learning_rate": 3.7616336725453197e-06, + "loss": 0.7405538, + "num_input_tokens_seen": 179860155, + "router_z_loss_clip": 0.70068359, + "router_z_loss_mlp": 0.14215088, + "step": 6282, + "time_per_iteration": 2.9407646656036377 + }, + { + "auxiliary_loss_clip": 0.01137821, + "auxiliary_loss_mlp": 0.010391, + "balance_loss_clip": 1.06026816, + "balance_loss_mlp": 1.02266145, + "epoch": 0.18231675468632116, + "flos": 28548049512960.0, + "grad_norm": 1.9207876483741866, + "language_loss": 0.69929767, + "learning_rate": 3.761544672926704e-06, + "loss": 0.72106683, + "num_input_tokens_seen": 179884795, + "router_z_loss_clip": 0.77587891, + "router_z_loss_mlp": 0.16442871, + "step": 6283, + "time_per_iteration": 2.843296527862549 + }, + { + "auxiliary_loss_clip": 0.01130075, + "auxiliary_loss_mlp": 0.01044291, + "balance_loss_clip": 1.05617404, + "balance_loss_mlp": 1.02814984, + "epoch": 0.18234577215483722, + "flos": 24784970094720.0, + "grad_norm": 2.6508480903810914, + "language_loss": 0.82841945, + "learning_rate": 3.761455657749414e-06, + "loss": 0.8501631, + "num_input_tokens_seen": 179901810, + "router_z_loss_clip": 0.73828125, + "router_z_loss_mlp": 0.16149902, + "step": 6284, + "time_per_iteration": 2.5567612648010254 + }, + { + "auxiliary_loss_clip": 0.01137426, + "auxiliary_loss_mlp": 0.01045642, + "balance_loss_clip": 1.06069255, + "balance_loss_mlp": 1.02700973, + "epoch": 0.18237478962335327, + "flos": 23031236465280.0, + "grad_norm": 2.8377279173347763, + "language_loss": 0.97265983, + "learning_rate": 3.7613666270142347e-06, + "loss": 0.99449056, + "num_input_tokens_seen": 179915765, + "router_z_loss_clip": 0.76660156, + "router_z_loss_mlp": 0.1862793, + "step": 6285, + "time_per_iteration": 2.537221670150757 + }, + { + "auxiliary_loss_clip": 0.01135704, + "auxiliary_loss_mlp": 0.01045159, + "balance_loss_clip": 1.05888391, + "balance_loss_mlp": 1.02856517, + "epoch": 0.18240380709186932, + "flos": 43985471875200.0, + "grad_norm": 1.9835795702017627, + "language_loss": 0.86959684, + "learning_rate": 3.7612775807219523e-06, + "loss": 0.89140546, + "num_input_tokens_seen": 179934435, + "router_z_loss_clip": 0.76806641, + "router_z_loss_mlp": 0.16607666, + "step": 6286, + "time_per_iteration": 2.641106128692627 + }, + { + "auxiliary_loss_clip": 0.01128778, + "auxiliary_loss_mlp": 0.01040059, + "balance_loss_clip": 1.05760002, + "balance_loss_mlp": 1.02498531, + "epoch": 0.18243282456038534, + "flos": 17378179130880.0, + "grad_norm": 2.593734390707952, + "language_loss": 0.70523626, + "learning_rate": 3.761188518873354e-06, + "loss": 0.72692466, + "num_input_tokens_seen": 179947990, + "router_z_loss_clip": 0.71142578, + "router_z_loss_mlp": 0.15075684, + "step": 6287, + "time_per_iteration": 2.5714540481567383 + }, + { + "auxiliary_loss_clip": 0.01130311, + "auxiliary_loss_mlp": 0.01044431, + "balance_loss_clip": 1.06066978, + "balance_loss_mlp": 1.02983427, + "epoch": 0.1824618420289014, + "flos": 9722635505280.0, + "grad_norm": 2.22202464173325, + "language_loss": 0.73676974, + "learning_rate": 3.761099441469225e-06, + "loss": 0.75851715, + "num_input_tokens_seen": 179959645, + "router_z_loss_clip": 0.69677734, + "router_z_loss_mlp": 0.14593506, + "step": 6288, + "time_per_iteration": 2.492523193359375 + }, + { + "auxiliary_loss_clip": 0.01132021, + "auxiliary_loss_mlp": 0.01038664, + "balance_loss_clip": 1.05735707, + "balance_loss_mlp": 1.02320266, + "epoch": 0.18249085949741745, + "flos": 11282773806720.0, + "grad_norm": 2.7763144166056115, + "language_loss": 0.91142249, + "learning_rate": 3.761010348510354e-06, + "loss": 0.93312937, + "num_input_tokens_seen": 179969945, + "router_z_loss_clip": 0.74658203, + "router_z_loss_mlp": 0.15466309, + "step": 6289, + "time_per_iteration": 2.514214277267456 + }, + { + "auxiliary_loss_clip": 0.01140376, + "auxiliary_loss_mlp": 0.01042707, + "balance_loss_clip": 1.0627979, + "balance_loss_mlp": 1.02486122, + "epoch": 0.1825198769659335, + "flos": 12852932002560.0, + "grad_norm": 3.367949279821754, + "language_loss": 0.937141, + "learning_rate": 3.7609212399975273e-06, + "loss": 0.95897186, + "num_input_tokens_seen": 179982260, + "router_z_loss_clip": 0.77539062, + "router_z_loss_mlp": 0.17840576, + "step": 6290, + "time_per_iteration": 2.4954006671905518 + }, + { + "auxiliary_loss_clip": 0.01126627, + "auxiliary_loss_mlp": 0.01039855, + "balance_loss_clip": 1.05726576, + "balance_loss_mlp": 1.02494144, + "epoch": 0.18254889443444955, + "flos": 12380455480320.0, + "grad_norm": 2.9217436827741663, + "language_loss": 0.86031735, + "learning_rate": 3.7608321159315315e-06, + "loss": 0.88198215, + "num_input_tokens_seen": 179993775, + "router_z_loss_clip": 0.69433594, + "router_z_loss_mlp": 0.14916992, + "step": 6291, + "time_per_iteration": 2.517019271850586 + }, + { + "auxiliary_loss_clip": 0.01125638, + "auxiliary_loss_mlp": 0.01040258, + "balance_loss_clip": 1.05699348, + "balance_loss_mlp": 1.02634025, + "epoch": 0.18257791190296557, + "flos": 13398917708160.0, + "grad_norm": 2.190570390605613, + "language_loss": 0.91651481, + "learning_rate": 3.7607429763131535e-06, + "loss": 0.93817377, + "num_input_tokens_seen": 180006425, + "router_z_loss_clip": 0.68701172, + "router_z_loss_mlp": 0.13916016, + "step": 6292, + "time_per_iteration": 2.5051369667053223 + }, + { + "auxiliary_loss_clip": 0.01133276, + "auxiliary_loss_mlp": 0.01044004, + "balance_loss_clip": 1.05916536, + "balance_loss_mlp": 1.02820301, + "epoch": 0.18260692937148162, + "flos": 23690269249920.0, + "grad_norm": 2.5391919980934365, + "language_loss": 0.72129905, + "learning_rate": 3.760653821143181e-06, + "loss": 0.74307185, + "num_input_tokens_seen": 180022700, + "router_z_loss_clip": 0.74072266, + "router_z_loss_mlp": 0.15802002, + "step": 6293, + "time_per_iteration": 2.5542330741882324 + }, + { + "auxiliary_loss_clip": 0.01139162, + "auxiliary_loss_mlp": 0.01056919, + "balance_loss_clip": 1.06353855, + "balance_loss_mlp": 1.03939474, + "epoch": 0.18263594683999768, + "flos": 16100226074880.0, + "grad_norm": 3.1238651753979947, + "language_loss": 0.79564482, + "learning_rate": 3.7605646504224017e-06, + "loss": 0.81760561, + "num_input_tokens_seen": 180034550, + "router_z_loss_clip": 0.7578125, + "router_z_loss_mlp": 0.17529297, + "step": 6294, + "time_per_iteration": 2.4969797134399414 + }, + { + "auxiliary_loss_clip": 0.01041668, + "auxiliary_loss_mlp": 0.01000318, + "balance_loss_clip": 1.02344131, + "balance_loss_mlp": 0.99945372, + "epoch": 0.18266496430851373, + "flos": 56390204949120.0, + "grad_norm": 0.6977939868639178, + "language_loss": 0.46288818, + "learning_rate": 3.7604754641516026e-06, + "loss": 0.48330805, + "num_input_tokens_seen": 180089620, + "router_z_loss_clip": 0.18261719, + "router_z_loss_mlp": 0.00866699, + "step": 6295, + "time_per_iteration": 3.019845962524414 + }, + { + "auxiliary_loss_clip": 0.01041505, + "auxiliary_loss_mlp": 0.01002774, + "balance_loss_clip": 1.02325892, + "balance_loss_mlp": 1.00191009, + "epoch": 0.18269398177702978, + "flos": 69357330288000.0, + "grad_norm": 0.6242714029860115, + "language_loss": 0.42394263, + "learning_rate": 3.7603862623315723e-06, + "loss": 0.44438541, + "num_input_tokens_seen": 180146355, + "router_z_loss_clip": 0.18261719, + "router_z_loss_mlp": 0.00866699, + "step": 6296, + "time_per_iteration": 3.0506386756896973 + }, + { + "auxiliary_loss_clip": 0.01141999, + "auxiliary_loss_mlp": 0.01043797, + "balance_loss_clip": 1.06192458, + "balance_loss_mlp": 1.02804351, + "epoch": 0.18272299924554583, + "flos": 20444196240000.0, + "grad_norm": 1.962049416126671, + "language_loss": 0.84993017, + "learning_rate": 3.760297044963098e-06, + "loss": 0.87178808, + "num_input_tokens_seen": 180165855, + "router_z_loss_clip": 0.80078125, + "router_z_loss_mlp": 0.1574707, + "step": 6297, + "time_per_iteration": 2.5486509799957275 + }, + { + "auxiliary_loss_clip": 0.01132281, + "auxiliary_loss_mlp": 0.01037518, + "balance_loss_clip": 1.05910599, + "balance_loss_mlp": 1.02174616, + "epoch": 0.18275201671406185, + "flos": 34670674368000.0, + "grad_norm": 2.685086145530198, + "language_loss": 0.94659716, + "learning_rate": 3.760207812046968e-06, + "loss": 0.9682951, + "num_input_tokens_seen": 180181675, + "router_z_loss_clip": 0.73144531, + "router_z_loss_mlp": 0.15783691, + "step": 6298, + "time_per_iteration": 2.657825231552124 + }, + { + "auxiliary_loss_clip": 0.01147798, + "auxiliary_loss_mlp": 0.0104714, + "balance_loss_clip": 1.0621798, + "balance_loss_mlp": 1.0291748, + "epoch": 0.1827810341825779, + "flos": 28031509981440.0, + "grad_norm": 2.3845197433638834, + "language_loss": 0.96951199, + "learning_rate": 3.7601185635839702e-06, + "loss": 0.99146128, + "num_input_tokens_seen": 180196365, + "router_z_loss_clip": 0.85498047, + "router_z_loss_mlp": 0.17944336, + "step": 6299, + "time_per_iteration": 2.5733439922332764 + }, + { + "auxiliary_loss_clip": 0.010354, + "auxiliary_loss_mlp": 0.01001202, + "balance_loss_clip": 1.01723886, + "balance_loss_mlp": 1.00037944, + "epoch": 0.18281005165109396, + "flos": 51098516392320.0, + "grad_norm": 0.6574227275159854, + "language_loss": 0.45485675, + "learning_rate": 3.760029299574893e-06, + "loss": 0.4752228, + "num_input_tokens_seen": 180253645, + "router_z_loss_clip": 0.18164062, + "router_z_loss_mlp": 0.00823975, + "step": 6300, + "time_per_iteration": 3.06992769241333 + }, + { + "auxiliary_loss_clip": 0.01034205, + "auxiliary_loss_mlp": 0.01003126, + "balance_loss_clip": 1.0161016, + "balance_loss_mlp": 1.00230336, + "epoch": 0.18283906911961, + "flos": 68677291025280.0, + "grad_norm": 0.6601392372854544, + "language_loss": 0.44574296, + "learning_rate": 3.759940020020525e-06, + "loss": 0.46611625, + "num_input_tokens_seen": 180312265, + "router_z_loss_clip": 0.18164062, + "router_z_loss_mlp": 0.00823975, + "step": 6301, + "time_per_iteration": 3.03326153755188 + }, + { + "auxiliary_loss_clip": 0.01145107, + "auxiliary_loss_mlp": 0.01055846, + "balance_loss_clip": 1.06476104, + "balance_loss_mlp": 1.03901958, + "epoch": 0.18286808658812606, + "flos": 34801286175360.0, + "grad_norm": 1.431618307396819, + "language_loss": 0.85321504, + "learning_rate": 3.759850724921654e-06, + "loss": 0.87522459, + "num_input_tokens_seen": 180346215, + "router_z_loss_clip": 0.80517578, + "router_z_loss_mlp": 0.16827393, + "step": 6302, + "time_per_iteration": 3.090298652648926 + }, + { + "auxiliary_loss_clip": 0.01130166, + "auxiliary_loss_mlp": 0.01042745, + "balance_loss_clip": 1.0579412, + "balance_loss_mlp": 1.02867222, + "epoch": 0.18289710405664208, + "flos": 17051320344960.0, + "grad_norm": 2.617238966042782, + "language_loss": 0.86523682, + "learning_rate": 3.75976141427907e-06, + "loss": 0.88696587, + "num_input_tokens_seen": 180359105, + "router_z_loss_clip": 0.72314453, + "router_z_loss_mlp": 0.14074707, + "step": 6303, + "time_per_iteration": 2.472736358642578 + }, + { + "auxiliary_loss_clip": 0.01030794, + "auxiliary_loss_mlp": 0.01004032, + "balance_loss_clip": 1.01280808, + "balance_loss_mlp": 1.00331652, + "epoch": 0.18292612152515814, + "flos": 69624793935360.0, + "grad_norm": 0.6749894479892465, + "language_loss": 0.45969158, + "learning_rate": 3.759672088093561e-06, + "loss": 0.48003983, + "num_input_tokens_seen": 180420235, + "router_z_loss_clip": 0.1796875, + "router_z_loss_mlp": 0.00714111, + "step": 6304, + "time_per_iteration": 3.1617984771728516 + }, + { + "auxiliary_loss_clip": 0.01030269, + "auxiliary_loss_mlp": 0.01005472, + "balance_loss_clip": 1.0124898, + "balance_loss_mlp": 1.00464964, + "epoch": 0.1829551389936742, + "flos": 74766195987840.0, + "grad_norm": 0.6511072443153438, + "language_loss": 0.48225811, + "learning_rate": 3.7595827463659155e-06, + "loss": 0.50261551, + "num_input_tokens_seen": 180484395, + "router_z_loss_clip": 0.17773438, + "router_z_loss_mlp": 0.00823975, + "step": 6305, + "time_per_iteration": 3.0694212913513184 + }, + { + "auxiliary_loss_clip": 0.01129891, + "auxiliary_loss_mlp": 0.01040674, + "balance_loss_clip": 1.0583235, + "balance_loss_mlp": 1.02565956, + "epoch": 0.18298415646219024, + "flos": 28760927466240.0, + "grad_norm": 2.3400341171206023, + "language_loss": 0.65017855, + "learning_rate": 3.7594933890969232e-06, + "loss": 0.67188418, + "num_input_tokens_seen": 180502065, + "router_z_loss_clip": 0.71533203, + "router_z_loss_mlp": 0.15002441, + "step": 6306, + "time_per_iteration": 2.5839219093322754 + }, + { + "auxiliary_loss_clip": 0.01131946, + "auxiliary_loss_mlp": 0.01039874, + "balance_loss_clip": 1.05820274, + "balance_loss_mlp": 1.02398324, + "epoch": 0.1830131739307063, + "flos": 11466133758720.0, + "grad_norm": 3.342381830381334, + "language_loss": 0.77947092, + "learning_rate": 3.759404016287374e-06, + "loss": 0.80118906, + "num_input_tokens_seen": 180512525, + "router_z_loss_clip": 0.73730469, + "router_z_loss_mlp": 0.15893555, + "step": 6307, + "time_per_iteration": 2.4948036670684814 + }, + { + "auxiliary_loss_clip": 0.01129826, + "auxiliary_loss_mlp": 0.0103957, + "balance_loss_clip": 1.05734861, + "balance_loss_mlp": 1.02454388, + "epoch": 0.18304219139922234, + "flos": 74732293549440.0, + "grad_norm": 2.1605701712582785, + "language_loss": 0.91729259, + "learning_rate": 3.759314627938056e-06, + "loss": 0.93898654, + "num_input_tokens_seen": 180533805, + "router_z_loss_clip": 0.72363281, + "router_z_loss_mlp": 0.15026855, + "step": 6308, + "time_per_iteration": 2.9905169010162354 + }, + { + "auxiliary_loss_clip": 0.01130295, + "auxiliary_loss_mlp": 0.01041145, + "balance_loss_clip": 1.05780482, + "balance_loss_mlp": 1.02634525, + "epoch": 0.18307120886773837, + "flos": 34891130090880.0, + "grad_norm": 1.9072835347478856, + "language_loss": 0.70660591, + "learning_rate": 3.7592252240497598e-06, + "loss": 0.72832036, + "num_input_tokens_seen": 180552535, + "router_z_loss_clip": 0.72412109, + "router_z_loss_mlp": 0.14794922, + "step": 6309, + "time_per_iteration": 2.8128163814544678 + }, + { + "auxiliary_loss_clip": 0.01134943, + "auxiliary_loss_mlp": 0.01035087, + "balance_loss_clip": 1.05967903, + "balance_loss_mlp": 1.01950634, + "epoch": 0.18310022633625442, + "flos": 15989692947840.0, + "grad_norm": 2.780143629445739, + "language_loss": 0.94301838, + "learning_rate": 3.7591358046232744e-06, + "loss": 0.9647187, + "num_input_tokens_seen": 180564270, + "router_z_loss_clip": 0.75292969, + "router_z_loss_mlp": 0.15594482, + "step": 6310, + "time_per_iteration": 2.5248663425445557 + }, + { + "auxiliary_loss_clip": 0.01135406, + "auxiliary_loss_mlp": 0.01044424, + "balance_loss_clip": 1.06212997, + "balance_loss_mlp": 1.02811587, + "epoch": 0.18312924380477047, + "flos": 15552839738880.0, + "grad_norm": 2.566104979833941, + "language_loss": 0.7536937, + "learning_rate": 3.7590463696593888e-06, + "loss": 0.77549195, + "num_input_tokens_seen": 180576640, + "router_z_loss_clip": 0.73291016, + "router_z_loss_mlp": 0.16320801, + "step": 6311, + "time_per_iteration": 2.539395332336426 + }, + { + "auxiliary_loss_clip": 0.01034313, + "auxiliary_loss_mlp": 0.01014127, + "balance_loss_clip": 1.01627302, + "balance_loss_mlp": 1.01319122, + "epoch": 0.18315826127328652, + "flos": 65471725578240.0, + "grad_norm": 0.6575628260375349, + "language_loss": 0.46862817, + "learning_rate": 3.7589569191588945e-06, + "loss": 0.48911262, + "num_input_tokens_seen": 180638815, + "router_z_loss_clip": 0.18066406, + "router_z_loss_mlp": 0.00933838, + "step": 6312, + "time_per_iteration": 3.1287105083465576 + }, + { + "auxiliary_loss_clip": 0.01034394, + "auxiliary_loss_mlp": 0.01011728, + "balance_loss_clip": 1.01630688, + "balance_loss_mlp": 1.01085174, + "epoch": 0.18318727874180257, + "flos": 61713027619200.0, + "grad_norm": 0.7317990598560393, + "language_loss": 0.49289051, + "learning_rate": 3.7588674531225815e-06, + "loss": 0.51335174, + "num_input_tokens_seen": 180693490, + "router_z_loss_clip": 0.18066406, + "router_z_loss_mlp": 0.00878906, + "step": 6313, + "time_per_iteration": 2.989485502243042 + }, + { + "auxiliary_loss_clip": 0.01128827, + "auxiliary_loss_mlp": 0.01043329, + "balance_loss_clip": 1.05585682, + "balance_loss_mlp": 1.02832031, + "epoch": 0.18321629621031862, + "flos": 45583352392320.0, + "grad_norm": 1.845139175346646, + "language_loss": 0.83135343, + "learning_rate": 3.7587779715512386e-06, + "loss": 0.85307491, + "num_input_tokens_seen": 180720790, + "router_z_loss_clip": 0.72949219, + "router_z_loss_mlp": 0.15014648, + "step": 6314, + "time_per_iteration": 2.924813747406006 + }, + { + "auxiliary_loss_clip": 0.01126278, + "auxiliary_loss_mlp": 0.01034867, + "balance_loss_clip": 1.05820131, + "balance_loss_mlp": 1.0203712, + "epoch": 0.18324531367883465, + "flos": 13801835543040.0, + "grad_norm": 3.1730580031195026, + "language_loss": 0.94364166, + "learning_rate": 3.758688474445657e-06, + "loss": 0.96525306, + "num_input_tokens_seen": 180732655, + "router_z_loss_clip": 0.68017578, + "router_z_loss_mlp": 0.14501953, + "step": 6315, + "time_per_iteration": 2.5179295539855957 + }, + { + "auxiliary_loss_clip": 0.01132453, + "auxiliary_loss_mlp": 0.01046552, + "balance_loss_clip": 1.06062388, + "balance_loss_mlp": 1.03054249, + "epoch": 0.1832743311473507, + "flos": 23981037845760.0, + "grad_norm": 2.412593774496726, + "language_loss": 0.70958579, + "learning_rate": 3.7585989618066276e-06, + "loss": 0.73137581, + "num_input_tokens_seen": 180746185, + "router_z_loss_clip": 0.71777344, + "router_z_loss_mlp": 0.16015625, + "step": 6316, + "time_per_iteration": 2.5741491317749023 + }, + { + "auxiliary_loss_clip": 0.01032318, + "auxiliary_loss_mlp": 0.0100599, + "balance_loss_clip": 1.01446986, + "balance_loss_mlp": 1.00517309, + "epoch": 0.18330334861586675, + "flos": 60464772132480.0, + "grad_norm": 0.67987379242178, + "language_loss": 0.47264916, + "learning_rate": 3.7585094336349405e-06, + "loss": 0.49303222, + "num_input_tokens_seen": 180807185, + "router_z_loss_clip": 0.17871094, + "router_z_loss_mlp": 0.00817871, + "step": 6317, + "time_per_iteration": 3.3189005851745605 + }, + { + "auxiliary_loss_clip": 0.01032201, + "auxiliary_loss_mlp": 0.01004607, + "balance_loss_clip": 1.01423895, + "balance_loss_mlp": 1.00373125, + "epoch": 0.1833323660843828, + "flos": 74765908679040.0, + "grad_norm": 0.6903583646103164, + "language_loss": 0.4705632, + "learning_rate": 3.7584198899313863e-06, + "loss": 0.49093127, + "num_input_tokens_seen": 180865900, + "router_z_loss_clip": 0.1796875, + "router_z_loss_mlp": 0.00878906, + "step": 6318, + "time_per_iteration": 3.055767297744751 + }, + { + "auxiliary_loss_clip": 0.01030699, + "auxiliary_loss_mlp": 0.01005627, + "balance_loss_clip": 1.01285291, + "balance_loss_mlp": 1.00478065, + "epoch": 0.18336138355289885, + "flos": 74780956477440.0, + "grad_norm": 0.7296057837630823, + "language_loss": 0.50365365, + "learning_rate": 3.758330330696756e-06, + "loss": 0.52401692, + "num_input_tokens_seen": 180927820, + "router_z_loss_clip": 0.17773438, + "router_z_loss_mlp": 0.00848389, + "step": 6319, + "time_per_iteration": 3.1421167850494385 + }, + { + "auxiliary_loss_clip": 0.01030028, + "auxiliary_loss_mlp": 0.01003066, + "balance_loss_clip": 1.01211715, + "balance_loss_mlp": 1.00227356, + "epoch": 0.18339040102141488, + "flos": 67404796836480.0, + "grad_norm": 0.6272245360914033, + "language_loss": 0.45115709, + "learning_rate": 3.7582407559318404e-06, + "loss": 0.47148803, + "num_input_tokens_seen": 180990645, + "router_z_loss_clip": 0.1796875, + "router_z_loss_mlp": 0.00793457, + "step": 6320, + "time_per_iteration": 3.2340123653411865 + }, + { + "auxiliary_loss_clip": 0.01029208, + "auxiliary_loss_mlp": 0.01000291, + "balance_loss_clip": 1.01132393, + "balance_loss_mlp": 0.9994747, + "epoch": 0.18341941848993093, + "flos": 59157947520000.0, + "grad_norm": 0.6615686345717088, + "language_loss": 0.4457415, + "learning_rate": 3.7581511656374313e-06, + "loss": 0.4660365, + "num_input_tokens_seen": 181052840, + "router_z_loss_clip": 0.1796875, + "router_z_loss_mlp": 0.00817871, + "step": 6321, + "time_per_iteration": 3.036426544189453 + }, + { + "auxiliary_loss_clip": 0.01029165, + "auxiliary_loss_mlp": 0.01001356, + "balance_loss_clip": 1.01133728, + "balance_loss_mlp": 1.00056314, + "epoch": 0.18344843595844698, + "flos": 74760018848640.0, + "grad_norm": 0.7456539251258355, + "language_loss": 0.52408183, + "learning_rate": 3.7580615598143198e-06, + "loss": 0.5443871, + "num_input_tokens_seen": 181107145, + "router_z_loss_clip": 0.17871094, + "router_z_loss_mlp": 0.00793457, + "step": 6322, + "time_per_iteration": 3.0446419715881348 + }, + { + "auxiliary_loss_clip": 0.01136002, + "auxiliary_loss_mlp": 0.01042244, + "balance_loss_clip": 1.05906641, + "balance_loss_mlp": 1.02531028, + "epoch": 0.18347745342696303, + "flos": 23872084917120.0, + "grad_norm": 2.425014992033444, + "language_loss": 0.99792564, + "learning_rate": 3.7579719384632973e-06, + "loss": 1.01970816, + "num_input_tokens_seen": 181120990, + "router_z_loss_clip": 0.77001953, + "router_z_loss_mlp": 0.16931152, + "step": 6323, + "time_per_iteration": 2.513509511947632 + }, + { + "auxiliary_loss_clip": 0.01027949, + "auxiliary_loss_mlp": 0.01007203, + "balance_loss_clip": 1.01016235, + "balance_loss_mlp": 1.00640476, + "epoch": 0.18350647089547908, + "flos": 54847050802560.0, + "grad_norm": 0.666034392612274, + "language_loss": 0.47749633, + "learning_rate": 3.757882301585155e-06, + "loss": 0.49784783, + "num_input_tokens_seen": 181177925, + "router_z_loss_clip": 0.17773438, + "router_z_loss_mlp": 0.00799561, + "step": 6324, + "time_per_iteration": 3.012572765350342 + }, + { + "auxiliary_loss_clip": 0.01026313, + "auxiliary_loss_mlp": 0.01003347, + "balance_loss_clip": 1.00847828, + "balance_loss_mlp": 1.00263202, + "epoch": 0.18353548836399514, + "flos": 62179686138240.0, + "grad_norm": 0.7062296408507696, + "language_loss": 0.49625805, + "learning_rate": 3.7577926491806846e-06, + "loss": 0.51655465, + "num_input_tokens_seen": 181232850, + "router_z_loss_clip": 0.17871094, + "router_z_loss_mlp": 0.00714111, + "step": 6325, + "time_per_iteration": 2.9550063610076904 + }, + { + "auxiliary_loss_clip": 0.01123545, + "auxiliary_loss_mlp": 0.01033225, + "balance_loss_clip": 1.05621552, + "balance_loss_mlp": 1.01986814, + "epoch": 0.18356450583251116, + "flos": 19057505736960.0, + "grad_norm": 2.574601818758727, + "language_loss": 0.86743671, + "learning_rate": 3.7577029812506787e-06, + "loss": 0.88900435, + "num_input_tokens_seen": 181247770, + "router_z_loss_clip": 0.67431641, + "router_z_loss_mlp": 0.13342285, + "step": 6326, + "time_per_iteration": 2.5538721084594727 + }, + { + "auxiliary_loss_clip": 0.0113443, + "auxiliary_loss_mlp": 0.01035669, + "balance_loss_clip": 1.05665636, + "balance_loss_mlp": 1.01958764, + "epoch": 0.1835935233010272, + "flos": 30513368206080.0, + "grad_norm": 1.8542264787997578, + "language_loss": 0.82954097, + "learning_rate": 3.757613297795928e-06, + "loss": 0.85124195, + "num_input_tokens_seen": 181264700, + "router_z_loss_clip": 0.77832031, + "router_z_loss_mlp": 0.1607666, + "step": 6327, + "time_per_iteration": 2.6080520153045654 + }, + { + "auxiliary_loss_clip": 0.01025003, + "auxiliary_loss_mlp": 0.01003913, + "balance_loss_clip": 1.0071702, + "balance_loss_mlp": 1.00314415, + "epoch": 0.18362254076954326, + "flos": 59635056896640.0, + "grad_norm": 0.6380881942933759, + "language_loss": 0.44276136, + "learning_rate": 3.7575235988172266e-06, + "loss": 0.46305054, + "num_input_tokens_seen": 181325930, + "router_z_loss_clip": 0.17773438, + "router_z_loss_mlp": 0.00765991, + "step": 6328, + "time_per_iteration": 3.0570225715637207 + }, + { + "auxiliary_loss_clip": 0.01132995, + "auxiliary_loss_mlp": 0.01039619, + "balance_loss_clip": 1.05810821, + "balance_loss_mlp": 1.02334142, + "epoch": 0.18365155823805931, + "flos": 12706955130240.0, + "grad_norm": 4.3372035058738785, + "language_loss": 0.75878972, + "learning_rate": 3.757433884315365e-06, + "loss": 0.78051591, + "num_input_tokens_seen": 181337180, + "router_z_loss_clip": 0.74951172, + "router_z_loss_mlp": 0.16278076, + "step": 6329, + "time_per_iteration": 2.477606773376465 + }, + { + "auxiliary_loss_clip": 0.01129634, + "auxiliary_loss_mlp": 0.01045739, + "balance_loss_clip": 1.05412221, + "balance_loss_mlp": 1.02997327, + "epoch": 0.18368057570657537, + "flos": 29678589152640.0, + "grad_norm": 1.9464909784515032, + "language_loss": 0.97874892, + "learning_rate": 3.757344154291136e-06, + "loss": 1.00050259, + "num_input_tokens_seen": 181357175, + "router_z_loss_clip": 0.75488281, + "router_z_loss_mlp": 0.15759277, + "step": 6330, + "time_per_iteration": 7.478971004486084 + }, + { + "auxiliary_loss_clip": 0.01023593, + "auxiliary_loss_mlp": 0.01002875, + "balance_loss_clip": 1.00563383, + "balance_loss_mlp": 1.00206459, + "epoch": 0.18370959317509142, + "flos": 74779160797440.0, + "grad_norm": 0.6561866426576449, + "language_loss": 0.50226259, + "learning_rate": 3.7572544087453325e-06, + "loss": 0.52252734, + "num_input_tokens_seen": 181424960, + "router_z_loss_clip": 0.1796875, + "router_z_loss_mlp": 0.00811768, + "step": 6331, + "time_per_iteration": 5.468651533126831 + }, + { + "auxiliary_loss_clip": 0.01024045, + "auxiliary_loss_mlp": 0.0100475, + "balance_loss_clip": 1.00600123, + "balance_loss_mlp": 1.00396907, + "epoch": 0.18373861064360744, + "flos": 70771315127040.0, + "grad_norm": 0.7291144915498192, + "language_loss": 0.49754077, + "learning_rate": 3.7571646476787474e-06, + "loss": 0.5178287, + "num_input_tokens_seen": 181487860, + "router_z_loss_clip": 0.18066406, + "router_z_loss_mlp": 0.0078125, + "step": 6332, + "time_per_iteration": 3.1603810787200928 + }, + { + "auxiliary_loss_clip": 0.01129716, + "auxiliary_loss_mlp": 0.01038153, + "balance_loss_clip": 1.05493999, + "balance_loss_mlp": 1.02077293, + "epoch": 0.1837676281121235, + "flos": 11686769049600.0, + "grad_norm": 2.688193859809901, + "language_loss": 0.73094487, + "learning_rate": 3.757074871092173e-06, + "loss": 0.75262356, + "num_input_tokens_seen": 181500145, + "router_z_loss_clip": 0.74853516, + "router_z_loss_mlp": 0.17382812, + "step": 6333, + "time_per_iteration": 2.5311193466186523 + }, + { + "auxiliary_loss_clip": 0.01023914, + "auxiliary_loss_mlp": 0.01000862, + "balance_loss_clip": 1.00574434, + "balance_loss_mlp": 1.00001609, + "epoch": 0.18379664558063954, + "flos": 62946235307520.0, + "grad_norm": 0.7539990330131122, + "language_loss": 0.46189833, + "learning_rate": 3.7569850789864017e-06, + "loss": 0.48214611, + "num_input_tokens_seen": 181551440, + "router_z_loss_clip": 0.18164062, + "router_z_loss_mlp": 0.00848389, + "step": 6334, + "time_per_iteration": 2.9214115142822266 + }, + { + "auxiliary_loss_clip": 0.01023207, + "auxiliary_loss_mlp": 0.010007, + "balance_loss_clip": 1.00514364, + "balance_loss_mlp": 0.99986571, + "epoch": 0.1838256630491556, + "flos": 67036999524480.0, + "grad_norm": 0.8560011314333781, + "language_loss": 0.47426465, + "learning_rate": 3.756895271362227e-06, + "loss": 0.49450374, + "num_input_tokens_seen": 181610960, + "router_z_loss_clip": 0.18066406, + "router_z_loss_mlp": 0.00836182, + "step": 6335, + "time_per_iteration": 3.033613920211792 + }, + { + "auxiliary_loss_clip": 0.01119843, + "auxiliary_loss_mlp": 0.01032407, + "balance_loss_clip": 1.05159903, + "balance_loss_mlp": 1.01902628, + "epoch": 0.18385468051767165, + "flos": 24528280527360.0, + "grad_norm": 2.5086726238240025, + "language_loss": 0.92122614, + "learning_rate": 3.7568054482204433e-06, + "loss": 0.94274867, + "num_input_tokens_seen": 181624170, + "router_z_loss_clip": 0.68164062, + "router_z_loss_mlp": 0.13378906, + "step": 6336, + "time_per_iteration": 2.623255729675293 + }, + { + "auxiliary_loss_clip": 0.01130909, + "auxiliary_loss_mlp": 0.01037284, + "balance_loss_clip": 1.05469656, + "balance_loss_mlp": 1.022645, + "epoch": 0.18388369798618767, + "flos": 33327149034240.0, + "grad_norm": 2.7569207826123234, + "language_loss": 1.00613976, + "learning_rate": 3.7567156095618426e-06, + "loss": 1.02782166, + "num_input_tokens_seen": 181644370, + "router_z_loss_clip": 0.76171875, + "router_z_loss_mlp": 0.14660645, + "step": 6337, + "time_per_iteration": 2.608022689819336 + }, + { + "auxiliary_loss_clip": 0.01133552, + "auxiliary_loss_mlp": 0.01045162, + "balance_loss_clip": 1.05747819, + "balance_loss_mlp": 1.02886605, + "epoch": 0.18391271545470372, + "flos": 31240954097280.0, + "grad_norm": 3.4278039477733118, + "language_loss": 0.79588789, + "learning_rate": 3.7566257553872182e-06, + "loss": 0.81767499, + "num_input_tokens_seen": 181659240, + "router_z_loss_clip": 0.76074219, + "router_z_loss_mlp": 0.16308594, + "step": 6338, + "time_per_iteration": 2.6462953090667725 + }, + { + "auxiliary_loss_clip": 0.01126152, + "auxiliary_loss_mlp": 0.01042956, + "balance_loss_clip": 1.0554347, + "balance_loss_mlp": 1.02667236, + "epoch": 0.18394173292321978, + "flos": 28360451756160.0, + "grad_norm": 2.079111375705109, + "language_loss": 1.03102529, + "learning_rate": 3.7565358856973648e-06, + "loss": 1.05271637, + "num_input_tokens_seen": 181677935, + "router_z_loss_clip": 0.70800781, + "router_z_loss_mlp": 0.16296387, + "step": 6339, + "time_per_iteration": 2.6179587841033936 + }, + { + "auxiliary_loss_clip": 0.01137332, + "auxiliary_loss_mlp": 0.01040714, + "balance_loss_clip": 1.05749547, + "balance_loss_mlp": 1.0234524, + "epoch": 0.18397075039173583, + "flos": 17158728988800.0, + "grad_norm": 2.6855136914183455, + "language_loss": 0.94969726, + "learning_rate": 3.7564460004930754e-06, + "loss": 0.97147775, + "num_input_tokens_seen": 181691315, + "router_z_loss_clip": 0.79882812, + "router_z_loss_mlp": 0.17260742, + "step": 6340, + "time_per_iteration": 2.5401384830474854 + }, + { + "auxiliary_loss_clip": 0.01136195, + "auxiliary_loss_mlp": 0.01041445, + "balance_loss_clip": 1.05696309, + "balance_loss_mlp": 1.02422488, + "epoch": 0.18399976786025188, + "flos": 28178025557760.0, + "grad_norm": 1.9963648033667143, + "language_loss": 0.82554477, + "learning_rate": 3.7563560997751447e-06, + "loss": 0.84732121, + "num_input_tokens_seen": 181708105, + "router_z_loss_clip": 0.79248047, + "router_z_loss_mlp": 0.17224121, + "step": 6341, + "time_per_iteration": 2.560976982116699 + }, + { + "auxiliary_loss_clip": 0.01133496, + "auxiliary_loss_mlp": 0.01039156, + "balance_loss_clip": 1.05939841, + "balance_loss_mlp": 1.0229677, + "epoch": 0.18402878532876793, + "flos": 14277328807680.0, + "grad_norm": 2.7083412320114495, + "language_loss": 0.77198541, + "learning_rate": 3.756266183544366e-06, + "loss": 0.7937119, + "num_input_tokens_seen": 181720545, + "router_z_loss_clip": 0.74121094, + "router_z_loss_mlp": 0.1619873, + "step": 6342, + "time_per_iteration": 2.5026028156280518 + }, + { + "auxiliary_loss_clip": 0.01133082, + "auxiliary_loss_mlp": 0.01045233, + "balance_loss_clip": 1.05929112, + "balance_loss_mlp": 1.02875829, + "epoch": 0.18405780279728395, + "flos": 20846216234880.0, + "grad_norm": 2.1020196837327925, + "language_loss": 0.90243214, + "learning_rate": 3.7561762518015334e-06, + "loss": 0.92421532, + "num_input_tokens_seen": 181735740, + "router_z_loss_clip": 0.73876953, + "router_z_loss_mlp": 0.16479492, + "step": 6343, + "time_per_iteration": 2.55963134765625 + }, + { + "auxiliary_loss_clip": 0.01139881, + "auxiliary_loss_mlp": 0.0103532, + "balance_loss_clip": 1.05979323, + "balance_loss_mlp": 1.01709318, + "epoch": 0.1840868202658, + "flos": 13762656783360.0, + "grad_norm": 3.224305202323561, + "language_loss": 0.95954758, + "learning_rate": 3.7560863045474414e-06, + "loss": 0.98129958, + "num_input_tokens_seen": 181745725, + "router_z_loss_clip": 0.80175781, + "router_z_loss_mlp": 0.18212891, + "step": 6344, + "time_per_iteration": 2.5102932453155518 + }, + { + "auxiliary_loss_clip": 0.01134457, + "auxiliary_loss_mlp": 0.01042895, + "balance_loss_clip": 1.05871558, + "balance_loss_mlp": 1.02623606, + "epoch": 0.18411583773431606, + "flos": 23216535751680.0, + "grad_norm": 2.2519063048359262, + "language_loss": 0.71517932, + "learning_rate": 3.755996341782885e-06, + "loss": 0.73695278, + "num_input_tokens_seen": 181764640, + "router_z_loss_clip": 0.75683594, + "router_z_loss_mlp": 0.16662598, + "step": 6345, + "time_per_iteration": 2.653963565826416 + }, + { + "auxiliary_loss_clip": 0.01125027, + "auxiliary_loss_mlp": 0.0103594, + "balance_loss_clip": 1.05475318, + "balance_loss_mlp": 1.02177155, + "epoch": 0.1841448552028321, + "flos": 27818344719360.0, + "grad_norm": 1.8784853256346037, + "language_loss": 0.75713313, + "learning_rate": 3.755906363508658e-06, + "loss": 0.77874279, + "num_input_tokens_seen": 181784705, + "router_z_loss_clip": 0.70263672, + "router_z_loss_mlp": 0.1418457, + "step": 6346, + "time_per_iteration": 2.608874559402466 + }, + { + "auxiliary_loss_clip": 0.01129317, + "auxiliary_loss_mlp": 0.01036892, + "balance_loss_clip": 1.05691147, + "balance_loss_mlp": 1.02174616, + "epoch": 0.18417387267134816, + "flos": 16246957132800.0, + "grad_norm": 2.3918937672120464, + "language_loss": 0.78766179, + "learning_rate": 3.7558163697255562e-06, + "loss": 0.80932391, + "num_input_tokens_seen": 181800060, + "router_z_loss_clip": 0.72460938, + "router_z_loss_mlp": 0.15155029, + "step": 6347, + "time_per_iteration": 2.4950501918792725 + }, + { + "auxiliary_loss_clip": 0.01128556, + "auxiliary_loss_mlp": 0.01047209, + "balance_loss_clip": 1.05788541, + "balance_loss_mlp": 1.03196263, + "epoch": 0.1842028901398642, + "flos": 33029951904000.0, + "grad_norm": 1.9402402105644188, + "language_loss": 0.67848587, + "learning_rate": 3.755726360434373e-06, + "loss": 0.70024347, + "num_input_tokens_seen": 181816035, + "router_z_loss_clip": 0.70654297, + "router_z_loss_mlp": 0.15258789, + "step": 6348, + "time_per_iteration": 2.6678121089935303 + }, + { + "auxiliary_loss_clip": 0.01132243, + "auxiliary_loss_mlp": 0.01044838, + "balance_loss_clip": 1.05389619, + "balance_loss_mlp": 1.02763617, + "epoch": 0.18423190760838024, + "flos": 27378905731200.0, + "grad_norm": 2.577719276595196, + "language_loss": 0.78507328, + "learning_rate": 3.755636335635904e-06, + "loss": 0.80684412, + "num_input_tokens_seen": 181836565, + "router_z_loss_clip": 0.78271484, + "router_z_loss_mlp": 0.17181396, + "step": 6349, + "time_per_iteration": 2.586122751235962 + }, + { + "auxiliary_loss_clip": 0.01128643, + "auxiliary_loss_mlp": 0.0103637, + "balance_loss_clip": 1.05404401, + "balance_loss_mlp": 1.02062225, + "epoch": 0.1842609250768963, + "flos": 27883952910720.0, + "grad_norm": 2.1269076544672303, + "language_loss": 0.74741459, + "learning_rate": 3.755546295330945e-06, + "loss": 0.76906472, + "num_input_tokens_seen": 181852210, + "router_z_loss_clip": 0.74707031, + "router_z_loss_mlp": 0.15740967, + "step": 6350, + "time_per_iteration": 2.62020206451416 + }, + { + "auxiliary_loss_clip": 0.01124416, + "auxiliary_loss_mlp": 0.01043799, + "balance_loss_clip": 1.05397117, + "balance_loss_mlp": 1.02855229, + "epoch": 0.18428994254541234, + "flos": 27301302397440.0, + "grad_norm": 2.056959211574652, + "language_loss": 0.78773642, + "learning_rate": 3.75545623952029e-06, + "loss": 0.8094185, + "num_input_tokens_seen": 181865785, + "router_z_loss_clip": 0.70507812, + "router_z_loss_mlp": 0.15246582, + "step": 6351, + "time_per_iteration": 2.62349271774292 + }, + { + "auxiliary_loss_clip": 0.01125838, + "auxiliary_loss_mlp": 0.01043031, + "balance_loss_clip": 1.05310953, + "balance_loss_mlp": 1.02769482, + "epoch": 0.1843189600139284, + "flos": 29563279516800.0, + "grad_norm": 2.1605621187380937, + "language_loss": 0.84747767, + "learning_rate": 3.7553661682047357e-06, + "loss": 0.86916637, + "num_input_tokens_seen": 181881200, + "router_z_loss_clip": 0.72607422, + "router_z_loss_mlp": 0.15350342, + "step": 6352, + "time_per_iteration": 2.6380555629730225 + }, + { + "auxiliary_loss_clip": 0.01030271, + "auxiliary_loss_mlp": 0.01008607, + "balance_loss_clip": 1.01195014, + "balance_loss_mlp": 1.00781465, + "epoch": 0.18434797748244444, + "flos": 74784547837440.0, + "grad_norm": 0.6238703936236252, + "language_loss": 0.46797851, + "learning_rate": 3.755276081385077e-06, + "loss": 0.48836726, + "num_input_tokens_seen": 181952110, + "router_z_loss_clip": 0.18359375, + "router_z_loss_mlp": 0.00793457, + "step": 6353, + "time_per_iteration": 3.2816343307495117 + }, + { + "auxiliary_loss_clip": 0.01029836, + "auxiliary_loss_mlp": 0.0100512, + "balance_loss_clip": 1.01157236, + "balance_loss_mlp": 1.00440431, + "epoch": 0.18437699495096047, + "flos": 64928397479040.0, + "grad_norm": 0.6998329250082913, + "language_loss": 0.46452391, + "learning_rate": 3.7551859790621092e-06, + "loss": 0.48487347, + "num_input_tokens_seen": 182017670, + "router_z_loss_clip": 0.18261719, + "router_z_loss_mlp": 0.00714111, + "step": 6354, + "time_per_iteration": 3.115999221801758 + }, + { + "auxiliary_loss_clip": 0.01031297, + "auxiliary_loss_mlp": 0.01001012, + "balance_loss_clip": 1.01290822, + "balance_loss_mlp": 1.00029683, + "epoch": 0.18440601241947652, + "flos": 74448387429120.0, + "grad_norm": 0.6595915390533804, + "language_loss": 0.49093175, + "learning_rate": 3.755095861236629e-06, + "loss": 0.51125485, + "num_input_tokens_seen": 182085255, + "router_z_loss_clip": 0.18359375, + "router_z_loss_mlp": 0.00714111, + "step": 6355, + "time_per_iteration": 3.1435494422912598 + }, + { + "auxiliary_loss_clip": 0.01122174, + "auxiliary_loss_mlp": 0.01039555, + "balance_loss_clip": 1.05104125, + "balance_loss_mlp": 1.02568531, + "epoch": 0.18443502988799257, + "flos": 29348498142720.0, + "grad_norm": 1.562050400112734, + "language_loss": 0.71690297, + "learning_rate": 3.755005727909432e-06, + "loss": 0.73852026, + "num_input_tokens_seen": 182104955, + "router_z_loss_clip": 0.71142578, + "router_z_loss_mlp": 0.1385498, + "step": 6356, + "time_per_iteration": 2.590618371963501 + }, + { + "auxiliary_loss_clip": 0.01120291, + "auxiliary_loss_mlp": 0.01050261, + "balance_loss_clip": 1.05015659, + "balance_loss_mlp": 1.03546751, + "epoch": 0.18446404735650862, + "flos": 26686691758080.0, + "grad_norm": 3.023950751401884, + "language_loss": 0.71663606, + "learning_rate": 3.7549155790813144e-06, + "loss": 0.73834157, + "num_input_tokens_seen": 182119450, + "router_z_loss_clip": 0.70117188, + "router_z_loss_mlp": 0.14794922, + "step": 6357, + "time_per_iteration": 2.5889265537261963 + }, + { + "auxiliary_loss_clip": 0.01126931, + "auxiliary_loss_mlp": 0.01044399, + "balance_loss_clip": 1.05398822, + "balance_loss_mlp": 1.02954531, + "epoch": 0.18449306482502467, + "flos": 12851710940160.0, + "grad_norm": 2.8633451666369742, + "language_loss": 0.77124256, + "learning_rate": 3.754825414753072e-06, + "loss": 0.79295582, + "num_input_tokens_seen": 182131300, + "router_z_loss_clip": 0.72900391, + "router_z_loss_mlp": 0.14849854, + "step": 6358, + "time_per_iteration": 2.4697766304016113 + }, + { + "auxiliary_loss_clip": 0.01114915, + "auxiliary_loss_mlp": 0.01034915, + "balance_loss_clip": 1.04904294, + "balance_loss_mlp": 1.02082479, + "epoch": 0.18452208229354072, + "flos": 19238782700160.0, + "grad_norm": 2.121894461975605, + "language_loss": 0.79189974, + "learning_rate": 3.754735234925501e-06, + "loss": 0.813398, + "num_input_tokens_seen": 182147885, + "router_z_loss_clip": 0.65869141, + "router_z_loss_mlp": 0.14086914, + "step": 6359, + "time_per_iteration": 2.513551950454712 + }, + { + "auxiliary_loss_clip": 0.01027768, + "auxiliary_loss_mlp": 0.01001173, + "balance_loss_clip": 1.00957966, + "balance_loss_mlp": 1.00042844, + "epoch": 0.18455109976205675, + "flos": 59776616396160.0, + "grad_norm": 0.6590170679187791, + "language_loss": 0.46791571, + "learning_rate": 3.754645039599399e-06, + "loss": 0.48820516, + "num_input_tokens_seen": 182205440, + "router_z_loss_clip": 0.18164062, + "router_z_loss_mlp": 0.00744629, + "step": 6360, + "time_per_iteration": 2.959609270095825 + }, + { + "auxiliary_loss_clip": 0.01129054, + "auxiliary_loss_mlp": 0.01035389, + "balance_loss_clip": 1.05327868, + "balance_loss_mlp": 1.01777601, + "epoch": 0.1845801172305728, + "flos": 55394541901440.0, + "grad_norm": 2.015194658082886, + "language_loss": 0.77357709, + "learning_rate": 3.754554828775562e-06, + "loss": 0.79522157, + "num_input_tokens_seen": 182224455, + "router_z_loss_clip": 0.75732422, + "router_z_loss_mlp": 0.17614746, + "step": 6361, + "time_per_iteration": 2.7983734607696533 + }, + { + "auxiliary_loss_clip": 0.01027054, + "auxiliary_loss_mlp": 0.00998593, + "balance_loss_clip": 1.00878429, + "balance_loss_mlp": 0.99781209, + "epoch": 0.18460913469908885, + "flos": 74784835146240.0, + "grad_norm": 0.646296278024722, + "language_loss": 0.47136796, + "learning_rate": 3.7544646024547863e-06, + "loss": 0.49162441, + "num_input_tokens_seen": 182291470, + "router_z_loss_clip": 0.18261719, + "router_z_loss_mlp": 0.00778198, + "step": 6362, + "time_per_iteration": 3.235365152359009 + }, + { + "auxiliary_loss_clip": 0.01113926, + "auxiliary_loss_mlp": 0.01038186, + "balance_loss_clip": 1.04662716, + "balance_loss_mlp": 1.02301049, + "epoch": 0.1846381521676049, + "flos": 20808438105600.0, + "grad_norm": 2.0615077766967738, + "language_loss": 0.79262412, + "learning_rate": 3.7543743606378698e-06, + "loss": 0.81414515, + "num_input_tokens_seen": 182306055, + "router_z_loss_clip": 0.67236328, + "router_z_loss_mlp": 0.15167236, + "step": 6363, + "time_per_iteration": 2.494483232498169 + }, + { + "auxiliary_loss_clip": 0.01026671, + "auxiliary_loss_mlp": 0.01002167, + "balance_loss_clip": 1.00862777, + "balance_loss_mlp": 1.0013926, + "epoch": 0.18466716963612095, + "flos": 64735771818240.0, + "grad_norm": 0.6012822396269129, + "language_loss": 0.46163291, + "learning_rate": 3.7542841033256086e-06, + "loss": 0.48192132, + "num_input_tokens_seen": 182365425, + "router_z_loss_clip": 0.1796875, + "router_z_loss_mlp": 0.00772095, + "step": 6364, + "time_per_iteration": 3.013317108154297 + }, + { + "auxiliary_loss_clip": 0.01121756, + "auxiliary_loss_mlp": 0.0104403, + "balance_loss_clip": 1.04943371, + "balance_loss_mlp": 1.02825236, + "epoch": 0.184696187104637, + "flos": 13187619953280.0, + "grad_norm": 2.3604791760695645, + "language_loss": 0.8177613, + "learning_rate": 3.7541938305188007e-06, + "loss": 0.83941919, + "num_input_tokens_seen": 182379450, + "router_z_loss_clip": 0.72363281, + "router_z_loss_mlp": 0.15765381, + "step": 6365, + "time_per_iteration": 2.515982151031494 + }, + { + "auxiliary_loss_clip": 0.01122237, + "auxiliary_loss_mlp": 0.01045441, + "balance_loss_clip": 1.04899395, + "balance_loss_mlp": 1.02961624, + "epoch": 0.18472520457315303, + "flos": 24353037048960.0, + "grad_norm": 2.4842185611403678, + "language_loss": 0.7076965, + "learning_rate": 3.7541035422182424e-06, + "loss": 0.72937328, + "num_input_tokens_seen": 182396590, + "router_z_loss_clip": 0.73291016, + "router_z_loss_mlp": 0.1583252, + "step": 6366, + "time_per_iteration": 2.6231958866119385 + }, + { + "auxiliary_loss_clip": 0.01124271, + "auxiliary_loss_mlp": 0.01041941, + "balance_loss_clip": 1.05066228, + "balance_loss_mlp": 1.0264914, + "epoch": 0.18475422204166908, + "flos": 29345409573120.0, + "grad_norm": 2.17362116661981, + "language_loss": 0.97476268, + "learning_rate": 3.7540132384247323e-06, + "loss": 0.99642479, + "num_input_tokens_seen": 182413650, + "router_z_loss_clip": 0.73486328, + "router_z_loss_mlp": 0.15454102, + "step": 6367, + "time_per_iteration": 2.671656370162964 + }, + { + "auxiliary_loss_clip": 0.01027652, + "auxiliary_loss_mlp": 0.01004405, + "balance_loss_clip": 1.00960422, + "balance_loss_mlp": 1.00353479, + "epoch": 0.18478323951018513, + "flos": 67889627637120.0, + "grad_norm": 0.6039690187715032, + "language_loss": 0.44817984, + "learning_rate": 3.753922919139067e-06, + "loss": 0.46850038, + "num_input_tokens_seen": 182481655, + "router_z_loss_clip": 0.18066406, + "router_z_loss_mlp": 0.00872803, + "step": 6368, + "time_per_iteration": 3.2278757095336914 + }, + { + "auxiliary_loss_clip": 0.01128864, + "auxiliary_loss_mlp": 0.01044169, + "balance_loss_clip": 1.05237782, + "balance_loss_mlp": 1.02725863, + "epoch": 0.18481225697870118, + "flos": 74731359795840.0, + "grad_norm": 2.624022821674026, + "language_loss": 0.70800811, + "learning_rate": 3.7538325843620456e-06, + "loss": 0.72973841, + "num_input_tokens_seen": 182502055, + "router_z_loss_clip": 0.76367188, + "router_z_loss_mlp": 0.16900635, + "step": 6369, + "time_per_iteration": 3.0489673614501953 + }, + { + "auxiliary_loss_clip": 0.01027916, + "auxiliary_loss_mlp": 0.0100044, + "balance_loss_clip": 1.00985122, + "balance_loss_mlp": 0.99945658, + "epoch": 0.18484127444721724, + "flos": 70707284156160.0, + "grad_norm": 0.7517732192752055, + "language_loss": 0.46469301, + "learning_rate": 3.7537422340944643e-06, + "loss": 0.48497659, + "num_input_tokens_seen": 182554665, + "router_z_loss_clip": 0.18066406, + "router_z_loss_mlp": 0.00982666, + "step": 6370, + "time_per_iteration": 3.018007516860962 + }, + { + "auxiliary_loss_clip": 0.01130304, + "auxiliary_loss_mlp": 0.01037923, + "balance_loss_clip": 1.05241442, + "balance_loss_mlp": 1.02161551, + "epoch": 0.18487029191573326, + "flos": 16546955523840.0, + "grad_norm": 2.2531917711640586, + "language_loss": 0.67834228, + "learning_rate": 3.7536518683371226e-06, + "loss": 0.70002455, + "num_input_tokens_seen": 182572315, + "router_z_loss_clip": 0.77783203, + "router_z_loss_mlp": 0.16308594, + "step": 6371, + "time_per_iteration": 2.733975887298584 + }, + { + "auxiliary_loss_clip": 0.01123205, + "auxiliary_loss_mlp": 0.01040709, + "balance_loss_clip": 1.04875994, + "balance_loss_mlp": 1.0242703, + "epoch": 0.1848993093842493, + "flos": 23800299586560.0, + "grad_norm": 2.306303600646984, + "language_loss": 0.70046544, + "learning_rate": 3.7535614870908177e-06, + "loss": 0.72210455, + "num_input_tokens_seen": 182593320, + "router_z_loss_clip": 0.74462891, + "router_z_loss_mlp": 0.16442871, + "step": 6372, + "time_per_iteration": 2.6756770610809326 + }, + { + "auxiliary_loss_clip": 0.01126131, + "auxiliary_loss_mlp": 0.01041987, + "balance_loss_clip": 1.05279827, + "balance_loss_mlp": 1.02563119, + "epoch": 0.18492832685276536, + "flos": 41129818767360.0, + "grad_norm": 2.434920440661761, + "language_loss": 0.80156267, + "learning_rate": 3.753471090356348e-06, + "loss": 0.82324386, + "num_input_tokens_seen": 182613910, + "router_z_loss_clip": 0.73193359, + "router_z_loss_mlp": 0.16351318, + "step": 6373, + "time_per_iteration": 2.7105767726898193 + }, + { + "auxiliary_loss_clip": 0.01127683, + "auxiliary_loss_mlp": 0.01046887, + "balance_loss_clip": 1.04986906, + "balance_loss_mlp": 1.02951241, + "epoch": 0.18495734432128141, + "flos": 34526349521280.0, + "grad_norm": 2.5354953957155666, + "language_loss": 0.7853452, + "learning_rate": 3.753380678134512e-06, + "loss": 0.80709088, + "num_input_tokens_seen": 182631700, + "router_z_loss_clip": 0.77783203, + "router_z_loss_mlp": 0.17376709, + "step": 6374, + "time_per_iteration": 2.6377735137939453 + }, + { + "auxiliary_loss_clip": 0.01119891, + "auxiliary_loss_mlp": 0.01035037, + "balance_loss_clip": 1.04904342, + "balance_loss_mlp": 1.02078593, + "epoch": 0.18498636178979747, + "flos": 26352219288960.0, + "grad_norm": 2.1051077524703916, + "language_loss": 0.78347051, + "learning_rate": 3.753290250426109e-06, + "loss": 0.80501986, + "num_input_tokens_seen": 182647175, + "router_z_loss_clip": 0.70898438, + "router_z_loss_mlp": 0.14245605, + "step": 6375, + "time_per_iteration": 2.637014865875244 + }, + { + "auxiliary_loss_clip": 0.0113257, + "auxiliary_loss_mlp": 0.01040196, + "balance_loss_clip": 1.05256832, + "balance_loss_mlp": 1.02336967, + "epoch": 0.18501537925831352, + "flos": 21790307352960.0, + "grad_norm": 2.1509599866278006, + "language_loss": 0.71671832, + "learning_rate": 3.753199807231936e-06, + "loss": 0.73844594, + "num_input_tokens_seen": 182661845, + "router_z_loss_clip": 0.79980469, + "router_z_loss_mlp": 0.16827393, + "step": 6376, + "time_per_iteration": 2.5339903831481934 + }, + { + "auxiliary_loss_clip": 0.01121827, + "auxiliary_loss_mlp": 0.01040266, + "balance_loss_clip": 1.04905605, + "balance_loss_mlp": 1.02611578, + "epoch": 0.18504439672682954, + "flos": 45617503248000.0, + "grad_norm": 2.8634097694340164, + "language_loss": 0.75813186, + "learning_rate": 3.7531093485527938e-06, + "loss": 0.77975273, + "num_input_tokens_seen": 182679175, + "router_z_loss_clip": 0.7277832, + "router_z_loss_mlp": 0.14154053, + "step": 6377, + "time_per_iteration": 2.7104039192199707 + }, + { + "auxiliary_loss_clip": 0.01030762, + "auxiliary_loss_mlp": 0.0100107, + "balance_loss_clip": 1.01243734, + "balance_loss_mlp": 1.00012827, + "epoch": 0.1850734141953456, + "flos": 63494124433920.0, + "grad_norm": 0.7076351856785112, + "language_loss": 0.48944408, + "learning_rate": 3.7530188743894797e-06, + "loss": 0.50976241, + "num_input_tokens_seen": 182736030, + "router_z_loss_clip": 0.18359375, + "router_z_loss_mlp": 0.00939941, + "step": 6378, + "time_per_iteration": 2.960847854614258 + }, + { + "auxiliary_loss_clip": 0.01119444, + "auxiliary_loss_mlp": 0.01038112, + "balance_loss_clip": 1.04953337, + "balance_loss_mlp": 1.02383697, + "epoch": 0.18510243166386164, + "flos": 44813175949440.0, + "grad_norm": 1.8964075987416003, + "language_loss": 0.79121208, + "learning_rate": 3.752928384742794e-06, + "loss": 0.81278771, + "num_input_tokens_seen": 182754860, + "router_z_loss_clip": 0.69921875, + "router_z_loss_mlp": 0.14263916, + "step": 6379, + "time_per_iteration": 2.6174685955047607 + }, + { + "auxiliary_loss_clip": 0.01122771, + "auxiliary_loss_mlp": 0.01033863, + "balance_loss_clip": 1.05281234, + "balance_loss_mlp": 1.01904535, + "epoch": 0.1851314491323777, + "flos": 74733658266240.0, + "grad_norm": 2.1355764276010536, + "language_loss": 0.66452217, + "learning_rate": 3.7528378796135354e-06, + "loss": 0.6860885, + "num_input_tokens_seen": 182779185, + "router_z_loss_clip": 0.70019531, + "router_z_loss_mlp": 0.14825439, + "step": 6380, + "time_per_iteration": 2.9305317401885986 + }, + { + "auxiliary_loss_clip": 0.01127562, + "auxiliary_loss_mlp": 0.01035857, + "balance_loss_clip": 1.05348742, + "balance_loss_mlp": 1.02127767, + "epoch": 0.18516046660089375, + "flos": 10956597379200.0, + "grad_norm": 2.738840556647191, + "language_loss": 0.76006293, + "learning_rate": 3.7527473590025034e-06, + "loss": 0.78169703, + "num_input_tokens_seen": 182791460, + "router_z_loss_clip": 0.73925781, + "router_z_loss_mlp": 0.14562988, + "step": 6381, + "time_per_iteration": 2.4801924228668213 + }, + { + "auxiliary_loss_clip": 0.01127539, + "auxiliary_loss_mlp": 0.01047085, + "balance_loss_clip": 1.05115581, + "balance_loss_mlp": 1.02887595, + "epoch": 0.18518948406940977, + "flos": 12013017304320.0, + "grad_norm": 3.990971189514512, + "language_loss": 0.82725126, + "learning_rate": 3.752656822910497e-06, + "loss": 0.84899747, + "num_input_tokens_seen": 182802510, + "router_z_loss_clip": 0.76367188, + "router_z_loss_mlp": 0.18218994, + "step": 6382, + "time_per_iteration": 2.491818428039551 + }, + { + "auxiliary_loss_clip": 0.01122159, + "auxiliary_loss_mlp": 0.01043454, + "balance_loss_clip": 1.04881251, + "balance_loss_mlp": 1.02644897, + "epoch": 0.18521850153792582, + "flos": 50398542103680.0, + "grad_norm": 2.659461769979293, + "language_loss": 0.93684292, + "learning_rate": 3.752566271338317e-06, + "loss": 0.95849907, + "num_input_tokens_seen": 182825130, + "router_z_loss_clip": 0.73291016, + "router_z_loss_mlp": 0.16998291, + "step": 6383, + "time_per_iteration": 2.8110151290893555 + }, + { + "auxiliary_loss_clip": 0.01125679, + "auxiliary_loss_mlp": 0.01040113, + "balance_loss_clip": 1.05234075, + "balance_loss_mlp": 1.02501535, + "epoch": 0.18524751900644187, + "flos": 13837961646720.0, + "grad_norm": 2.223884928427601, + "language_loss": 0.75518197, + "learning_rate": 3.7524757042867618e-06, + "loss": 0.77683985, + "num_input_tokens_seen": 182839605, + "router_z_loss_clip": 0.73388672, + "router_z_loss_mlp": 0.15112305, + "step": 6384, + "time_per_iteration": 2.6175661087036133 + }, + { + "auxiliary_loss_clip": 0.01125273, + "auxiliary_loss_mlp": 0.01044935, + "balance_loss_clip": 1.04972637, + "balance_loss_mlp": 1.02880561, + "epoch": 0.18527653647495793, + "flos": 12485888876160.0, + "grad_norm": 2.4811683686551382, + "language_loss": 0.76785266, + "learning_rate": 3.7523851217566325e-06, + "loss": 0.78955472, + "num_input_tokens_seen": 182850825, + "router_z_loss_clip": 0.75585938, + "router_z_loss_mlp": 0.16137695, + "step": 6385, + "time_per_iteration": 2.5080983638763428 + }, + { + "auxiliary_loss_clip": 0.01114183, + "auxiliary_loss_mlp": 0.01034478, + "balance_loss_clip": 1.04749024, + "balance_loss_mlp": 1.02067351, + "epoch": 0.18530555394347398, + "flos": 10299360274560.0, + "grad_norm": 2.052640161422278, + "language_loss": 0.66922212, + "learning_rate": 3.7522945237487286e-06, + "loss": 0.69070882, + "num_input_tokens_seen": 182861690, + "router_z_loss_clip": 0.66650391, + "router_z_loss_mlp": 0.13793945, + "step": 6386, + "time_per_iteration": 2.516023874282837 + }, + { + "auxiliary_loss_clip": 0.01120893, + "auxiliary_loss_mlp": 0.01043592, + "balance_loss_clip": 1.0480113, + "balance_loss_mlp": 1.02801776, + "epoch": 0.18533457141199003, + "flos": 16718859037440.0, + "grad_norm": 2.7273986425241445, + "language_loss": 0.82513183, + "learning_rate": 3.7522039102638506e-06, + "loss": 0.84677672, + "num_input_tokens_seen": 182873830, + "router_z_loss_clip": 0.73046875, + "router_z_loss_mlp": 0.15570068, + "step": 6387, + "time_per_iteration": 2.4533398151397705 + }, + { + "auxiliary_loss_clip": 0.01138391, + "auxiliary_loss_mlp": 0.01058211, + "balance_loss_clip": 1.05789137, + "balance_loss_mlp": 1.04215348, + "epoch": 0.18536358888050605, + "flos": 20041422059520.0, + "grad_norm": 2.5442591361127147, + "language_loss": 1.02285981, + "learning_rate": 3.7521132813027984e-06, + "loss": 1.04482591, + "num_input_tokens_seen": 182890515, + "router_z_loss_clip": 0.80517578, + "router_z_loss_mlp": 0.16052246, + "step": 6388, + "time_per_iteration": 2.5227572917938232 + }, + { + "auxiliary_loss_clip": 0.01118802, + "auxiliary_loss_mlp": 0.01040216, + "balance_loss_clip": 1.04895556, + "balance_loss_mlp": 1.02467728, + "epoch": 0.1853926063490221, + "flos": 74736746835840.0, + "grad_norm": 1.810408868477032, + "language_loss": 0.83322257, + "learning_rate": 3.752022636866372e-06, + "loss": 0.85481274, + "num_input_tokens_seen": 182914930, + "router_z_loss_clip": 0.69824219, + "router_z_loss_mlp": 0.15533447, + "step": 6389, + "time_per_iteration": 2.9241995811462402 + }, + { + "auxiliary_loss_clip": 0.01125636, + "auxiliary_loss_mlp": 0.01042635, + "balance_loss_clip": 1.05166686, + "balance_loss_mlp": 1.02796674, + "epoch": 0.18542162381753816, + "flos": 41385179531520.0, + "grad_norm": 4.0490312553574075, + "language_loss": 0.73920554, + "learning_rate": 3.7519319769553735e-06, + "loss": 0.76088828, + "num_input_tokens_seen": 182935525, + "router_z_loss_clip": 0.74023438, + "router_z_loss_mlp": 0.14660645, + "step": 6390, + "time_per_iteration": 2.694378137588501 + }, + { + "auxiliary_loss_clip": 0.01039981, + "auxiliary_loss_mlp": 0.01015048, + "balance_loss_clip": 1.02118349, + "balance_loss_mlp": 1.01412964, + "epoch": 0.1854506412860542, + "flos": 64015799610240.0, + "grad_norm": 0.6876373713273235, + "language_loss": 0.47641623, + "learning_rate": 3.751841301570603e-06, + "loss": 0.49696657, + "num_input_tokens_seen": 182991195, + "router_z_loss_clip": 0.1875, + "router_z_loss_mlp": 0.00915527, + "step": 6391, + "time_per_iteration": 3.068136692047119 + }, + { + "auxiliary_loss_clip": 0.01120753, + "auxiliary_loss_mlp": 0.01043723, + "balance_loss_clip": 1.05020344, + "balance_loss_mlp": 1.02764797, + "epoch": 0.18547965875457026, + "flos": 29272439093760.0, + "grad_norm": 2.271336483177184, + "language_loss": 0.75990725, + "learning_rate": 3.751750610712861e-06, + "loss": 0.78155208, + "num_input_tokens_seen": 183007285, + "router_z_loss_clip": 0.70605469, + "router_z_loss_mlp": 0.16082764, + "step": 6392, + "time_per_iteration": 2.568236827850342 + }, + { + "auxiliary_loss_clip": 0.01127421, + "auxiliary_loss_mlp": 0.01045558, + "balance_loss_clip": 1.05222178, + "balance_loss_mlp": 1.02922678, + "epoch": 0.1855086762230863, + "flos": 16100513383680.0, + "grad_norm": 2.3341632477619987, + "language_loss": 0.78793126, + "learning_rate": 3.7516599043829485e-06, + "loss": 0.80966103, + "num_input_tokens_seen": 183023425, + "router_z_loss_clip": 0.75244141, + "router_z_loss_mlp": 0.16308594, + "step": 6393, + "time_per_iteration": 2.514191150665283 + }, + { + "auxiliary_loss_clip": 0.01126242, + "auxiliary_loss_mlp": 0.01043036, + "balance_loss_clip": 1.05240703, + "balance_loss_mlp": 1.02896905, + "epoch": 0.18553769369160233, + "flos": 13435546602240.0, + "grad_norm": 2.19409658899077, + "language_loss": 0.69999689, + "learning_rate": 3.751569182581667e-06, + "loss": 0.7216897, + "num_input_tokens_seen": 183033950, + "router_z_loss_clip": 0.73876953, + "router_z_loss_mlp": 0.14068604, + "step": 6394, + "time_per_iteration": 2.453932762145996 + }, + { + "auxiliary_loss_clip": 0.01038579, + "auxiliary_loss_mlp": 0.01000911, + "balance_loss_clip": 1.01961088, + "balance_loss_mlp": 0.99998677, + "epoch": 0.18556671116011839, + "flos": 74767057914240.0, + "grad_norm": 0.6383290449790461, + "language_loss": 0.47091201, + "learning_rate": 3.751478445309818e-06, + "loss": 0.4913069, + "num_input_tokens_seen": 183094520, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.00921631, + "step": 6395, + "time_per_iteration": 3.2382071018218994 + }, + { + "auxiliary_loss_clip": 0.01119935, + "auxiliary_loss_mlp": 0.01050253, + "balance_loss_clip": 1.04957712, + "balance_loss_mlp": 1.0354594, + "epoch": 0.18559572862863444, + "flos": 25877228814720.0, + "grad_norm": 2.101677791179285, + "language_loss": 0.76359153, + "learning_rate": 3.751387692568202e-06, + "loss": 0.78529346, + "num_input_tokens_seen": 183108615, + "router_z_loss_clip": 0.70361328, + "router_z_loss_mlp": 0.14807129, + "step": 6396, + "time_per_iteration": 2.5995383262634277 + }, + { + "auxiliary_loss_clip": 0.0103872, + "auxiliary_loss_mlp": 0.00999525, + "balance_loss_clip": 1.01986885, + "balance_loss_mlp": 0.99848157, + "epoch": 0.1856247460971505, + "flos": 66166704898560.0, + "grad_norm": 0.6318263451880232, + "language_loss": 0.45557699, + "learning_rate": 3.7512969243576222e-06, + "loss": 0.47595942, + "num_input_tokens_seen": 183171230, + "router_z_loss_clip": 0.18847656, + "router_z_loss_mlp": 0.01043701, + "step": 6397, + "time_per_iteration": 3.134788990020752 + }, + { + "auxiliary_loss_clip": 0.01037702, + "auxiliary_loss_mlp": 0.00998702, + "balance_loss_clip": 1.01866794, + "balance_loss_mlp": 0.99767733, + "epoch": 0.18565376356566654, + "flos": 63571763681280.0, + "grad_norm": 0.6476139809880518, + "language_loss": 0.48682413, + "learning_rate": 3.7512061406788783e-06, + "loss": 0.5071882, + "num_input_tokens_seen": 183228935, + "router_z_loss_clip": 0.19042969, + "router_z_loss_mlp": 0.01025391, + "step": 6398, + "time_per_iteration": 3.0101308822631836 + }, + { + "auxiliary_loss_clip": 0.01037267, + "auxiliary_loss_mlp": 0.01001864, + "balance_loss_clip": 1.01831388, + "balance_loss_mlp": 1.00092208, + "epoch": 0.18568278103418256, + "flos": 57436569066240.0, + "grad_norm": 0.6678616025403413, + "language_loss": 0.48375061, + "learning_rate": 3.751115341532774e-06, + "loss": 0.50414193, + "num_input_tokens_seen": 183283170, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.00939941, + "step": 6399, + "time_per_iteration": 2.8881754875183105 + }, + { + "auxiliary_loss_clip": 0.01123468, + "auxiliary_loss_mlp": 0.010409, + "balance_loss_clip": 1.04938674, + "balance_loss_mlp": 1.02501547, + "epoch": 0.18571179850269862, + "flos": 16062771168000.0, + "grad_norm": 3.937373681950306, + "language_loss": 0.71956992, + "learning_rate": 3.7510245269201094e-06, + "loss": 0.74121362, + "num_input_tokens_seen": 183296055, + "router_z_loss_clip": 0.74169922, + "router_z_loss_mlp": 0.15887451, + "step": 6400, + "time_per_iteration": 4.734126806259155 + }, + { + "auxiliary_loss_clip": 0.01116117, + "auxiliary_loss_mlp": 0.01033764, + "balance_loss_clip": 1.04723847, + "balance_loss_mlp": 1.01984596, + "epoch": 0.18574081597121467, + "flos": 20111591278080.0, + "grad_norm": 4.88756567857266, + "language_loss": 0.74182224, + "learning_rate": 3.750933696841688e-06, + "loss": 0.76332104, + "num_input_tokens_seen": 183308225, + "router_z_loss_clip": 0.68896484, + "router_z_loss_mlp": 0.13928223, + "step": 6401, + "time_per_iteration": 7.611396312713623 + }, + { + "auxiliary_loss_clip": 0.01124924, + "auxiliary_loss_mlp": 0.01043639, + "balance_loss_clip": 1.05032301, + "balance_loss_mlp": 1.02890468, + "epoch": 0.18576983343973072, + "flos": 30478786387200.0, + "grad_norm": 2.9655778264879142, + "language_loss": 0.89941955, + "learning_rate": 3.750842851298312e-06, + "loss": 0.92110515, + "num_input_tokens_seen": 183323355, + "router_z_loss_clip": 0.74658203, + "router_z_loss_mlp": 0.1473999, + "step": 6402, + "time_per_iteration": 4.953094005584717 + }, + { + "auxiliary_loss_clip": 0.01127643, + "auxiliary_loss_mlp": 0.01048104, + "balance_loss_clip": 1.0535996, + "balance_loss_mlp": 1.0338521, + "epoch": 0.18579885090824677, + "flos": 25149140133120.0, + "grad_norm": 2.2428205909862444, + "language_loss": 0.87017655, + "learning_rate": 3.7507519902907833e-06, + "loss": 0.89193404, + "num_input_tokens_seen": 183338615, + "router_z_loss_clip": 0.73974609, + "router_z_loss_mlp": 0.1427002, + "step": 6403, + "time_per_iteration": 2.5492103099823 + }, + { + "auxiliary_loss_clip": 0.01126508, + "auxiliary_loss_mlp": 0.0104641, + "balance_loss_clip": 1.05102968, + "balance_loss_mlp": 1.02951169, + "epoch": 0.18582786837676282, + "flos": 11538637361280.0, + "grad_norm": 3.1304187967454458, + "language_loss": 0.92357779, + "learning_rate": 3.7506611138199044e-06, + "loss": 0.94530702, + "num_input_tokens_seen": 183350115, + "router_z_loss_clip": 0.75439453, + "router_z_loss_mlp": 0.16906738, + "step": 6404, + "time_per_iteration": 2.5307815074920654 + }, + { + "auxiliary_loss_clip": 0.01123822, + "auxiliary_loss_mlp": 0.01044689, + "balance_loss_clip": 1.05007362, + "balance_loss_mlp": 1.02865529, + "epoch": 0.18585688584527885, + "flos": 12416006966400.0, + "grad_norm": 2.8004831601356415, + "language_loss": 1.07381034, + "learning_rate": 3.7505702218864784e-06, + "loss": 1.09549546, + "num_input_tokens_seen": 183361080, + "router_z_loss_clip": 0.73779297, + "router_z_loss_mlp": 0.16027832, + "step": 6405, + "time_per_iteration": 2.480041265487671 + }, + { + "auxiliary_loss_clip": 0.01126738, + "auxiliary_loss_mlp": 0.01049433, + "balance_loss_clip": 1.05227888, + "balance_loss_mlp": 1.03337598, + "epoch": 0.1858859033137949, + "flos": 35038076630400.0, + "grad_norm": 2.252892763975003, + "language_loss": 0.95635587, + "learning_rate": 3.750479314491308e-06, + "loss": 0.97811753, + "num_input_tokens_seen": 183384035, + "router_z_loss_clip": 0.74511719, + "router_z_loss_mlp": 0.16040039, + "step": 6406, + "time_per_iteration": 2.6906683444976807 + }, + { + "auxiliary_loss_clip": 0.01048502, + "auxiliary_loss_mlp": 0.01016379, + "balance_loss_clip": 1.02921653, + "balance_loss_mlp": 1.01531184, + "epoch": 0.18591492078231095, + "flos": 66520783215360.0, + "grad_norm": 0.61767610714731, + "language_loss": 0.43903282, + "learning_rate": 3.750388391635195e-06, + "loss": 0.45968163, + "num_input_tokens_seen": 183441240, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.01068115, + "step": 6407, + "time_per_iteration": 3.126268148422241 + }, + { + "auxiliary_loss_clip": 0.01128332, + "auxiliary_loss_mlp": 0.01040393, + "balance_loss_clip": 1.05445206, + "balance_loss_mlp": 1.02479422, + "epoch": 0.185943938250827, + "flos": 11789365271040.0, + "grad_norm": 4.61155449679527, + "language_loss": 0.95076913, + "learning_rate": 3.750297453318944e-06, + "loss": 0.9724564, + "num_input_tokens_seen": 183449125, + "router_z_loss_clip": 0.73779297, + "router_z_loss_mlp": 0.15588379, + "step": 6408, + "time_per_iteration": 2.500558376312256 + }, + { + "auxiliary_loss_clip": 0.01124798, + "auxiliary_loss_mlp": 0.01042801, + "balance_loss_clip": 1.05212116, + "balance_loss_mlp": 1.02728546, + "epoch": 0.18597295571934305, + "flos": 16355945975040.0, + "grad_norm": 2.273119324629515, + "language_loss": 0.83797848, + "learning_rate": 3.750206499543358e-06, + "loss": 0.85965449, + "num_input_tokens_seen": 183461685, + "router_z_loss_clip": 0.72753906, + "router_z_loss_mlp": 0.15527344, + "step": 6409, + "time_per_iteration": 2.4927635192871094 + }, + { + "auxiliary_loss_clip": 0.01054575, + "auxiliary_loss_mlp": 0.0100336, + "balance_loss_clip": 1.03546667, + "balance_loss_mlp": 1.00204873, + "epoch": 0.1860019731878591, + "flos": 62441834572800.0, + "grad_norm": 0.6467278230860118, + "language_loss": 0.50348628, + "learning_rate": 3.750115530309239e-06, + "loss": 0.52406561, + "num_input_tokens_seen": 183522720, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.01312256, + "step": 6410, + "time_per_iteration": 3.056727409362793 + }, + { + "auxiliary_loss_clip": 0.01123927, + "auxiliary_loss_mlp": 0.01033464, + "balance_loss_clip": 1.05110383, + "balance_loss_mlp": 1.01901031, + "epoch": 0.18603099065637513, + "flos": 18217375557120.0, + "grad_norm": 2.6853619842965504, + "language_loss": 0.9267261, + "learning_rate": 3.7500245456173927e-06, + "loss": 0.9483, + "num_input_tokens_seen": 183534625, + "router_z_loss_clip": 0.7265625, + "router_z_loss_mlp": 0.14465332, + "step": 6411, + "time_per_iteration": 2.4847183227539062 + }, + { + "auxiliary_loss_clip": 0.01124267, + "auxiliary_loss_mlp": 0.01036183, + "balance_loss_clip": 1.05117822, + "balance_loss_mlp": 1.02078068, + "epoch": 0.18606000812489118, + "flos": 14714505239040.0, + "grad_norm": 2.672361943577674, + "language_loss": 0.73076713, + "learning_rate": 3.749933545468621e-06, + "loss": 0.75237167, + "num_input_tokens_seen": 183547345, + "router_z_loss_clip": 0.73193359, + "router_z_loss_mlp": 0.15411377, + "step": 6412, + "time_per_iteration": 2.52817440032959 + }, + { + "auxiliary_loss_clip": 0.01129466, + "auxiliary_loss_mlp": 0.01043922, + "balance_loss_clip": 1.05336607, + "balance_loss_mlp": 1.02797151, + "epoch": 0.18608902559340723, + "flos": 26862833076480.0, + "grad_norm": 2.73553261917718, + "language_loss": 0.92651367, + "learning_rate": 3.7498425298637276e-06, + "loss": 0.94824761, + "num_input_tokens_seen": 183561495, + "router_z_loss_clip": 0.76123047, + "router_z_loss_mlp": 0.15960693, + "step": 6413, + "time_per_iteration": 2.609746217727661 + }, + { + "auxiliary_loss_clip": 0.01115308, + "auxiliary_loss_mlp": 0.01032152, + "balance_loss_clip": 1.04849494, + "balance_loss_mlp": 1.01922393, + "epoch": 0.18611804306192328, + "flos": 22632376867200.0, + "grad_norm": 1.9899042879824835, + "language_loss": 0.69488829, + "learning_rate": 3.749751498803517e-06, + "loss": 0.71636289, + "num_input_tokens_seen": 183577220, + "router_z_loss_clip": 0.66796875, + "router_z_loss_mlp": 0.12945557, + "step": 6414, + "time_per_iteration": 2.497690200805664 + }, + { + "auxiliary_loss_clip": 0.01130373, + "auxiliary_loss_mlp": 0.01039533, + "balance_loss_clip": 1.05554605, + "balance_loss_mlp": 1.02368379, + "epoch": 0.18614706053043933, + "flos": 28543991276160.0, + "grad_norm": 7.132186228628395, + "language_loss": 0.8476997, + "learning_rate": 3.7496604522887933e-06, + "loss": 0.86939871, + "num_input_tokens_seen": 183593415, + "router_z_loss_clip": 0.74902344, + "router_z_loss_mlp": 0.15844727, + "step": 6415, + "time_per_iteration": 2.5839738845825195 + }, + { + "auxiliary_loss_clip": 0.01131946, + "auxiliary_loss_mlp": 0.01046845, + "balance_loss_clip": 1.05395627, + "balance_loss_mlp": 1.02925563, + "epoch": 0.18617607799895536, + "flos": 56928250771200.0, + "grad_norm": 2.061792618951138, + "language_loss": 1.04821301, + "learning_rate": 3.7495693903203603e-06, + "loss": 1.07000089, + "num_input_tokens_seen": 183618400, + "router_z_loss_clip": 0.77929688, + "router_z_loss_mlp": 0.17602539, + "step": 6416, + "time_per_iteration": 2.8811991214752197 + }, + { + "auxiliary_loss_clip": 0.01128424, + "auxiliary_loss_mlp": 0.01038739, + "balance_loss_clip": 1.05319333, + "balance_loss_mlp": 1.02278841, + "epoch": 0.1862050954674714, + "flos": 36205388818560.0, + "grad_norm": 1.6496260859574483, + "language_loss": 0.79393077, + "learning_rate": 3.749478312899023e-06, + "loss": 0.81560242, + "num_input_tokens_seen": 183638995, + "router_z_loss_clip": 0.75244141, + "router_z_loss_mlp": 0.15960693, + "step": 6417, + "time_per_iteration": 2.6174843311309814 + }, + { + "auxiliary_loss_clip": 0.01122172, + "auxiliary_loss_mlp": 0.01038902, + "balance_loss_clip": 1.05160141, + "balance_loss_mlp": 1.02435243, + "epoch": 0.18623411293598746, + "flos": 56890867691520.0, + "grad_norm": 1.9898414778885525, + "language_loss": 0.66236311, + "learning_rate": 3.749387220025585e-06, + "loss": 0.68397385, + "num_input_tokens_seen": 183658755, + "router_z_loss_clip": 0.70605469, + "router_z_loss_mlp": 0.14532471, + "step": 6418, + "time_per_iteration": 2.82855224609375 + }, + { + "auxiliary_loss_clip": 0.01055763, + "auxiliary_loss_mlp": 0.01007749, + "balance_loss_clip": 1.03678536, + "balance_loss_mlp": 1.00635433, + "epoch": 0.1862631304045035, + "flos": 64890763004160.0, + "grad_norm": 0.6353771133603151, + "language_loss": 0.45558339, + "learning_rate": 3.749296111700851e-06, + "loss": 0.47621846, + "num_input_tokens_seen": 183721035, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.01397705, + "step": 6419, + "time_per_iteration": 3.058304786682129 + }, + { + "auxiliary_loss_clip": 0.01119425, + "auxiliary_loss_mlp": 0.01035996, + "balance_loss_clip": 1.05064917, + "balance_loss_mlp": 1.01996839, + "epoch": 0.18629214787301956, + "flos": 23434764831360.0, + "grad_norm": 2.0064107004672396, + "language_loss": 0.6836338, + "learning_rate": 3.7492049879256258e-06, + "loss": 0.70518804, + "num_input_tokens_seen": 183738425, + "router_z_loss_clip": 0.68798828, + "router_z_loss_mlp": 0.16015625, + "step": 6420, + "time_per_iteration": 2.4881949424743652 + }, + { + "auxiliary_loss_clip": 0.01134424, + "auxiliary_loss_mlp": 0.01037382, + "balance_loss_clip": 1.05728948, + "balance_loss_mlp": 1.02161658, + "epoch": 0.18632116534153562, + "flos": 19711151481600.0, + "grad_norm": 2.6585813772456994, + "language_loss": 1.11835945, + "learning_rate": 3.7491138487007144e-06, + "loss": 1.14007747, + "num_input_tokens_seen": 183752490, + "router_z_loss_clip": 0.77294922, + "router_z_loss_mlp": 0.15765381, + "step": 6421, + "time_per_iteration": 2.5204808712005615 + }, + { + "auxiliary_loss_clip": 0.011258, + "auxiliary_loss_mlp": 0.01037303, + "balance_loss_clip": 1.05478966, + "balance_loss_mlp": 1.02284932, + "epoch": 0.18635018281005164, + "flos": 15700432723200.0, + "grad_norm": 1.9560392526109491, + "language_loss": 0.6468606, + "learning_rate": 3.749022694026922e-06, + "loss": 0.6684916, + "num_input_tokens_seen": 183767930, + "router_z_loss_clip": 0.71044922, + "router_z_loss_mlp": 0.14453125, + "step": 6422, + "time_per_iteration": 2.448981761932373 + }, + { + "auxiliary_loss_clip": 0.01121068, + "auxiliary_loss_mlp": 0.01036417, + "balance_loss_clip": 1.05418181, + "balance_loss_mlp": 1.02270246, + "epoch": 0.1863792002785677, + "flos": 17267789658240.0, + "grad_norm": 2.6171913964750426, + "language_loss": 0.76300585, + "learning_rate": 3.7489315239050533e-06, + "loss": 0.78458071, + "num_input_tokens_seen": 183780460, + "router_z_loss_clip": 0.66894531, + "router_z_loss_mlp": 0.13726807, + "step": 6423, + "time_per_iteration": 2.5107314586639404 + }, + { + "auxiliary_loss_clip": 0.01132477, + "auxiliary_loss_mlp": 0.01046534, + "balance_loss_clip": 1.05749667, + "balance_loss_mlp": 1.03020835, + "epoch": 0.18640821774708374, + "flos": 36648993784320.0, + "grad_norm": 2.3009510928734205, + "language_loss": 0.83625621, + "learning_rate": 3.7488403383359134e-06, + "loss": 0.85804629, + "num_input_tokens_seen": 183800805, + "router_z_loss_clip": 0.74951172, + "router_z_loss_mlp": 0.16320801, + "step": 6424, + "time_per_iteration": 2.656438112258911 + }, + { + "auxiliary_loss_clip": 0.01129387, + "auxiliary_loss_mlp": 0.0104073, + "balance_loss_clip": 1.05722535, + "balance_loss_mlp": 1.02610934, + "epoch": 0.1864372352155998, + "flos": 10955484057600.0, + "grad_norm": 2.349875685468396, + "language_loss": 0.71600413, + "learning_rate": 3.748749137320308e-06, + "loss": 0.73770535, + "num_input_tokens_seen": 183811185, + "router_z_loss_clip": 0.72167969, + "router_z_loss_mlp": 0.14624023, + "step": 6425, + "time_per_iteration": 2.5156280994415283 + }, + { + "auxiliary_loss_clip": 0.0113037, + "auxiliary_loss_mlp": 0.01047207, + "balance_loss_clip": 1.0573554, + "balance_loss_mlp": 1.03159618, + "epoch": 0.18646625268411585, + "flos": 18363711565440.0, + "grad_norm": 2.5777547651902295, + "language_loss": 0.74937886, + "learning_rate": 3.7486579208590426e-06, + "loss": 0.7711547, + "num_input_tokens_seen": 183825655, + "router_z_loss_clip": 0.72949219, + "router_z_loss_mlp": 0.15612793, + "step": 6426, + "time_per_iteration": 2.5473015308380127 + }, + { + "auxiliary_loss_clip": 0.0112773, + "auxiliary_loss_mlp": 0.01039082, + "balance_loss_clip": 1.05951691, + "balance_loss_mlp": 1.02375185, + "epoch": 0.1864952701526319, + "flos": 33612710158080.0, + "grad_norm": 3.7331962481124834, + "language_loss": 0.81719542, + "learning_rate": 3.7485666889529234e-06, + "loss": 0.83886361, + "num_input_tokens_seen": 183841125, + "router_z_loss_clip": 0.68261719, + "router_z_loss_mlp": 0.15338135, + "step": 6427, + "time_per_iteration": 2.703305721282959 + }, + { + "auxiliary_loss_clip": 0.01138648, + "auxiliary_loss_mlp": 0.01039993, + "balance_loss_clip": 1.06173742, + "balance_loss_mlp": 1.02470994, + "epoch": 0.18652428762114792, + "flos": 15700899600000.0, + "grad_norm": 2.831223321582156, + "language_loss": 0.75182915, + "learning_rate": 3.748475441602755e-06, + "loss": 0.77361554, + "num_input_tokens_seen": 183854640, + "router_z_loss_clip": 0.76953125, + "router_z_loss_mlp": 0.15270996, + "step": 6428, + "time_per_iteration": 2.5250697135925293 + }, + { + "auxiliary_loss_clip": 0.01067762, + "auxiliary_loss_mlp": 0.01022915, + "balance_loss_clip": 1.04768729, + "balance_loss_mlp": 1.02175319, + "epoch": 0.18655330508966397, + "flos": 64269544262400.0, + "grad_norm": 0.7191160458224312, + "language_loss": 0.53019208, + "learning_rate": 3.7483841788093438e-06, + "loss": 0.55109882, + "num_input_tokens_seen": 183916125, + "router_z_loss_clip": 0.20117188, + "router_z_loss_mlp": 0.01159668, + "step": 6429, + "time_per_iteration": 3.107835292816162 + }, + { + "auxiliary_loss_clip": 0.01067087, + "auxiliary_loss_mlp": 0.01013917, + "balance_loss_clip": 1.047351, + "balance_loss_mlp": 1.01262975, + "epoch": 0.18658232255818002, + "flos": 58942088737920.0, + "grad_norm": 0.6637190828904705, + "language_loss": 0.45433921, + "learning_rate": 3.7482929005734966e-06, + "loss": 0.4751493, + "num_input_tokens_seen": 183973985, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.01287842, + "step": 6430, + "time_per_iteration": 3.0076799392700195 + }, + { + "auxiliary_loss_clip": 0.01129297, + "auxiliary_loss_mlp": 0.01033489, + "balance_loss_clip": 1.05967665, + "balance_loss_mlp": 1.02017915, + "epoch": 0.18661134002669608, + "flos": 16902542211840.0, + "grad_norm": 3.4845660971278845, + "language_loss": 0.78904665, + "learning_rate": 3.748201606896019e-06, + "loss": 0.81067455, + "num_input_tokens_seen": 183985465, + "router_z_loss_clip": 0.69604492, + "router_z_loss_mlp": 0.13311768, + "step": 6431, + "time_per_iteration": 2.496321678161621 + }, + { + "auxiliary_loss_clip": 0.0112701, + "auxiliary_loss_mlp": 0.01039438, + "balance_loss_clip": 1.06052399, + "balance_loss_mlp": 1.02619362, + "epoch": 0.18664035749521213, + "flos": 16355802320640.0, + "grad_norm": 2.5632062270531337, + "language_loss": 0.77461398, + "learning_rate": 3.748110297777717e-06, + "loss": 0.79627848, + "num_input_tokens_seen": 183999345, + "router_z_loss_clip": 0.66503906, + "router_z_loss_mlp": 0.13262939, + "step": 6432, + "time_per_iteration": 2.5741279125213623 + }, + { + "auxiliary_loss_clip": 0.01061912, + "auxiliary_loss_mlp": 0.00999477, + "balance_loss_clip": 1.04275584, + "balance_loss_mlp": 0.99812376, + "epoch": 0.18666937496372815, + "flos": 69989215368960.0, + "grad_norm": 0.6769884211979292, + "language_loss": 0.49857807, + "learning_rate": 3.7480189732193973e-06, + "loss": 0.51919192, + "num_input_tokens_seen": 184060220, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.0135498, + "step": 6433, + "time_per_iteration": 3.14345121383667 + }, + { + "auxiliary_loss_clip": 0.01129274, + "auxiliary_loss_mlp": 0.01035069, + "balance_loss_clip": 1.06012964, + "balance_loss_mlp": 1.02065659, + "epoch": 0.1866983924322442, + "flos": 13911255348480.0, + "grad_norm": 2.8921865303234373, + "language_loss": 0.59632242, + "learning_rate": 3.7479276332218675e-06, + "loss": 0.61796582, + "num_input_tokens_seen": 184073510, + "router_z_loss_clip": 0.69189453, + "router_z_loss_mlp": 0.14422607, + "step": 6434, + "time_per_iteration": 2.5525999069213867 + }, + { + "auxiliary_loss_clip": 0.011283, + "auxiliary_loss_mlp": 0.01044908, + "balance_loss_clip": 1.05815852, + "balance_loss_mlp": 1.03040588, + "epoch": 0.18672740990076026, + "flos": 11867902358400.0, + "grad_norm": 2.3610954917367546, + "language_loss": 0.86136687, + "learning_rate": 3.747836277785933e-06, + "loss": 0.88309896, + "num_input_tokens_seen": 184086205, + "router_z_loss_clip": 0.70166016, + "router_z_loss_mlp": 0.14489746, + "step": 6435, + "time_per_iteration": 2.5170185565948486 + }, + { + "auxiliary_loss_clip": 0.01061067, + "auxiliary_loss_mlp": 0.00999577, + "balance_loss_clip": 1.0419066, + "balance_loss_mlp": 0.9982416, + "epoch": 0.1867564273692763, + "flos": 57439693549440.0, + "grad_norm": 0.6416549172017657, + "language_loss": 0.47331879, + "learning_rate": 3.747744906912401e-06, + "loss": 0.49392524, + "num_input_tokens_seen": 184153225, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.0133667, + "step": 6436, + "time_per_iteration": 3.194561719894409 + }, + { + "auxiliary_loss_clip": 0.01135373, + "auxiliary_loss_mlp": 0.01043796, + "balance_loss_clip": 1.06049144, + "balance_loss_mlp": 1.02755415, + "epoch": 0.18678544483779236, + "flos": 34891381486080.0, + "grad_norm": 2.2191903443838976, + "language_loss": 0.96293306, + "learning_rate": 3.747653520602079e-06, + "loss": 0.98472476, + "num_input_tokens_seen": 184170150, + "router_z_loss_clip": 0.74926758, + "router_z_loss_mlp": 0.16247559, + "step": 6437, + "time_per_iteration": 2.689575433731079 + }, + { + "auxiliary_loss_clip": 0.01129878, + "auxiliary_loss_mlp": 0.01038216, + "balance_loss_clip": 1.05815363, + "balance_loss_mlp": 1.0231185, + "epoch": 0.1868144623063084, + "flos": 24492154423680.0, + "grad_norm": 2.4990674346196027, + "language_loss": 0.93761832, + "learning_rate": 3.7475621188557743e-06, + "loss": 0.95929933, + "num_input_tokens_seen": 184185535, + "router_z_loss_clip": 0.71679688, + "router_z_loss_mlp": 0.15100098, + "step": 6438, + "time_per_iteration": 2.6269965171813965 + }, + { + "auxiliary_loss_clip": 0.0112685, + "auxiliary_loss_mlp": 0.01040744, + "balance_loss_clip": 1.05740261, + "balance_loss_mlp": 1.02627206, + "epoch": 0.18684347977482443, + "flos": 31715369953920.0, + "grad_norm": 2.5934643571463734, + "language_loss": 0.84558105, + "learning_rate": 3.7474707016742933e-06, + "loss": 0.867257, + "num_input_tokens_seen": 184201550, + "router_z_loss_clip": 0.69433594, + "router_z_loss_mlp": 0.14471436, + "step": 6439, + "time_per_iteration": 2.5907928943634033 + }, + { + "auxiliary_loss_clip": 0.01131044, + "auxiliary_loss_mlp": 0.01034114, + "balance_loss_clip": 1.05931783, + "balance_loss_mlp": 1.0181818, + "epoch": 0.18687249724334049, + "flos": 30492792691200.0, + "grad_norm": 2.129294087625292, + "language_loss": 0.9252398, + "learning_rate": 3.7473792690584444e-06, + "loss": 0.94689143, + "num_input_tokens_seen": 184219125, + "router_z_loss_clip": 0.71728516, + "router_z_loss_mlp": 0.15942383, + "step": 6440, + "time_per_iteration": 2.715698480606079 + }, + { + "auxiliary_loss_clip": 0.01125177, + "auxiliary_loss_mlp": 0.01029125, + "balance_loss_clip": 1.05789554, + "balance_loss_mlp": 1.01582122, + "epoch": 0.18690151471185654, + "flos": 23581244494080.0, + "grad_norm": 1.734721467493844, + "language_loss": 0.67735219, + "learning_rate": 3.747287821009034e-06, + "loss": 0.69889522, + "num_input_tokens_seen": 184237010, + "router_z_loss_clip": 0.67236328, + "router_z_loss_mlp": 0.13311768, + "step": 6441, + "time_per_iteration": 2.5560412406921387 + }, + { + "auxiliary_loss_clip": 0.01138193, + "auxiliary_loss_mlp": 0.0104137, + "balance_loss_clip": 1.0644716, + "balance_loss_mlp": 1.02433455, + "epoch": 0.1869305321803726, + "flos": 32409164125440.0, + "grad_norm": 3.6716296528972383, + "language_loss": 0.93214917, + "learning_rate": 3.7471963575268707e-06, + "loss": 0.9539448, + "num_input_tokens_seen": 184254870, + "router_z_loss_clip": 0.73730469, + "router_z_loss_mlp": 0.17053223, + "step": 6442, + "time_per_iteration": 2.5566275119781494 + }, + { + "auxiliary_loss_clip": 0.01129249, + "auxiliary_loss_mlp": 0.0103813, + "balance_loss_clip": 1.05933821, + "balance_loss_mlp": 1.02383721, + "epoch": 0.18695954964888864, + "flos": 12344149808640.0, + "grad_norm": 2.5236687352772127, + "language_loss": 0.71175539, + "learning_rate": 3.747104878612763e-06, + "loss": 0.73342919, + "num_input_tokens_seen": 184268155, + "router_z_loss_clip": 0.69824219, + "router_z_loss_mlp": 0.14300537, + "step": 6443, + "time_per_iteration": 2.4808502197265625 + }, + { + "auxiliary_loss_clip": 0.01134157, + "auxiliary_loss_mlp": 0.01043672, + "balance_loss_clip": 1.06234264, + "balance_loss_mlp": 1.02912831, + "epoch": 0.18698856711740466, + "flos": 13479789179520.0, + "grad_norm": 9.81992204981198, + "language_loss": 0.7325654, + "learning_rate": 3.7470133842675173e-06, + "loss": 0.75434369, + "num_input_tokens_seen": 184281635, + "router_z_loss_clip": 0.71801758, + "router_z_loss_mlp": 0.14538574, + "step": 6444, + "time_per_iteration": 2.5171701908111572 + }, + { + "auxiliary_loss_clip": 0.0113904, + "auxiliary_loss_mlp": 0.01046311, + "balance_loss_clip": 1.06389785, + "balance_loss_mlp": 1.03121948, + "epoch": 0.18701758458592072, + "flos": 47265803481600.0, + "grad_norm": 2.0773959508408364, + "language_loss": 0.9522838, + "learning_rate": 3.7469218744919423e-06, + "loss": 0.97413731, + "num_input_tokens_seen": 184302495, + "router_z_loss_clip": 0.75097656, + "router_z_loss_mlp": 0.15106201, + "step": 6445, + "time_per_iteration": 2.780184745788574 + }, + { + "auxiliary_loss_clip": 0.01138641, + "auxiliary_loss_mlp": 0.01041846, + "balance_loss_clip": 1.06074345, + "balance_loss_mlp": 1.02553201, + "epoch": 0.18704660205443677, + "flos": 74737177799040.0, + "grad_norm": 1.6915668558059342, + "language_loss": 0.7653147, + "learning_rate": 3.7468303492868466e-06, + "loss": 0.78711951, + "num_input_tokens_seen": 184338970, + "router_z_loss_clip": 0.77929688, + "router_z_loss_mlp": 0.16308594, + "step": 6446, + "time_per_iteration": 3.0196878910064697 + }, + { + "auxiliary_loss_clip": 0.01142303, + "auxiliary_loss_mlp": 0.01048586, + "balance_loss_clip": 1.06775117, + "balance_loss_mlp": 1.03266001, + "epoch": 0.18707561952295282, + "flos": 32482098691200.0, + "grad_norm": 2.695388648669891, + "language_loss": 0.90042758, + "learning_rate": 3.746738808653038e-06, + "loss": 0.92233646, + "num_input_tokens_seen": 184356650, + "router_z_loss_clip": 0.74462891, + "router_z_loss_mlp": 0.15930176, + "step": 6447, + "time_per_iteration": 2.660738468170166 + }, + { + "auxiliary_loss_clip": 0.011354, + "auxiliary_loss_mlp": 0.01041534, + "balance_loss_clip": 1.06268358, + "balance_loss_mlp": 1.02706826, + "epoch": 0.18710463699146887, + "flos": 16283837422080.0, + "grad_norm": 2.6769013621190303, + "language_loss": 0.9157666, + "learning_rate": 3.7466472525913266e-06, + "loss": 0.93753588, + "num_input_tokens_seen": 184367955, + "router_z_loss_clip": 0.72705078, + "router_z_loss_mlp": 0.14465332, + "step": 6448, + "time_per_iteration": 2.515390396118164 + }, + { + "auxiliary_loss_clip": 0.01129723, + "auxiliary_loss_mlp": 0.01037258, + "balance_loss_clip": 1.06158829, + "balance_loss_mlp": 1.02347767, + "epoch": 0.18713365445998492, + "flos": 22448837347200.0, + "grad_norm": 4.130577496031813, + "language_loss": 0.97787601, + "learning_rate": 3.746555681102519e-06, + "loss": 0.99954581, + "num_input_tokens_seen": 184383145, + "router_z_loss_clip": 0.68115234, + "router_z_loss_mlp": 0.13806152, + "step": 6449, + "time_per_iteration": 2.5573012828826904 + }, + { + "auxiliary_loss_clip": 0.01130932, + "auxiliary_loss_mlp": 0.0105239, + "balance_loss_clip": 1.05989504, + "balance_loss_mlp": 1.03623736, + "epoch": 0.18716267192850095, + "flos": 21252689516160.0, + "grad_norm": 1.6475106198309495, + "language_loss": 0.62136698, + "learning_rate": 3.7464640941874247e-06, + "loss": 0.64320022, + "num_input_tokens_seen": 184401385, + "router_z_loss_clip": 0.70996094, + "router_z_loss_mlp": 0.16149902, + "step": 6450, + "time_per_iteration": 2.5432870388031006 + }, + { + "auxiliary_loss_clip": 0.01137711, + "auxiliary_loss_mlp": 0.01044553, + "balance_loss_clip": 1.06122446, + "balance_loss_mlp": 1.02907372, + "epoch": 0.187191689397017, + "flos": 12413708496000.0, + "grad_norm": 2.6471476746537665, + "language_loss": 0.88448721, + "learning_rate": 3.746372491846853e-06, + "loss": 0.90630984, + "num_input_tokens_seen": 184413160, + "router_z_loss_clip": 0.76513672, + "router_z_loss_mlp": 0.15478516, + "step": 6451, + "time_per_iteration": 2.515209674835205 + }, + { + "auxiliary_loss_clip": 0.01134444, + "auxiliary_loss_mlp": 0.01041752, + "balance_loss_clip": 1.06172395, + "balance_loss_mlp": 1.02831662, + "epoch": 0.18722070686553305, + "flos": 30586482339840.0, + "grad_norm": 2.0090528279947946, + "language_loss": 0.69078797, + "learning_rate": 3.746280874081613e-06, + "loss": 0.71254992, + "num_input_tokens_seen": 184431410, + "router_z_loss_clip": 0.72705078, + "router_z_loss_mlp": 0.13446045, + "step": 6452, + "time_per_iteration": 2.6114211082458496 + }, + { + "auxiliary_loss_clip": 0.01143362, + "auxiliary_loss_mlp": 0.0104451, + "balance_loss_clip": 1.06561339, + "balance_loss_mlp": 1.02919781, + "epoch": 0.1872497243340491, + "flos": 11831129809920.0, + "grad_norm": 2.2127871632304155, + "language_loss": 0.64094901, + "learning_rate": 3.7461892408925137e-06, + "loss": 0.66282773, + "num_input_tokens_seen": 184445055, + "router_z_loss_clip": 0.77587891, + "router_z_loss_mlp": 0.15332031, + "step": 6453, + "time_per_iteration": 2.496591329574585 + }, + { + "auxiliary_loss_clip": 0.01073112, + "auxiliary_loss_mlp": 0.01028752, + "balance_loss_clip": 1.05290508, + "balance_loss_mlp": 1.02745295, + "epoch": 0.18727874180256515, + "flos": 74781315613440.0, + "grad_norm": 0.6096673054694329, + "language_loss": 0.45648944, + "learning_rate": 3.746097592280364e-06, + "loss": 0.47750807, + "num_input_tokens_seen": 184514825, + "router_z_loss_clip": 0.20214844, + "router_z_loss_mlp": 0.01300049, + "step": 6454, + "time_per_iteration": 3.2481796741485596 + }, + { + "auxiliary_loss_clip": 0.01142376, + "auxiliary_loss_mlp": 0.01042522, + "balance_loss_clip": 1.06465149, + "balance_loss_mlp": 1.02623773, + "epoch": 0.1873077592710812, + "flos": 27228224177280.0, + "grad_norm": 1.7147177091887056, + "language_loss": 0.84860671, + "learning_rate": 3.7460059282459743e-06, + "loss": 0.87045574, + "num_input_tokens_seen": 184533310, + "router_z_loss_clip": 0.77685547, + "router_z_loss_mlp": 0.16271973, + "step": 6455, + "time_per_iteration": 2.615804433822632 + }, + { + "auxiliary_loss_clip": 0.01066556, + "auxiliary_loss_mlp": 0.01010489, + "balance_loss_clip": 1.0465678, + "balance_loss_mlp": 1.00928545, + "epoch": 0.18733677673959723, + "flos": 74777401031040.0, + "grad_norm": 0.6948124544403733, + "language_loss": 0.48891628, + "learning_rate": 3.745914248790153e-06, + "loss": 0.50968671, + "num_input_tokens_seen": 184593675, + "router_z_loss_clip": 0.19921875, + "router_z_loss_mlp": 0.01202393, + "step": 6456, + "time_per_iteration": 3.1394712924957275 + }, + { + "auxiliary_loss_clip": 0.01134742, + "auxiliary_loss_mlp": 0.01032661, + "balance_loss_clip": 1.06041849, + "balance_loss_mlp": 1.01810575, + "epoch": 0.18736579420811328, + "flos": 15993894839040.0, + "grad_norm": 2.092358588148484, + "language_loss": 0.63321412, + "learning_rate": 3.745822553913711e-06, + "loss": 0.65488815, + "num_input_tokens_seen": 184613025, + "router_z_loss_clip": 0.74414062, + "router_z_loss_mlp": 0.14532471, + "step": 6457, + "time_per_iteration": 2.6441891193389893 + }, + { + "auxiliary_loss_clip": 0.01125582, + "auxiliary_loss_mlp": 0.01036391, + "balance_loss_clip": 1.06088781, + "balance_loss_mlp": 1.02379632, + "epoch": 0.18739481167662933, + "flos": 37334599655040.0, + "grad_norm": 1.9065205479065834, + "language_loss": 0.69237041, + "learning_rate": 3.745730843617458e-06, + "loss": 0.71399015, + "num_input_tokens_seen": 184632220, + "router_z_loss_clip": 0.64648438, + "router_z_loss_mlp": 0.1260376, + "step": 6458, + "time_per_iteration": 2.6946256160736084 + }, + { + "auxiliary_loss_clip": 0.01061102, + "auxiliary_loss_mlp": 0.01005365, + "balance_loss_clip": 1.04161859, + "balance_loss_mlp": 1.00404191, + "epoch": 0.18742382914514538, + "flos": 74773701930240.0, + "grad_norm": 0.7458575735321114, + "language_loss": 0.53927016, + "learning_rate": 3.7456391179022033e-06, + "loss": 0.55993474, + "num_input_tokens_seen": 184689715, + "router_z_loss_clip": 0.1953125, + "router_z_loss_mlp": 0.01324463, + "step": 6459, + "time_per_iteration": 3.096113443374634 + }, + { + "auxiliary_loss_clip": 0.01145859, + "auxiliary_loss_mlp": 0.01048584, + "balance_loss_clip": 1.06305695, + "balance_loss_mlp": 1.03018999, + "epoch": 0.18745284661366143, + "flos": 12266977438080.0, + "grad_norm": 3.2034139005546196, + "language_loss": 0.95456761, + "learning_rate": 3.745547376768758e-06, + "loss": 0.97651201, + "num_input_tokens_seen": 184701450, + "router_z_loss_clip": 0.82910156, + "router_z_loss_mlp": 0.18389893, + "step": 6460, + "time_per_iteration": 2.520806074142456 + }, + { + "auxiliary_loss_clip": 0.01058993, + "auxiliary_loss_mlp": 0.01008044, + "balance_loss_clip": 1.03968287, + "balance_loss_mlp": 1.00670862, + "epoch": 0.18748186408217746, + "flos": 64961830062720.0, + "grad_norm": 0.6972494681385044, + "language_loss": 0.51399052, + "learning_rate": 3.7454556202179318e-06, + "loss": 0.53466094, + "num_input_tokens_seen": 184759690, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.0133667, + "step": 6461, + "time_per_iteration": 3.0442399978637695 + }, + { + "auxiliary_loss_clip": 0.01057509, + "auxiliary_loss_mlp": 0.01006373, + "balance_loss_clip": 1.03833079, + "balance_loss_mlp": 1.00501966, + "epoch": 0.1875108815506935, + "flos": 61342141737600.0, + "grad_norm": 0.6229011783217794, + "language_loss": 0.46793005, + "learning_rate": 3.745363848250535e-06, + "loss": 0.48856887, + "num_input_tokens_seen": 184822995, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.0135498, + "step": 6462, + "time_per_iteration": 3.2660717964172363 + }, + { + "auxiliary_loss_clip": 0.01129263, + "auxiliary_loss_mlp": 0.01029557, + "balance_loss_clip": 1.06094086, + "balance_loss_mlp": 1.01666451, + "epoch": 0.18753989901920956, + "flos": 32450749096320.0, + "grad_norm": 2.041039999879287, + "language_loss": 0.6914627, + "learning_rate": 3.7452720608673785e-06, + "loss": 0.71305084, + "num_input_tokens_seen": 184843275, + "router_z_loss_clip": 0.68310547, + "router_z_loss_mlp": 0.12884521, + "step": 6463, + "time_per_iteration": 2.6093103885650635 + }, + { + "auxiliary_loss_clip": 0.01133647, + "auxiliary_loss_mlp": 0.01035328, + "balance_loss_clip": 1.06315076, + "balance_loss_mlp": 1.02115989, + "epoch": 0.1875689164877256, + "flos": 19018470631680.0, + "grad_norm": 2.341135820588534, + "language_loss": 0.81159908, + "learning_rate": 3.745180258069273e-06, + "loss": 0.83328885, + "num_input_tokens_seen": 184855355, + "router_z_loss_clip": 0.70556641, + "router_z_loss_mlp": 0.14172363, + "step": 6464, + "time_per_iteration": 2.6210484504699707 + }, + { + "auxiliary_loss_clip": 0.01134468, + "auxiliary_loss_mlp": 0.01042957, + "balance_loss_clip": 1.06203258, + "balance_loss_mlp": 1.02678585, + "epoch": 0.18759793395624166, + "flos": 19201866497280.0, + "grad_norm": 3.8485745011920476, + "language_loss": 0.718952, + "learning_rate": 3.745088439857029e-06, + "loss": 0.74072623, + "num_input_tokens_seen": 184868350, + "router_z_loss_clip": 0.72460938, + "router_z_loss_mlp": 0.16168213, + "step": 6465, + "time_per_iteration": 2.5697388648986816 + }, + { + "auxiliary_loss_clip": 0.01135569, + "auxiliary_loss_mlp": 0.01038994, + "balance_loss_clip": 1.06493831, + "balance_loss_mlp": 1.02508283, + "epoch": 0.18762695142475772, + "flos": 11907907130880.0, + "grad_norm": 2.311616400732501, + "language_loss": 0.78647041, + "learning_rate": 3.744996606231458e-06, + "loss": 0.80821604, + "num_input_tokens_seen": 184879375, + "router_z_loss_clip": 0.70556641, + "router_z_loss_mlp": 0.13928223, + "step": 6466, + "time_per_iteration": 2.489064931869507 + }, + { + "auxiliary_loss_clip": 0.01136398, + "auxiliary_loss_mlp": 0.01034388, + "balance_loss_clip": 1.06332469, + "balance_loss_mlp": 1.01983237, + "epoch": 0.18765596889327374, + "flos": 74735094810240.0, + "grad_norm": 2.3311269195259454, + "language_loss": 0.83929914, + "learning_rate": 3.744904757193371e-06, + "loss": 0.86100698, + "num_input_tokens_seen": 184903265, + "router_z_loss_clip": 0.73046875, + "router_z_loss_mlp": 0.14544678, + "step": 6467, + "time_per_iteration": 2.950279712677002 + }, + { + "auxiliary_loss_clip": 0.01052875, + "auxiliary_loss_mlp": 0.01034909, + "balance_loss_clip": 1.0334537, + "balance_loss_mlp": 1.033705, + "epoch": 0.1876849863617898, + "flos": 58645178916480.0, + "grad_norm": 0.693453296205454, + "language_loss": 0.48016036, + "learning_rate": 3.7448128927435788e-06, + "loss": 0.50103819, + "num_input_tokens_seen": 184961320, + "router_z_loss_clip": 0.19433594, + "router_z_loss_mlp": 0.01202393, + "step": 6468, + "time_per_iteration": 3.2264041900634766 + }, + { + "auxiliary_loss_clip": 0.01128931, + "auxiliary_loss_mlp": 0.01038551, + "balance_loss_clip": 1.06111526, + "balance_loss_mlp": 1.02608752, + "epoch": 0.18771400383030584, + "flos": 11542839252480.0, + "grad_norm": 2.8166850259499148, + "language_loss": 0.81449449, + "learning_rate": 3.744721012882893e-06, + "loss": 0.8361693, + "num_input_tokens_seen": 184974175, + "router_z_loss_clip": 0.67871094, + "router_z_loss_mlp": 0.12469482, + "step": 6469, + "time_per_iteration": 2.5012307167053223 + }, + { + "auxiliary_loss_clip": 0.01138997, + "auxiliary_loss_mlp": 0.01042983, + "balance_loss_clip": 1.06394327, + "balance_loss_mlp": 1.02891028, + "epoch": 0.1877430212988219, + "flos": 14019669573120.0, + "grad_norm": 6.551978551431994, + "language_loss": 0.77688229, + "learning_rate": 3.7446291176121255e-06, + "loss": 0.79870212, + "num_input_tokens_seen": 184987370, + "router_z_loss_clip": 0.75048828, + "router_z_loss_mlp": 0.14080811, + "step": 6470, + "time_per_iteration": 2.566222906112671 + }, + { + "auxiliary_loss_clip": 0.01129894, + "auxiliary_loss_mlp": 0.0102573, + "balance_loss_clip": 1.06070054, + "balance_loss_mlp": 1.01243258, + "epoch": 0.18777203876733795, + "flos": 27685655879040.0, + "grad_norm": 2.0780650485964967, + "language_loss": 0.80045736, + "learning_rate": 3.7445372069320877e-06, + "loss": 0.82201362, + "num_input_tokens_seen": 185008320, + "router_z_loss_clip": 0.69091797, + "router_z_loss_mlp": 0.13293457, + "step": 6471, + "time_per_iteration": 4.97592568397522 + }, + { + "auxiliary_loss_clip": 0.01141705, + "auxiliary_loss_mlp": 0.01040688, + "balance_loss_clip": 1.06327438, + "balance_loss_mlp": 1.02515519, + "epoch": 0.187801056235854, + "flos": 16537761642240.0, + "grad_norm": 2.1879853240836127, + "language_loss": 0.7655412, + "learning_rate": 3.7444452808435906e-06, + "loss": 0.78736514, + "num_input_tokens_seen": 185020805, + "router_z_loss_clip": 0.78417969, + "router_z_loss_mlp": 0.15527344, + "step": 6472, + "time_per_iteration": 4.967042446136475 + }, + { + "auxiliary_loss_clip": 0.01051103, + "auxiliary_loss_mlp": 0.01002265, + "balance_loss_clip": 1.03221369, + "balance_loss_mlp": 1.00116825, + "epoch": 0.18783007370437002, + "flos": 66824911670400.0, + "grad_norm": 0.6689292450623807, + "language_loss": 0.47621071, + "learning_rate": 3.7443533393474478e-06, + "loss": 0.49674436, + "num_input_tokens_seen": 185081065, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.01098633, + "step": 6473, + "time_per_iteration": 5.476801156997681 + }, + { + "auxiliary_loss_clip": 0.01131955, + "auxiliary_loss_mlp": 0.01035737, + "balance_loss_clip": 1.05896497, + "balance_loss_mlp": 1.02053785, + "epoch": 0.18785909117288607, + "flos": 41019321553920.0, + "grad_norm": 4.266487347392046, + "language_loss": 0.92599064, + "learning_rate": 3.74426138244447e-06, + "loss": 0.9476676, + "num_input_tokens_seen": 185096625, + "router_z_loss_clip": 0.72949219, + "router_z_loss_mlp": 0.15197754, + "step": 6474, + "time_per_iteration": 2.7092301845550537 + }, + { + "auxiliary_loss_clip": 0.01136356, + "auxiliary_loss_mlp": 0.01039938, + "balance_loss_clip": 1.06032634, + "balance_loss_mlp": 1.02420855, + "epoch": 0.18788810864140212, + "flos": 18003527936640.0, + "grad_norm": 2.8806960086641387, + "language_loss": 0.88292873, + "learning_rate": 3.7441694101354697e-06, + "loss": 0.9046917, + "num_input_tokens_seen": 185114335, + "router_z_loss_clip": 0.76025391, + "router_z_loss_mlp": 0.1572876, + "step": 6475, + "time_per_iteration": 2.5168421268463135 + }, + { + "auxiliary_loss_clip": 0.01138923, + "auxiliary_loss_mlp": 0.01038405, + "balance_loss_clip": 1.06419575, + "balance_loss_mlp": 1.02183521, + "epoch": 0.18791712610991818, + "flos": 25513489739520.0, + "grad_norm": 1.873121325459647, + "language_loss": 0.63232911, + "learning_rate": 3.7440774224212595e-06, + "loss": 0.65410233, + "num_input_tokens_seen": 185129305, + "router_z_loss_clip": 0.74658203, + "router_z_loss_mlp": 0.16564941, + "step": 6476, + "time_per_iteration": 2.578510046005249 + }, + { + "auxiliary_loss_clip": 0.01135562, + "auxiliary_loss_mlp": 0.01038851, + "balance_loss_clip": 1.05868423, + "balance_loss_mlp": 1.02318072, + "epoch": 0.18794614357843423, + "flos": 22048361637120.0, + "grad_norm": 2.775125410854115, + "language_loss": 0.8244307, + "learning_rate": 3.7439854193026523e-06, + "loss": 0.84617484, + "num_input_tokens_seen": 185144715, + "router_z_loss_clip": 0.76904297, + "router_z_loss_mlp": 0.15679932, + "step": 6477, + "time_per_iteration": 2.535907030105591 + }, + { + "auxiliary_loss_clip": 0.01146047, + "auxiliary_loss_mlp": 0.0105169, + "balance_loss_clip": 1.06312954, + "balance_loss_mlp": 1.03407097, + "epoch": 0.18797516104695025, + "flos": 34638534673920.0, + "grad_norm": 2.303211389140196, + "language_loss": 0.95968169, + "learning_rate": 3.743893400780459e-06, + "loss": 0.98165905, + "num_input_tokens_seen": 185163750, + "router_z_loss_clip": 0.82861328, + "router_z_loss_mlp": 0.17602539, + "step": 6478, + "time_per_iteration": 2.662113904953003 + }, + { + "auxiliary_loss_clip": 0.0112767, + "auxiliary_loss_mlp": 0.01033858, + "balance_loss_clip": 1.0565958, + "balance_loss_mlp": 1.01816988, + "epoch": 0.1880041785154663, + "flos": 24018456839040.0, + "grad_norm": 2.026365812046232, + "language_loss": 0.79552591, + "learning_rate": 3.7438013668554945e-06, + "loss": 0.81714118, + "num_input_tokens_seen": 185179065, + "router_z_loss_clip": 0.71044922, + "router_z_loss_mlp": 0.15698242, + "step": 6479, + "time_per_iteration": 2.5314338207244873 + }, + { + "auxiliary_loss_clip": 0.01141049, + "auxiliary_loss_mlp": 0.01043283, + "balance_loss_clip": 1.06306577, + "balance_loss_mlp": 1.02761924, + "epoch": 0.18803319598398235, + "flos": 74732760426240.0, + "grad_norm": 1.8985168420745309, + "language_loss": 0.92908496, + "learning_rate": 3.7437093175285702e-06, + "loss": 0.95092827, + "num_input_tokens_seen": 185204255, + "router_z_loss_clip": 0.78027344, + "router_z_loss_mlp": 0.15661621, + "step": 6480, + "time_per_iteration": 2.9208362102508545 + }, + { + "auxiliary_loss_clip": 0.01132884, + "auxiliary_loss_mlp": 0.01045376, + "balance_loss_clip": 1.05766749, + "balance_loss_mlp": 1.02975345, + "epoch": 0.1880622134524984, + "flos": 22230859662720.0, + "grad_norm": 2.2659980474190156, + "language_loss": 0.85912102, + "learning_rate": 3.7436172528005e-06, + "loss": 0.8809036, + "num_input_tokens_seen": 185221540, + "router_z_loss_clip": 0.75195312, + "router_z_loss_mlp": 0.15631104, + "step": 6481, + "time_per_iteration": 2.5642237663269043 + }, + { + "auxiliary_loss_clip": 0.01127254, + "auxiliary_loss_mlp": 0.01035298, + "balance_loss_clip": 1.05780363, + "balance_loss_mlp": 1.02073646, + "epoch": 0.18809123092101446, + "flos": 10663314831360.0, + "grad_norm": 2.804519056882058, + "language_loss": 0.83998632, + "learning_rate": 3.743525172672097e-06, + "loss": 0.86161184, + "num_input_tokens_seen": 185232130, + "router_z_loss_clip": 0.69458008, + "router_z_loss_mlp": 0.14544678, + "step": 6482, + "time_per_iteration": 2.5180530548095703 + }, + { + "auxiliary_loss_clip": 0.01126483, + "auxiliary_loss_mlp": 0.01040221, + "balance_loss_clip": 1.0568198, + "balance_loss_mlp": 1.02529633, + "epoch": 0.1881202483895305, + "flos": 26937204186240.0, + "grad_norm": 2.1760314948684902, + "language_loss": 1.09693384, + "learning_rate": 3.743433077144173e-06, + "loss": 1.11860085, + "num_input_tokens_seen": 185250475, + "router_z_loss_clip": 0.6953125, + "router_z_loss_mlp": 0.14910889, + "step": 6483, + "time_per_iteration": 2.5859427452087402 + }, + { + "auxiliary_loss_clip": 0.01131098, + "auxiliary_loss_mlp": 0.01034257, + "balance_loss_clip": 1.05809069, + "balance_loss_mlp": 1.0199461, + "epoch": 0.18814926585804653, + "flos": 31210969219200.0, + "grad_norm": 1.9969858164597731, + "language_loss": 0.77501923, + "learning_rate": 3.7433409662175434e-06, + "loss": 0.79667282, + "num_input_tokens_seen": 185271605, + "router_z_loss_clip": 0.73046875, + "router_z_loss_mlp": 0.14318848, + "step": 6484, + "time_per_iteration": 2.6509478092193604 + }, + { + "auxiliary_loss_clip": 0.01124989, + "auxiliary_loss_mlp": 0.01032935, + "balance_loss_clip": 1.05750251, + "balance_loss_mlp": 1.01908898, + "epoch": 0.18817828332656258, + "flos": 43136363295360.0, + "grad_norm": 1.8029444127230405, + "language_loss": 0.72495306, + "learning_rate": 3.743248839893021e-06, + "loss": 0.74653232, + "num_input_tokens_seen": 185292130, + "router_z_loss_clip": 0.67382812, + "router_z_loss_mlp": 0.13848877, + "step": 6485, + "time_per_iteration": 2.756511688232422 + }, + { + "auxiliary_loss_clip": 0.01128116, + "auxiliary_loss_mlp": 0.01031766, + "balance_loss_clip": 1.05671299, + "balance_loss_mlp": 1.01748526, + "epoch": 0.18820730079507864, + "flos": 25295117005440.0, + "grad_norm": 2.3248326548165568, + "language_loss": 0.85242444, + "learning_rate": 3.743156698171419e-06, + "loss": 0.87402326, + "num_input_tokens_seen": 185308730, + "router_z_loss_clip": 0.71386719, + "router_z_loss_mlp": 0.14282227, + "step": 6486, + "time_per_iteration": 2.5693721771240234 + }, + { + "auxiliary_loss_clip": 0.01135473, + "auxiliary_loss_mlp": 0.01036721, + "balance_loss_clip": 1.05858934, + "balance_loss_mlp": 1.02122998, + "epoch": 0.1882363182635947, + "flos": 22561776685440.0, + "grad_norm": 1.9495118126346718, + "language_loss": 0.7228778, + "learning_rate": 3.7430645410535513e-06, + "loss": 0.74459976, + "num_input_tokens_seen": 185327600, + "router_z_loss_clip": 0.76904297, + "router_z_loss_mlp": 0.15472412, + "step": 6487, + "time_per_iteration": 2.6388068199157715 + }, + { + "auxiliary_loss_clip": 0.01129676, + "auxiliary_loss_mlp": 0.01036395, + "balance_loss_clip": 1.05831957, + "balance_loss_mlp": 1.02222741, + "epoch": 0.18826533573211074, + "flos": 17341478409600.0, + "grad_norm": 2.2343149704502543, + "language_loss": 0.79474211, + "learning_rate": 3.742972368540233e-06, + "loss": 0.81640279, + "num_input_tokens_seen": 185341170, + "router_z_loss_clip": 0.71362305, + "router_z_loss_mlp": 0.14178467, + "step": 6488, + "time_per_iteration": 2.501159906387329 + }, + { + "auxiliary_loss_clip": 0.01125627, + "auxiliary_loss_mlp": 0.01035827, + "balance_loss_clip": 1.05461359, + "balance_loss_mlp": 1.02109265, + "epoch": 0.1882943532006268, + "flos": 33758651116800.0, + "grad_norm": 2.5903726644166483, + "language_loss": 0.86294115, + "learning_rate": 3.7428801806322774e-06, + "loss": 0.8845557, + "num_input_tokens_seen": 185357070, + "router_z_loss_clip": 0.71044922, + "router_z_loss_mlp": 0.1473999, + "step": 6489, + "time_per_iteration": 2.6623480319976807 + }, + { + "auxiliary_loss_clip": 0.01056968, + "auxiliary_loss_mlp": 0.01005796, + "balance_loss_clip": 1.03809714, + "balance_loss_mlp": 1.00471139, + "epoch": 0.18832337066914281, + "flos": 58298319233280.0, + "grad_norm": 0.6799219067026897, + "language_loss": 0.47529012, + "learning_rate": 3.7427879773304986e-06, + "loss": 0.49591777, + "num_input_tokens_seen": 185425485, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.01086426, + "step": 6490, + "time_per_iteration": 3.326231002807617 + }, + { + "auxiliary_loss_clip": 0.01131117, + "auxiliary_loss_mlp": 0.01033958, + "balance_loss_clip": 1.05634642, + "balance_loss_mlp": 1.01983726, + "epoch": 0.18835238813765887, + "flos": 29234265914880.0, + "grad_norm": 19.098679221607263, + "language_loss": 1.01593184, + "learning_rate": 3.7426957586357108e-06, + "loss": 1.03758252, + "num_input_tokens_seen": 185438845, + "router_z_loss_clip": 0.74707031, + "router_z_loss_mlp": 0.14123535, + "step": 6491, + "time_per_iteration": 2.5797526836395264 + }, + { + "auxiliary_loss_clip": 0.01132397, + "auxiliary_loss_mlp": 0.01041689, + "balance_loss_clip": 1.05883694, + "balance_loss_mlp": 1.02666235, + "epoch": 0.18838140560617492, + "flos": 27336243352320.0, + "grad_norm": 2.3771861495640865, + "language_loss": 1.02491987, + "learning_rate": 3.7426035245487296e-06, + "loss": 1.04666066, + "num_input_tokens_seen": 185452280, + "router_z_loss_clip": 0.73535156, + "router_z_loss_mlp": 0.15032959, + "step": 6492, + "time_per_iteration": 2.652493715286255 + }, + { + "auxiliary_loss_clip": 0.01130947, + "auxiliary_loss_mlp": 0.01043355, + "balance_loss_clip": 1.05330658, + "balance_loss_mlp": 1.02741718, + "epoch": 0.18841042307469097, + "flos": 26719657464960.0, + "grad_norm": 2.269107108022175, + "language_loss": 0.95094311, + "learning_rate": 3.742511275070368e-06, + "loss": 0.97268617, + "num_input_tokens_seen": 185468625, + "router_z_loss_clip": 0.77539062, + "router_z_loss_mlp": 0.15942383, + "step": 6493, + "time_per_iteration": 2.6043972969055176 + }, + { + "auxiliary_loss_clip": 0.01132189, + "auxiliary_loss_mlp": 0.01047921, + "balance_loss_clip": 1.05745327, + "balance_loss_mlp": 1.03262043, + "epoch": 0.18843944054320702, + "flos": 15699355315200.0, + "grad_norm": 2.2183876492788857, + "language_loss": 0.75710714, + "learning_rate": 3.7424190102014423e-06, + "loss": 0.77890825, + "num_input_tokens_seen": 185483505, + "router_z_loss_clip": 0.74755859, + "router_z_loss_mlp": 0.15301514, + "step": 6494, + "time_per_iteration": 2.5060155391693115 + }, + { + "auxiliary_loss_clip": 0.01121999, + "auxiliary_loss_mlp": 0.01035148, + "balance_loss_clip": 1.05393207, + "balance_loss_mlp": 1.02226126, + "epoch": 0.18846845801172304, + "flos": 24825836793600.0, + "grad_norm": 2.0656766280549896, + "language_loss": 0.66619462, + "learning_rate": 3.7423267299427667e-06, + "loss": 0.68776608, + "num_input_tokens_seen": 185499470, + "router_z_loss_clip": 0.68115234, + "router_z_loss_mlp": 0.12872314, + "step": 6495, + "time_per_iteration": 2.5801496505737305 + }, + { + "auxiliary_loss_clip": 0.01054044, + "auxiliary_loss_mlp": 0.01001335, + "balance_loss_clip": 1.03518653, + "balance_loss_mlp": 1.00036359, + "epoch": 0.1884974754802391, + "flos": 59907473642880.0, + "grad_norm": 0.8199476340125473, + "language_loss": 0.4860068, + "learning_rate": 3.7422344342951564e-06, + "loss": 0.50656056, + "num_input_tokens_seen": 185545560, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.00970459, + "step": 6496, + "time_per_iteration": 2.8988864421844482 + }, + { + "auxiliary_loss_clip": 0.01119823, + "auxiliary_loss_mlp": 0.01037093, + "balance_loss_clip": 1.05285454, + "balance_loss_mlp": 1.02440333, + "epoch": 0.18852649294875515, + "flos": 16357167037440.0, + "grad_norm": 2.460996208908758, + "language_loss": 0.66244423, + "learning_rate": 3.742142123259427e-06, + "loss": 0.68401337, + "num_input_tokens_seen": 185558690, + "router_z_loss_clip": 0.66992188, + "router_z_loss_mlp": 0.12695312, + "step": 6497, + "time_per_iteration": 2.52221941947937 + }, + { + "auxiliary_loss_clip": 0.01051354, + "auxiliary_loss_mlp": 0.01003882, + "balance_loss_clip": 1.03259385, + "balance_loss_mlp": 1.00294673, + "epoch": 0.1885555104172712, + "flos": 57221751778560.0, + "grad_norm": 0.7196010573539542, + "language_loss": 0.46065265, + "learning_rate": 3.7420497968363922e-06, + "loss": 0.48120505, + "num_input_tokens_seen": 185620775, + "router_z_loss_clip": 0.1875, + "router_z_loss_mlp": 0.00933838, + "step": 6498, + "time_per_iteration": 3.07641339302063 + }, + { + "auxiliary_loss_clip": 0.01135895, + "auxiliary_loss_mlp": 0.01036128, + "balance_loss_clip": 1.05775166, + "balance_loss_mlp": 1.01985574, + "epoch": 0.18858452788578725, + "flos": 27635559384960.0, + "grad_norm": 2.6751264828764296, + "language_loss": 0.87091547, + "learning_rate": 3.7419574550268694e-06, + "loss": 0.8926357, + "num_input_tokens_seen": 185640720, + "router_z_loss_clip": 0.78173828, + "router_z_loss_mlp": 0.16265869, + "step": 6499, + "time_per_iteration": 2.663886547088623 + }, + { + "auxiliary_loss_clip": 0.01127579, + "auxiliary_loss_mlp": 0.01034327, + "balance_loss_clip": 1.05426097, + "balance_loss_mlp": 1.01962233, + "epoch": 0.1886135453543033, + "flos": 16466622756480.0, + "grad_norm": 2.0849899802306173, + "language_loss": 0.69939518, + "learning_rate": 3.7418650978316737e-06, + "loss": 0.72101426, + "num_input_tokens_seen": 185655870, + "router_z_loss_clip": 0.73242188, + "router_z_loss_mlp": 0.14709473, + "step": 6500, + "time_per_iteration": 2.504739761352539 + }, + { + "auxiliary_loss_clip": 0.0104957, + "auxiliary_loss_mlp": 0.01001348, + "balance_loss_clip": 1.03088212, + "balance_loss_mlp": 1.0003289, + "epoch": 0.18864256282281933, + "flos": 67790155898880.0, + "grad_norm": 0.6725568810215228, + "language_loss": 0.48003852, + "learning_rate": 3.7417727252516204e-06, + "loss": 0.50054771, + "num_input_tokens_seen": 185723505, + "router_z_loss_clip": 0.1875, + "router_z_loss_mlp": 0.01019287, + "step": 6501, + "time_per_iteration": 3.2507283687591553 + }, + { + "auxiliary_loss_clip": 0.01129038, + "auxiliary_loss_mlp": 0.01042527, + "balance_loss_clip": 1.0539372, + "balance_loss_mlp": 1.02604008, + "epoch": 0.18867158029133538, + "flos": 24966426625920.0, + "grad_norm": 2.6604577853085245, + "language_loss": 0.81280458, + "learning_rate": 3.7416803372875256e-06, + "loss": 0.83452022, + "num_input_tokens_seen": 185742225, + "router_z_loss_clip": 0.75146484, + "router_z_loss_mlp": 0.16479492, + "step": 6502, + "time_per_iteration": 2.593985080718994 + }, + { + "auxiliary_loss_clip": 0.01131584, + "auxiliary_loss_mlp": 0.01035231, + "balance_loss_clip": 1.05556965, + "balance_loss_mlp": 1.02003169, + "epoch": 0.18870059775985143, + "flos": 15881350550400.0, + "grad_norm": 2.2836031916923196, + "language_loss": 0.75480902, + "learning_rate": 3.741587933940205e-06, + "loss": 0.77647722, + "num_input_tokens_seen": 185755155, + "router_z_loss_clip": 0.75927734, + "router_z_loss_mlp": 0.15216064, + "step": 6503, + "time_per_iteration": 2.465730905532837 + }, + { + "auxiliary_loss_clip": 0.01128083, + "auxiliary_loss_mlp": 0.01042552, + "balance_loss_clip": 1.05539429, + "balance_loss_mlp": 1.02835464, + "epoch": 0.18872961522836748, + "flos": 35878781427840.0, + "grad_norm": 2.39359850423464, + "language_loss": 0.57269615, + "learning_rate": 3.7414955152104754e-06, + "loss": 0.59440255, + "num_input_tokens_seen": 185772685, + "router_z_loss_clip": 0.7265625, + "router_z_loss_mlp": 0.1418457, + "step": 6504, + "time_per_iteration": 2.6410152912139893 + }, + { + "auxiliary_loss_clip": 0.01134037, + "auxiliary_loss_mlp": 0.0103414, + "balance_loss_clip": 1.05622768, + "balance_loss_mlp": 1.01835048, + "epoch": 0.18875863269688353, + "flos": 18302377092480.0, + "grad_norm": 2.3999147792878555, + "language_loss": 0.84694743, + "learning_rate": 3.7414030810991523e-06, + "loss": 0.86862922, + "num_input_tokens_seen": 185789565, + "router_z_loss_clip": 0.77734375, + "router_z_loss_mlp": 0.15777588, + "step": 6505, + "time_per_iteration": 2.5122034549713135 + }, + { + "auxiliary_loss_clip": 0.01132322, + "auxiliary_loss_mlp": 0.0105113, + "balance_loss_clip": 1.05523694, + "balance_loss_mlp": 1.03401172, + "epoch": 0.18878765016539958, + "flos": 11648847265920.0, + "grad_norm": 2.7378629281537883, + "language_loss": 0.84116697, + "learning_rate": 3.741310631607053e-06, + "loss": 0.86300147, + "num_input_tokens_seen": 185800930, + "router_z_loss_clip": 0.77099609, + "router_z_loss_mlp": 0.17108154, + "step": 6506, + "time_per_iteration": 2.4753570556640625 + }, + { + "auxiliary_loss_clip": 0.01048809, + "auxiliary_loss_mlp": 0.01001863, + "balance_loss_clip": 1.0300951, + "balance_loss_mlp": 1.00084352, + "epoch": 0.1888166676339156, + "flos": 68251822427520.0, + "grad_norm": 0.6413687680415664, + "language_loss": 0.4757255, + "learning_rate": 3.7412181667349933e-06, + "loss": 0.49623224, + "num_input_tokens_seen": 185865620, + "router_z_loss_clip": 0.1875, + "router_z_loss_mlp": 0.01019287, + "step": 6507, + "time_per_iteration": 3.1875252723693848 + }, + { + "auxiliary_loss_clip": 0.01129636, + "auxiliary_loss_mlp": 0.0103943, + "balance_loss_clip": 1.05572534, + "balance_loss_mlp": 1.02462995, + "epoch": 0.18884568510243166, + "flos": 15882140649600.0, + "grad_norm": 2.2163230358975916, + "language_loss": 0.77494431, + "learning_rate": 3.74112568648379e-06, + "loss": 0.79663497, + "num_input_tokens_seen": 185882120, + "router_z_loss_clip": 0.73876953, + "router_z_loss_mlp": 0.14794922, + "step": 6508, + "time_per_iteration": 2.524371385574341 + }, + { + "auxiliary_loss_clip": 0.0112512, + "auxiliary_loss_mlp": 0.01031762, + "balance_loss_clip": 1.05571914, + "balance_loss_mlp": 1.01703334, + "epoch": 0.1888747025709477, + "flos": 11757441058560.0, + "grad_norm": 3.7677263367936833, + "language_loss": 0.78120768, + "learning_rate": 3.74103319085426e-06, + "loss": 0.80277652, + "num_input_tokens_seen": 185893295, + "router_z_loss_clip": 0.69384766, + "router_z_loss_mlp": 0.14715576, + "step": 6509, + "time_per_iteration": 2.48404860496521 + }, + { + "auxiliary_loss_clip": 0.01130173, + "auxiliary_loss_mlp": 0.01034858, + "balance_loss_clip": 1.05787075, + "balance_loss_mlp": 1.02132154, + "epoch": 0.18890372003946376, + "flos": 12159030090240.0, + "grad_norm": 2.038326787330808, + "language_loss": 0.57167637, + "learning_rate": 3.740940679847221e-06, + "loss": 0.59332669, + "num_input_tokens_seen": 185904960, + "router_z_loss_clip": 0.72314453, + "router_z_loss_mlp": 0.13525391, + "step": 6510, + "time_per_iteration": 2.527430295944214 + }, + { + "auxiliary_loss_clip": 0.01127474, + "auxiliary_loss_mlp": 0.01038149, + "balance_loss_clip": 1.05604482, + "balance_loss_mlp": 1.02413011, + "epoch": 0.18893273750797981, + "flos": 17194495956480.0, + "grad_norm": 2.036175542057041, + "language_loss": 0.75455749, + "learning_rate": 3.7408481534634887e-06, + "loss": 0.77621377, + "num_input_tokens_seen": 185917515, + "router_z_loss_clip": 0.71484375, + "router_z_loss_mlp": 0.14013672, + "step": 6511, + "time_per_iteration": 2.542100429534912 + }, + { + "auxiliary_loss_clip": 0.01124785, + "auxiliary_loss_mlp": 0.01041339, + "balance_loss_clip": 1.05436254, + "balance_loss_mlp": 1.02683711, + "epoch": 0.18896175497649584, + "flos": 29417230817280.0, + "grad_norm": 2.2513425784284635, + "language_loss": 0.70239443, + "learning_rate": 3.7407556117038813e-06, + "loss": 0.72405571, + "num_input_tokens_seen": 185931520, + "router_z_loss_clip": 0.70458984, + "router_z_loss_mlp": 0.1451416, + "step": 6512, + "time_per_iteration": 2.5978822708129883 + }, + { + "auxiliary_loss_clip": 0.01126849, + "auxiliary_loss_mlp": 0.01035129, + "balance_loss_clip": 1.05550814, + "balance_loss_mlp": 1.02185547, + "epoch": 0.1889907724450119, + "flos": 16685965157760.0, + "grad_norm": 4.178566518305836, + "language_loss": 0.74137449, + "learning_rate": 3.7406630545692164e-06, + "loss": 0.76299429, + "num_input_tokens_seen": 185946565, + "router_z_loss_clip": 0.71337891, + "router_z_loss_mlp": 0.13269043, + "step": 6513, + "time_per_iteration": 2.5163497924804688 + }, + { + "auxiliary_loss_clip": 0.01119933, + "auxiliary_loss_mlp": 0.01033129, + "balance_loss_clip": 1.0547924, + "balance_loss_mlp": 1.02065969, + "epoch": 0.18901978991352794, + "flos": 13071807527040.0, + "grad_norm": 4.398400079602095, + "language_loss": 0.85792178, + "learning_rate": 3.7405704820603105e-06, + "loss": 0.87945241, + "num_input_tokens_seen": 185958560, + "router_z_loss_clip": 0.65234375, + "router_z_loss_mlp": 0.12475586, + "step": 6514, + "time_per_iteration": 2.4801177978515625 + }, + { + "auxiliary_loss_clip": 0.01133839, + "auxiliary_loss_mlp": 0.01050269, + "balance_loss_clip": 1.0574708, + "balance_loss_mlp": 1.03447962, + "epoch": 0.189048807382044, + "flos": 12342749178240.0, + "grad_norm": 2.558429530826267, + "language_loss": 0.80582994, + "learning_rate": 3.7404778941779816e-06, + "loss": 0.82767105, + "num_input_tokens_seen": 185969935, + "router_z_loss_clip": 0.76318359, + "router_z_loss_mlp": 0.15795898, + "step": 6515, + "time_per_iteration": 2.5376291275024414 + }, + { + "auxiliary_loss_clip": 0.01050993, + "auxiliary_loss_mlp": 0.01013763, + "balance_loss_clip": 1.03185499, + "balance_loss_mlp": 1.01275015, + "epoch": 0.18907782485056004, + "flos": 69345733173120.0, + "grad_norm": 0.6537872345882708, + "language_loss": 0.51264369, + "learning_rate": 3.740385290923048e-06, + "loss": 0.53329134, + "num_input_tokens_seen": 186033810, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.01013184, + "step": 6516, + "time_per_iteration": 3.195436716079712 + }, + { + "auxiliary_loss_clip": 0.0113705, + "auxiliary_loss_mlp": 0.01045493, + "balance_loss_clip": 1.06035042, + "balance_loss_mlp": 1.02880394, + "epoch": 0.1891068423190761, + "flos": 25112439411840.0, + "grad_norm": 2.83158262797178, + "language_loss": 0.85852492, + "learning_rate": 3.740292672296327e-06, + "loss": 0.88035029, + "num_input_tokens_seen": 186056415, + "router_z_loss_clip": 0.76611328, + "router_z_loss_mlp": 0.16693115, + "step": 6517, + "time_per_iteration": 2.81965970993042 + }, + { + "auxiliary_loss_clip": 0.01126703, + "auxiliary_loss_mlp": 0.01051934, + "balance_loss_clip": 1.05347228, + "balance_loss_mlp": 1.03514934, + "epoch": 0.18913585978759212, + "flos": 33067694119680.0, + "grad_norm": 2.296710394212624, + "language_loss": 0.71510011, + "learning_rate": 3.7402000382986373e-06, + "loss": 0.7368865, + "num_input_tokens_seen": 186074315, + "router_z_loss_clip": 0.73291016, + "router_z_loss_mlp": 0.16784668, + "step": 6518, + "time_per_iteration": 2.6314890384674072 + }, + { + "auxiliary_loss_clip": 0.01049632, + "auxiliary_loss_mlp": 0.01003948, + "balance_loss_clip": 1.03080523, + "balance_loss_mlp": 1.00281572, + "epoch": 0.18916487725610817, + "flos": 74784009133440.0, + "grad_norm": 0.6102435169631499, + "language_loss": 0.44061479, + "learning_rate": 3.7401073889307966e-06, + "loss": 0.46115059, + "num_input_tokens_seen": 186144145, + "router_z_loss_clip": 0.18847656, + "router_z_loss_mlp": 0.01135254, + "step": 6519, + "time_per_iteration": 3.359067678451538 + }, + { + "auxiliary_loss_clip": 0.0112618, + "auxiliary_loss_mlp": 0.01037003, + "balance_loss_clip": 1.05465412, + "balance_loss_mlp": 1.02155972, + "epoch": 0.18919389472462422, + "flos": 33430535354880.0, + "grad_norm": 1.782945167013538, + "language_loss": 0.96560866, + "learning_rate": 3.7400147241936227e-06, + "loss": 0.98724049, + "num_input_tokens_seen": 186163425, + "router_z_loss_clip": 0.71484375, + "router_z_loss_mlp": 0.15441895, + "step": 6520, + "time_per_iteration": 2.6650917530059814 + }, + { + "auxiliary_loss_clip": 0.01119568, + "auxiliary_loss_mlp": 0.01032525, + "balance_loss_clip": 1.05267906, + "balance_loss_mlp": 1.01894069, + "epoch": 0.18922291219314027, + "flos": 16245771984000.0, + "grad_norm": 2.9635187978060538, + "language_loss": 0.82404172, + "learning_rate": 3.7399220440879355e-06, + "loss": 0.8455627, + "num_input_tokens_seen": 186176670, + "router_z_loss_clip": 0.66943359, + "router_z_loss_mlp": 0.13580322, + "step": 6521, + "time_per_iteration": 2.549504518508911 + }, + { + "auxiliary_loss_clip": 0.01125108, + "auxiliary_loss_mlp": 0.01040843, + "balance_loss_clip": 1.05397058, + "balance_loss_mlp": 1.02477908, + "epoch": 0.18925192966165633, + "flos": 74731826672640.0, + "grad_norm": 1.9858731876036146, + "language_loss": 0.63188571, + "learning_rate": 3.7398293486145524e-06, + "loss": 0.65354526, + "num_input_tokens_seen": 186199935, + "router_z_loss_clip": 0.71191406, + "router_z_loss_mlp": 0.16064453, + "step": 6522, + "time_per_iteration": 3.0084421634674072 + }, + { + "auxiliary_loss_clip": 0.01128741, + "auxiliary_loss_mlp": 0.01046661, + "balance_loss_clip": 1.05678642, + "balance_loss_mlp": 1.03109837, + "epoch": 0.18928094713017235, + "flos": 28140965700480.0, + "grad_norm": 2.1606810955145486, + "language_loss": 0.78125024, + "learning_rate": 3.739736637774292e-06, + "loss": 0.80300421, + "num_input_tokens_seen": 186215060, + "router_z_loss_clip": 0.71923828, + "router_z_loss_mlp": 0.15557861, + "step": 6523, + "time_per_iteration": 2.6146838665008545 + }, + { + "auxiliary_loss_clip": 0.01129385, + "auxiliary_loss_mlp": 0.0103667, + "balance_loss_clip": 1.05301094, + "balance_loss_mlp": 1.02020693, + "epoch": 0.1893099645986884, + "flos": 47044593573120.0, + "grad_norm": 2.545349452445039, + "language_loss": 0.84393477, + "learning_rate": 3.7396439115679744e-06, + "loss": 0.86559534, + "num_input_tokens_seen": 186232635, + "router_z_loss_clip": 0.76416016, + "router_z_loss_mlp": 0.16455078, + "step": 6524, + "time_per_iteration": 2.7008562088012695 + }, + { + "auxiliary_loss_clip": 0.01131666, + "auxiliary_loss_mlp": 0.01040974, + "balance_loss_clip": 1.05531001, + "balance_loss_mlp": 1.0239923, + "epoch": 0.18933898206720445, + "flos": 16539377754240.0, + "grad_norm": 2.330572353461467, + "language_loss": 0.66367662, + "learning_rate": 3.7395511699964173e-06, + "loss": 0.68540299, + "num_input_tokens_seen": 186246215, + "router_z_loss_clip": 0.76464844, + "router_z_loss_mlp": 0.1697998, + "step": 6525, + "time_per_iteration": 2.5913424491882324 + }, + { + "auxiliary_loss_clip": 0.01130407, + "auxiliary_loss_mlp": 0.01047811, + "balance_loss_clip": 1.05765057, + "balance_loss_mlp": 1.03156924, + "epoch": 0.1893679995357205, + "flos": 30656005113600.0, + "grad_norm": 2.256599316438756, + "language_loss": 0.75806141, + "learning_rate": 3.73945841306044e-06, + "loss": 0.77984357, + "num_input_tokens_seen": 186261635, + "router_z_loss_clip": 0.72802734, + "router_z_loss_mlp": 0.16247559, + "step": 6526, + "time_per_iteration": 2.664095401763916 + }, + { + "auxiliary_loss_clip": 0.0113637, + "auxiliary_loss_mlp": 0.01040582, + "balance_loss_clip": 1.05902624, + "balance_loss_mlp": 1.0236181, + "epoch": 0.18939701700423656, + "flos": 33577805116800.0, + "grad_norm": 2.2928008449405524, + "language_loss": 0.99551952, + "learning_rate": 3.7393656407608626e-06, + "loss": 1.01728892, + "num_input_tokens_seen": 186281960, + "router_z_loss_clip": 0.77392578, + "router_z_loss_mlp": 0.16955566, + "step": 6527, + "time_per_iteration": 2.6662585735321045 + }, + { + "auxiliary_loss_clip": 0.01049124, + "auxiliary_loss_mlp": 0.01013564, + "balance_loss_clip": 1.03026628, + "balance_loss_mlp": 1.01253927, + "epoch": 0.1894260344727526, + "flos": 64713508364160.0, + "grad_norm": 0.6427445374717212, + "language_loss": 0.46367997, + "learning_rate": 3.7392728530985045e-06, + "loss": 0.48430684, + "num_input_tokens_seen": 186345930, + "router_z_loss_clip": 0.18847656, + "router_z_loss_mlp": 0.01025391, + "step": 6528, + "time_per_iteration": 3.17500638961792 + }, + { + "auxiliary_loss_clip": 0.0112547, + "auxiliary_loss_mlp": 0.01042338, + "balance_loss_clip": 1.05263507, + "balance_loss_mlp": 1.02583945, + "epoch": 0.18945505194126863, + "flos": 22376118263040.0, + "grad_norm": 2.1493409557635514, + "language_loss": 0.86761284, + "learning_rate": 3.739180050074184e-06, + "loss": 0.88929093, + "num_input_tokens_seen": 186360795, + "router_z_loss_clip": 0.72802734, + "router_z_loss_mlp": 0.16479492, + "step": 6529, + "time_per_iteration": 2.5226831436157227 + }, + { + "auxiliary_loss_clip": 0.01047939, + "auxiliary_loss_mlp": 0.01002298, + "balance_loss_clip": 1.02921367, + "balance_loss_mlp": 1.00132692, + "epoch": 0.18948406940978468, + "flos": 68535264648960.0, + "grad_norm": 0.6385903848044963, + "language_loss": 0.48001969, + "learning_rate": 3.739087231688722e-06, + "loss": 0.50052202, + "num_input_tokens_seen": 186418360, + "router_z_loss_clip": 0.1875, + "router_z_loss_mlp": 0.00970459, + "step": 6530, + "time_per_iteration": 3.097214698791504 + }, + { + "auxiliary_loss_clip": 0.01128526, + "auxiliary_loss_mlp": 0.01036289, + "balance_loss_clip": 1.05387712, + "balance_loss_mlp": 1.02032089, + "epoch": 0.18951308687830073, + "flos": 25660472192640.0, + "grad_norm": 2.4518770756269377, + "language_loss": 0.82323319, + "learning_rate": 3.7389943979429374e-06, + "loss": 0.8448813, + "num_input_tokens_seen": 186433070, + "router_z_loss_clip": 0.74609375, + "router_z_loss_mlp": 0.15966797, + "step": 6531, + "time_per_iteration": 2.5821807384490967 + }, + { + "auxiliary_loss_clip": 0.01123724, + "auxiliary_loss_mlp": 0.01035851, + "balance_loss_clip": 1.05324078, + "balance_loss_mlp": 1.02198148, + "epoch": 0.1895421043468168, + "flos": 19530987840000.0, + "grad_norm": 2.512441269655393, + "language_loss": 0.6918447, + "learning_rate": 3.738901548837651e-06, + "loss": 0.71344042, + "num_input_tokens_seen": 186446755, + "router_z_loss_clip": 0.70410156, + "router_z_loss_mlp": 0.13873291, + "step": 6532, + "time_per_iteration": 2.528458595275879 + }, + { + "auxiliary_loss_clip": 0.01048306, + "auxiliary_loss_mlp": 0.01011102, + "balance_loss_clip": 1.02946448, + "balance_loss_mlp": 1.01012444, + "epoch": 0.18957112181533284, + "flos": 60357794451840.0, + "grad_norm": 0.7123883664727403, + "language_loss": 0.53288209, + "learning_rate": 3.738808684373682e-06, + "loss": 0.55347621, + "num_input_tokens_seen": 186508145, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.00976562, + "step": 6533, + "time_per_iteration": 3.0868051052093506 + }, + { + "auxiliary_loss_clip": 0.01124627, + "auxiliary_loss_mlp": 0.01043356, + "balance_loss_clip": 1.05272067, + "balance_loss_mlp": 1.02849698, + "epoch": 0.1896001392838489, + "flos": 12232251964800.0, + "grad_norm": 3.3345684390386885, + "language_loss": 1.09625554, + "learning_rate": 3.7387158045518517e-06, + "loss": 1.1179353, + "num_input_tokens_seen": 186519375, + "router_z_loss_clip": 0.72021484, + "router_z_loss_mlp": 0.14862061, + "step": 6534, + "time_per_iteration": 2.4837050437927246 + }, + { + "auxiliary_loss_clip": 0.01120564, + "auxiliary_loss_mlp": 0.01038833, + "balance_loss_clip": 1.0519681, + "balance_loss_mlp": 1.02545202, + "epoch": 0.1896291567523649, + "flos": 43247111904000.0, + "grad_norm": 2.628901191099681, + "language_loss": 0.8793453, + "learning_rate": 3.7386229093729787e-06, + "loss": 0.90093923, + "num_input_tokens_seen": 186537580, + "router_z_loss_clip": 0.68554688, + "router_z_loss_mlp": 0.13378906, + "step": 6535, + "time_per_iteration": 2.7264692783355713 + }, + { + "auxiliary_loss_clip": 0.01123407, + "auxiliary_loss_mlp": 0.01042976, + "balance_loss_clip": 1.05622005, + "balance_loss_mlp": 1.02836704, + "epoch": 0.18965817422088097, + "flos": 18508395548160.0, + "grad_norm": 2.3878118588904917, + "language_loss": 0.80627537, + "learning_rate": 3.738529998837886e-06, + "loss": 0.82793927, + "num_input_tokens_seen": 186551035, + "router_z_loss_clip": 0.67138672, + "router_z_loss_mlp": 0.14605713, + "step": 6536, + "time_per_iteration": 2.5008349418640137 + }, + { + "auxiliary_loss_clip": 0.01126656, + "auxiliary_loss_mlp": 0.01042866, + "balance_loss_clip": 1.05649853, + "balance_loss_mlp": 1.02887118, + "epoch": 0.18968719168939702, + "flos": 11356785780480.0, + "grad_norm": 3.020917955697821, + "language_loss": 0.97654998, + "learning_rate": 3.7384370729473922e-06, + "loss": 0.99824512, + "num_input_tokens_seen": 186561495, + "router_z_loss_clip": 0.70117188, + "router_z_loss_mlp": 0.13995361, + "step": 6537, + "time_per_iteration": 2.513906717300415 + }, + { + "auxiliary_loss_clip": 0.01126505, + "auxiliary_loss_mlp": 0.01042296, + "balance_loss_clip": 1.0558815, + "balance_loss_mlp": 1.02753162, + "epoch": 0.18971620915791307, + "flos": 16646678657280.0, + "grad_norm": 2.6668841406047306, + "language_loss": 0.67091411, + "learning_rate": 3.738344131702318e-06, + "loss": 0.69260216, + "num_input_tokens_seen": 186574660, + "router_z_loss_clip": 0.70605469, + "router_z_loss_mlp": 0.14752197, + "step": 6538, + "time_per_iteration": 2.4878501892089844 + }, + { + "auxiliary_loss_clip": 0.01125691, + "auxiliary_loss_mlp": 0.01041177, + "balance_loss_clip": 1.05623865, + "balance_loss_mlp": 1.02587032, + "epoch": 0.18974522662642912, + "flos": 42667226737920.0, + "grad_norm": 2.9791059284876122, + "language_loss": 0.9471041, + "learning_rate": 3.738251175103486e-06, + "loss": 0.96877277, + "num_input_tokens_seen": 186596180, + "router_z_loss_clip": 0.69384766, + "router_z_loss_mlp": 0.15307617, + "step": 6539, + "time_per_iteration": 2.6644606590270996 + }, + { + "auxiliary_loss_clip": 0.01132777, + "auxiliary_loss_mlp": 0.01041768, + "balance_loss_clip": 1.06026399, + "balance_loss_mlp": 1.026366, + "epoch": 0.18977424409494514, + "flos": 20479137194880.0, + "grad_norm": 2.686530092818462, + "language_loss": 0.89596111, + "learning_rate": 3.738158203151716e-06, + "loss": 0.91770661, + "num_input_tokens_seen": 186608745, + "router_z_loss_clip": 0.72509766, + "router_z_loss_mlp": 0.1539917, + "step": 6540, + "time_per_iteration": 2.5470330715179443 + }, + { + "auxiliary_loss_clip": 0.01129682, + "auxiliary_loss_mlp": 0.01035552, + "balance_loss_clip": 1.05587697, + "balance_loss_mlp": 1.02013791, + "epoch": 0.1898032615634612, + "flos": 13583283240960.0, + "grad_norm": 2.774436681111558, + "language_loss": 0.7639128, + "learning_rate": 3.7380652158478295e-06, + "loss": 0.78556514, + "num_input_tokens_seen": 186622140, + "router_z_loss_clip": 0.73779297, + "router_z_loss_mlp": 0.15411377, + "step": 6541, + "time_per_iteration": 2.536428451538086 + }, + { + "auxiliary_loss_clip": 0.01133706, + "auxiliary_loss_mlp": 0.01043368, + "balance_loss_clip": 1.05900168, + "balance_loss_mlp": 1.02915859, + "epoch": 0.18983227903197725, + "flos": 24309512743680.0, + "grad_norm": 2.0048781739732173, + "language_loss": 0.86859763, + "learning_rate": 3.7379722131926474e-06, + "loss": 0.89036846, + "num_input_tokens_seen": 186638800, + "router_z_loss_clip": 0.74707031, + "router_z_loss_mlp": 0.14215088, + "step": 6542, + "time_per_iteration": 2.5638747215270996 + }, + { + "auxiliary_loss_clip": 0.01127765, + "auxiliary_loss_mlp": 0.01039791, + "balance_loss_clip": 1.05787015, + "balance_loss_mlp": 1.0256052, + "epoch": 0.1898612965004933, + "flos": 40289904069120.0, + "grad_norm": 2.498763467817508, + "language_loss": 0.75101864, + "learning_rate": 3.7378791951869913e-06, + "loss": 0.77269423, + "num_input_tokens_seen": 186656005, + "router_z_loss_clip": 0.69946289, + "router_z_loss_mlp": 0.1418457, + "step": 6543, + "time_per_iteration": 9.966556072235107 + }, + { + "auxiliary_loss_clip": 0.01128756, + "auxiliary_loss_mlp": 0.01030952, + "balance_loss_clip": 1.05794251, + "balance_loss_mlp": 1.017869, + "epoch": 0.18989031396900935, + "flos": 25988983004160.0, + "grad_norm": 2.817500317227317, + "language_loss": 0.75083756, + "learning_rate": 3.737786161831683e-06, + "loss": 0.77243465, + "num_input_tokens_seen": 186678790, + "router_z_loss_clip": 0.70751953, + "router_z_loss_mlp": 0.13092041, + "step": 6544, + "time_per_iteration": 5.0664191246032715 + }, + { + "auxiliary_loss_clip": 0.01052054, + "auxiliary_loss_mlp": 0.01011692, + "balance_loss_clip": 1.03305697, + "balance_loss_mlp": 1.01068485, + "epoch": 0.1899193314375254, + "flos": 60216306779520.0, + "grad_norm": 0.6210855983274713, + "language_loss": 0.44802946, + "learning_rate": 3.7376931131275447e-06, + "loss": 0.46866691, + "num_input_tokens_seen": 186741460, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.0100708, + "step": 6545, + "time_per_iteration": 3.1409096717834473 + }, + { + "auxiliary_loss_clip": 0.01051726, + "auxiliary_loss_mlp": 0.01008193, + "balance_loss_clip": 1.03282869, + "balance_loss_mlp": 1.00733423, + "epoch": 0.18994834890604143, + "flos": 61934129786880.0, + "grad_norm": 0.6881919270779607, + "language_loss": 0.49617827, + "learning_rate": 3.7376000490753974e-06, + "loss": 0.5167774, + "num_input_tokens_seen": 186801435, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.00860596, + "step": 6546, + "time_per_iteration": 3.0371999740600586 + }, + { + "auxiliary_loss_clip": 0.01127701, + "auxiliary_loss_mlp": 0.01037516, + "balance_loss_clip": 1.05926156, + "balance_loss_mlp": 1.02269793, + "epoch": 0.18997736637455748, + "flos": 11684291011200.0, + "grad_norm": 3.0185641094925004, + "language_loss": 0.8858794, + "learning_rate": 3.7375069696760627e-06, + "loss": 0.90753162, + "num_input_tokens_seen": 186812960, + "router_z_loss_clip": 0.68408203, + "router_z_loss_mlp": 0.14801025, + "step": 6547, + "time_per_iteration": 2.609098434448242 + }, + { + "auxiliary_loss_clip": 0.01130413, + "auxiliary_loss_mlp": 0.01038236, + "balance_loss_clip": 1.05642533, + "balance_loss_mlp": 1.02320397, + "epoch": 0.19000638384307353, + "flos": 17012105671680.0, + "grad_norm": 4.758892472752411, + "language_loss": 0.74478149, + "learning_rate": 3.737413874930364e-06, + "loss": 0.76646799, + "num_input_tokens_seen": 186826330, + "router_z_loss_clip": 0.74072266, + "router_z_loss_mlp": 0.15039062, + "step": 6548, + "time_per_iteration": 2.5183448791503906 + }, + { + "auxiliary_loss_clip": 0.01133574, + "auxiliary_loss_mlp": 0.01035962, + "balance_loss_clip": 1.0578481, + "balance_loss_mlp": 1.01837838, + "epoch": 0.19003540131158958, + "flos": 36167395207680.0, + "grad_norm": 2.6130790422433843, + "language_loss": 0.93435591, + "learning_rate": 3.7373207648391226e-06, + "loss": 0.95605123, + "num_input_tokens_seen": 186842965, + "router_z_loss_clip": 0.7578125, + "router_z_loss_mlp": 0.17590332, + "step": 6549, + "time_per_iteration": 2.6422951221466064 + }, + { + "auxiliary_loss_clip": 0.01140473, + "auxiliary_loss_mlp": 0.01046403, + "balance_loss_clip": 1.05808055, + "balance_loss_mlp": 1.02799153, + "epoch": 0.19006441878010563, + "flos": 44266471971840.0, + "grad_norm": 2.907599342522183, + "language_loss": 0.82612199, + "learning_rate": 3.7372276394031614e-06, + "loss": 0.84799075, + "num_input_tokens_seen": 186860500, + "router_z_loss_clip": 0.82470703, + "router_z_loss_mlp": 0.184021, + "step": 6550, + "time_per_iteration": 2.738814353942871 + }, + { + "auxiliary_loss_clip": 0.01138349, + "auxiliary_loss_mlp": 0.01050847, + "balance_loss_clip": 1.05871463, + "balance_loss_mlp": 1.0343188, + "epoch": 0.19009343624862168, + "flos": 19275411594240.0, + "grad_norm": 2.0692055246056023, + "language_loss": 0.83900446, + "learning_rate": 3.7371344986233025e-06, + "loss": 0.86089647, + "num_input_tokens_seen": 186874780, + "router_z_loss_clip": 0.79589844, + "router_z_loss_mlp": 0.16546631, + "step": 6551, + "time_per_iteration": 2.5062737464904785 + }, + { + "auxiliary_loss_clip": 0.01130194, + "auxiliary_loss_mlp": 0.01042279, + "balance_loss_clip": 1.05643594, + "balance_loss_mlp": 1.02646637, + "epoch": 0.1901224537171377, + "flos": 35802291415680.0, + "grad_norm": 2.4005041511106704, + "language_loss": 0.76957524, + "learning_rate": 3.737041342500369e-06, + "loss": 0.79129994, + "num_input_tokens_seen": 186895035, + "router_z_loss_clip": 0.73730469, + "router_z_loss_mlp": 0.15808105, + "step": 6552, + "time_per_iteration": 2.710188627243042 + }, + { + "auxiliary_loss_clip": 0.01052316, + "auxiliary_loss_mlp": 0.01017745, + "balance_loss_clip": 1.03282046, + "balance_loss_mlp": 1.01682687, + "epoch": 0.19015147118565376, + "flos": 73938958790400.0, + "grad_norm": 0.7148722558364289, + "language_loss": 0.47349143, + "learning_rate": 3.7369481710351833e-06, + "loss": 0.494192, + "num_input_tokens_seen": 186961900, + "router_z_loss_clip": 0.1953125, + "router_z_loss_mlp": 0.00915527, + "step": 6553, + "time_per_iteration": 3.2855186462402344 + }, + { + "auxiliary_loss_clip": 0.0113133, + "auxiliary_loss_mlp": 0.01047662, + "balance_loss_clip": 1.0584023, + "balance_loss_mlp": 1.03256369, + "epoch": 0.1901804886541698, + "flos": 28029786128640.0, + "grad_norm": 2.1589245622416096, + "language_loss": 0.93770903, + "learning_rate": 3.7368549842285684e-06, + "loss": 0.959499, + "num_input_tokens_seen": 186979585, + "router_z_loss_clip": 0.72851562, + "router_z_loss_mlp": 0.15100098, + "step": 6554, + "time_per_iteration": 2.6115715503692627 + }, + { + "auxiliary_loss_clip": 0.01131996, + "auxiliary_loss_mlp": 0.01042356, + "balance_loss_clip": 1.06077886, + "balance_loss_mlp": 1.02843845, + "epoch": 0.19020950612268586, + "flos": 34053301359360.0, + "grad_norm": 3.1057687719581235, + "language_loss": 0.88608503, + "learning_rate": 3.7367617820813474e-06, + "loss": 0.90782869, + "num_input_tokens_seen": 187008615, + "router_z_loss_clip": 0.71289062, + "router_z_loss_mlp": 0.13934326, + "step": 6555, + "time_per_iteration": 2.859693765640259 + }, + { + "auxiliary_loss_clip": 0.01048936, + "auxiliary_loss_mlp": 0.01002635, + "balance_loss_clip": 1.02965772, + "balance_loss_mlp": 1.00176513, + "epoch": 0.1902385235912019, + "flos": 62411526472320.0, + "grad_norm": 0.6890720930735509, + "language_loss": 0.50437355, + "learning_rate": 3.736668564594344e-06, + "loss": 0.52488929, + "num_input_tokens_seen": 187067830, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.00872803, + "step": 6556, + "time_per_iteration": 3.1053922176361084 + }, + { + "auxiliary_loss_clip": 0.0114447, + "auxiliary_loss_mlp": 0.01047402, + "balance_loss_clip": 1.06307507, + "balance_loss_mlp": 1.03042674, + "epoch": 0.19026754105971794, + "flos": 20843989591680.0, + "grad_norm": 3.795350883892436, + "language_loss": 0.85755718, + "learning_rate": 3.7365753317683808e-06, + "loss": 0.87947583, + "num_input_tokens_seen": 187083890, + "router_z_loss_clip": 0.81347656, + "router_z_loss_mlp": 0.1697998, + "step": 6557, + "time_per_iteration": 2.6036853790283203 + }, + { + "auxiliary_loss_clip": 0.01131263, + "auxiliary_loss_mlp": 0.01037961, + "balance_loss_clip": 1.0554322, + "balance_loss_mlp": 1.023471, + "epoch": 0.190296558528234, + "flos": 74740769159040.0, + "grad_norm": 1.556567172452482, + "language_loss": 0.80436593, + "learning_rate": 3.736482083604281e-06, + "loss": 0.82605815, + "num_input_tokens_seen": 187116285, + "router_z_loss_clip": 0.75976562, + "router_z_loss_mlp": 0.14483643, + "step": 6558, + "time_per_iteration": 2.949777126312256 + }, + { + "auxiliary_loss_clip": 0.01137407, + "auxiliary_loss_mlp": 0.01042129, + "balance_loss_clip": 1.05903554, + "balance_loss_mlp": 1.02605915, + "epoch": 0.19032557599675004, + "flos": 28687238714880.0, + "grad_norm": 2.7711393555866, + "language_loss": 0.84208292, + "learning_rate": 3.7363888201028696e-06, + "loss": 0.86387825, + "num_input_tokens_seen": 187132870, + "router_z_loss_clip": 0.78417969, + "router_z_loss_mlp": 0.16082764, + "step": 6559, + "time_per_iteration": 2.6099915504455566 + }, + { + "auxiliary_loss_clip": 0.01048845, + "auxiliary_loss_mlp": 0.01016345, + "balance_loss_clip": 1.02923775, + "balance_loss_mlp": 1.01548088, + "epoch": 0.1903545934652661, + "flos": 74777975648640.0, + "grad_norm": 0.6471865245679744, + "language_loss": 0.50989044, + "learning_rate": 3.7362955412649688e-06, + "loss": 0.53054231, + "num_input_tokens_seen": 187199455, + "router_z_loss_clip": 0.19628906, + "router_z_loss_mlp": 0.00866699, + "step": 6560, + "time_per_iteration": 3.2194697856903076 + }, + { + "auxiliary_loss_clip": 0.01134709, + "auxiliary_loss_mlp": 0.01045232, + "balance_loss_clip": 1.05852246, + "balance_loss_mlp": 1.03011584, + "epoch": 0.19038361093378214, + "flos": 37792677801600.0, + "grad_norm": 7.59548237750211, + "language_loss": 0.65104359, + "learning_rate": 3.7362022470914034e-06, + "loss": 0.67284298, + "num_input_tokens_seen": 187221780, + "router_z_loss_clip": 0.76123047, + "router_z_loss_mlp": 0.15124512, + "step": 6561, + "time_per_iteration": 2.74239182472229 + }, + { + "auxiliary_loss_clip": 0.01047142, + "auxiliary_loss_mlp": 0.01010461, + "balance_loss_clip": 1.02773213, + "balance_loss_mlp": 1.00962055, + "epoch": 0.1904126284022982, + "flos": 53323756876800.0, + "grad_norm": 0.6595780274142676, + "language_loss": 0.48580363, + "learning_rate": 3.7361089375829973e-06, + "loss": 0.5063796, + "num_input_tokens_seen": 187283200, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.00842285, + "step": 6562, + "time_per_iteration": 3.0312681198120117 + }, + { + "auxiliary_loss_clip": 0.01045657, + "auxiliary_loss_mlp": 0.01006718, + "balance_loss_clip": 1.02657068, + "balance_loss_mlp": 1.00584817, + "epoch": 0.19044164587081422, + "flos": 62442911980800.0, + "grad_norm": 0.6172787941725568, + "language_loss": 0.48908761, + "learning_rate": 3.736015612740575e-06, + "loss": 0.50961131, + "num_input_tokens_seen": 187345580, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.00872803, + "step": 6563, + "time_per_iteration": 3.0778541564941406 + }, + { + "auxiliary_loss_clip": 0.01131662, + "auxiliary_loss_mlp": 0.01036521, + "balance_loss_clip": 1.05787849, + "balance_loss_mlp": 1.02217376, + "epoch": 0.19047066333933027, + "flos": 26899892933760.0, + "grad_norm": 2.034419823174379, + "language_loss": 0.9149102, + "learning_rate": 3.7359222725649604e-06, + "loss": 0.93659204, + "num_input_tokens_seen": 187360060, + "router_z_loss_clip": 0.73730469, + "router_z_loss_mlp": 0.14349365, + "step": 6564, + "time_per_iteration": 2.598639965057373 + }, + { + "auxiliary_loss_clip": 0.01043369, + "auxiliary_loss_mlp": 0.01005337, + "balance_loss_clip": 1.02414095, + "balance_loss_mlp": 1.00441873, + "epoch": 0.19049968080784632, + "flos": 63281713357440.0, + "grad_norm": 0.655873387221935, + "language_loss": 0.45691895, + "learning_rate": 3.735828917056977e-06, + "loss": 0.47740602, + "num_input_tokens_seen": 187424760, + "router_z_loss_clip": 0.19238281, + "router_z_loss_mlp": 0.00915527, + "step": 6565, + "time_per_iteration": 3.2137680053710938 + }, + { + "auxiliary_loss_clip": 0.01129, + "auxiliary_loss_mlp": 0.01034729, + "balance_loss_clip": 1.05702615, + "balance_loss_mlp": 1.01971424, + "epoch": 0.19052869827636237, + "flos": 24199805629440.0, + "grad_norm": 1.8370945712778464, + "language_loss": 0.72172362, + "learning_rate": 3.7357355462174504e-06, + "loss": 0.74336088, + "num_input_tokens_seen": 187439065, + "router_z_loss_clip": 0.71923828, + "router_z_loss_mlp": 0.15020752, + "step": 6566, + "time_per_iteration": 2.57267689704895 + }, + { + "auxiliary_loss_clip": 0.01126563, + "auxiliary_loss_mlp": 0.01031614, + "balance_loss_clip": 1.05536103, + "balance_loss_mlp": 1.01717794, + "epoch": 0.19055771574487843, + "flos": 37483305960960.0, + "grad_norm": 2.429013701893632, + "language_loss": 0.97368228, + "learning_rate": 3.735642160047205e-06, + "loss": 0.99526411, + "num_input_tokens_seen": 187458220, + "router_z_loss_clip": 0.71240234, + "router_z_loss_mlp": 0.14428711, + "step": 6567, + "time_per_iteration": 2.6892318725585938 + }, + { + "auxiliary_loss_clip": 0.01041201, + "auxiliary_loss_mlp": 0.01001305, + "balance_loss_clip": 1.02202177, + "balance_loss_mlp": 1.00039923, + "epoch": 0.19058673321339448, + "flos": 74768566285440.0, + "grad_norm": 0.6316911722729553, + "language_loss": 0.44321692, + "learning_rate": 3.735548758547066e-06, + "loss": 0.46364197, + "num_input_tokens_seen": 187524120, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.0090332, + "step": 6568, + "time_per_iteration": 3.2960479259490967 + }, + { + "auxiliary_loss_clip": 0.01124508, + "auxiliary_loss_mlp": 0.01034641, + "balance_loss_clip": 1.05458891, + "balance_loss_mlp": 1.02137852, + "epoch": 0.1906157506819105, + "flos": 22776953109120.0, + "grad_norm": 2.6771945610360772, + "language_loss": 0.76703411, + "learning_rate": 3.735455341717858e-06, + "loss": 0.7886256, + "num_input_tokens_seen": 187538340, + "router_z_loss_clip": 0.69970703, + "router_z_loss_mlp": 0.13256836, + "step": 6569, + "time_per_iteration": 2.486572027206421 + }, + { + "auxiliary_loss_clip": 0.01134109, + "auxiliary_loss_mlp": 0.01043601, + "balance_loss_clip": 1.05487275, + "balance_loss_mlp": 1.02667379, + "epoch": 0.19064476815042655, + "flos": 16391281979520.0, + "grad_norm": 2.903385536961027, + "language_loss": 0.82873595, + "learning_rate": 3.735361909560406e-06, + "loss": 0.8505131, + "num_input_tokens_seen": 187551825, + "router_z_loss_clip": 0.79296875, + "router_z_loss_mlp": 0.16925049, + "step": 6570, + "time_per_iteration": 2.537416458129883 + }, + { + "auxiliary_loss_clip": 0.01128709, + "auxiliary_loss_mlp": 0.01040974, + "balance_loss_clip": 1.05436587, + "balance_loss_mlp": 1.02647257, + "epoch": 0.1906737856189426, + "flos": 38215488792960.0, + "grad_norm": 2.831827432850751, + "language_loss": 0.71454573, + "learning_rate": 3.7352684620755356e-06, + "loss": 0.73624259, + "num_input_tokens_seen": 187571125, + "router_z_loss_clip": 0.74414062, + "router_z_loss_mlp": 0.14508057, + "step": 6571, + "time_per_iteration": 2.687911033630371 + }, + { + "auxiliary_loss_clip": 0.01038574, + "auxiliary_loss_mlp": 0.01004838, + "balance_loss_clip": 1.01972246, + "balance_loss_mlp": 1.00381839, + "epoch": 0.19070280308745866, + "flos": 62339202437760.0, + "grad_norm": 0.6743189346855759, + "language_loss": 0.45174533, + "learning_rate": 3.735174999264072e-06, + "loss": 0.47217947, + "num_input_tokens_seen": 187630460, + "router_z_loss_clip": 0.18847656, + "router_z_loss_mlp": 0.01019287, + "step": 6572, + "time_per_iteration": 3.0511934757232666 + }, + { + "auxiliary_loss_clip": 0.01038174, + "auxiliary_loss_mlp": 0.01004227, + "balance_loss_clip": 1.01934242, + "balance_loss_mlp": 1.00324905, + "epoch": 0.1907318205559747, + "flos": 74773630103040.0, + "grad_norm": 0.6412793091941464, + "language_loss": 0.48319355, + "learning_rate": 3.7350815211268405e-06, + "loss": 0.50361753, + "num_input_tokens_seen": 187691940, + "router_z_loss_clip": 0.18847656, + "router_z_loss_mlp": 0.00976562, + "step": 6573, + "time_per_iteration": 3.100681781768799 + }, + { + "auxiliary_loss_clip": 0.01129231, + "auxiliary_loss_mlp": 0.01032173, + "balance_loss_clip": 1.05404615, + "balance_loss_mlp": 1.0167768, + "epoch": 0.19076083802449073, + "flos": 23324231704320.0, + "grad_norm": 1.7936653770877404, + "language_loss": 0.67329335, + "learning_rate": 3.734988027664667e-06, + "loss": 0.69490743, + "num_input_tokens_seen": 187707385, + "router_z_loss_clip": 0.75244141, + "router_z_loss_mlp": 0.15380859, + "step": 6574, + "time_per_iteration": 2.5324923992156982 + }, + { + "auxiliary_loss_clip": 0.01119516, + "auxiliary_loss_mlp": 0.01034305, + "balance_loss_clip": 1.05148065, + "balance_loss_mlp": 1.02157927, + "epoch": 0.19078985549300678, + "flos": 16249650652800.0, + "grad_norm": 2.943275737241229, + "language_loss": 0.78734422, + "learning_rate": 3.7348945188783772e-06, + "loss": 0.80888247, + "num_input_tokens_seen": 187723315, + "router_z_loss_clip": 0.68017578, + "router_z_loss_mlp": 0.12738037, + "step": 6575, + "time_per_iteration": 2.579267740249634 + }, + { + "auxiliary_loss_clip": 0.01127694, + "auxiliary_loss_mlp": 0.01032716, + "balance_loss_clip": 1.05496454, + "balance_loss_mlp": 1.0189532, + "epoch": 0.19081887296152283, + "flos": 12742398875520.0, + "grad_norm": 2.632330747244202, + "language_loss": 0.98073214, + "learning_rate": 3.7348009947687966e-06, + "loss": 1.00233626, + "num_input_tokens_seen": 187734475, + "router_z_loss_clip": 0.72802734, + "router_z_loss_mlp": 0.13757324, + "step": 6576, + "time_per_iteration": 2.4915809631347656 + }, + { + "auxiliary_loss_clip": 0.01037146, + "auxiliary_loss_mlp": 0.0101161, + "balance_loss_clip": 1.01821744, + "balance_loss_mlp": 1.01069188, + "epoch": 0.19084789043003889, + "flos": 67833677226240.0, + "grad_norm": 0.6952861447896659, + "language_loss": 0.47509974, + "learning_rate": 3.7347074553367515e-06, + "loss": 0.49558729, + "num_input_tokens_seen": 187793645, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.00915527, + "step": 6577, + "time_per_iteration": 3.0662853717803955 + }, + { + "auxiliary_loss_clip": 0.01129861, + "auxiliary_loss_mlp": 0.01036117, + "balance_loss_clip": 1.05541277, + "balance_loss_mlp": 1.02025604, + "epoch": 0.19087690789855494, + "flos": 22593629070720.0, + "grad_norm": 2.411315273250537, + "language_loss": 0.70744252, + "learning_rate": 3.734613900583069e-06, + "loss": 0.72910231, + "num_input_tokens_seen": 187805175, + "router_z_loss_clip": 0.74462891, + "router_z_loss_mlp": 0.15856934, + "step": 6578, + "time_per_iteration": 2.5252957344055176 + }, + { + "auxiliary_loss_clip": 0.01120431, + "auxiliary_loss_mlp": 0.01034266, + "balance_loss_clip": 1.05043578, + "balance_loss_mlp": 1.01996076, + "epoch": 0.190905925367071, + "flos": 42558561118080.0, + "grad_norm": 2.0123249089489206, + "language_loss": 1.02353692, + "learning_rate": 3.734520330508574e-06, + "loss": 1.04508388, + "num_input_tokens_seen": 187826980, + "router_z_loss_clip": 0.70019531, + "router_z_loss_mlp": 0.14300537, + "step": 6579, + "time_per_iteration": 2.7162463665008545 + }, + { + "auxiliary_loss_clip": 0.01130253, + "auxiliary_loss_mlp": 0.01040636, + "balance_loss_clip": 1.05438495, + "balance_loss_mlp": 1.02395844, + "epoch": 0.190934942835587, + "flos": 11612254285440.0, + "grad_norm": 2.6754915861445925, + "language_loss": 0.87068677, + "learning_rate": 3.7344267451140938e-06, + "loss": 0.89239562, + "num_input_tokens_seen": 187838195, + "router_z_loss_clip": 0.75927734, + "router_z_loss_mlp": 0.16674805, + "step": 6580, + "time_per_iteration": 2.482132911682129 + }, + { + "auxiliary_loss_clip": 0.01037224, + "auxiliary_loss_mlp": 0.01021955, + "balance_loss_clip": 1.0185101, + "balance_loss_mlp": 1.02103698, + "epoch": 0.19096396030410306, + "flos": 53862559862400.0, + "grad_norm": 0.7420346630854507, + "language_loss": 0.5211249, + "learning_rate": 3.7343331444004542e-06, + "loss": 0.54171669, + "num_input_tokens_seen": 187898375, + "router_z_loss_clip": 0.1875, + "router_z_loss_mlp": 0.00915527, + "step": 6581, + "time_per_iteration": 3.083246946334839 + }, + { + "auxiliary_loss_clip": 0.01037279, + "auxiliary_loss_mlp": 0.01017817, + "balance_loss_clip": 1.01856482, + "balance_loss_mlp": 1.01688766, + "epoch": 0.19099297777261912, + "flos": 74778622093440.0, + "grad_norm": 0.647362572904935, + "language_loss": 0.46030462, + "learning_rate": 3.734239528368483e-06, + "loss": 0.48085558, + "num_input_tokens_seen": 187964160, + "router_z_loss_clip": 0.1875, + "router_z_loss_mlp": 0.00927734, + "step": 6582, + "time_per_iteration": 3.1496331691741943 + }, + { + "auxiliary_loss_clip": 0.0103811, + "auxiliary_loss_mlp": 0.01007857, + "balance_loss_clip": 1.01926088, + "balance_loss_mlp": 1.00691569, + "epoch": 0.19102199524113517, + "flos": 56206521774720.0, + "grad_norm": 0.6594977138387892, + "language_loss": 0.43841085, + "learning_rate": 3.7341458970190065e-06, + "loss": 0.4588705, + "num_input_tokens_seen": 188026835, + "router_z_loss_clip": 0.18847656, + "router_z_loss_mlp": 0.00939941, + "step": 6583, + "time_per_iteration": 3.0920212268829346 + }, + { + "auxiliary_loss_clip": 0.01129352, + "auxiliary_loss_mlp": 0.01045665, + "balance_loss_clip": 1.05554712, + "balance_loss_mlp": 1.03006625, + "epoch": 0.19105101270965122, + "flos": 32348368356480.0, + "grad_norm": 1.4652254713889363, + "language_loss": 0.88700485, + "learning_rate": 3.7340522503528512e-06, + "loss": 0.908755, + "num_input_tokens_seen": 188055020, + "router_z_loss_clip": 0.73828125, + "router_z_loss_mlp": 0.15582275, + "step": 6584, + "time_per_iteration": 2.775998830795288 + }, + { + "auxiliary_loss_clip": 0.01125081, + "auxiliary_loss_mlp": 0.01037178, + "balance_loss_clip": 1.05317032, + "balance_loss_mlp": 1.02277756, + "epoch": 0.19108003017816727, + "flos": 21352592217600.0, + "grad_norm": 2.375653011084097, + "language_loss": 0.85383463, + "learning_rate": 3.7339585883708457e-06, + "loss": 0.87545723, + "num_input_tokens_seen": 188069215, + "router_z_loss_clip": 0.71826172, + "router_z_loss_mlp": 0.144104, + "step": 6585, + "time_per_iteration": 2.4963958263397217 + }, + { + "auxiliary_loss_clip": 0.0103835, + "auxiliary_loss_mlp": 0.01000532, + "balance_loss_clip": 1.01935971, + "balance_loss_mlp": 0.99960232, + "epoch": 0.1911090476466833, + "flos": 72625238766720.0, + "grad_norm": 0.6655440952287052, + "language_loss": 0.52810836, + "learning_rate": 3.7338649110738158e-06, + "loss": 0.5484972, + "num_input_tokens_seen": 188129955, + "router_z_loss_clip": 0.19042969, + "router_z_loss_mlp": 0.00927734, + "step": 6586, + "time_per_iteration": 3.1090359687805176 + }, + { + "auxiliary_loss_clip": 0.01037736, + "auxiliary_loss_mlp": 0.01001736, + "balance_loss_clip": 1.0187788, + "balance_loss_mlp": 1.00077677, + "epoch": 0.19113806511519935, + "flos": 65672395885440.0, + "grad_norm": 0.6294545471593536, + "language_loss": 0.4518702, + "learning_rate": 3.733771218462589e-06, + "loss": 0.47226495, + "num_input_tokens_seen": 188200245, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.00958252, + "step": 6587, + "time_per_iteration": 3.2315053939819336 + }, + { + "auxiliary_loss_clip": 0.01036701, + "auxiliary_loss_mlp": 0.01001769, + "balance_loss_clip": 1.01788449, + "balance_loss_mlp": 1.00080299, + "epoch": 0.1911670825837154, + "flos": 60288379418880.0, + "grad_norm": 0.6786890748189902, + "language_loss": 0.45638958, + "learning_rate": 3.7336775105379937e-06, + "loss": 0.47677433, + "num_input_tokens_seen": 188263695, + "router_z_loss_clip": 0.18847656, + "router_z_loss_mlp": 0.00964355, + "step": 6588, + "time_per_iteration": 3.1591641902923584 + }, + { + "auxiliary_loss_clip": 0.0103619, + "auxiliary_loss_mlp": 0.01001654, + "balance_loss_clip": 1.01745713, + "balance_loss_mlp": 1.00078964, + "epoch": 0.19119610005223145, + "flos": 60506393016960.0, + "grad_norm": 0.9192496731849418, + "language_loss": 0.47341287, + "learning_rate": 3.7335837873008567e-06, + "loss": 0.49379134, + "num_input_tokens_seen": 188322555, + "router_z_loss_clip": 0.1875, + "router_z_loss_mlp": 0.00866699, + "step": 6589, + "time_per_iteration": 2.985020875930786 + }, + { + "auxiliary_loss_clip": 0.01128689, + "auxiliary_loss_mlp": 0.0103916, + "balance_loss_clip": 1.05393863, + "balance_loss_mlp": 1.02364469, + "epoch": 0.1912251175207475, + "flos": 37265328276480.0, + "grad_norm": 1.8484903426896735, + "language_loss": 0.79034579, + "learning_rate": 3.7334900487520063e-06, + "loss": 0.8120243, + "num_input_tokens_seen": 188341815, + "router_z_loss_clip": 0.74755859, + "router_z_loss_mlp": 0.1552124, + "step": 6590, + "time_per_iteration": 2.6749930381774902 + }, + { + "auxiliary_loss_clip": 0.01129766, + "auxiliary_loss_mlp": 0.01033953, + "balance_loss_clip": 1.05374503, + "balance_loss_mlp": 1.01902199, + "epoch": 0.19125413498926352, + "flos": 12277320554880.0, + "grad_norm": 3.842603430726161, + "language_loss": 0.75335717, + "learning_rate": 3.7333962948922705e-06, + "loss": 0.77499425, + "num_input_tokens_seen": 188356515, + "router_z_loss_clip": 0.76025391, + "router_z_loss_mlp": 0.14910889, + "step": 6591, + "time_per_iteration": 2.5014753341674805 + }, + { + "auxiliary_loss_clip": 0.01130616, + "auxiliary_loss_mlp": 0.01037612, + "balance_loss_clip": 1.05485582, + "balance_loss_mlp": 1.02239442, + "epoch": 0.19128315245777958, + "flos": 70757629067520.0, + "grad_norm": 2.2878263799029233, + "language_loss": 0.92285681, + "learning_rate": 3.7333025257224772e-06, + "loss": 0.94453913, + "num_input_tokens_seen": 188380815, + "router_z_loss_clip": 0.75830078, + "router_z_loss_mlp": 0.15222168, + "step": 6592, + "time_per_iteration": 2.858656883239746 + }, + { + "auxiliary_loss_clip": 0.01134872, + "auxiliary_loss_mlp": 0.01051626, + "balance_loss_clip": 1.05611825, + "balance_loss_mlp": 1.03494287, + "epoch": 0.19131216992629563, + "flos": 15264908317440.0, + "grad_norm": 2.790515144921797, + "language_loss": 0.97401726, + "learning_rate": 3.733208741243454e-06, + "loss": 0.99588221, + "num_input_tokens_seen": 188395565, + "router_z_loss_clip": 0.78662109, + "router_z_loss_mlp": 0.16680908, + "step": 6593, + "time_per_iteration": 2.493903875350952 + }, + { + "auxiliary_loss_clip": 0.01123231, + "auxiliary_loss_mlp": 0.01046855, + "balance_loss_clip": 1.05296648, + "balance_loss_mlp": 1.03176892, + "epoch": 0.19134118739481168, + "flos": 36095933099520.0, + "grad_norm": 2.0957660548185046, + "language_loss": 0.80869234, + "learning_rate": 3.733114941456031e-06, + "loss": 0.8303932, + "num_input_tokens_seen": 188411615, + "router_z_loss_clip": 0.70336914, + "router_z_loss_mlp": 0.15075684, + "step": 6594, + "time_per_iteration": 2.651095151901245 + }, + { + "auxiliary_loss_clip": 0.01122178, + "auxiliary_loss_mlp": 0.01045648, + "balance_loss_clip": 1.05228436, + "balance_loss_mlp": 1.03078818, + "epoch": 0.19137020486332773, + "flos": 23871007509120.0, + "grad_norm": 1.9162227916976737, + "language_loss": 0.71674502, + "learning_rate": 3.7330211263610354e-06, + "loss": 0.73842323, + "num_input_tokens_seen": 188425620, + "router_z_loss_clip": 0.69921875, + "router_z_loss_mlp": 0.14849854, + "step": 6595, + "time_per_iteration": 2.542799711227417 + }, + { + "auxiliary_loss_clip": 0.01131274, + "auxiliary_loss_mlp": 0.01048202, + "balance_loss_clip": 1.0526365, + "balance_loss_mlp": 1.03081524, + "epoch": 0.19139922233184378, + "flos": 20953409397120.0, + "grad_norm": 3.057053589013328, + "language_loss": 0.95852542, + "learning_rate": 3.7329272959592948e-06, + "loss": 0.98032022, + "num_input_tokens_seen": 188437815, + "router_z_loss_clip": 0.78515625, + "router_z_loss_mlp": 0.17370605, + "step": 6596, + "time_per_iteration": 2.5303499698638916 + }, + { + "auxiliary_loss_clip": 0.01036561, + "auxiliary_loss_mlp": 0.01049339, + "balance_loss_clip": 1.01784313, + "balance_loss_mlp": 1.04846883, + "epoch": 0.1914282398003598, + "flos": 74776934154240.0, + "grad_norm": 0.6496684785084355, + "language_loss": 0.47221345, + "learning_rate": 3.7328334502516396e-06, + "loss": 0.49307245, + "num_input_tokens_seen": 188504755, + "router_z_loss_clip": 0.1875, + "router_z_loss_mlp": 0.00872803, + "step": 6597, + "time_per_iteration": 3.248533010482788 + }, + { + "auxiliary_loss_clip": 0.01132539, + "auxiliary_loss_mlp": 0.01052954, + "balance_loss_clip": 1.05439734, + "balance_loss_mlp": 1.03589499, + "epoch": 0.19145725726887586, + "flos": 20002817917440.0, + "grad_norm": 2.4206242119176724, + "language_loss": 0.88624787, + "learning_rate": 3.732739589238898e-06, + "loss": 0.90810281, + "num_input_tokens_seen": 188517895, + "router_z_loss_clip": 0.78173828, + "router_z_loss_mlp": 0.1706543, + "step": 6598, + "time_per_iteration": 2.5453109741210938 + }, + { + "auxiliary_loss_clip": 0.01036767, + "auxiliary_loss_mlp": 0.01020084, + "balance_loss_clip": 1.01825523, + "balance_loss_mlp": 1.0192374, + "epoch": 0.1914862747373919, + "flos": 74770541533440.0, + "grad_norm": 0.6538467395027725, + "language_loss": 0.50542879, + "learning_rate": 3.7326457129219e-06, + "loss": 0.52599728, + "num_input_tokens_seen": 188581265, + "router_z_loss_clip": 0.18554688, + "router_z_loss_mlp": 0.00848389, + "step": 6599, + "time_per_iteration": 3.1774816513061523 + }, + { + "auxiliary_loss_clip": 0.01130784, + "auxiliary_loss_mlp": 0.01044452, + "balance_loss_clip": 1.05270696, + "balance_loss_mlp": 1.02815652, + "epoch": 0.19151529220590796, + "flos": 16575719339520.0, + "grad_norm": 2.75138957276358, + "language_loss": 0.53856087, + "learning_rate": 3.7325518213014727e-06, + "loss": 0.56031322, + "num_input_tokens_seen": 188595440, + "router_z_loss_clip": 0.78027344, + "router_z_loss_mlp": 0.16308594, + "step": 6600, + "time_per_iteration": 2.5418026447296143 + }, + { + "auxiliary_loss_clip": 0.01124416, + "auxiliary_loss_mlp": 0.01028798, + "balance_loss_clip": 1.05266571, + "balance_loss_mlp": 1.01590586, + "epoch": 0.191544309674424, + "flos": 24945348320640.0, + "grad_norm": 3.7992622478616656, + "language_loss": 0.92793441, + "learning_rate": 3.7324579143784474e-06, + "loss": 0.94946659, + "num_input_tokens_seen": 188613935, + "router_z_loss_clip": 0.71679688, + "router_z_loss_mlp": 0.12890625, + "step": 6601, + "time_per_iteration": 2.621067762374878 + }, + { + "auxiliary_loss_clip": 0.0113404, + "auxiliary_loss_mlp": 0.01035059, + "balance_loss_clip": 1.05515742, + "balance_loss_mlp": 1.01937115, + "epoch": 0.19157332714294004, + "flos": 14968393545600.0, + "grad_norm": 3.095755071509999, + "language_loss": 1.12583566, + "learning_rate": 3.732363992153653e-06, + "loss": 1.14752662, + "num_input_tokens_seen": 188624715, + "router_z_loss_clip": 0.78857422, + "router_z_loss_mlp": 0.15679932, + "step": 6602, + "time_per_iteration": 2.525949239730835 + }, + { + "auxiliary_loss_clip": 0.01131799, + "auxiliary_loss_mlp": 0.01039579, + "balance_loss_clip": 1.05608594, + "balance_loss_mlp": 1.02359915, + "epoch": 0.1916023446114561, + "flos": 28144521146880.0, + "grad_norm": 3.245081969659062, + "language_loss": 0.74664056, + "learning_rate": 3.732270054627918e-06, + "loss": 0.76835436, + "num_input_tokens_seen": 188643260, + "router_z_loss_clip": 0.75830078, + "router_z_loss_mlp": 0.15985107, + "step": 6603, + "time_per_iteration": 2.5878195762634277 + }, + { + "auxiliary_loss_clip": 0.01047393, + "auxiliary_loss_mlp": 0.01005569, + "balance_loss_clip": 1.02880049, + "balance_loss_mlp": 1.00474668, + "epoch": 0.19163136207997214, + "flos": 61720102598400.0, + "grad_norm": 0.6877352099762782, + "language_loss": 0.51815486, + "learning_rate": 3.7321761018020738e-06, + "loss": 0.53868449, + "num_input_tokens_seen": 188703380, + "router_z_loss_clip": 0.18554688, + "router_z_loss_mlp": 0.00823975, + "step": 6604, + "time_per_iteration": 3.0797183513641357 + }, + { + "auxiliary_loss_clip": 0.01045798, + "auxiliary_loss_mlp": 0.01007466, + "balance_loss_clip": 1.02711773, + "balance_loss_mlp": 1.00657237, + "epoch": 0.1916603795484882, + "flos": 55700612668800.0, + "grad_norm": 0.6481976460220935, + "language_loss": 0.46891719, + "learning_rate": 3.7320821336769484e-06, + "loss": 0.48944983, + "num_input_tokens_seen": 188766235, + "router_z_loss_clip": 0.1875, + "router_z_loss_mlp": 0.00891113, + "step": 6605, + "time_per_iteration": 3.137194871902466 + }, + { + "auxiliary_loss_clip": 0.01131584, + "auxiliary_loss_mlp": 0.0103543, + "balance_loss_clip": 1.05466127, + "balance_loss_mlp": 1.01854444, + "epoch": 0.19168939701700424, + "flos": 41864623292160.0, + "grad_norm": 2.1006472695940532, + "language_loss": 0.81915307, + "learning_rate": 3.7319881502533734e-06, + "loss": 0.84082323, + "num_input_tokens_seen": 188788130, + "router_z_loss_clip": 0.77001953, + "router_z_loss_mlp": 0.16876221, + "step": 6606, + "time_per_iteration": 2.686779022216797 + }, + { + "auxiliary_loss_clip": 0.01129374, + "auxiliary_loss_mlp": 0.01043193, + "balance_loss_clip": 1.05462456, + "balance_loss_mlp": 1.02869725, + "epoch": 0.1917184144855203, + "flos": 39890110717440.0, + "grad_norm": 2.4399781354624674, + "language_loss": 0.87477291, + "learning_rate": 3.7318941515321784e-06, + "loss": 0.89649856, + "num_input_tokens_seen": 188803490, + "router_z_loss_clip": 0.74707031, + "router_z_loss_mlp": 0.1449585, + "step": 6607, + "time_per_iteration": 2.5983023643493652 + }, + { + "auxiliary_loss_clip": 0.01040491, + "auxiliary_loss_mlp": 0.0100094, + "balance_loss_clip": 1.02202272, + "balance_loss_mlp": 1.0000819, + "epoch": 0.19174743195403632, + "flos": 61707065961600.0, + "grad_norm": 0.6848217962734529, + "language_loss": 0.43101183, + "learning_rate": 3.7318001375141926e-06, + "loss": 0.45142615, + "num_input_tokens_seen": 188860050, + "router_z_loss_clip": 0.18457031, + "router_z_loss_mlp": 0.00860596, + "step": 6608, + "time_per_iteration": 2.95304012298584 + }, + { + "auxiliary_loss_clip": 0.01133604, + "auxiliary_loss_mlp": 0.01039303, + "balance_loss_clip": 1.05524182, + "balance_loss_mlp": 1.02421117, + "epoch": 0.19177644942255237, + "flos": 31863070679040.0, + "grad_norm": 2.0777911380529708, + "language_loss": 0.92669082, + "learning_rate": 3.731706108200248e-06, + "loss": 0.94841993, + "num_input_tokens_seen": 188877325, + "router_z_loss_clip": 0.78271484, + "router_z_loss_mlp": 0.15093994, + "step": 6609, + "time_per_iteration": 2.6467247009277344 + }, + { + "auxiliary_loss_clip": 0.01038936, + "auxiliary_loss_mlp": 0.01004342, + "balance_loss_clip": 1.02057314, + "balance_loss_mlp": 1.00350118, + "epoch": 0.19180546689106842, + "flos": 66162826229760.0, + "grad_norm": 0.6499371573445785, + "language_loss": 0.45966384, + "learning_rate": 3.7316120635911733e-06, + "loss": 0.48009664, + "num_input_tokens_seen": 188938340, + "router_z_loss_clip": 0.18359375, + "router_z_loss_mlp": 0.00842285, + "step": 6610, + "time_per_iteration": 3.0172176361083984 + }, + { + "auxiliary_loss_clip": 0.01125337, + "auxiliary_loss_mlp": 0.01038345, + "balance_loss_clip": 1.05279732, + "balance_loss_mlp": 1.02396226, + "epoch": 0.19183448435958447, + "flos": 31573271750400.0, + "grad_norm": 1.8682064093270654, + "language_loss": 0.8882606, + "learning_rate": 3.731518003687801e-06, + "loss": 0.90989745, + "num_input_tokens_seen": 188954410, + "router_z_loss_clip": 0.7265625, + "router_z_loss_mlp": 0.1439209, + "step": 6611, + "time_per_iteration": 2.660579204559326 + }, + { + "auxiliary_loss_clip": 0.01130248, + "auxiliary_loss_mlp": 0.01044178, + "balance_loss_clip": 1.05546021, + "balance_loss_mlp": 1.02971184, + "epoch": 0.19186350182810052, + "flos": 22631766336000.0, + "grad_norm": 1.9561786010638225, + "language_loss": 0.82201695, + "learning_rate": 3.7314239284909606e-06, + "loss": 0.84376121, + "num_input_tokens_seen": 188969430, + "router_z_loss_clip": 0.74853516, + "router_z_loss_mlp": 0.14483643, + "step": 6612, + "time_per_iteration": 2.5211524963378906 + }, + { + "auxiliary_loss_clip": 0.01128065, + "auxiliary_loss_mlp": 0.01041748, + "balance_loss_clip": 1.05169702, + "balance_loss_mlp": 1.02499294, + "epoch": 0.19189251929661658, + "flos": 26099372476800.0, + "grad_norm": 2.460657815124301, + "language_loss": 0.64586437, + "learning_rate": 3.7313298380014838e-06, + "loss": 0.66756248, + "num_input_tokens_seen": 188987035, + "router_z_loss_clip": 0.76464844, + "router_z_loss_mlp": 0.16741943, + "step": 6613, + "time_per_iteration": 4.835782527923584 + }, + { + "auxiliary_loss_clip": 0.0112291, + "auxiliary_loss_mlp": 0.01045671, + "balance_loss_clip": 1.05183315, + "balance_loss_mlp": 1.03139603, + "epoch": 0.1919215367651326, + "flos": 27921407817600.0, + "grad_norm": 1.8850404604323376, + "language_loss": 0.63863897, + "learning_rate": 3.731235732220201e-06, + "loss": 0.66032481, + "num_input_tokens_seen": 189000300, + "router_z_loss_clip": 0.71191406, + "router_z_loss_mlp": 0.1427002, + "step": 6614, + "time_per_iteration": 5.124691963195801 + }, + { + "auxiliary_loss_clip": 0.01120359, + "auxiliary_loss_mlp": 0.01044177, + "balance_loss_clip": 1.05088818, + "balance_loss_mlp": 1.03043246, + "epoch": 0.19195055423364865, + "flos": 12790520121600.0, + "grad_norm": 2.5160065002392242, + "language_loss": 0.86654204, + "learning_rate": 3.7311416111479436e-06, + "loss": 0.88818741, + "num_input_tokens_seen": 189014850, + "router_z_loss_clip": 0.69433594, + "router_z_loss_mlp": 0.13745117, + "step": 6615, + "time_per_iteration": 7.387006759643555 + }, + { + "auxiliary_loss_clip": 0.01136291, + "auxiliary_loss_mlp": 0.01045718, + "balance_loss_clip": 1.05879402, + "balance_loss_mlp": 1.0288614, + "epoch": 0.1919795717021647, + "flos": 30732423298560.0, + "grad_norm": 2.983108646998142, + "language_loss": 0.91464633, + "learning_rate": 3.7310474747855434e-06, + "loss": 0.9364664, + "num_input_tokens_seen": 189032640, + "router_z_loss_clip": 0.77490234, + "router_z_loss_mlp": 0.16870117, + "step": 6616, + "time_per_iteration": 2.6165804862976074 + }, + { + "auxiliary_loss_clip": 0.01132906, + "auxiliary_loss_mlp": 0.01040305, + "balance_loss_clip": 1.05463505, + "balance_loss_mlp": 1.02510548, + "epoch": 0.19200858917068075, + "flos": 23725892563200.0, + "grad_norm": 1.9260554907181218, + "language_loss": 0.74978912, + "learning_rate": 3.730953323133831e-06, + "loss": 0.77152121, + "num_input_tokens_seen": 189048555, + "router_z_loss_clip": 0.78173828, + "router_z_loss_mlp": 0.15197754, + "step": 6617, + "time_per_iteration": 2.549168825149536 + }, + { + "auxiliary_loss_clip": 0.01134189, + "auxiliary_loss_mlp": 0.0103828, + "balance_loss_clip": 1.05610657, + "balance_loss_mlp": 1.02170956, + "epoch": 0.1920376066391968, + "flos": 26165914421760.0, + "grad_norm": 2.664921745878617, + "language_loss": 1.11868143, + "learning_rate": 3.7308591561936383e-06, + "loss": 1.14040613, + "num_input_tokens_seen": 189063310, + "router_z_loss_clip": 0.78076172, + "router_z_loss_mlp": 0.16577148, + "step": 6618, + "time_per_iteration": 2.54512882232666 + }, + { + "auxiliary_loss_clip": 0.01128177, + "auxiliary_loss_mlp": 0.01041866, + "balance_loss_clip": 1.05401874, + "balance_loss_mlp": 1.0261364, + "epoch": 0.19206662410771283, + "flos": 74730857005440.0, + "grad_norm": 2.271564000918662, + "language_loss": 0.61308694, + "learning_rate": 3.7307649739657974e-06, + "loss": 0.63478732, + "num_input_tokens_seen": 189083930, + "router_z_loss_clip": 0.7421875, + "router_z_loss_mlp": 0.15734863, + "step": 6619, + "time_per_iteration": 2.975116491317749 + }, + { + "auxiliary_loss_clip": 0.01038141, + "auxiliary_loss_mlp": 0.0100967, + "balance_loss_clip": 1.01981854, + "balance_loss_mlp": 1.0086745, + "epoch": 0.19209564157622888, + "flos": 74485878249600.0, + "grad_norm": 0.7330825949225156, + "language_loss": 0.53105927, + "learning_rate": 3.7306707764511395e-06, + "loss": 0.55153739, + "num_input_tokens_seen": 189149295, + "router_z_loss_clip": 0.18359375, + "router_z_loss_mlp": 0.00994873, + "step": 6620, + "time_per_iteration": 3.2149417400360107 + }, + { + "auxiliary_loss_clip": 0.01134863, + "auxiliary_loss_mlp": 0.01040636, + "balance_loss_clip": 1.05462933, + "balance_loss_mlp": 1.02314806, + "epoch": 0.19212465904474493, + "flos": 13474114830720.0, + "grad_norm": 2.6530620589502436, + "language_loss": 0.77924865, + "learning_rate": 3.7305765636504977e-06, + "loss": 0.80100363, + "num_input_tokens_seen": 189162340, + "router_z_loss_clip": 0.80175781, + "router_z_loss_mlp": 0.17480469, + "step": 6621, + "time_per_iteration": 2.481351137161255 + }, + { + "auxiliary_loss_clip": 0.01036437, + "auxiliary_loss_mlp": 0.01006263, + "balance_loss_clip": 1.01808929, + "balance_loss_mlp": 1.00525558, + "epoch": 0.19215367651326098, + "flos": 62114472996480.0, + "grad_norm": 0.6571091958054168, + "language_loss": 0.50119674, + "learning_rate": 3.7304823355647034e-06, + "loss": 0.52162373, + "num_input_tokens_seen": 189226640, + "router_z_loss_clip": 0.18359375, + "router_z_loss_mlp": 0.0100708, + "step": 6622, + "time_per_iteration": 3.181493043899536 + }, + { + "auxiliary_loss_clip": 0.01125794, + "auxiliary_loss_mlp": 0.01035102, + "balance_loss_clip": 1.0548296, + "balance_loss_mlp": 1.02073145, + "epoch": 0.19218269398177704, + "flos": 26827963948800.0, + "grad_norm": 4.159073636153599, + "language_loss": 0.92187786, + "learning_rate": 3.7303880921945884e-06, + "loss": 0.94348687, + "num_input_tokens_seen": 189242570, + "router_z_loss_clip": 0.70996094, + "router_z_loss_mlp": 0.14355469, + "step": 6623, + "time_per_iteration": 2.580296754837036 + }, + { + "auxiliary_loss_clip": 0.01131805, + "auxiliary_loss_mlp": 0.01042243, + "balance_loss_clip": 1.05217218, + "balance_loss_mlp": 1.02589381, + "epoch": 0.1922117114502931, + "flos": 32044634951040.0, + "grad_norm": 2.355881043870893, + "language_loss": 0.78338033, + "learning_rate": 3.730293833540985e-06, + "loss": 0.80512083, + "num_input_tokens_seen": 189258585, + "router_z_loss_clip": 0.79589844, + "router_z_loss_mlp": 0.16345215, + "step": 6624, + "time_per_iteration": 2.6362547874450684 + }, + { + "auxiliary_loss_clip": 0.01038587, + "auxiliary_loss_mlp": 0.01002556, + "balance_loss_clip": 1.02002311, + "balance_loss_mlp": 1.00151849, + "epoch": 0.1922407289188091, + "flos": 56822030254080.0, + "grad_norm": 0.7360412414657507, + "language_loss": 0.5225023, + "learning_rate": 3.7301995596047274e-06, + "loss": 0.54291368, + "num_input_tokens_seen": 189314485, + "router_z_loss_clip": 0.18554688, + "router_z_loss_mlp": 0.01037598, + "step": 6625, + "time_per_iteration": 2.9287667274475098 + }, + { + "auxiliary_loss_clip": 0.01121881, + "auxiliary_loss_mlp": 0.01033828, + "balance_loss_clip": 1.0516504, + "balance_loss_mlp": 1.01962435, + "epoch": 0.19226974638732516, + "flos": 24965959749120.0, + "grad_norm": 2.843877318406453, + "language_loss": 0.8530919, + "learning_rate": 3.7301052703866463e-06, + "loss": 0.87464899, + "num_input_tokens_seen": 189330830, + "router_z_loss_clip": 0.70361328, + "router_z_loss_mlp": 0.14196777, + "step": 6626, + "time_per_iteration": 2.604015588760376 + }, + { + "auxiliary_loss_clip": 0.0112251, + "auxiliary_loss_mlp": 0.0103931, + "balance_loss_clip": 1.0543288, + "balance_loss_mlp": 1.02440333, + "epoch": 0.19229876385584121, + "flos": 74732042154240.0, + "grad_norm": 2.1110475387764898, + "language_loss": 0.57353038, + "learning_rate": 3.730010965887576e-06, + "loss": 0.59514856, + "num_input_tokens_seen": 189353705, + "router_z_loss_clip": 0.68261719, + "router_z_loss_mlp": 0.14886475, + "step": 6627, + "time_per_iteration": 2.915144920349121 + }, + { + "auxiliary_loss_clip": 0.01038018, + "auxiliary_loss_mlp": 0.01001132, + "balance_loss_clip": 1.01956105, + "balance_loss_mlp": 1.0001843, + "epoch": 0.19232778132435727, + "flos": 62197858419840.0, + "grad_norm": 0.6504817259863562, + "language_loss": 0.53168648, + "learning_rate": 3.7299166461083483e-06, + "loss": 0.55207807, + "num_input_tokens_seen": 189420585, + "router_z_loss_clip": 0.18457031, + "router_z_loss_mlp": 0.00946045, + "step": 6628, + "time_per_iteration": 3.223928928375244 + }, + { + "auxiliary_loss_clip": 0.01039212, + "auxiliary_loss_mlp": 0.01001112, + "balance_loss_clip": 1.02072906, + "balance_loss_mlp": 1.00011051, + "epoch": 0.19235679879287332, + "flos": 74769607779840.0, + "grad_norm": 0.6667601012276033, + "language_loss": 0.48528594, + "learning_rate": 3.7298223110497966e-06, + "loss": 0.5056892, + "num_input_tokens_seen": 189482005, + "router_z_loss_clip": 0.18554688, + "router_z_loss_mlp": 0.01000977, + "step": 6629, + "time_per_iteration": 3.103620767593384 + }, + { + "auxiliary_loss_clip": 0.01036578, + "auxiliary_loss_mlp": 0.01004456, + "balance_loss_clip": 1.01835048, + "balance_loss_mlp": 1.00347233, + "epoch": 0.19238581626138937, + "flos": 59740274810880.0, + "grad_norm": 0.6886511194125023, + "language_loss": 0.51485747, + "learning_rate": 3.7297279607127548e-06, + "loss": 0.53526783, + "num_input_tokens_seen": 189543585, + "router_z_loss_clip": 0.18261719, + "router_z_loss_mlp": 0.00982666, + "step": 6630, + "time_per_iteration": 3.0154964923858643 + }, + { + "auxiliary_loss_clip": 0.01036314, + "auxiliary_loss_mlp": 0.01003049, + "balance_loss_clip": 1.01807928, + "balance_loss_mlp": 1.00214887, + "epoch": 0.1924148337299054, + "flos": 67740526281600.0, + "grad_norm": 0.6807620197547916, + "language_loss": 0.53194177, + "learning_rate": 3.7296335950980558e-06, + "loss": 0.55233538, + "num_input_tokens_seen": 189613050, + "router_z_loss_clip": 0.18261719, + "router_z_loss_mlp": 0.00897217, + "step": 6631, + "time_per_iteration": 3.2863385677337646 + }, + { + "auxiliary_loss_clip": 0.01129062, + "auxiliary_loss_mlp": 0.01033411, + "balance_loss_clip": 1.05272818, + "balance_loss_mlp": 1.01656032, + "epoch": 0.19244385119842145, + "flos": 31993284458880.0, + "grad_norm": 3.268499418265024, + "language_loss": 0.8824929, + "learning_rate": 3.7295392142065327e-06, + "loss": 0.90411758, + "num_input_tokens_seen": 189633925, + "router_z_loss_clip": 0.76318359, + "router_z_loss_mlp": 0.16864014, + "step": 6632, + "time_per_iteration": 2.7814812660217285 + }, + { + "auxiliary_loss_clip": 0.01035299, + "auxiliary_loss_mlp": 0.01001975, + "balance_loss_clip": 1.01694393, + "balance_loss_mlp": 1.00110435, + "epoch": 0.1924728686669375, + "flos": 62149593519360.0, + "grad_norm": 0.7531537325688857, + "language_loss": 0.4868657, + "learning_rate": 3.7294448180390194e-06, + "loss": 0.50723839, + "num_input_tokens_seen": 189685955, + "router_z_loss_clip": 0.18359375, + "router_z_loss_mlp": 0.00872803, + "step": 6633, + "time_per_iteration": 2.955648422241211 + }, + { + "auxiliary_loss_clip": 0.01122501, + "auxiliary_loss_mlp": 0.01033569, + "balance_loss_clip": 1.05249429, + "balance_loss_mlp": 1.01854873, + "epoch": 0.19250188613545355, + "flos": 27445519503360.0, + "grad_norm": 1.8282401519714186, + "language_loss": 0.56591445, + "learning_rate": 3.7293504065963494e-06, + "loss": 0.58747518, + "num_input_tokens_seen": 189701090, + "router_z_loss_clip": 0.70019531, + "router_z_loss_mlp": 0.15020752, + "step": 6634, + "time_per_iteration": 2.6246728897094727 + }, + { + "auxiliary_loss_clip": 0.01125041, + "auxiliary_loss_mlp": 0.01037636, + "balance_loss_clip": 1.05696607, + "balance_loss_mlp": 1.02373052, + "epoch": 0.1925309036039696, + "flos": 31095731410560.0, + "grad_norm": 2.5705950901332715, + "language_loss": 0.72052068, + "learning_rate": 3.729255979879357e-06, + "loss": 0.74214745, + "num_input_tokens_seen": 189721075, + "router_z_loss_clip": 0.68066406, + "router_z_loss_mlp": 0.13891602, + "step": 6635, + "time_per_iteration": 2.614708185195923 + }, + { + "auxiliary_loss_clip": 0.01135987, + "auxiliary_loss_mlp": 0.01049571, + "balance_loss_clip": 1.05625832, + "balance_loss_mlp": 1.0313971, + "epoch": 0.19255992107248562, + "flos": 40473012625920.0, + "grad_norm": 2.7269079627399595, + "language_loss": 0.81816703, + "learning_rate": 3.7291615378888763e-06, + "loss": 0.84002268, + "num_input_tokens_seen": 189740445, + "router_z_loss_clip": 0.79736328, + "router_z_loss_mlp": 0.18182373, + "step": 6636, + "time_per_iteration": 2.6380748748779297 + }, + { + "auxiliary_loss_clip": 0.0112734, + "auxiliary_loss_mlp": 0.01040093, + "balance_loss_clip": 1.05166936, + "balance_loss_mlp": 1.02341557, + "epoch": 0.19258893854100168, + "flos": 28904390386560.0, + "grad_norm": 1.7565747921653054, + "language_loss": 0.77577221, + "learning_rate": 3.729067080625741e-06, + "loss": 0.79744655, + "num_input_tokens_seen": 189755785, + "router_z_loss_clip": 0.75732422, + "router_z_loss_mlp": 0.16680908, + "step": 6637, + "time_per_iteration": 2.584378480911255 + }, + { + "auxiliary_loss_clip": 0.01128644, + "auxiliary_loss_mlp": 0.010367, + "balance_loss_clip": 1.05372334, + "balance_loss_mlp": 1.02163196, + "epoch": 0.19261795600951773, + "flos": 74739943146240.0, + "grad_norm": 1.7866879215333875, + "language_loss": 0.84030414, + "learning_rate": 3.7289726080907854e-06, + "loss": 0.86195755, + "num_input_tokens_seen": 189780460, + "router_z_loss_clip": 0.75048828, + "router_z_loss_mlp": 0.1506958, + "step": 6638, + "time_per_iteration": 2.9357335567474365 + }, + { + "auxiliary_loss_clip": 0.01132542, + "auxiliary_loss_mlp": 0.01046167, + "balance_loss_clip": 1.05111873, + "balance_loss_mlp": 1.02733207, + "epoch": 0.19264697347803378, + "flos": 29893514181120.0, + "grad_norm": 3.086909084280113, + "language_loss": 0.7827127, + "learning_rate": 3.728878120284844e-06, + "loss": 0.80449975, + "num_input_tokens_seen": 189798110, + "router_z_loss_clip": 0.81445312, + "router_z_loss_mlp": 0.18859863, + "step": 6639, + "time_per_iteration": 2.5865986347198486 + }, + { + "auxiliary_loss_clip": 0.01033547, + "auxiliary_loss_mlp": 0.01003495, + "balance_loss_clip": 1.01529121, + "balance_loss_mlp": 1.00255919, + "epoch": 0.19267599094654983, + "flos": 71086071029760.0, + "grad_norm": 0.6885580198933043, + "language_loss": 0.48391032, + "learning_rate": 3.728783617208752e-06, + "loss": 0.50428069, + "num_input_tokens_seen": 189860725, + "router_z_loss_clip": 0.18261719, + "router_z_loss_mlp": 0.00933838, + "step": 6640, + "time_per_iteration": 3.1469216346740723 + }, + { + "auxiliary_loss_clip": 0.0113327, + "auxiliary_loss_mlp": 0.01039468, + "balance_loss_clip": 1.05652797, + "balance_loss_mlp": 1.02289772, + "epoch": 0.19270500841506588, + "flos": 26426662225920.0, + "grad_norm": 2.0129172002489244, + "language_loss": 0.72525007, + "learning_rate": 3.7286890988633434e-06, + "loss": 0.74697745, + "num_input_tokens_seen": 189877890, + "router_z_loss_clip": 0.76757812, + "router_z_loss_mlp": 0.16558838, + "step": 6641, + "time_per_iteration": 2.654949426651001 + }, + { + "auxiliary_loss_clip": 0.01033286, + "auxiliary_loss_mlp": 0.01001989, + "balance_loss_clip": 1.01508951, + "balance_loss_mlp": 1.00113714, + "epoch": 0.1927340258835819, + "flos": 65511694437120.0, + "grad_norm": 0.6380459194052809, + "language_loss": 0.5560807, + "learning_rate": 3.7285945652494527e-06, + "loss": 0.57643354, + "num_input_tokens_seen": 189941385, + "router_z_loss_clip": 0.18164062, + "router_z_loss_mlp": 0.00854492, + "step": 6642, + "time_per_iteration": 3.0845720767974854 + }, + { + "auxiliary_loss_clip": 0.01134504, + "auxiliary_loss_mlp": 0.01050763, + "balance_loss_clip": 1.05520046, + "balance_loss_mlp": 1.03357887, + "epoch": 0.19276304335209796, + "flos": 29930143075200.0, + "grad_norm": 3.037016166137696, + "language_loss": 0.86280936, + "learning_rate": 3.728500016367915e-06, + "loss": 0.88466197, + "num_input_tokens_seen": 189956405, + "router_z_loss_clip": 0.79345703, + "router_z_loss_mlp": 0.17163086, + "step": 6643, + "time_per_iteration": 2.6215906143188477 + }, + { + "auxiliary_loss_clip": 0.01036731, + "auxiliary_loss_mlp": 0.01005077, + "balance_loss_clip": 1.01828301, + "balance_loss_mlp": 1.00418305, + "epoch": 0.192792060820614, + "flos": 69119531274240.0, + "grad_norm": 0.6856347458876939, + "language_loss": 0.48846501, + "learning_rate": 3.728405452219567e-06, + "loss": 0.50888306, + "num_input_tokens_seen": 190015095, + "router_z_loss_clip": 0.18457031, + "router_z_loss_mlp": 0.00891113, + "step": 6644, + "time_per_iteration": 3.0886850357055664 + }, + { + "auxiliary_loss_clip": 0.01138161, + "auxiliary_loss_mlp": 0.01041084, + "balance_loss_clip": 1.05782735, + "balance_loss_mlp": 1.02470481, + "epoch": 0.19282107828913006, + "flos": 16363020954240.0, + "grad_norm": 1.9430016282547409, + "language_loss": 0.7412672, + "learning_rate": 3.7283108728052416e-06, + "loss": 0.76305968, + "num_input_tokens_seen": 190028075, + "router_z_loss_clip": 0.8046875, + "router_z_loss_mlp": 0.16387939, + "step": 6645, + "time_per_iteration": 2.630552291870117 + }, + { + "auxiliary_loss_clip": 0.01122989, + "auxiliary_loss_mlp": 0.01035346, + "balance_loss_clip": 1.05169606, + "balance_loss_mlp": 1.02141678, + "epoch": 0.1928500957576461, + "flos": 74732329463040.0, + "grad_norm": 2.568352871173643, + "language_loss": 0.80392951, + "learning_rate": 3.728216278125775e-06, + "loss": 0.82551277, + "num_input_tokens_seen": 190050185, + "router_z_loss_clip": 0.71289062, + "router_z_loss_mlp": 0.13909912, + "step": 6646, + "time_per_iteration": 2.900588035583496 + }, + { + "auxiliary_loss_clip": 0.01127104, + "auxiliary_loss_mlp": 0.01033898, + "balance_loss_clip": 1.05520844, + "balance_loss_mlp": 1.01864481, + "epoch": 0.19287911322616216, + "flos": 26826455577600.0, + "grad_norm": 2.622633782551573, + "language_loss": 0.86571437, + "learning_rate": 3.7281216681820034e-06, + "loss": 0.88732433, + "num_input_tokens_seen": 190066495, + "router_z_loss_clip": 0.71923828, + "router_z_loss_mlp": 0.15240479, + "step": 6647, + "time_per_iteration": 2.5765678882598877 + }, + { + "auxiliary_loss_clip": 0.01123844, + "auxiliary_loss_mlp": 0.01037317, + "balance_loss_clip": 1.05133021, + "balance_loss_mlp": 1.02254665, + "epoch": 0.1929081306946782, + "flos": 12853362965760.0, + "grad_norm": 2.67345926740046, + "language_loss": 0.95341027, + "learning_rate": 3.7280270429747623e-06, + "loss": 0.9750219, + "num_input_tokens_seen": 190081075, + "router_z_loss_clip": 0.72607422, + "router_z_loss_mlp": 0.14764404, + "step": 6648, + "time_per_iteration": 2.4694974422454834 + }, + { + "auxiliary_loss_clip": 0.01036263, + "auxiliary_loss_mlp": 0.01002014, + "balance_loss_clip": 1.01791978, + "balance_loss_mlp": 1.00119746, + "epoch": 0.19293714816319424, + "flos": 74776503191040.0, + "grad_norm": 0.645535046659625, + "language_loss": 0.44241297, + "learning_rate": 3.7279324025048866e-06, + "loss": 0.46279573, + "num_input_tokens_seen": 190145165, + "router_z_loss_clip": 0.18359375, + "router_z_loss_mlp": 0.00817871, + "step": 6649, + "time_per_iteration": 3.160538911819458 + }, + { + "auxiliary_loss_clip": 0.01119164, + "auxiliary_loss_mlp": 0.0103571, + "balance_loss_clip": 1.04915643, + "balance_loss_mlp": 1.02213836, + "epoch": 0.1929661656317103, + "flos": 29673202112640.0, + "grad_norm": 2.3519479829403664, + "language_loss": 0.6696927, + "learning_rate": 3.727837746773213e-06, + "loss": 0.69124144, + "num_input_tokens_seen": 190160895, + "router_z_loss_clip": 0.69995117, + "router_z_loss_mlp": 0.13586426, + "step": 6650, + "time_per_iteration": 2.525317668914795 + }, + { + "auxiliary_loss_clip": 0.0103515, + "auxiliary_loss_mlp": 0.00999321, + "balance_loss_clip": 1.01691675, + "balance_loss_mlp": 0.99852222, + "epoch": 0.19299518310022634, + "flos": 70465067769600.0, + "grad_norm": 0.6956642405590882, + "language_loss": 0.47515363, + "learning_rate": 3.727743075780577e-06, + "loss": 0.49549836, + "num_input_tokens_seen": 190225085, + "router_z_loss_clip": 0.18261719, + "router_z_loss_mlp": 0.00799561, + "step": 6651, + "time_per_iteration": 3.09438157081604 + }, + { + "auxiliary_loss_clip": 0.01127828, + "auxiliary_loss_mlp": 0.01041843, + "balance_loss_clip": 1.05188966, + "balance_loss_mlp": 1.02504611, + "epoch": 0.1930242005687424, + "flos": 22928999379840.0, + "grad_norm": 2.537714726380516, + "language_loss": 0.84869957, + "learning_rate": 3.7276483895278144e-06, + "loss": 0.87039626, + "num_input_tokens_seen": 190239105, + "router_z_loss_clip": 0.75976562, + "router_z_loss_mlp": 0.16802979, + "step": 6652, + "time_per_iteration": 2.55279803276062 + }, + { + "auxiliary_loss_clip": 0.01127209, + "auxiliary_loss_mlp": 0.01042625, + "balance_loss_clip": 1.05090475, + "balance_loss_mlp": 1.02514863, + "epoch": 0.19305321803725842, + "flos": 27263811576960.0, + "grad_norm": 2.6465282693955863, + "language_loss": 0.78834194, + "learning_rate": 3.7275536880157635e-06, + "loss": 0.8100403, + "num_input_tokens_seen": 190254435, + "router_z_loss_clip": 0.76269531, + "router_z_loss_mlp": 0.17480469, + "step": 6653, + "time_per_iteration": 2.5816619396209717 + }, + { + "auxiliary_loss_clip": 0.0113736, + "auxiliary_loss_mlp": 0.01049922, + "balance_loss_clip": 1.05850697, + "balance_loss_mlp": 1.03126574, + "epoch": 0.19308223550577447, + "flos": 21176774121600.0, + "grad_norm": 2.2688962737750216, + "language_loss": 0.78502727, + "learning_rate": 3.7274589712452586e-06, + "loss": 0.80690008, + "num_input_tokens_seen": 190269070, + "router_z_loss_clip": 0.78759766, + "router_z_loss_mlp": 0.18670654, + "step": 6654, + "time_per_iteration": 2.5242393016815186 + }, + { + "auxiliary_loss_clip": 0.01125764, + "auxiliary_loss_mlp": 0.01046502, + "balance_loss_clip": 1.05308843, + "balance_loss_mlp": 1.03113616, + "epoch": 0.19311125297429052, + "flos": 26060516939520.0, + "grad_norm": 2.6555024026266083, + "language_loss": 0.91282535, + "learning_rate": 3.727364239217137e-06, + "loss": 0.93454802, + "num_input_tokens_seen": 190283555, + "router_z_loss_clip": 0.7265625, + "router_z_loss_mlp": 0.15344238, + "step": 6655, + "time_per_iteration": 2.599729061126709 + }, + { + "auxiliary_loss_clip": 0.01034453, + "auxiliary_loss_mlp": 0.01000286, + "balance_loss_clip": 1.01625335, + "balance_loss_mlp": 0.99955255, + "epoch": 0.19314027044280657, + "flos": 66750468733440.0, + "grad_norm": 0.6605543462121264, + "language_loss": 0.47960281, + "learning_rate": 3.7272694919322354e-06, + "loss": 0.4999502, + "num_input_tokens_seen": 190346815, + "router_z_loss_clip": 0.18164062, + "router_z_loss_mlp": 0.00732422, + "step": 6656, + "time_per_iteration": 3.1293082237243652 + }, + { + "auxiliary_loss_clip": 0.01128998, + "auxiliary_loss_mlp": 0.01048267, + "balance_loss_clip": 1.05116129, + "balance_loss_mlp": 1.03331208, + "epoch": 0.19316928791132262, + "flos": 31789848804480.0, + "grad_norm": 2.4311251945294328, + "language_loss": 0.84637487, + "learning_rate": 3.7271747293913904e-06, + "loss": 0.86814749, + "num_input_tokens_seen": 190361695, + "router_z_loss_clip": 0.77832031, + "router_z_loss_mlp": 0.14959717, + "step": 6657, + "time_per_iteration": 2.6385021209716797 + }, + { + "auxiliary_loss_clip": 0.01129297, + "auxiliary_loss_mlp": 0.01049518, + "balance_loss_clip": 1.05276644, + "balance_loss_mlp": 1.03414583, + "epoch": 0.19319830537983868, + "flos": 25660292624640.0, + "grad_norm": 3.2774862999923675, + "language_loss": 0.75924671, + "learning_rate": 3.72707995159544e-06, + "loss": 0.78103495, + "num_input_tokens_seen": 190380310, + "router_z_loss_clip": 0.76513672, + "router_z_loss_mlp": 0.15356445, + "step": 6658, + "time_per_iteration": 2.644390821456909 + }, + { + "auxiliary_loss_clip": 0.01124583, + "auxiliary_loss_mlp": 0.01041973, + "balance_loss_clip": 1.05157709, + "balance_loss_mlp": 1.02737606, + "epoch": 0.1932273228483547, + "flos": 33320612759040.0, + "grad_norm": 3.3004587593719927, + "language_loss": 0.95772511, + "learning_rate": 3.7269851585452205e-06, + "loss": 0.97939062, + "num_input_tokens_seen": 190396625, + "router_z_loss_clip": 0.73046875, + "router_z_loss_mlp": 0.14611816, + "step": 6659, + "time_per_iteration": 2.574054002761841 + }, + { + "auxiliary_loss_clip": 0.01124136, + "auxiliary_loss_mlp": 0.01051181, + "balance_loss_clip": 1.05081177, + "balance_loss_mlp": 1.03500462, + "epoch": 0.19325634031687075, + "flos": 15844901224320.0, + "grad_norm": 3.273920247897115, + "language_loss": 0.85887241, + "learning_rate": 3.726890350241569e-06, + "loss": 0.88062561, + "num_input_tokens_seen": 190410485, + "router_z_loss_clip": 0.73291016, + "router_z_loss_mlp": 0.16168213, + "step": 6660, + "time_per_iteration": 2.5333821773529053 + }, + { + "auxiliary_loss_clip": 0.01124015, + "auxiliary_loss_mlp": 0.01039907, + "balance_loss_clip": 1.04968476, + "balance_loss_mlp": 1.02442765, + "epoch": 0.1932853577853868, + "flos": 28176158050560.0, + "grad_norm": 2.4964111770430115, + "language_loss": 0.76051003, + "learning_rate": 3.7267955266853226e-06, + "loss": 0.78214926, + "num_input_tokens_seen": 190425185, + "router_z_loss_clip": 0.74414062, + "router_z_loss_mlp": 0.15490723, + "step": 6661, + "time_per_iteration": 2.572154998779297 + }, + { + "auxiliary_loss_clip": 0.01120335, + "auxiliary_loss_mlp": 0.01043082, + "balance_loss_clip": 1.04785693, + "balance_loss_mlp": 1.0290215, + "epoch": 0.19331437525390285, + "flos": 16648151114880.0, + "grad_norm": 2.579845279935649, + "language_loss": 0.78598511, + "learning_rate": 3.72670068787732e-06, + "loss": 0.80761933, + "num_input_tokens_seen": 190438430, + "router_z_loss_clip": 0.72363281, + "router_z_loss_mlp": 0.140625, + "step": 6662, + "time_per_iteration": 2.506650924682617 + }, + { + "auxiliary_loss_clip": 0.01038141, + "auxiliary_loss_mlp": 0.01015208, + "balance_loss_clip": 1.01963711, + "balance_loss_mlp": 1.01437938, + "epoch": 0.1933433927224189, + "flos": 59200071194880.0, + "grad_norm": 0.7333102966335968, + "language_loss": 0.47956473, + "learning_rate": 3.7266058338183985e-06, + "loss": 0.50009823, + "num_input_tokens_seen": 190501905, + "router_z_loss_clip": 0.18554688, + "router_z_loss_mlp": 0.00830078, + "step": 6663, + "time_per_iteration": 3.0756828784942627 + }, + { + "auxiliary_loss_clip": 0.01129519, + "auxiliary_loss_mlp": 0.01045328, + "balance_loss_clip": 1.05235577, + "balance_loss_mlp": 1.02789378, + "epoch": 0.19337241019093493, + "flos": 33394265596800.0, + "grad_norm": 2.0035432526775376, + "language_loss": 0.85613954, + "learning_rate": 3.7265109645093952e-06, + "loss": 0.87788802, + "num_input_tokens_seen": 190517540, + "router_z_loss_clip": 0.77197266, + "router_z_loss_mlp": 0.17443848, + "step": 6664, + "time_per_iteration": 2.6290969848632812 + }, + { + "auxiliary_loss_clip": 0.01126365, + "auxiliary_loss_mlp": 0.0104817, + "balance_loss_clip": 1.04877138, + "balance_loss_mlp": 1.03187442, + "epoch": 0.19340142765945098, + "flos": 20370722970240.0, + "grad_norm": 2.1581770765342903, + "language_loss": 0.87435216, + "learning_rate": 3.726416079951148e-06, + "loss": 0.89609754, + "num_input_tokens_seen": 190532560, + "router_z_loss_clip": 0.77636719, + "router_z_loss_mlp": 0.1630249, + "step": 6665, + "time_per_iteration": 2.5339555740356445 + }, + { + "auxiliary_loss_clip": 0.01133352, + "auxiliary_loss_mlp": 0.01043706, + "balance_loss_clip": 1.0512445, + "balance_loss_mlp": 1.02768457, + "epoch": 0.19343044512796703, + "flos": 14313275343360.0, + "grad_norm": 2.715044101944827, + "language_loss": 0.92258155, + "learning_rate": 3.726321180144496e-06, + "loss": 0.94435215, + "num_input_tokens_seen": 190543930, + "router_z_loss_clip": 0.82226562, + "router_z_loss_mlp": 0.16027832, + "step": 6666, + "time_per_iteration": 2.5160646438598633 + }, + { + "auxiliary_loss_clip": 0.01133629, + "auxiliary_loss_mlp": 0.01050992, + "balance_loss_clip": 1.05520701, + "balance_loss_mlp": 1.03200746, + "epoch": 0.19345946259648308, + "flos": 10115641186560.0, + "grad_norm": 2.304631628675666, + "language_loss": 0.80106628, + "learning_rate": 3.7262262650902762e-06, + "loss": 0.8229124, + "num_input_tokens_seen": 190556125, + "router_z_loss_clip": 0.78417969, + "router_z_loss_mlp": 0.19000244, + "step": 6667, + "time_per_iteration": 2.526759386062622 + }, + { + "auxiliary_loss_clip": 0.01122351, + "auxiliary_loss_mlp": 0.01042216, + "balance_loss_clip": 1.05316198, + "balance_loss_mlp": 1.02856612, + "epoch": 0.19348848006499914, + "flos": 16538192605440.0, + "grad_norm": 2.5382495179764275, + "language_loss": 0.78583688, + "learning_rate": 3.726131334789328e-06, + "loss": 0.80748248, + "num_input_tokens_seen": 190568295, + "router_z_loss_clip": 0.69189453, + "router_z_loss_mlp": 0.13653564, + "step": 6668, + "time_per_iteration": 2.480952501296997 + }, + { + "auxiliary_loss_clip": 0.01037914, + "auxiliary_loss_mlp": 0.01001022, + "balance_loss_clip": 1.01954043, + "balance_loss_mlp": 1.00015759, + "epoch": 0.1935174975335152, + "flos": 65874140622720.0, + "grad_norm": 0.6518820441551786, + "language_loss": 0.49285156, + "learning_rate": 3.726036389242489e-06, + "loss": 0.51324093, + "num_input_tokens_seen": 190632740, + "router_z_loss_clip": 0.18359375, + "router_z_loss_mlp": 0.00866699, + "step": 6669, + "time_per_iteration": 3.1288726329803467 + }, + { + "auxiliary_loss_clip": 0.01120023, + "auxiliary_loss_mlp": 0.01033635, + "balance_loss_clip": 1.04979181, + "balance_loss_mlp": 1.01863861, + "epoch": 0.1935465150020312, + "flos": 31460763375360.0, + "grad_norm": 2.4909548191164457, + "language_loss": 0.84287655, + "learning_rate": 3.725941428450599e-06, + "loss": 0.86441308, + "num_input_tokens_seen": 190647705, + "router_z_loss_clip": 0.70214844, + "router_z_loss_mlp": 0.14996338, + "step": 6670, + "time_per_iteration": 2.584581136703491 + }, + { + "auxiliary_loss_clip": 0.01037616, + "auxiliary_loss_mlp": 0.01002836, + "balance_loss_clip": 1.01927185, + "balance_loss_mlp": 1.00188828, + "epoch": 0.19357553247054726, + "flos": 63503857019520.0, + "grad_norm": 0.7371049022978087, + "language_loss": 0.5026924, + "learning_rate": 3.7258464524144946e-06, + "loss": 0.52309692, + "num_input_tokens_seen": 190706095, + "router_z_loss_clip": 0.18359375, + "router_z_loss_mlp": 0.00946045, + "step": 6671, + "time_per_iteration": 3.0710909366607666 + }, + { + "auxiliary_loss_clip": 0.01131148, + "auxiliary_loss_mlp": 0.01036456, + "balance_loss_clip": 1.05407596, + "balance_loss_mlp": 1.01899767, + "epoch": 0.19360454993906331, + "flos": 21461293751040.0, + "grad_norm": 2.2070413885724527, + "language_loss": 0.81416613, + "learning_rate": 3.725751461135017e-06, + "loss": 0.83584219, + "num_input_tokens_seen": 190720950, + "router_z_loss_clip": 0.77050781, + "router_z_loss_mlp": 0.17468262, + "step": 6672, + "time_per_iteration": 2.5460426807403564 + }, + { + "auxiliary_loss_clip": 0.01035879, + "auxiliary_loss_mlp": 0.01000368, + "balance_loss_clip": 1.01752663, + "balance_loss_mlp": 0.99941427, + "epoch": 0.19363356740757937, + "flos": 74768602199040.0, + "grad_norm": 0.6656759097541668, + "language_loss": 0.45575017, + "learning_rate": 3.7256564546130036e-06, + "loss": 0.47611266, + "num_input_tokens_seen": 190783085, + "router_z_loss_clip": 0.18359375, + "router_z_loss_mlp": 0.00952148, + "step": 6673, + "time_per_iteration": 3.1078243255615234 + }, + { + "auxiliary_loss_clip": 0.01119302, + "auxiliary_loss_mlp": 0.01041636, + "balance_loss_clip": 1.04908514, + "balance_loss_mlp": 1.02749801, + "epoch": 0.19366258487609542, + "flos": 10261797626880.0, + "grad_norm": 2.427461309914846, + "language_loss": 0.81774139, + "learning_rate": 3.7255614328492943e-06, + "loss": 0.83935082, + "num_input_tokens_seen": 190792860, + "router_z_loss_clip": 0.70214844, + "router_z_loss_mlp": 0.14129639, + "step": 6674, + "time_per_iteration": 2.497802257537842 + }, + { + "auxiliary_loss_clip": 0.01127426, + "auxiliary_loss_mlp": 0.01037829, + "balance_loss_clip": 1.05215847, + "balance_loss_mlp": 1.02022219, + "epoch": 0.19369160234461147, + "flos": 42123647243520.0, + "grad_norm": 2.638854641932758, + "language_loss": 0.6606577, + "learning_rate": 3.7254663958447285e-06, + "loss": 0.68231022, + "num_input_tokens_seen": 190814865, + "router_z_loss_clip": 0.75292969, + "router_z_loss_mlp": 0.17614746, + "step": 6675, + "time_per_iteration": 2.7397024631500244 + }, + { + "auxiliary_loss_clip": 0.01117167, + "auxiliary_loss_mlp": 0.01033585, + "balance_loss_clip": 1.048334, + "balance_loss_mlp": 1.01944661, + "epoch": 0.1937206198131275, + "flos": 16172765591040.0, + "grad_norm": 2.615762712294519, + "language_loss": 0.54186571, + "learning_rate": 3.7253713436001447e-06, + "loss": 0.56337321, + "num_input_tokens_seen": 190826695, + "router_z_loss_clip": 0.68847656, + "router_z_loss_mlp": 0.14123535, + "step": 6676, + "time_per_iteration": 2.475595235824585 + }, + { + "auxiliary_loss_clip": 0.01124277, + "auxiliary_loss_mlp": 0.01041695, + "balance_loss_clip": 1.05029678, + "balance_loss_mlp": 1.02625751, + "epoch": 0.19374963728164354, + "flos": 30005088802560.0, + "grad_norm": 1.956395455732982, + "language_loss": 0.84284091, + "learning_rate": 3.725276276116383e-06, + "loss": 0.86450064, + "num_input_tokens_seen": 190844010, + "router_z_loss_clip": 0.73876953, + "router_z_loss_mlp": 0.15454102, + "step": 6677, + "time_per_iteration": 2.613553524017334 + }, + { + "auxiliary_loss_clip": 0.01120806, + "auxiliary_loss_mlp": 0.01036048, + "balance_loss_clip": 1.05005169, + "balance_loss_mlp": 1.02164149, + "epoch": 0.1937786547501596, + "flos": 21245327228160.0, + "grad_norm": 2.141081148523569, + "language_loss": 0.84530795, + "learning_rate": 3.7251811933942835e-06, + "loss": 0.86687648, + "num_input_tokens_seen": 190858365, + "router_z_loss_clip": 0.70751953, + "router_z_loss_mlp": 0.14398193, + "step": 6678, + "time_per_iteration": 2.5481064319610596 + }, + { + "auxiliary_loss_clip": 0.01130931, + "auxiliary_loss_mlp": 0.01041273, + "balance_loss_clip": 1.05239987, + "balance_loss_mlp": 1.02290845, + "epoch": 0.19380767221867565, + "flos": 14750020811520.0, + "grad_norm": 2.2813270825450966, + "language_loss": 0.90618825, + "learning_rate": 3.725086095434685e-06, + "loss": 0.92791021, + "num_input_tokens_seen": 190871005, + "router_z_loss_clip": 0.78515625, + "router_z_loss_mlp": 0.18359375, + "step": 6679, + "time_per_iteration": 2.5085926055908203 + }, + { + "auxiliary_loss_clip": 0.01127673, + "auxiliary_loss_mlp": 0.01040593, + "balance_loss_clip": 1.05306411, + "balance_loss_mlp": 1.02466631, + "epoch": 0.1938366896871917, + "flos": 13727715828480.0, + "grad_norm": 2.2140894464413097, + "language_loss": 0.78970993, + "learning_rate": 3.7249909822384284e-06, + "loss": 0.81139255, + "num_input_tokens_seen": 190882780, + "router_z_loss_clip": 0.74609375, + "router_z_loss_mlp": 0.15917969, + "step": 6680, + "time_per_iteration": 2.4923079013824463 + }, + { + "auxiliary_loss_clip": 0.01031836, + "auxiliary_loss_mlp": 0.01001409, + "balance_loss_clip": 1.01375663, + "balance_loss_mlp": 1.00051486, + "epoch": 0.19386570715570772, + "flos": 64008221840640.0, + "grad_norm": 0.6864111092106713, + "language_loss": 0.49576136, + "learning_rate": 3.7248958538063536e-06, + "loss": 0.51609385, + "num_input_tokens_seen": 190942190, + "router_z_loss_clip": 0.18066406, + "router_z_loss_mlp": 0.00891113, + "step": 6681, + "time_per_iteration": 3.066909074783325 + }, + { + "auxiliary_loss_clip": 0.01032287, + "auxiliary_loss_mlp": 0.01001677, + "balance_loss_clip": 1.01416516, + "balance_loss_mlp": 1.00075865, + "epoch": 0.19389472462422377, + "flos": 46708687537920.0, + "grad_norm": 0.6927985728345125, + "language_loss": 0.44874471, + "learning_rate": 3.7248007101393002e-06, + "loss": 0.46908435, + "num_input_tokens_seen": 190993195, + "router_z_loss_clip": 0.18164062, + "router_z_loss_mlp": 0.00915527, + "step": 6682, + "time_per_iteration": 2.867734432220459 + }, + { + "auxiliary_loss_clip": 0.0111814, + "auxiliary_loss_mlp": 0.01036779, + "balance_loss_clip": 1.04730546, + "balance_loss_mlp": 1.02138889, + "epoch": 0.19392374209273983, + "flos": 28324181998080.0, + "grad_norm": 2.1618545578872976, + "language_loss": 0.93972778, + "learning_rate": 3.7247055512381094e-06, + "loss": 0.96127701, + "num_input_tokens_seen": 191011830, + "router_z_loss_clip": 0.70800781, + "router_z_loss_mlp": 0.15386963, + "step": 6683, + "time_per_iteration": 2.587144374847412 + }, + { + "auxiliary_loss_clip": 0.01131054, + "auxiliary_loss_mlp": 0.01049295, + "balance_loss_clip": 1.05293453, + "balance_loss_mlp": 1.03129387, + "epoch": 0.19395275956125588, + "flos": 28178312866560.0, + "grad_norm": 1.9982164871487047, + "language_loss": 0.77705967, + "learning_rate": 3.724610377103621e-06, + "loss": 0.79886317, + "num_input_tokens_seen": 191028315, + "router_z_loss_clip": 0.78125, + "router_z_loss_mlp": 0.18017578, + "step": 6684, + "time_per_iteration": 4.829049348831177 + }, + { + "auxiliary_loss_clip": 0.01131262, + "auxiliary_loss_mlp": 0.0104344, + "balance_loss_clip": 1.05375767, + "balance_loss_mlp": 1.0266912, + "epoch": 0.19398177702977193, + "flos": 19675851390720.0, + "grad_norm": 3.6764882677509823, + "language_loss": 0.76352549, + "learning_rate": 3.7245151877366762e-06, + "loss": 0.78527254, + "num_input_tokens_seen": 191040630, + "router_z_loss_clip": 0.77490234, + "router_z_loss_mlp": 0.16748047, + "step": 6685, + "time_per_iteration": 5.1652727127075195 + }, + { + "auxiliary_loss_clip": 0.0112597, + "auxiliary_loss_mlp": 0.01038205, + "balance_loss_clip": 1.05309856, + "balance_loss_mlp": 1.02307177, + "epoch": 0.19401079449828798, + "flos": 14241705494400.0, + "grad_norm": 2.698507567323565, + "language_loss": 0.87442786, + "learning_rate": 3.7244199831381147e-06, + "loss": 0.89606965, + "num_input_tokens_seen": 191052515, + "router_z_loss_clip": 0.72924805, + "router_z_loss_mlp": 0.15142822, + "step": 6686, + "time_per_iteration": 4.836379289627075 + }, + { + "auxiliary_loss_clip": 0.01132952, + "auxiliary_loss_mlp": 0.01046636, + "balance_loss_clip": 1.05667937, + "balance_loss_mlp": 1.03080499, + "epoch": 0.194039811966804, + "flos": 27158952798720.0, + "grad_norm": 2.6210705326222836, + "language_loss": 1.09540391, + "learning_rate": 3.724324763308779e-06, + "loss": 1.11719966, + "num_input_tokens_seen": 191066655, + "router_z_loss_clip": 0.76416016, + "router_z_loss_mlp": 0.15844727, + "step": 6687, + "time_per_iteration": 2.595644235610962 + }, + { + "auxiliary_loss_clip": 0.01118531, + "auxiliary_loss_mlp": 0.01045637, + "balance_loss_clip": 1.05181456, + "balance_loss_mlp": 1.03138542, + "epoch": 0.19406882943532006, + "flos": 12304216863360.0, + "grad_norm": 2.7327375804386618, + "language_loss": 0.78374648, + "learning_rate": 3.7242295282495086e-06, + "loss": 0.80538809, + "num_input_tokens_seen": 191078655, + "router_z_loss_clip": 0.66748047, + "router_z_loss_mlp": 0.14239502, + "step": 6688, + "time_per_iteration": 2.4687201976776123 + }, + { + "auxiliary_loss_clip": 0.01137218, + "auxiliary_loss_mlp": 0.01045235, + "balance_loss_clip": 1.05817723, + "balance_loss_mlp": 1.02865255, + "epoch": 0.1940978469038361, + "flos": 11578570306560.0, + "grad_norm": 3.1388489688928325, + "language_loss": 0.74921513, + "learning_rate": 3.724134277961146e-06, + "loss": 0.77103966, + "num_input_tokens_seen": 191091670, + "router_z_loss_clip": 0.79052734, + "router_z_loss_mlp": 0.16577148, + "step": 6689, + "time_per_iteration": 2.48429012298584 + }, + { + "auxiliary_loss_clip": 0.01132259, + "auxiliary_loss_mlp": 0.01050758, + "balance_loss_clip": 1.05516672, + "balance_loss_mlp": 1.03321075, + "epoch": 0.19412686437235216, + "flos": 25622370840960.0, + "grad_norm": 2.5865622958372407, + "language_loss": 1.01794243, + "learning_rate": 3.724039012444531e-06, + "loss": 1.03977263, + "num_input_tokens_seen": 191106000, + "router_z_loss_clip": 0.77148438, + "router_z_loss_mlp": 0.17553711, + "step": 6690, + "time_per_iteration": 2.5489535331726074 + }, + { + "auxiliary_loss_clip": 0.01043554, + "auxiliary_loss_mlp": 0.01034223, + "balance_loss_clip": 1.02452016, + "balance_loss_mlp": 1.03344822, + "epoch": 0.1941558818408682, + "flos": 62047464174720.0, + "grad_norm": 0.7122124445297445, + "language_loss": 0.48230833, + "learning_rate": 3.7239437317005055e-06, + "loss": 0.50308609, + "num_input_tokens_seen": 191161260, + "router_z_loss_clip": 0.19042969, + "router_z_loss_mlp": 0.00775146, + "step": 6691, + "time_per_iteration": 3.0758824348449707 + }, + { + "auxiliary_loss_clip": 0.01118206, + "auxiliary_loss_mlp": 0.01043316, + "balance_loss_clip": 1.05263567, + "balance_loss_mlp": 1.03003037, + "epoch": 0.19418489930938426, + "flos": 16245843811200.0, + "grad_norm": 2.7616615336859898, + "language_loss": 0.78303313, + "learning_rate": 3.7238484357299127e-06, + "loss": 0.80464834, + "num_input_tokens_seen": 191173775, + "router_z_loss_clip": 0.65576172, + "router_z_loss_mlp": 0.13287354, + "step": 6692, + "time_per_iteration": 2.4945507049560547 + }, + { + "auxiliary_loss_clip": 0.01121965, + "auxiliary_loss_mlp": 0.01042936, + "balance_loss_clip": 1.05304635, + "balance_loss_mlp": 1.02862453, + "epoch": 0.1942139167779003, + "flos": 24201493568640.0, + "grad_norm": 2.104119759931237, + "language_loss": 0.74551356, + "learning_rate": 3.7237531245335914e-06, + "loss": 0.76716256, + "num_input_tokens_seen": 191191875, + "router_z_loss_clip": 0.68994141, + "router_z_loss_mlp": 0.14324951, + "step": 6693, + "time_per_iteration": 2.592036724090576 + }, + { + "auxiliary_loss_clip": 0.01133545, + "auxiliary_loss_mlp": 0.01038903, + "balance_loss_clip": 1.05751729, + "balance_loss_mlp": 1.02354836, + "epoch": 0.19424293424641634, + "flos": 30002036146560.0, + "grad_norm": 2.6614456087387106, + "language_loss": 0.81787819, + "learning_rate": 3.723657798112386e-06, + "loss": 0.83960259, + "num_input_tokens_seen": 191206055, + "router_z_loss_clip": 0.76025391, + "router_z_loss_mlp": 0.15356445, + "step": 6694, + "time_per_iteration": 2.5715675354003906 + }, + { + "auxiliary_loss_clip": 0.01126497, + "auxiliary_loss_mlp": 0.01044282, + "balance_loss_clip": 1.05493355, + "balance_loss_mlp": 1.02899373, + "epoch": 0.1942719517149324, + "flos": 34124904144000.0, + "grad_norm": 1.733481599451738, + "language_loss": 0.87378144, + "learning_rate": 3.723562456467137e-06, + "loss": 0.89548916, + "num_input_tokens_seen": 191223865, + "router_z_loss_clip": 0.71533203, + "router_z_loss_mlp": 0.1529541, + "step": 6695, + "time_per_iteration": 2.6881260871887207 + }, + { + "auxiliary_loss_clip": 0.01042287, + "auxiliary_loss_mlp": 0.0100141, + "balance_loss_clip": 1.02370048, + "balance_loss_mlp": 1.00053394, + "epoch": 0.19430096918344844, + "flos": 74777329203840.0, + "grad_norm": 0.6622924818935481, + "language_loss": 0.47454232, + "learning_rate": 3.7234670995986877e-06, + "loss": 0.49497932, + "num_input_tokens_seen": 191289200, + "router_z_loss_clip": 0.18554688, + "router_z_loss_mlp": 0.00878906, + "step": 6696, + "time_per_iteration": 3.1837620735168457 + }, + { + "auxiliary_loss_clip": 0.0104155, + "auxiliary_loss_mlp": 0.01000522, + "balance_loss_clip": 1.02295434, + "balance_loss_mlp": 0.99962777, + "epoch": 0.1943299866519645, + "flos": 64734083879040.0, + "grad_norm": 0.6993722368651081, + "language_loss": 0.47814873, + "learning_rate": 3.7233717275078787e-06, + "loss": 0.4985694, + "num_input_tokens_seen": 191351365, + "router_z_loss_clip": 0.18554688, + "router_z_loss_mlp": 0.00891113, + "step": 6697, + "time_per_iteration": 3.1704516410827637 + }, + { + "auxiliary_loss_clip": 0.01127722, + "auxiliary_loss_mlp": 0.01036344, + "balance_loss_clip": 1.05669856, + "balance_loss_mlp": 1.02135897, + "epoch": 0.19435900412048052, + "flos": 27885281713920.0, + "grad_norm": 1.964190447004258, + "language_loss": 0.81803238, + "learning_rate": 3.723276340195554e-06, + "loss": 0.83967304, + "num_input_tokens_seen": 191369160, + "router_z_loss_clip": 0.70947266, + "router_z_loss_mlp": 0.14971924, + "step": 6698, + "time_per_iteration": 2.6576988697052 + }, + { + "auxiliary_loss_clip": 0.010408, + "auxiliary_loss_mlp": 0.00998653, + "balance_loss_clip": 1.0220356, + "balance_loss_mlp": 0.99780047, + "epoch": 0.19438802158899657, + "flos": 59701562928000.0, + "grad_norm": 0.7770722416740431, + "language_loss": 0.51685762, + "learning_rate": 3.7231809376625542e-06, + "loss": 0.53725213, + "num_input_tokens_seen": 191424155, + "router_z_loss_clip": 0.1875, + "router_z_loss_mlp": 0.00854492, + "step": 6699, + "time_per_iteration": 2.9901297092437744 + }, + { + "auxiliary_loss_clip": 0.01125879, + "auxiliary_loss_mlp": 0.01039998, + "balance_loss_clip": 1.05378318, + "balance_loss_mlp": 1.02483451, + "epoch": 0.19441703905751262, + "flos": 12086813796480.0, + "grad_norm": 2.6302390731007486, + "language_loss": 0.69958138, + "learning_rate": 3.723085519909724e-06, + "loss": 0.72124016, + "num_input_tokens_seen": 191436615, + "router_z_loss_clip": 0.72021484, + "router_z_loss_mlp": 0.1517334, + "step": 6700, + "time_per_iteration": 2.5055835247039795 + }, + { + "auxiliary_loss_clip": 0.01038085, + "auxiliary_loss_mlp": 0.01000571, + "balance_loss_clip": 1.01945758, + "balance_loss_mlp": 0.99982035, + "epoch": 0.19444605652602867, + "flos": 74775749005440.0, + "grad_norm": 0.6144676494528262, + "language_loss": 0.46194449, + "learning_rate": 3.7229900869379048e-06, + "loss": 0.4823311, + "num_input_tokens_seen": 191498940, + "router_z_loss_clip": 0.18652344, + "router_z_loss_mlp": 0.00750732, + "step": 6701, + "time_per_iteration": 3.1318929195404053 + }, + { + "auxiliary_loss_clip": 0.01128204, + "auxiliary_loss_mlp": 0.0103418, + "balance_loss_clip": 1.05298471, + "balance_loss_mlp": 1.02014351, + "epoch": 0.19447507399454472, + "flos": 23470567712640.0, + "grad_norm": 3.916685147452459, + "language_loss": 0.9981339, + "learning_rate": 3.72289463874794e-06, + "loss": 1.01975775, + "num_input_tokens_seen": 191512015, + "router_z_loss_clip": 0.75292969, + "router_z_loss_mlp": 0.14038086, + "step": 6702, + "time_per_iteration": 2.56021785736084 + }, + { + "auxiliary_loss_clip": 0.01036499, + "auxiliary_loss_mlp": 0.01002233, + "balance_loss_clip": 1.01799989, + "balance_loss_mlp": 1.00140476, + "epoch": 0.19450409146306077, + "flos": 71310584989440.0, + "grad_norm": 0.6396762367553495, + "language_loss": 0.43976355, + "learning_rate": 3.7227991753406727e-06, + "loss": 0.4601509, + "num_input_tokens_seen": 191580040, + "router_z_loss_clip": 0.18554688, + "router_z_loss_mlp": 0.00830078, + "step": 6703, + "time_per_iteration": 3.13956618309021 + }, + { + "auxiliary_loss_clip": 0.01122711, + "auxiliary_loss_mlp": 0.010439, + "balance_loss_clip": 1.05320001, + "balance_loss_mlp": 1.03011382, + "epoch": 0.1945331089315768, + "flos": 13625155520640.0, + "grad_norm": 2.283115981538331, + "language_loss": 0.73017466, + "learning_rate": 3.722703696716946e-06, + "loss": 0.75184077, + "num_input_tokens_seen": 191594375, + "router_z_loss_clip": 0.6953125, + "router_z_loss_mlp": 0.13787842, + "step": 6704, + "time_per_iteration": 2.523810386657715 + }, + { + "auxiliary_loss_clip": 0.01117688, + "auxiliary_loss_mlp": 0.01038652, + "balance_loss_clip": 1.04906321, + "balance_loss_mlp": 1.02446604, + "epoch": 0.19456212640009285, + "flos": 16207455150720.0, + "grad_norm": 2.990191899677835, + "language_loss": 0.71320653, + "learning_rate": 3.722608202877603e-06, + "loss": 0.73476988, + "num_input_tokens_seen": 191605220, + "router_z_loss_clip": 0.68603516, + "router_z_loss_mlp": 0.14190674, + "step": 6705, + "time_per_iteration": 2.4854509830474854 + }, + { + "auxiliary_loss_clip": 0.01127131, + "auxiliary_loss_mlp": 0.01041934, + "balance_loss_clip": 1.05425835, + "balance_loss_mlp": 1.02627563, + "epoch": 0.1945911438686089, + "flos": 25477579117440.0, + "grad_norm": 2.121090682279583, + "language_loss": 0.7406742, + "learning_rate": 3.722512693823487e-06, + "loss": 0.76236486, + "num_input_tokens_seen": 191620190, + "router_z_loss_clip": 0.72851562, + "router_z_loss_mlp": 0.15667725, + "step": 6706, + "time_per_iteration": 2.660414695739746 + }, + { + "auxiliary_loss_clip": 0.01124902, + "auxiliary_loss_mlp": 0.01045804, + "balance_loss_clip": 1.05264115, + "balance_loss_mlp": 1.03145111, + "epoch": 0.19462016133712495, + "flos": 28176948149760.0, + "grad_norm": 2.383567596528669, + "language_loss": 0.82260251, + "learning_rate": 3.7224171695554423e-06, + "loss": 0.84430957, + "num_input_tokens_seen": 191635515, + "router_z_loss_clip": 0.72314453, + "router_z_loss_mlp": 0.14349365, + "step": 6707, + "time_per_iteration": 2.609893560409546 + }, + { + "auxiliary_loss_clip": 0.01037893, + "auxiliary_loss_mlp": 0.01019034, + "balance_loss_clip": 1.01906586, + "balance_loss_mlp": 1.01825297, + "epoch": 0.194649178805641, + "flos": 58796542828800.0, + "grad_norm": 0.6695032366965619, + "language_loss": 0.48129755, + "learning_rate": 3.7223216300743117e-06, + "loss": 0.50186682, + "num_input_tokens_seen": 191697065, + "router_z_loss_clip": 0.18847656, + "router_z_loss_mlp": 0.0078125, + "step": 6708, + "time_per_iteration": 3.099867820739746 + }, + { + "auxiliary_loss_clip": 0.01036523, + "auxiliary_loss_mlp": 0.01011071, + "balance_loss_clip": 1.01793742, + "balance_loss_mlp": 1.01027834, + "epoch": 0.19467819627415706, + "flos": 63456274477440.0, + "grad_norm": 0.6732500851558991, + "language_loss": 0.43443811, + "learning_rate": 3.7222260753809403e-06, + "loss": 0.45491409, + "num_input_tokens_seen": 191754840, + "router_z_loss_clip": 0.18554688, + "router_z_loss_mlp": 0.00793457, + "step": 6709, + "time_per_iteration": 3.0993571281433105 + }, + { + "auxiliary_loss_clip": 0.01124018, + "auxiliary_loss_mlp": 0.01031022, + "balance_loss_clip": 1.0516845, + "balance_loss_mlp": 1.01685977, + "epoch": 0.19470721374267308, + "flos": 38284113726720.0, + "grad_norm": 1.9621043146937813, + "language_loss": 0.74532819, + "learning_rate": 3.7221305054761705e-06, + "loss": 0.7668786, + "num_input_tokens_seen": 191775525, + "router_z_loss_clip": 0.72412109, + "router_z_loss_mlp": 0.14172363, + "step": 6710, + "time_per_iteration": 2.6615843772888184 + }, + { + "auxiliary_loss_clip": 0.01131312, + "auxiliary_loss_mlp": 0.01047641, + "balance_loss_clip": 1.05596733, + "balance_loss_mlp": 1.03082621, + "epoch": 0.19473623121118913, + "flos": 14166687939840.0, + "grad_norm": 2.46464967780961, + "language_loss": 0.66236681, + "learning_rate": 3.7220349203608476e-06, + "loss": 0.68415636, + "num_input_tokens_seen": 191786490, + "router_z_loss_clip": 0.75244141, + "router_z_loss_mlp": 0.16821289, + "step": 6711, + "time_per_iteration": 2.5357213020324707 + }, + { + "auxiliary_loss_clip": 0.01036314, + "auxiliary_loss_mlp": 0.01001433, + "balance_loss_clip": 1.01766634, + "balance_loss_mlp": 1.00065792, + "epoch": 0.19476524867970518, + "flos": 74780310032640.0, + "grad_norm": 0.6167094279225881, + "language_loss": 0.44546869, + "learning_rate": 3.7219393200358153e-06, + "loss": 0.46584615, + "num_input_tokens_seen": 191855245, + "router_z_loss_clip": 0.18652344, + "router_z_loss_mlp": 0.00775146, + "step": 6712, + "time_per_iteration": 3.3059256076812744 + }, + { + "auxiliary_loss_clip": 0.01131427, + "auxiliary_loss_mlp": 0.01035131, + "balance_loss_clip": 1.05440915, + "balance_loss_mlp": 1.01808429, + "epoch": 0.19479426614822123, + "flos": 22486759130880.0, + "grad_norm": 2.5785365725585283, + "language_loss": 0.74176395, + "learning_rate": 3.7218437045019185e-06, + "loss": 0.76342952, + "num_input_tokens_seen": 191870145, + "router_z_loss_clip": 0.76904297, + "router_z_loss_mlp": 0.17047119, + "step": 6713, + "time_per_iteration": 2.4728891849517822 + }, + { + "auxiliary_loss_clip": 0.01123642, + "auxiliary_loss_mlp": 0.01035078, + "balance_loss_clip": 1.050632, + "balance_loss_mlp": 1.02000439, + "epoch": 0.19482328361673729, + "flos": 31394472825600.0, + "grad_norm": 1.8021836485383327, + "language_loss": 0.74777836, + "learning_rate": 3.721748073760001e-06, + "loss": 0.76936555, + "num_input_tokens_seen": 191889295, + "router_z_loss_clip": 0.72949219, + "router_z_loss_mlp": 0.15063477, + "step": 6714, + "time_per_iteration": 2.5938775539398193 + }, + { + "auxiliary_loss_clip": 0.01121878, + "auxiliary_loss_mlp": 0.01036379, + "balance_loss_clip": 1.05245447, + "balance_loss_mlp": 1.02253914, + "epoch": 0.1948523010852533, + "flos": 32736346133760.0, + "grad_norm": 1.6713840141924212, + "language_loss": 0.64656198, + "learning_rate": 3.7216524278109076e-06, + "loss": 0.66814458, + "num_input_tokens_seen": 191907490, + "router_z_loss_clip": 0.69433594, + "router_z_loss_mlp": 0.13848877, + "step": 6715, + "time_per_iteration": 2.609550714492798 + }, + { + "auxiliary_loss_clip": 0.01128627, + "auxiliary_loss_mlp": 0.01049018, + "balance_loss_clip": 1.05350947, + "balance_loss_mlp": 1.032287, + "epoch": 0.19488131855376936, + "flos": 15952920399360.0, + "grad_norm": 2.787301484894736, + "language_loss": 0.70258635, + "learning_rate": 3.7215567666554834e-06, + "loss": 0.72436273, + "num_input_tokens_seen": 191923760, + "router_z_loss_clip": 0.75097656, + "router_z_loss_mlp": 0.16741943, + "step": 6716, + "time_per_iteration": 2.5586740970611572 + }, + { + "auxiliary_loss_clip": 0.0103533, + "auxiliary_loss_mlp": 0.01001529, + "balance_loss_clip": 1.01701856, + "balance_loss_mlp": 1.00061691, + "epoch": 0.1949103360222854, + "flos": 59264278755840.0, + "grad_norm": 0.6785898441799204, + "language_loss": 0.48217842, + "learning_rate": 3.7214610902945735e-06, + "loss": 0.50254703, + "num_input_tokens_seen": 191982295, + "router_z_loss_clip": 0.18359375, + "router_z_loss_mlp": 0.00909424, + "step": 6717, + "time_per_iteration": 3.079690456390381 + }, + { + "auxiliary_loss_clip": 0.0112213, + "auxiliary_loss_mlp": 0.01040883, + "balance_loss_clip": 1.05090928, + "balance_loss_mlp": 1.02583849, + "epoch": 0.19493935349080146, + "flos": 74737931984640.0, + "grad_norm": 1.6080022889464296, + "language_loss": 0.71635044, + "learning_rate": 3.7213653987290227e-06, + "loss": 0.7379806, + "num_input_tokens_seen": 192009545, + "router_z_loss_clip": 0.71191406, + "router_z_loss_mlp": 0.15045166, + "step": 6718, + "time_per_iteration": 2.971146583557129 + }, + { + "auxiliary_loss_clip": 0.01123426, + "auxiliary_loss_mlp": 0.0103126, + "balance_loss_clip": 1.05077469, + "balance_loss_mlp": 1.01654315, + "epoch": 0.19496837095931752, + "flos": 22746285872640.0, + "grad_norm": 1.774630281325971, + "language_loss": 0.73794246, + "learning_rate": 3.7212696919596757e-06, + "loss": 0.7594893, + "num_input_tokens_seen": 192028310, + "router_z_loss_clip": 0.72705078, + "router_z_loss_mlp": 0.14703369, + "step": 6719, + "time_per_iteration": 2.5459413528442383 + }, + { + "auxiliary_loss_clip": 0.01125911, + "auxiliary_loss_mlp": 0.01036415, + "balance_loss_clip": 1.0530231, + "balance_loss_mlp": 1.02221704, + "epoch": 0.19499738842783357, + "flos": 21317866744320.0, + "grad_norm": 2.32690978062578, + "language_loss": 0.87485182, + "learning_rate": 3.7211739699873786e-06, + "loss": 0.89647508, + "num_input_tokens_seen": 192044180, + "router_z_loss_clip": 0.72900391, + "router_z_loss_mlp": 0.14190674, + "step": 6720, + "time_per_iteration": 2.5839569568634033 + }, + { + "auxiliary_loss_clip": 0.01131338, + "auxiliary_loss_mlp": 0.0103445, + "balance_loss_clip": 1.05753326, + "balance_loss_mlp": 1.01907206, + "epoch": 0.1950264058963496, + "flos": 14931872392320.0, + "grad_norm": 2.321024958318261, + "language_loss": 0.91224366, + "learning_rate": 3.7210782328129764e-06, + "loss": 0.93390155, + "num_input_tokens_seen": 192056835, + "router_z_loss_clip": 0.73779297, + "router_z_loss_mlp": 0.15380859, + "step": 6721, + "time_per_iteration": 2.700545310974121 + }, + { + "auxiliary_loss_clip": 0.01127264, + "auxiliary_loss_mlp": 0.01035603, + "balance_loss_clip": 1.05794215, + "balance_loss_mlp": 1.02214396, + "epoch": 0.19505542336486564, + "flos": 27119199421440.0, + "grad_norm": 1.9934588427105386, + "language_loss": 0.82818627, + "learning_rate": 3.720982480437315e-06, + "loss": 0.84981495, + "num_input_tokens_seen": 192072945, + "router_z_loss_clip": 0.69287109, + "router_z_loss_mlp": 0.13452148, + "step": 6722, + "time_per_iteration": 2.6229734420776367 + }, + { + "auxiliary_loss_clip": 0.01035028, + "auxiliary_loss_mlp": 0.01004682, + "balance_loss_clip": 1.01678288, + "balance_loss_mlp": 1.00376415, + "epoch": 0.1950844408333817, + "flos": 62693424408960.0, + "grad_norm": 0.7530357918252366, + "language_loss": 0.5231508, + "learning_rate": 3.7208867128612393e-06, + "loss": 0.54354793, + "num_input_tokens_seen": 192125760, + "router_z_loss_clip": 0.18261719, + "router_z_loss_mlp": 0.00915527, + "step": 6723, + "time_per_iteration": 3.013551950454712 + }, + { + "auxiliary_loss_clip": 0.01131617, + "auxiliary_loss_mlp": 0.01041425, + "balance_loss_clip": 1.05642653, + "balance_loss_mlp": 1.02653015, + "epoch": 0.19511345830189775, + "flos": 28944071936640.0, + "grad_norm": 1.6755326792127887, + "language_loss": 0.76500338, + "learning_rate": 3.7207909300855964e-06, + "loss": 0.78673381, + "num_input_tokens_seen": 192145865, + "router_z_loss_clip": 0.75146484, + "router_z_loss_mlp": 0.14886475, + "step": 6724, + "time_per_iteration": 2.7457728385925293 + }, + { + "auxiliary_loss_clip": 0.01136394, + "auxiliary_loss_mlp": 0.01043878, + "balance_loss_clip": 1.05849028, + "balance_loss_mlp": 1.02734423, + "epoch": 0.1951424757704138, + "flos": 74728702189440.0, + "grad_norm": 2.7096790556543198, + "language_loss": 0.93413687, + "learning_rate": 3.720695132111231e-06, + "loss": 0.95593965, + "num_input_tokens_seen": 192167900, + "router_z_loss_clip": 0.77929688, + "router_z_loss_mlp": 0.16534424, + "step": 6725, + "time_per_iteration": 3.0026421546936035 + }, + { + "auxiliary_loss_clip": 0.01124092, + "auxiliary_loss_mlp": 0.01035867, + "balance_loss_clip": 1.05508125, + "balance_loss_mlp": 1.02200902, + "epoch": 0.19517149323892985, + "flos": 28069252197120.0, + "grad_norm": 1.6432316699503886, + "language_loss": 0.69304776, + "learning_rate": 3.7205993189389905e-06, + "loss": 0.71464729, + "num_input_tokens_seen": 192183325, + "router_z_loss_clip": 0.68994141, + "router_z_loss_mlp": 0.1385498, + "step": 6726, + "time_per_iteration": 2.6527035236358643 + }, + { + "auxiliary_loss_clip": 0.01130673, + "auxiliary_loss_mlp": 0.01044882, + "balance_loss_clip": 1.0556668, + "balance_loss_mlp": 1.02884817, + "epoch": 0.19520051070744587, + "flos": 21902277024000.0, + "grad_norm": 2.055438992383273, + "language_loss": 0.90716833, + "learning_rate": 3.7205034905697207e-06, + "loss": 0.9289239, + "num_input_tokens_seen": 192202335, + "router_z_loss_clip": 0.74951172, + "router_z_loss_mlp": 0.16033936, + "step": 6727, + "time_per_iteration": 2.785132646560669 + }, + { + "auxiliary_loss_clip": 0.01036935, + "auxiliary_loss_mlp": 0.0100613, + "balance_loss_clip": 1.01851606, + "balance_loss_mlp": 1.00517654, + "epoch": 0.19522952817596192, + "flos": 67448716191360.0, + "grad_norm": 0.6446573741688711, + "language_loss": 0.48985523, + "learning_rate": 3.7204076470042677e-06, + "loss": 0.51028585, + "num_input_tokens_seen": 192267885, + "router_z_loss_clip": 0.18457031, + "router_z_loss_mlp": 0.00952148, + "step": 6728, + "time_per_iteration": 3.2222859859466553 + }, + { + "auxiliary_loss_clip": 0.0112602, + "auxiliary_loss_mlp": 0.01036348, + "balance_loss_clip": 1.05575597, + "balance_loss_mlp": 1.02388477, + "epoch": 0.19525854564447798, + "flos": 30112138310400.0, + "grad_norm": 1.600149510440936, + "language_loss": 0.88233137, + "learning_rate": 3.720311788243478e-06, + "loss": 0.90395498, + "num_input_tokens_seen": 192290955, + "router_z_loss_clip": 0.70214844, + "router_z_loss_mlp": 0.12469482, + "step": 6729, + "time_per_iteration": 2.8467283248901367 + }, + { + "auxiliary_loss_clip": 0.01124901, + "auxiliary_loss_mlp": 0.0103613, + "balance_loss_clip": 1.05506003, + "balance_loss_mlp": 1.02350616, + "epoch": 0.19528756311299403, + "flos": 46126536837120.0, + "grad_norm": 2.6258575037479104, + "language_loss": 1.02764964, + "learning_rate": 3.720215914288198e-06, + "loss": 1.04926002, + "num_input_tokens_seen": 192308635, + "router_z_loss_clip": 0.69824219, + "router_z_loss_mlp": 0.12634277, + "step": 6730, + "time_per_iteration": 2.823347330093384 + }, + { + "auxiliary_loss_clip": 0.01128122, + "auxiliary_loss_mlp": 0.01038706, + "balance_loss_clip": 1.05411959, + "balance_loss_mlp": 1.02422261, + "epoch": 0.19531658058151008, + "flos": 24238122462720.0, + "grad_norm": 2.220243023446471, + "language_loss": 0.8560375, + "learning_rate": 3.720120025139276e-06, + "loss": 0.87770575, + "num_input_tokens_seen": 192323045, + "router_z_loss_clip": 0.74072266, + "router_z_loss_mlp": 0.14489746, + "step": 6731, + "time_per_iteration": 2.595097541809082 + }, + { + "auxiliary_loss_clip": 0.01134971, + "auxiliary_loss_mlp": 0.01038554, + "balance_loss_clip": 1.05672264, + "balance_loss_mlp": 1.02094722, + "epoch": 0.1953455980500261, + "flos": 21426209141760.0, + "grad_norm": 2.1302510190437123, + "language_loss": 0.78240991, + "learning_rate": 3.720024120797557e-06, + "loss": 0.80414522, + "num_input_tokens_seen": 192337565, + "router_z_loss_clip": 0.78222656, + "router_z_loss_mlp": 0.17596436, + "step": 6732, + "time_per_iteration": 2.5859768390655518 + }, + { + "auxiliary_loss_clip": 0.01132223, + "auxiliary_loss_mlp": 0.01043453, + "balance_loss_clip": 1.05642903, + "balance_loss_mlp": 1.02678764, + "epoch": 0.19537461551854216, + "flos": 40405357359360.0, + "grad_norm": 2.019419073333657, + "language_loss": 0.86603183, + "learning_rate": 3.719928201263889e-06, + "loss": 0.88778853, + "num_input_tokens_seen": 192359205, + "router_z_loss_clip": 0.75732422, + "router_z_loss_mlp": 0.16674805, + "step": 6733, + "time_per_iteration": 2.725332736968994 + }, + { + "auxiliary_loss_clip": 0.01036209, + "auxiliary_loss_mlp": 0.01000602, + "balance_loss_clip": 1.01786327, + "balance_loss_mlp": 0.99960661, + "epoch": 0.1954036329870582, + "flos": 74705184737280.0, + "grad_norm": 0.6716020960311976, + "language_loss": 0.48281091, + "learning_rate": 3.71983226653912e-06, + "loss": 0.50317901, + "num_input_tokens_seen": 192416725, + "router_z_loss_clip": 0.18359375, + "router_z_loss_mlp": 0.00994873, + "step": 6734, + "time_per_iteration": 3.2393574714660645 + }, + { + "auxiliary_loss_clip": 0.01037112, + "auxiliary_loss_mlp": 0.00999753, + "balance_loss_clip": 1.01870918, + "balance_loss_mlp": 0.9987337, + "epoch": 0.19543265045557426, + "flos": 74781818403840.0, + "grad_norm": 0.6164178323092488, + "language_loss": 0.47823453, + "learning_rate": 3.7197363166240957e-06, + "loss": 0.49860317, + "num_input_tokens_seen": 192485435, + "router_z_loss_clip": 0.18457031, + "router_z_loss_mlp": 0.01019287, + "step": 6735, + "time_per_iteration": 3.2449374198913574 + }, + { + "auxiliary_loss_clip": 0.0103569, + "auxiliary_loss_mlp": 0.0099999, + "balance_loss_clip": 1.01747501, + "balance_loss_mlp": 0.9989233, + "epoch": 0.1954616679240903, + "flos": 64141772607360.0, + "grad_norm": 0.6826555025906147, + "language_loss": 0.47013569, + "learning_rate": 3.7196403515196647e-06, + "loss": 0.49049246, + "num_input_tokens_seen": 192535675, + "router_z_loss_clip": 0.18261719, + "router_z_loss_mlp": 0.01068115, + "step": 6736, + "time_per_iteration": 2.888868808746338 + }, + { + "auxiliary_loss_clip": 0.01129041, + "auxiliary_loss_mlp": 0.01034245, + "balance_loss_clip": 1.05608892, + "balance_loss_mlp": 1.01827145, + "epoch": 0.19549068539260636, + "flos": 17193310807680.0, + "grad_norm": 2.8725187573344524, + "language_loss": 0.83693343, + "learning_rate": 3.719544371226674e-06, + "loss": 0.85856628, + "num_input_tokens_seen": 192548055, + "router_z_loss_clip": 0.72900391, + "router_z_loss_mlp": 0.15991211, + "step": 6737, + "time_per_iteration": 2.5000929832458496 + }, + { + "auxiliary_loss_clip": 0.01033837, + "auxiliary_loss_mlp": 0.01003033, + "balance_loss_clip": 1.01542771, + "balance_loss_mlp": 1.00209701, + "epoch": 0.19551970286112239, + "flos": 70426104491520.0, + "grad_norm": 0.7186969369627099, + "language_loss": 0.51430333, + "learning_rate": 3.719448375745972e-06, + "loss": 0.53467202, + "num_input_tokens_seen": 192605645, + "router_z_loss_clip": 0.18359375, + "router_z_loss_mlp": 0.00933838, + "step": 6738, + "time_per_iteration": 3.1251516342163086 + }, + { + "auxiliary_loss_clip": 0.01033312, + "auxiliary_loss_mlp": 0.01005473, + "balance_loss_clip": 1.01482749, + "balance_loss_mlp": 1.00456131, + "epoch": 0.19554872032963844, + "flos": 59517736099200.0, + "grad_norm": 0.6444799369106574, + "language_loss": 0.464926, + "learning_rate": 3.7193523650784054e-06, + "loss": 0.48531383, + "num_input_tokens_seen": 192665315, + "router_z_loss_clip": 0.18554688, + "router_z_loss_mlp": 0.00909424, + "step": 6739, + "time_per_iteration": 2.996161699295044 + }, + { + "auxiliary_loss_clip": 0.01133763, + "auxiliary_loss_mlp": 0.01038885, + "balance_loss_clip": 1.05491722, + "balance_loss_mlp": 1.0227443, + "epoch": 0.1955777377981545, + "flos": 74735274378240.0, + "grad_norm": 2.0363249467341507, + "language_loss": 0.70736516, + "learning_rate": 3.7192563392248235e-06, + "loss": 0.72909164, + "num_input_tokens_seen": 192691100, + "router_z_loss_clip": 0.78857422, + "router_z_loss_mlp": 0.16143799, + "step": 6740, + "time_per_iteration": 2.9861490726470947 + }, + { + "auxiliary_loss_clip": 0.01033439, + "auxiliary_loss_mlp": 0.01004717, + "balance_loss_clip": 1.0151844, + "balance_loss_mlp": 1.00376892, + "epoch": 0.19560675526667054, + "flos": 54751062683520.0, + "grad_norm": 0.9469222187962328, + "language_loss": 0.47754496, + "learning_rate": 3.7191602981860737e-06, + "loss": 0.49792653, + "num_input_tokens_seen": 192750150, + "router_z_loss_clip": 0.18261719, + "router_z_loss_mlp": 0.00946045, + "step": 6741, + "time_per_iteration": 3.0967955589294434 + }, + { + "auxiliary_loss_clip": 0.01120839, + "auxiliary_loss_mlp": 0.01030967, + "balance_loss_clip": 1.05088568, + "balance_loss_mlp": 1.01691782, + "epoch": 0.1956357727351866, + "flos": 41823110148480.0, + "grad_norm": 12.485600928419885, + "language_loss": 0.95957547, + "learning_rate": 3.7190642419630043e-06, + "loss": 0.98109353, + "num_input_tokens_seen": 192767790, + "router_z_loss_clip": 0.70019531, + "router_z_loss_mlp": 0.14056396, + "step": 6742, + "time_per_iteration": 2.7451858520507812 + }, + { + "auxiliary_loss_clip": 0.0113045, + "auxiliary_loss_mlp": 0.01038184, + "balance_loss_clip": 1.05780506, + "balance_loss_mlp": 1.02247846, + "epoch": 0.19566479020370262, + "flos": 25402130599680.0, + "grad_norm": 3.12582591576273, + "language_loss": 0.63275886, + "learning_rate": 3.7189681705564645e-06, + "loss": 0.65444517, + "num_input_tokens_seen": 192783605, + "router_z_loss_clip": 0.7265625, + "router_z_loss_mlp": 0.15716553, + "step": 6743, + "time_per_iteration": 2.5832977294921875 + }, + { + "auxiliary_loss_clip": 0.01134211, + "auxiliary_loss_mlp": 0.01042452, + "balance_loss_clip": 1.05599749, + "balance_loss_mlp": 1.02535748, + "epoch": 0.19569380767221867, + "flos": 28760747898240.0, + "grad_norm": 2.0413464414701945, + "language_loss": 0.95680606, + "learning_rate": 3.718872083967302e-06, + "loss": 0.97857273, + "num_input_tokens_seen": 192802860, + "router_z_loss_clip": 0.78271484, + "router_z_loss_mlp": 0.1708374, + "step": 6744, + "time_per_iteration": 2.7140989303588867 + }, + { + "auxiliary_loss_clip": 0.01125812, + "auxiliary_loss_mlp": 0.01039436, + "balance_loss_clip": 1.05638683, + "balance_loss_mlp": 1.02355766, + "epoch": 0.19572282514073472, + "flos": 11720596682880.0, + "grad_norm": 2.446345952388403, + "language_loss": 0.77144074, + "learning_rate": 3.7187759821963657e-06, + "loss": 0.79309326, + "num_input_tokens_seen": 192813230, + "router_z_loss_clip": 0.69433594, + "router_z_loss_mlp": 0.15893555, + "step": 6745, + "time_per_iteration": 2.5039801597595215 + }, + { + "auxiliary_loss_clip": 0.01122357, + "auxiliary_loss_mlp": 0.01032437, + "balance_loss_clip": 1.05228925, + "balance_loss_mlp": 1.018525, + "epoch": 0.19575184260925077, + "flos": 33328477837440.0, + "grad_norm": 1.9666395029063557, + "language_loss": 0.72858459, + "learning_rate": 3.7186798652445043e-06, + "loss": 0.7501325, + "num_input_tokens_seen": 192834315, + "router_z_loss_clip": 0.70117188, + "router_z_loss_mlp": 0.13916016, + "step": 6746, + "time_per_iteration": 2.6488537788391113 + }, + { + "auxiliary_loss_clip": 0.01035846, + "auxiliary_loss_mlp": 0.01004707, + "balance_loss_clip": 1.01774085, + "balance_loss_mlp": 1.00371766, + "epoch": 0.19578086007776682, + "flos": 73317416826240.0, + "grad_norm": 0.6903563331933361, + "language_loss": 0.58253944, + "learning_rate": 3.7185837331125665e-06, + "loss": 0.60294497, + "num_input_tokens_seen": 192900960, + "router_z_loss_clip": 0.18164062, + "router_z_loss_mlp": 0.0098877, + "step": 6747, + "time_per_iteration": 3.1809628009796143 + }, + { + "auxiliary_loss_clip": 0.01118239, + "auxiliary_loss_mlp": 0.01039133, + "balance_loss_clip": 1.05090559, + "balance_loss_mlp": 1.02494121, + "epoch": 0.19580987754628287, + "flos": 19530125913600.0, + "grad_norm": 2.9436201367779193, + "language_loss": 0.90984142, + "learning_rate": 3.7184875858014022e-06, + "loss": 0.9314152, + "num_input_tokens_seen": 192912735, + "router_z_loss_clip": 0.67285156, + "router_z_loss_mlp": 0.1418457, + "step": 6748, + "time_per_iteration": 2.5797533988952637 + }, + { + "auxiliary_loss_clip": 0.01127073, + "auxiliary_loss_mlp": 0.01037879, + "balance_loss_clip": 1.05616868, + "balance_loss_mlp": 1.02414596, + "epoch": 0.1958388950147989, + "flos": 37664439269760.0, + "grad_norm": 2.1552121855641055, + "language_loss": 0.81667453, + "learning_rate": 3.7183914233118603e-06, + "loss": 0.83832407, + "num_input_tokens_seen": 192929295, + "router_z_loss_clip": 0.70947266, + "router_z_loss_mlp": 0.1373291, + "step": 6749, + "time_per_iteration": 2.678734540939331 + }, + { + "auxiliary_loss_clip": 0.01126076, + "auxiliary_loss_mlp": 0.01042191, + "balance_loss_clip": 1.05363739, + "balance_loss_mlp": 1.02634263, + "epoch": 0.19586791248331495, + "flos": 60066771482880.0, + "grad_norm": 2.282894344500764, + "language_loss": 0.86628211, + "learning_rate": 3.71829524564479e-06, + "loss": 0.88796484, + "num_input_tokens_seen": 192952070, + "router_z_loss_clip": 0.72363281, + "router_z_loss_mlp": 0.15856934, + "step": 6750, + "time_per_iteration": 2.829653263092041 + }, + { + "auxiliary_loss_clip": 0.01035468, + "auxiliary_loss_mlp": 0.01001697, + "balance_loss_clip": 1.01749897, + "balance_loss_mlp": 1.00078475, + "epoch": 0.195896929951831, + "flos": 66093159801600.0, + "grad_norm": 0.6390262719028759, + "language_loss": 0.52753872, + "learning_rate": 3.71819905280104e-06, + "loss": 0.54791039, + "num_input_tokens_seen": 193015025, + "router_z_loss_clip": 0.1796875, + "router_z_loss_mlp": 0.00909424, + "step": 6751, + "time_per_iteration": 3.147676467895508 + }, + { + "auxiliary_loss_clip": 0.01033869, + "auxiliary_loss_mlp": 0.01002053, + "balance_loss_clip": 1.0158689, + "balance_loss_mlp": 1.00117075, + "epoch": 0.19592594742034705, + "flos": 67882660398720.0, + "grad_norm": 0.6656053127484431, + "language_loss": 0.48987621, + "learning_rate": 3.7181028447814613e-06, + "loss": 0.51023543, + "num_input_tokens_seen": 193081865, + "router_z_loss_clip": 0.1796875, + "router_z_loss_mlp": 0.0088501, + "step": 6752, + "time_per_iteration": 3.2493624687194824 + }, + { + "auxiliary_loss_clip": 0.01134439, + "auxiliary_loss_mlp": 0.01037083, + "balance_loss_clip": 1.05575967, + "balance_loss_mlp": 1.01939273, + "epoch": 0.1959549648888631, + "flos": 30147869364480.0, + "grad_norm": 2.143566255041449, + "language_loss": 0.94497573, + "learning_rate": 3.718006621586903e-06, + "loss": 0.9666909, + "num_input_tokens_seen": 193097280, + "router_z_loss_clip": 0.78613281, + "router_z_loss_mlp": 0.17681885, + "step": 6753, + "time_per_iteration": 2.607578992843628 + }, + { + "auxiliary_loss_clip": 0.01033513, + "auxiliary_loss_mlp": 0.01001796, + "balance_loss_clip": 1.01537776, + "balance_loss_mlp": 1.00093794, + "epoch": 0.19598398235737916, + "flos": 74765046752640.0, + "grad_norm": 0.6843562762886569, + "language_loss": 0.45599541, + "learning_rate": 3.717910383218215e-06, + "loss": 0.47634849, + "num_input_tokens_seen": 193152220, + "router_z_loss_clip": 0.18164062, + "router_z_loss_mlp": 0.00860596, + "step": 6754, + "time_per_iteration": 3.0590853691101074 + }, + { + "auxiliary_loss_clip": 0.01034384, + "auxiliary_loss_mlp": 0.01001009, + "balance_loss_clip": 1.01624238, + "balance_loss_mlp": 1.00004923, + "epoch": 0.19601299982589518, + "flos": 74769069075840.0, + "grad_norm": 0.6625936006561866, + "language_loss": 0.47239411, + "learning_rate": 3.717814129676247e-06, + "loss": 0.49274805, + "num_input_tokens_seen": 193210835, + "router_z_loss_clip": 0.18164062, + "router_z_loss_mlp": 0.00958252, + "step": 6755, + "time_per_iteration": 3.1363449096679688 + }, + { + "auxiliary_loss_clip": 0.01121035, + "auxiliary_loss_mlp": 0.01036689, + "balance_loss_clip": 1.05009103, + "balance_loss_mlp": 1.02163339, + "epoch": 0.19604201729441123, + "flos": 27921407817600.0, + "grad_norm": 3.780031809758613, + "language_loss": 0.71981716, + "learning_rate": 3.71771786096185e-06, + "loss": 0.7413944, + "num_input_tokens_seen": 193225640, + "router_z_loss_clip": 0.70898438, + "router_z_loss_mlp": 0.15063477, + "step": 6756, + "time_per_iteration": 7.332639694213867 + }, + { + "auxiliary_loss_clip": 0.01116461, + "auxiliary_loss_mlp": 0.01031473, + "balance_loss_clip": 1.04685092, + "balance_loss_mlp": 1.01751339, + "epoch": 0.19607103476292728, + "flos": 31938016406400.0, + "grad_norm": 1.7868849396611335, + "language_loss": 0.66404772, + "learning_rate": 3.7176215770758734e-06, + "loss": 0.68552703, + "num_input_tokens_seen": 193241785, + "router_z_loss_clip": 0.6953125, + "router_z_loss_mlp": 0.13952637, + "step": 6757, + "time_per_iteration": 5.033056259155273 + }, + { + "auxiliary_loss_clip": 0.01108075, + "auxiliary_loss_mlp": 0.010297, + "balance_loss_clip": 1.04621422, + "balance_loss_mlp": 1.0176121, + "epoch": 0.19610005223144333, + "flos": 11539750682880.0, + "grad_norm": 2.3139482956811803, + "language_loss": 0.78129375, + "learning_rate": 3.7175252780191683e-06, + "loss": 0.80267155, + "num_input_tokens_seen": 193254425, + "router_z_loss_clip": 0.61816406, + "router_z_loss_mlp": 0.12084961, + "step": 6758, + "time_per_iteration": 2.5883612632751465 + }, + { + "auxiliary_loss_clip": 0.01119182, + "auxiliary_loss_mlp": 0.01043325, + "balance_loss_clip": 1.05056214, + "balance_loss_mlp": 1.02902555, + "epoch": 0.19612906969995939, + "flos": 28940911539840.0, + "grad_norm": 1.9984467231669112, + "language_loss": 0.727745, + "learning_rate": 3.7174289637925843e-06, + "loss": 0.7493701, + "num_input_tokens_seen": 193268325, + "router_z_loss_clip": 0.68603516, + "router_z_loss_mlp": 0.14300537, + "step": 6759, + "time_per_iteration": 2.603377103805542 + }, + { + "auxiliary_loss_clip": 0.01034559, + "auxiliary_loss_mlp": 0.0100322, + "balance_loss_clip": 1.01628125, + "balance_loss_mlp": 1.00236821, + "epoch": 0.1961580871684754, + "flos": 74777472858240.0, + "grad_norm": 0.6493869943331009, + "language_loss": 0.49521488, + "learning_rate": 3.7173326343969734e-06, + "loss": 0.51559263, + "num_input_tokens_seen": 193333915, + "router_z_loss_clip": 0.18261719, + "router_z_loss_mlp": 0.00854492, + "step": 6760, + "time_per_iteration": 3.1675145626068115 + }, + { + "auxiliary_loss_clip": 0.011285, + "auxiliary_loss_mlp": 0.01039458, + "balance_loss_clip": 1.05379868, + "balance_loss_mlp": 1.02340674, + "epoch": 0.19618710463699146, + "flos": 35402174841600.0, + "grad_norm": 2.218226184763003, + "language_loss": 0.88642049, + "learning_rate": 3.7172362898331856e-06, + "loss": 0.90810007, + "num_input_tokens_seen": 193351530, + "router_z_loss_clip": 0.74755859, + "router_z_loss_mlp": 0.16033936, + "step": 6761, + "time_per_iteration": 2.6864960193634033 + }, + { + "auxiliary_loss_clip": 0.01124862, + "auxiliary_loss_mlp": 0.01038251, + "balance_loss_clip": 1.04950452, + "balance_loss_mlp": 1.02261019, + "epoch": 0.1962161221055075, + "flos": 24489496817280.0, + "grad_norm": 2.1187561241748902, + "language_loss": 1.06418836, + "learning_rate": 3.7171399301020714e-06, + "loss": 1.08581948, + "num_input_tokens_seen": 193367590, + "router_z_loss_clip": 0.75390625, + "router_z_loss_mlp": 0.15637207, + "step": 6762, + "time_per_iteration": 2.5460009574890137 + }, + { + "auxiliary_loss_clip": 0.01131337, + "auxiliary_loss_mlp": 0.01031343, + "balance_loss_clip": 1.05411327, + "balance_loss_mlp": 1.01545274, + "epoch": 0.19624513957402356, + "flos": 27195833088000.0, + "grad_norm": 2.8867352032032887, + "language_loss": 0.99190271, + "learning_rate": 3.7170435552044834e-06, + "loss": 1.01352954, + "num_input_tokens_seen": 193389145, + "router_z_loss_clip": 0.77246094, + "router_z_loss_mlp": 0.15893555, + "step": 6763, + "time_per_iteration": 2.66280198097229 + }, + { + "auxiliary_loss_clip": 0.01124461, + "auxiliary_loss_mlp": 0.01034272, + "balance_loss_clip": 1.05271459, + "balance_loss_mlp": 1.01916838, + "epoch": 0.19627415704253962, + "flos": 42297130955520.0, + "grad_norm": 1.7566797452459595, + "language_loss": 0.85211045, + "learning_rate": 3.7169471651412714e-06, + "loss": 0.87369776, + "num_input_tokens_seen": 193413790, + "router_z_loss_clip": 0.71728516, + "router_z_loss_mlp": 0.15106201, + "step": 6764, + "time_per_iteration": 2.7330775260925293 + }, + { + "auxiliary_loss_clip": 0.01037926, + "auxiliary_loss_mlp": 0.00998572, + "balance_loss_clip": 1.01983595, + "balance_loss_mlp": 0.99762994, + "epoch": 0.19630317451105567, + "flos": 68220329178240.0, + "grad_norm": 0.7017362257138066, + "language_loss": 0.47304952, + "learning_rate": 3.716850759913287e-06, + "loss": 0.49341452, + "num_input_tokens_seen": 193481410, + "router_z_loss_clip": 0.18164062, + "router_z_loss_mlp": 0.00939941, + "step": 6765, + "time_per_iteration": 3.2916715145111084 + }, + { + "auxiliary_loss_clip": 0.01037841, + "auxiliary_loss_mlp": 0.00997921, + "balance_loss_clip": 1.01974177, + "balance_loss_mlp": 0.99699098, + "epoch": 0.1963321919795717, + "flos": 65476107037440.0, + "grad_norm": 0.6132705731555202, + "language_loss": 0.45463073, + "learning_rate": 3.7167543395213824e-06, + "loss": 0.47498834, + "num_input_tokens_seen": 193545285, + "router_z_loss_clip": 0.18164062, + "router_z_loss_mlp": 0.00927734, + "step": 6766, + "time_per_iteration": 3.1839616298675537 + }, + { + "auxiliary_loss_clip": 0.01118466, + "auxiliary_loss_mlp": 0.0103604, + "balance_loss_clip": 1.04886031, + "balance_loss_mlp": 1.02259898, + "epoch": 0.19636120944808774, + "flos": 16355766407040.0, + "grad_norm": 2.229793944743806, + "language_loss": 0.80832648, + "learning_rate": 3.716657903966409e-06, + "loss": 0.82987154, + "num_input_tokens_seen": 193559045, + "router_z_loss_clip": 0.69628906, + "router_z_loss_mlp": 0.13439941, + "step": 6767, + "time_per_iteration": 2.5351972579956055 + }, + { + "auxiliary_loss_clip": 0.01126209, + "auxiliary_loss_mlp": 0.01039975, + "balance_loss_clip": 1.05520904, + "balance_loss_mlp": 1.02531266, + "epoch": 0.1963902269166038, + "flos": 13290575310720.0, + "grad_norm": 3.3247855624813774, + "language_loss": 0.8074683, + "learning_rate": 3.716561453249218e-06, + "loss": 0.82913017, + "num_input_tokens_seen": 193570200, + "router_z_loss_clip": 0.71044922, + "router_z_loss_mlp": 0.14654541, + "step": 6768, + "time_per_iteration": 2.5737318992614746 + }, + { + "auxiliary_loss_clip": 0.01123539, + "auxiliary_loss_mlp": 0.01043605, + "balance_loss_clip": 1.04934788, + "balance_loss_mlp": 1.02762449, + "epoch": 0.19641924438511985, + "flos": 68163621603840.0, + "grad_norm": 2.5564442837533528, + "language_loss": 0.82584679, + "learning_rate": 3.7164649873706617e-06, + "loss": 0.84751832, + "num_input_tokens_seen": 193590130, + "router_z_loss_clip": 0.74291992, + "router_z_loss_mlp": 0.15997314, + "step": 6769, + "time_per_iteration": 2.9578144550323486 + }, + { + "auxiliary_loss_clip": 0.01125257, + "auxiliary_loss_mlp": 0.01039128, + "balance_loss_clip": 1.05280113, + "balance_loss_mlp": 1.02436399, + "epoch": 0.1964482618536359, + "flos": 13035178632960.0, + "grad_norm": 1.9063626524268407, + "language_loss": 0.6743685, + "learning_rate": 3.716368506331592e-06, + "loss": 0.69601232, + "num_input_tokens_seen": 193604825, + "router_z_loss_clip": 0.72460938, + "router_z_loss_mlp": 0.14770508, + "step": 6770, + "time_per_iteration": 2.5071632862091064 + }, + { + "auxiliary_loss_clip": 0.01127622, + "auxiliary_loss_mlp": 0.01040263, + "balance_loss_clip": 1.05454743, + "balance_loss_mlp": 1.02497411, + "epoch": 0.19647727932215195, + "flos": 31825867167360.0, + "grad_norm": 2.4440039471811765, + "language_loss": 0.84951162, + "learning_rate": 3.7162720101328607e-06, + "loss": 0.87119043, + "num_input_tokens_seen": 193624230, + "router_z_loss_clip": 0.73046875, + "router_z_loss_mlp": 0.15289307, + "step": 6771, + "time_per_iteration": 2.614018678665161 + }, + { + "auxiliary_loss_clip": 0.01128043, + "auxiliary_loss_mlp": 0.01040016, + "balance_loss_clip": 1.05485761, + "balance_loss_mlp": 1.02634239, + "epoch": 0.19650629679066797, + "flos": 17013757697280.0, + "grad_norm": 2.1521622325648235, + "language_loss": 0.73611975, + "learning_rate": 3.716175498775321e-06, + "loss": 0.75780034, + "num_input_tokens_seen": 193641105, + "router_z_loss_clip": 0.73193359, + "router_z_loss_mlp": 0.13684082, + "step": 6772, + "time_per_iteration": 2.819206953048706 + }, + { + "auxiliary_loss_clip": 0.01116315, + "auxiliary_loss_mlp": 0.01032192, + "balance_loss_clip": 1.04996896, + "balance_loss_mlp": 1.01875067, + "epoch": 0.19653531425918402, + "flos": 23362333056000.0, + "grad_norm": 4.210082861558393, + "language_loss": 0.82693738, + "learning_rate": 3.716078972259825e-06, + "loss": 0.84842241, + "num_input_tokens_seen": 193655920, + "router_z_loss_clip": 0.66357422, + "router_z_loss_mlp": 0.13433838, + "step": 6773, + "time_per_iteration": 2.560420513153076 + }, + { + "auxiliary_loss_clip": 0.01035201, + "auxiliary_loss_mlp": 0.01012398, + "balance_loss_clip": 1.01716805, + "balance_loss_mlp": 1.0115931, + "epoch": 0.19656433172770008, + "flos": 74764615789440.0, + "grad_norm": 0.6744433845363876, + "language_loss": 0.47944814, + "learning_rate": 3.7159824305872247e-06, + "loss": 0.49992412, + "num_input_tokens_seen": 193711615, + "router_z_loss_clip": 0.18066406, + "router_z_loss_mlp": 0.00805664, + "step": 6774, + "time_per_iteration": 3.0466394424438477 + }, + { + "auxiliary_loss_clip": 0.01127616, + "auxiliary_loss_mlp": 0.01041384, + "balance_loss_clip": 1.05557132, + "balance_loss_mlp": 1.02561259, + "epoch": 0.19659334919621613, + "flos": 19457370915840.0, + "grad_norm": 2.2720555378361182, + "language_loss": 0.74042362, + "learning_rate": 3.7158858737583733e-06, + "loss": 0.76211363, + "num_input_tokens_seen": 193726740, + "router_z_loss_clip": 0.72119141, + "router_z_loss_mlp": 0.15765381, + "step": 6775, + "time_per_iteration": 2.493452548980713 + }, + { + "auxiliary_loss_clip": 0.01131979, + "auxiliary_loss_mlp": 0.01042393, + "balance_loss_clip": 1.0595212, + "balance_loss_mlp": 1.02463055, + "epoch": 0.19662236666473218, + "flos": 36314700883200.0, + "grad_norm": 2.113820888242252, + "language_loss": 0.84977758, + "learning_rate": 3.7157893017741234e-06, + "loss": 0.87152123, + "num_input_tokens_seen": 193744105, + "router_z_loss_clip": 0.72509766, + "router_z_loss_mlp": 0.17767334, + "step": 6776, + "time_per_iteration": 2.6390175819396973 + }, + { + "auxiliary_loss_clip": 0.0111892, + "auxiliary_loss_mlp": 0.01037759, + "balance_loss_clip": 1.05136871, + "balance_loss_mlp": 1.0243597, + "epoch": 0.1966513841332482, + "flos": 26172127474560.0, + "grad_norm": 2.7028653661175266, + "language_loss": 0.82968783, + "learning_rate": 3.7156927146353284e-06, + "loss": 0.85125464, + "num_input_tokens_seen": 193759185, + "router_z_loss_clip": 0.67578125, + "router_z_loss_mlp": 0.1340332, + "step": 6777, + "time_per_iteration": 2.5517280101776123 + }, + { + "auxiliary_loss_clip": 0.01135341, + "auxiliary_loss_mlp": 0.01040815, + "balance_loss_clip": 1.05909526, + "balance_loss_mlp": 1.0257951, + "epoch": 0.19668040160176425, + "flos": 31096126460160.0, + "grad_norm": 2.0230244784288685, + "language_loss": 0.90277433, + "learning_rate": 3.7155961123428407e-06, + "loss": 0.92453593, + "num_input_tokens_seen": 193783395, + "router_z_loss_clip": 0.76220703, + "router_z_loss_mlp": 0.15032959, + "step": 6778, + "time_per_iteration": 2.8189573287963867 + }, + { + "auxiliary_loss_clip": 0.01126175, + "auxiliary_loss_mlp": 0.01036213, + "balance_loss_clip": 1.05484009, + "balance_loss_mlp": 1.02170563, + "epoch": 0.1967094190702803, + "flos": 25514854456320.0, + "grad_norm": 1.947947072105074, + "language_loss": 0.98144519, + "learning_rate": 3.7154994948975143e-06, + "loss": 1.00306904, + "num_input_tokens_seen": 193800720, + "router_z_loss_clip": 0.71386719, + "router_z_loss_mlp": 0.1451416, + "step": 6779, + "time_per_iteration": 2.551603317260742 + }, + { + "auxiliary_loss_clip": 0.01033044, + "auxiliary_loss_mlp": 0.01000613, + "balance_loss_clip": 1.01492918, + "balance_loss_mlp": 0.99978417, + "epoch": 0.19673843653879636, + "flos": 64670881898880.0, + "grad_norm": 0.6677203925509253, + "language_loss": 0.49890918, + "learning_rate": 3.7154028623002016e-06, + "loss": 0.51924574, + "num_input_tokens_seen": 193860920, + "router_z_loss_clip": 0.18164062, + "router_z_loss_mlp": 0.00830078, + "step": 6780, + "time_per_iteration": 3.1224145889282227 + }, + { + "auxiliary_loss_clip": 0.01032578, + "auxiliary_loss_mlp": 0.00999092, + "balance_loss_clip": 1.01451373, + "balance_loss_mlp": 0.99826908, + "epoch": 0.1967674540073124, + "flos": 73792119991680.0, + "grad_norm": 0.6635514686951292, + "language_loss": 0.46470678, + "learning_rate": 3.7153062145517565e-06, + "loss": 0.4850235, + "num_input_tokens_seen": 193924980, + "router_z_loss_clip": 0.18066406, + "router_z_loss_mlp": 0.00823975, + "step": 6781, + "time_per_iteration": 3.1487185955047607 + }, + { + "auxiliary_loss_clip": 0.01031845, + "auxiliary_loss_mlp": 0.01000101, + "balance_loss_clip": 1.0136683, + "balance_loss_mlp": 0.99935019, + "epoch": 0.19679647147582846, + "flos": 74771295719040.0, + "grad_norm": 0.6674595037362845, + "language_loss": 0.49958944, + "learning_rate": 3.715209551653034e-06, + "loss": 0.51990891, + "num_input_tokens_seen": 193988520, + "router_z_loss_clip": 0.18164062, + "router_z_loss_mlp": 0.00750732, + "step": 6782, + "time_per_iteration": 3.24528169631958 + }, + { + "auxiliary_loss_clip": 0.01125794, + "auxiliary_loss_mlp": 0.01042726, + "balance_loss_clip": 1.05197918, + "balance_loss_mlp": 1.02642977, + "epoch": 0.19682548894434448, + "flos": 24126691495680.0, + "grad_norm": 2.159347315589776, + "language_loss": 0.74014425, + "learning_rate": 3.7151128736048855e-06, + "loss": 0.7618295, + "num_input_tokens_seen": 194004690, + "router_z_loss_clip": 0.73730469, + "router_z_loss_mlp": 0.16290283, + "step": 6783, + "time_per_iteration": 2.563326120376587 + }, + { + "auxiliary_loss_clip": 0.01125623, + "auxiliary_loss_mlp": 0.01041746, + "balance_loss_clip": 1.0535655, + "balance_loss_mlp": 1.02720809, + "epoch": 0.19685450641286054, + "flos": 39524683703040.0, + "grad_norm": 1.9806657942736527, + "language_loss": 0.78207177, + "learning_rate": 3.715016180408167e-06, + "loss": 0.80374551, + "num_input_tokens_seen": 194022895, + "router_z_loss_clip": 0.72119141, + "router_z_loss_mlp": 0.14532471, + "step": 6784, + "time_per_iteration": 2.6679539680480957 + }, + { + "auxiliary_loss_clip": 0.0112742, + "auxiliary_loss_mlp": 0.01039057, + "balance_loss_clip": 1.05385423, + "balance_loss_mlp": 1.02368546, + "epoch": 0.1968835238813766, + "flos": 28796083902720.0, + "grad_norm": 2.2188626251432826, + "language_loss": 0.84177434, + "learning_rate": 3.7149194720637313e-06, + "loss": 0.86343908, + "num_input_tokens_seen": 194040075, + "router_z_loss_clip": 0.73632812, + "router_z_loss_mlp": 0.15368652, + "step": 6785, + "time_per_iteration": 2.5991151332855225 + }, + { + "auxiliary_loss_clip": 0.01121986, + "auxiliary_loss_mlp": 0.01027984, + "balance_loss_clip": 1.05106497, + "balance_loss_mlp": 1.01397061, + "epoch": 0.19691254134989264, + "flos": 54480615229440.0, + "grad_norm": 2.9289440826599606, + "language_loss": 0.80955946, + "learning_rate": 3.714822748572432e-06, + "loss": 0.8310591, + "num_input_tokens_seen": 194058855, + "router_z_loss_clip": 0.70898438, + "router_z_loss_mlp": 0.14013672, + "step": 6786, + "time_per_iteration": 2.7971246242523193 + }, + { + "auxiliary_loss_clip": 0.01031615, + "auxiliary_loss_mlp": 0.01012625, + "balance_loss_clip": 1.01363122, + "balance_loss_mlp": 1.01183212, + "epoch": 0.1969415588184087, + "flos": 66388920387840.0, + "grad_norm": 0.6487620159406564, + "language_loss": 0.49613899, + "learning_rate": 3.7147260099351252e-06, + "loss": 0.51658136, + "num_input_tokens_seen": 194122215, + "router_z_loss_clip": 0.1796875, + "router_z_loss_mlp": 0.00793457, + "step": 6787, + "time_per_iteration": 3.0855305194854736 + }, + { + "auxiliary_loss_clip": 0.01132481, + "auxiliary_loss_mlp": 0.0104292, + "balance_loss_clip": 1.05860972, + "balance_loss_mlp": 1.02683318, + "epoch": 0.19697057628692474, + "flos": 31794445745280.0, + "grad_norm": 1.949149786025729, + "language_loss": 0.86573517, + "learning_rate": 3.7146292561526648e-06, + "loss": 0.8874892, + "num_input_tokens_seen": 194141705, + "router_z_loss_clip": 0.73974609, + "router_z_loss_mlp": 0.16101074, + "step": 6788, + "time_per_iteration": 2.634260416030884 + }, + { + "auxiliary_loss_clip": 0.01031568, + "auxiliary_loss_mlp": 0.01008368, + "balance_loss_clip": 1.0136745, + "balance_loss_mlp": 1.00766122, + "epoch": 0.19699959375544077, + "flos": 67323889451520.0, + "grad_norm": 0.6726814069437418, + "language_loss": 0.5002538, + "learning_rate": 3.714532487225904e-06, + "loss": 0.52065313, + "num_input_tokens_seen": 194202905, + "router_z_loss_clip": 0.1796875, + "router_z_loss_mlp": 0.00704956, + "step": 6789, + "time_per_iteration": 3.16694712638855 + }, + { + "auxiliary_loss_clip": 0.01121343, + "auxiliary_loss_mlp": 0.01040969, + "balance_loss_clip": 1.0524292, + "balance_loss_mlp": 1.02655041, + "epoch": 0.19702861122395682, + "flos": 28250636901120.0, + "grad_norm": 1.9593854124257961, + "language_loss": 0.77072519, + "learning_rate": 3.7144357031556986e-06, + "loss": 0.79234827, + "num_input_tokens_seen": 194222510, + "router_z_loss_clip": 0.68994141, + "router_z_loss_mlp": 0.14428711, + "step": 6790, + "time_per_iteration": 2.603614330291748 + }, + { + "auxiliary_loss_clip": 0.01127742, + "auxiliary_loss_mlp": 0.01040821, + "balance_loss_clip": 1.05577779, + "balance_loss_mlp": 1.02535391, + "epoch": 0.19705762869247287, + "flos": 32554781861760.0, + "grad_norm": 2.0589393709309296, + "language_loss": 0.8950147, + "learning_rate": 3.714338903942904e-06, + "loss": 0.91670024, + "num_input_tokens_seen": 194237620, + "router_z_loss_clip": 0.71923828, + "router_z_loss_mlp": 0.15466309, + "step": 6791, + "time_per_iteration": 2.6570630073547363 + }, + { + "auxiliary_loss_clip": 0.01134448, + "auxiliary_loss_mlp": 0.01045284, + "balance_loss_clip": 1.05697584, + "balance_loss_mlp": 1.02923834, + "epoch": 0.19708664616098892, + "flos": 27010821110400.0, + "grad_norm": 2.0480715102281977, + "language_loss": 0.90724844, + "learning_rate": 3.714242089588374e-06, + "loss": 0.9290458, + "num_input_tokens_seen": 194255135, + "router_z_loss_clip": 0.77490234, + "router_z_loss_mlp": 0.16040039, + "step": 6792, + "time_per_iteration": 2.578383207321167 + }, + { + "auxiliary_loss_clip": 0.01033752, + "auxiliary_loss_mlp": 0.01016694, + "balance_loss_clip": 1.01585245, + "balance_loss_mlp": 1.01599121, + "epoch": 0.19711566362950497, + "flos": 61380279002880.0, + "grad_norm": 0.6626624461546771, + "language_loss": 0.46535069, + "learning_rate": 3.714145260092964e-06, + "loss": 0.48585516, + "num_input_tokens_seen": 194315000, + "router_z_loss_clip": 0.1796875, + "router_z_loss_mlp": 0.00701904, + "step": 6793, + "time_per_iteration": 3.0739152431488037 + }, + { + "auxiliary_loss_clip": 0.01127794, + "auxiliary_loss_mlp": 0.01040048, + "balance_loss_clip": 1.05603051, + "balance_loss_mlp": 1.02475333, + "epoch": 0.197144681098021, + "flos": 39237111417600.0, + "grad_norm": 2.2058236508207916, + "language_loss": 0.62517381, + "learning_rate": 3.7140484154575294e-06, + "loss": 0.64685225, + "num_input_tokens_seen": 194333560, + "router_z_loss_clip": 0.71777344, + "router_z_loss_mlp": 0.15289307, + "step": 6794, + "time_per_iteration": 2.6750807762145996 + }, + { + "auxiliary_loss_clip": 0.01034241, + "auxiliary_loss_mlp": 0.01000963, + "balance_loss_clip": 1.01622939, + "balance_loss_mlp": 1.00020647, + "epoch": 0.19717369856653705, + "flos": 56796103612800.0, + "grad_norm": 0.6858253607992466, + "language_loss": 0.51703131, + "learning_rate": 3.7139515556829263e-06, + "loss": 0.53738338, + "num_input_tokens_seen": 194393075, + "router_z_loss_clip": 0.18066406, + "router_z_loss_mlp": 0.00756836, + "step": 6795, + "time_per_iteration": 3.0863733291625977 + }, + { + "auxiliary_loss_clip": 0.01117556, + "auxiliary_loss_mlp": 0.01030636, + "balance_loss_clip": 1.04996705, + "balance_loss_mlp": 1.0179168, + "epoch": 0.1972027160350531, + "flos": 26206709293440.0, + "grad_norm": 2.0963085742806706, + "language_loss": 0.57999331, + "learning_rate": 3.7138546807700085e-06, + "loss": 0.60147524, + "num_input_tokens_seen": 194410270, + "router_z_loss_clip": 0.67700195, + "router_z_loss_mlp": 0.12719727, + "step": 6796, + "time_per_iteration": 2.5607519149780273 + }, + { + "auxiliary_loss_clip": 0.01119269, + "auxiliary_loss_mlp": 0.01040405, + "balance_loss_clip": 1.0542208, + "balance_loss_mlp": 1.02682114, + "epoch": 0.19723173350356915, + "flos": 26392655024640.0, + "grad_norm": 2.1830177731002314, + "language_loss": 0.86831981, + "learning_rate": 3.7137577907196336e-06, + "loss": 0.88991654, + "num_input_tokens_seen": 194427010, + "router_z_loss_clip": 0.64990234, + "router_z_loss_mlp": 0.13574219, + "step": 6797, + "time_per_iteration": 2.5949881076812744 + }, + { + "auxiliary_loss_clip": 0.01123585, + "auxiliary_loss_mlp": 0.0103454, + "balance_loss_clip": 1.05681837, + "balance_loss_mlp": 1.0215404, + "epoch": 0.1972607509720852, + "flos": 20331220988160.0, + "grad_norm": 3.4355883534059006, + "language_loss": 0.72733998, + "learning_rate": 3.713660885532656e-06, + "loss": 0.74892128, + "num_input_tokens_seen": 194440035, + "router_z_loss_clip": 0.66845703, + "router_z_loss_mlp": 0.13012695, + "step": 6798, + "time_per_iteration": 2.4610114097595215 + }, + { + "auxiliary_loss_clip": 0.01115274, + "auxiliary_loss_mlp": 0.01030678, + "balance_loss_clip": 1.05032444, + "balance_loss_mlp": 1.0176723, + "epoch": 0.19728976844060125, + "flos": 15662295457920.0, + "grad_norm": 2.920281734933326, + "language_loss": 0.77719021, + "learning_rate": 3.713563965209932e-06, + "loss": 0.79864973, + "num_input_tokens_seen": 194451795, + "router_z_loss_clip": 0.64990234, + "router_z_loss_mlp": 0.12994385, + "step": 6799, + "time_per_iteration": 2.4774982929229736 + }, + { + "auxiliary_loss_clip": 0.01119322, + "auxiliary_loss_mlp": 0.01034024, + "balance_loss_clip": 1.05348039, + "balance_loss_mlp": 1.02061284, + "epoch": 0.19731878590911728, + "flos": 12089255921280.0, + "grad_norm": 2.4277170182675785, + "language_loss": 0.69992173, + "learning_rate": 3.7134670297523176e-06, + "loss": 0.72145516, + "num_input_tokens_seen": 194464480, + "router_z_loss_clip": 0.65820312, + "router_z_loss_mlp": 0.13415527, + "step": 6800, + "time_per_iteration": 2.467808723449707 + }, + { + "auxiliary_loss_clip": 0.01131457, + "auxiliary_loss_mlp": 0.01036132, + "balance_loss_clip": 1.05439568, + "balance_loss_mlp": 1.01958621, + "epoch": 0.19734780337763333, + "flos": 32995298257920.0, + "grad_norm": 2.362489742256319, + "language_loss": 1.18800604, + "learning_rate": 3.7133700791606693e-06, + "loss": 1.20968199, + "num_input_tokens_seen": 194485490, + "router_z_loss_clip": 0.77050781, + "router_z_loss_mlp": 0.16552734, + "step": 6801, + "time_per_iteration": 2.6337454319000244 + }, + { + "auxiliary_loss_clip": 0.01127752, + "auxiliary_loss_mlp": 0.01034355, + "balance_loss_clip": 1.05573475, + "balance_loss_mlp": 1.01797593, + "epoch": 0.19737682084614938, + "flos": 26824480329600.0, + "grad_norm": 2.2429017960549427, + "language_loss": 0.82364178, + "learning_rate": 3.7132731134358428e-06, + "loss": 0.84526283, + "num_input_tokens_seen": 194502270, + "router_z_loss_clip": 0.72070312, + "router_z_loss_mlp": 0.16369629, + "step": 6802, + "time_per_iteration": 2.5704360008239746 + }, + { + "auxiliary_loss_clip": 0.01127113, + "auxiliary_loss_mlp": 0.0104413, + "balance_loss_clip": 1.05452991, + "balance_loss_mlp": 1.02846026, + "epoch": 0.19740583831466543, + "flos": 28831922697600.0, + "grad_norm": 2.3068044547915547, + "language_loss": 0.86901724, + "learning_rate": 3.7131761325786947e-06, + "loss": 0.89072967, + "num_input_tokens_seen": 194517170, + "router_z_loss_clip": 0.72607422, + "router_z_loss_mlp": 0.15667725, + "step": 6803, + "time_per_iteration": 2.6034533977508545 + }, + { + "auxiliary_loss_clip": 0.01121961, + "auxiliary_loss_mlp": 0.01032312, + "balance_loss_clip": 1.05565405, + "balance_loss_mlp": 1.01960397, + "epoch": 0.19743485578318148, + "flos": 41317596092160.0, + "grad_norm": 1.8414202634105905, + "language_loss": 0.81371725, + "learning_rate": 3.7130791365900823e-06, + "loss": 0.83525997, + "num_input_tokens_seen": 194535835, + "router_z_loss_clip": 0.66357422, + "router_z_loss_mlp": 0.1270752, + "step": 6804, + "time_per_iteration": 2.704740524291992 + }, + { + "auxiliary_loss_clip": 0.0103508, + "auxiliary_loss_mlp": 0.01001712, + "balance_loss_clip": 1.01692581, + "balance_loss_mlp": 1.00089502, + "epoch": 0.1974638732516975, + "flos": 74762999677440.0, + "grad_norm": 0.8197740600472894, + "language_loss": 0.52992475, + "learning_rate": 3.7129821254708615e-06, + "loss": 0.55029273, + "num_input_tokens_seen": 194587335, + "router_z_loss_clip": 0.18164062, + "router_z_loss_mlp": 0.00817871, + "step": 6805, + "time_per_iteration": 3.0513205528259277 + }, + { + "auxiliary_loss_clip": 0.01128654, + "auxiliary_loss_mlp": 0.0104186, + "balance_loss_clip": 1.05644083, + "balance_loss_mlp": 1.02555203, + "epoch": 0.19749289072021356, + "flos": 26900431637760.0, + "grad_norm": 2.329200486708231, + "language_loss": 0.9013294, + "learning_rate": 3.712885099221889e-06, + "loss": 0.92303455, + "num_input_tokens_seen": 194602280, + "router_z_loss_clip": 0.72265625, + "router_z_loss_mlp": 0.16314697, + "step": 6806, + "time_per_iteration": 2.576841115951538 + }, + { + "auxiliary_loss_clip": 0.01117852, + "auxiliary_loss_mlp": 0.01034087, + "balance_loss_clip": 1.05045402, + "balance_loss_mlp": 1.02136767, + "epoch": 0.1975219081887296, + "flos": 14751062305920.0, + "grad_norm": 2.2853654014963114, + "language_loss": 0.81828189, + "learning_rate": 3.7127880578440226e-06, + "loss": 0.83980131, + "num_input_tokens_seen": 194616150, + "router_z_loss_clip": 0.67431641, + "router_z_loss_mlp": 0.12713623, + "step": 6807, + "time_per_iteration": 2.5959084033966064 + }, + { + "auxiliary_loss_clip": 0.01121416, + "auxiliary_loss_mlp": 0.01041125, + "balance_loss_clip": 1.05129886, + "balance_loss_mlp": 1.02673006, + "epoch": 0.19755092565724566, + "flos": 27859714208640.0, + "grad_norm": 2.5599797788365524, + "language_loss": 0.8637625, + "learning_rate": 3.7126910013381194e-06, + "loss": 0.8853879, + "num_input_tokens_seen": 194631985, + "router_z_loss_clip": 0.70166016, + "router_z_loss_mlp": 0.1439209, + "step": 6808, + "time_per_iteration": 2.6578071117401123 + }, + { + "auxiliary_loss_clip": 0.01122067, + "auxiliary_loss_mlp": 0.01032517, + "balance_loss_clip": 1.05452371, + "balance_loss_mlp": 1.01882005, + "epoch": 0.19757994312576171, + "flos": 12159963843840.0, + "grad_norm": 2.4311133981327355, + "language_loss": 0.81414342, + "learning_rate": 3.7125939297050356e-06, + "loss": 0.83568925, + "num_input_tokens_seen": 194647370, + "router_z_loss_clip": 0.67578125, + "router_z_loss_mlp": 0.13702393, + "step": 6809, + "time_per_iteration": 2.565782070159912 + }, + { + "auxiliary_loss_clip": 0.01124033, + "auxiliary_loss_mlp": 0.01039831, + "balance_loss_clip": 1.05599952, + "balance_loss_mlp": 1.02526987, + "epoch": 0.19760896059427777, + "flos": 10189904555520.0, + "grad_norm": 2.7013496702852855, + "language_loss": 0.68982631, + "learning_rate": 3.7124968429456295e-06, + "loss": 0.711465, + "num_input_tokens_seen": 194659285, + "router_z_loss_clip": 0.68017578, + "router_z_loss_mlp": 0.14562988, + "step": 6810, + "time_per_iteration": 2.5178258419036865 + }, + { + "auxiliary_loss_clip": 0.01033346, + "auxiliary_loss_mlp": 0.01009254, + "balance_loss_clip": 1.01496983, + "balance_loss_mlp": 1.00843132, + "epoch": 0.1976379780627938, + "flos": 73062271543680.0, + "grad_norm": 0.6859239188293033, + "language_loss": 0.50808543, + "learning_rate": 3.712399741060758e-06, + "loss": 0.5285114, + "num_input_tokens_seen": 194720360, + "router_z_loss_clip": 0.18359375, + "router_z_loss_mlp": 0.00823975, + "step": 6811, + "time_per_iteration": 3.2058308124542236 + }, + { + "auxiliary_loss_clip": 0.01125266, + "auxiliary_loss_mlp": 0.01047835, + "balance_loss_clip": 1.05551136, + "balance_loss_mlp": 1.03254008, + "epoch": 0.19766699553130984, + "flos": 42588617823360.0, + "grad_norm": 2.2890898837625118, + "language_loss": 0.87516296, + "learning_rate": 3.712302624051279e-06, + "loss": 0.89689392, + "num_input_tokens_seen": 194737575, + "router_z_loss_clip": 0.69726562, + "router_z_loss_mlp": 0.15283203, + "step": 6812, + "time_per_iteration": 2.7244794368743896 + }, + { + "auxiliary_loss_clip": 0.01032949, + "auxiliary_loss_mlp": 0.01012283, + "balance_loss_clip": 1.01484323, + "balance_loss_mlp": 1.01151395, + "epoch": 0.1976960129998259, + "flos": 61934812145280.0, + "grad_norm": 0.6679761781421223, + "language_loss": 0.49710286, + "learning_rate": 3.7122054919180506e-06, + "loss": 0.51755518, + "num_input_tokens_seen": 194800150, + "router_z_loss_clip": 0.18164062, + "router_z_loss_mlp": 0.00765991, + "step": 6813, + "time_per_iteration": 3.1856679916381836 + }, + { + "auxiliary_loss_clip": 0.01130513, + "auxiliary_loss_mlp": 0.01037474, + "balance_loss_clip": 1.05605066, + "balance_loss_mlp": 1.02282929, + "epoch": 0.19772503046834194, + "flos": 41130608866560.0, + "grad_norm": 2.263237405132544, + "language_loss": 0.92855513, + "learning_rate": 3.7121083446619306e-06, + "loss": 0.95023495, + "num_input_tokens_seen": 194817850, + "router_z_loss_clip": 0.74511719, + "router_z_loss_mlp": 0.1463623, + "step": 6814, + "time_per_iteration": 2.7600698471069336 + }, + { + "auxiliary_loss_clip": 0.01129396, + "auxiliary_loss_mlp": 0.01041245, + "balance_loss_clip": 1.05306053, + "balance_loss_mlp": 1.0261476, + "epoch": 0.197754047936858, + "flos": 33686075687040.0, + "grad_norm": 2.258307747885332, + "language_loss": 0.78652108, + "learning_rate": 3.7120111822837767e-06, + "loss": 0.80822748, + "num_input_tokens_seen": 194832875, + "router_z_loss_clip": 0.76318359, + "router_z_loss_mlp": 0.15081787, + "step": 6815, + "time_per_iteration": 2.6527740955352783 + }, + { + "auxiliary_loss_clip": 0.01033682, + "auxiliary_loss_mlp": 0.01005485, + "balance_loss_clip": 1.01533079, + "balance_loss_mlp": 1.0046382, + "epoch": 0.19778306540537405, + "flos": 51930530098560.0, + "grad_norm": 0.599845604424436, + "language_loss": 0.42839217, + "learning_rate": 3.711914004784447e-06, + "loss": 0.44878387, + "num_input_tokens_seen": 194894840, + "router_z_loss_clip": 0.18359375, + "router_z_loss_mlp": 0.00848389, + "step": 6816, + "time_per_iteration": 3.1367270946502686 + }, + { + "auxiliary_loss_clip": 0.01034263, + "auxiliary_loss_mlp": 0.01002902, + "balance_loss_clip": 1.01597071, + "balance_loss_mlp": 1.00209737, + "epoch": 0.19781208287389007, + "flos": 67400630858880.0, + "grad_norm": 0.6645268241612879, + "language_loss": 0.49150103, + "learning_rate": 3.7118168121648e-06, + "loss": 0.51187271, + "num_input_tokens_seen": 194956915, + "router_z_loss_clip": 0.18359375, + "router_z_loss_mlp": 0.00805664, + "step": 6817, + "time_per_iteration": 3.110936403274536 + }, + { + "auxiliary_loss_clip": 0.01117692, + "auxiliary_loss_mlp": 0.01037321, + "balance_loss_clip": 1.0507791, + "balance_loss_mlp": 1.02393365, + "epoch": 0.19784110034240612, + "flos": 41057710214400.0, + "grad_norm": 1.9689202570987632, + "language_loss": 0.78922379, + "learning_rate": 3.7117196044256946e-06, + "loss": 0.81077385, + "num_input_tokens_seen": 194976380, + "router_z_loss_clip": 0.66943359, + "router_z_loss_mlp": 0.13391113, + "step": 6818, + "time_per_iteration": 2.720611572265625 + }, + { + "auxiliary_loss_clip": 0.01125196, + "auxiliary_loss_mlp": 0.01043862, + "balance_loss_clip": 1.05389738, + "balance_loss_mlp": 1.02856731, + "epoch": 0.19787011781092217, + "flos": 22595496577920.0, + "grad_norm": 2.3590853704621213, + "language_loss": 0.82810634, + "learning_rate": 3.7116223815679893e-06, + "loss": 0.84979689, + "num_input_tokens_seen": 194989220, + "router_z_loss_clip": 0.71337891, + "router_z_loss_mlp": 0.15283203, + "step": 6819, + "time_per_iteration": 2.5991640090942383 + }, + { + "auxiliary_loss_clip": 0.01126401, + "auxiliary_loss_mlp": 0.01046992, + "balance_loss_clip": 1.05306292, + "balance_loss_mlp": 1.03141761, + "epoch": 0.19789913527943823, + "flos": 34562116488960.0, + "grad_norm": 2.408497644496094, + "language_loss": 0.954849, + "learning_rate": 3.711525143592542e-06, + "loss": 0.97658294, + "num_input_tokens_seen": 195003950, + "router_z_loss_clip": 0.73291016, + "router_z_loss_mlp": 0.15557861, + "step": 6820, + "time_per_iteration": 2.665419816970825 + }, + { + "auxiliary_loss_clip": 0.01118698, + "auxiliary_loss_mlp": 0.01032431, + "balance_loss_clip": 1.05048382, + "balance_loss_mlp": 1.01843619, + "epoch": 0.19792815274795428, + "flos": 28577103615360.0, + "grad_norm": 2.026627069896031, + "language_loss": 0.82310563, + "learning_rate": 3.7114278905002122e-06, + "loss": 0.84461695, + "num_input_tokens_seen": 195030060, + "router_z_loss_clip": 0.68310547, + "router_z_loss_mlp": 0.14001465, + "step": 6821, + "time_per_iteration": 2.7831103801727295 + }, + { + "auxiliary_loss_clip": 0.01114932, + "auxiliary_loss_mlp": 0.0104161, + "balance_loss_clip": 1.05104494, + "balance_loss_mlp": 1.02756131, + "epoch": 0.1979571702164703, + "flos": 16720295581440.0, + "grad_norm": 2.113469930667283, + "language_loss": 0.5656305, + "learning_rate": 3.7113306222918586e-06, + "loss": 0.58719587, + "num_input_tokens_seen": 195042560, + "router_z_loss_clip": 0.63964844, + "router_z_loss_mlp": 0.14056396, + "step": 6822, + "time_per_iteration": 2.557623863220215 + }, + { + "auxiliary_loss_clip": 0.0112627, + "auxiliary_loss_mlp": 0.01045462, + "balance_loss_clip": 1.05300593, + "balance_loss_mlp": 1.02964306, + "epoch": 0.19798618768498635, + "flos": 16538408087040.0, + "grad_norm": 3.0425909444277095, + "language_loss": 0.69207066, + "learning_rate": 3.7112333389683405e-06, + "loss": 0.71378797, + "num_input_tokens_seen": 195055205, + "router_z_loss_clip": 0.73339844, + "router_z_loss_mlp": 0.15808105, + "step": 6823, + "time_per_iteration": 2.5453364849090576 + }, + { + "auxiliary_loss_clip": 0.01122491, + "auxiliary_loss_mlp": 0.01033642, + "balance_loss_clip": 1.05051935, + "balance_loss_mlp": 1.01787651, + "epoch": 0.1980152051535024, + "flos": 17676130446720.0, + "grad_norm": 2.348585810045074, + "language_loss": 0.83723807, + "learning_rate": 3.7111360405305173e-06, + "loss": 0.8587994, + "num_input_tokens_seen": 195071145, + "router_z_loss_clip": 0.72045898, + "router_z_loss_mlp": 0.15771484, + "step": 6824, + "time_per_iteration": 2.5566840171813965 + }, + { + "auxiliary_loss_clip": 0.01122229, + "auxiliary_loss_mlp": 0.01040092, + "balance_loss_clip": 1.05409169, + "balance_loss_mlp": 1.02595997, + "epoch": 0.19804422262201846, + "flos": 19894008643200.0, + "grad_norm": 2.555379331013512, + "language_loss": 0.71770579, + "learning_rate": 3.7110387269792474e-06, + "loss": 0.73932898, + "num_input_tokens_seen": 195085480, + "router_z_loss_clip": 0.6809082, + "router_z_loss_mlp": 0.14135742, + "step": 6825, + "time_per_iteration": 2.601158618927002 + }, + { + "auxiliary_loss_clip": 0.010331, + "auxiliary_loss_mlp": 0.01007303, + "balance_loss_clip": 1.01495969, + "balance_loss_mlp": 1.00643849, + "epoch": 0.1980732400905345, + "flos": 69878287192320.0, + "grad_norm": 0.7170327920200895, + "language_loss": 0.46673661, + "learning_rate": 3.7109413983153922e-06, + "loss": 0.48714063, + "num_input_tokens_seen": 195137330, + "router_z_loss_clip": 0.18164062, + "router_z_loss_mlp": 0.00866699, + "step": 6826, + "time_per_iteration": 3.045855760574341 + }, + { + "auxiliary_loss_clip": 0.01123978, + "auxiliary_loss_mlp": 0.01042373, + "balance_loss_clip": 1.05271602, + "balance_loss_mlp": 1.02877092, + "epoch": 0.19810225755905056, + "flos": 17997997242240.0, + "grad_norm": 3.2825839028859427, + "language_loss": 1.03219748, + "learning_rate": 3.7108440545398095e-06, + "loss": 1.05386114, + "num_input_tokens_seen": 195153415, + "router_z_loss_clip": 0.71289062, + "router_z_loss_mlp": 0.13604736, + "step": 6827, + "time_per_iteration": 5.092571258544922 + }, + { + "auxiliary_loss_clip": 0.0103267, + "auxiliary_loss_mlp": 0.00999804, + "balance_loss_clip": 1.01473284, + "balance_loss_mlp": 0.99892759, + "epoch": 0.19813127502756658, + "flos": 59745625937280.0, + "grad_norm": 0.7648741979916738, + "language_loss": 0.54199958, + "learning_rate": 3.71074669565336e-06, + "loss": 0.56232429, + "num_input_tokens_seen": 195211290, + "router_z_loss_clip": 0.1796875, + "router_z_loss_mlp": 0.00878906, + "step": 6828, + "time_per_iteration": 7.711230039596558 + }, + { + "auxiliary_loss_clip": 0.01113596, + "auxiliary_loss_mlp": 0.01031784, + "balance_loss_clip": 1.05035067, + "balance_loss_mlp": 1.01816463, + "epoch": 0.19816029249608264, + "flos": 12086598314880.0, + "grad_norm": 3.521804234363953, + "language_loss": 0.684811, + "learning_rate": 3.710649321656904e-06, + "loss": 0.70626485, + "num_input_tokens_seen": 195222885, + "router_z_loss_clip": 0.63183594, + "router_z_loss_mlp": 0.1361084, + "step": 6829, + "time_per_iteration": 2.5169763565063477 + }, + { + "auxiliary_loss_clip": 0.01128667, + "auxiliary_loss_mlp": 0.01035891, + "balance_loss_clip": 1.05384314, + "balance_loss_mlp": 1.02009618, + "epoch": 0.1981893099645987, + "flos": 28623605771520.0, + "grad_norm": 1.811114509608128, + "language_loss": 0.64250839, + "learning_rate": 3.7105519325512997e-06, + "loss": 0.66415393, + "num_input_tokens_seen": 195242480, + "router_z_loss_clip": 0.74902344, + "router_z_loss_mlp": 0.15777588, + "step": 6830, + "time_per_iteration": 2.710993766784668 + }, + { + "auxiliary_loss_clip": 0.01129497, + "auxiliary_loss_mlp": 0.01052533, + "balance_loss_clip": 1.05348969, + "balance_loss_mlp": 1.03693438, + "epoch": 0.19821832743311474, + "flos": 14785823692800.0, + "grad_norm": 2.1596019956808563, + "language_loss": 0.8544929, + "learning_rate": 3.7104545283374097e-06, + "loss": 0.87631321, + "num_input_tokens_seen": 195255320, + "router_z_loss_clip": 0.76025391, + "router_z_loss_mlp": 0.15600586, + "step": 6831, + "time_per_iteration": 2.55936598777771 + }, + { + "auxiliary_loss_clip": 0.01130609, + "auxiliary_loss_mlp": 0.01055109, + "balance_loss_clip": 1.0553484, + "balance_loss_mlp": 1.03797865, + "epoch": 0.1982473449016308, + "flos": 16208209336320.0, + "grad_norm": 3.8557575683522867, + "language_loss": 0.70244503, + "learning_rate": 3.7103571090160926e-06, + "loss": 0.72430217, + "num_input_tokens_seen": 195269020, + "router_z_loss_clip": 0.75244141, + "router_z_loss_mlp": 0.17150879, + "step": 6832, + "time_per_iteration": 2.641740560531616 + }, + { + "auxiliary_loss_clip": 0.01032808, + "auxiliary_loss_mlp": 0.01023902, + "balance_loss_clip": 1.01432776, + "balance_loss_mlp": 1.0230794, + "epoch": 0.19827636237014684, + "flos": 59119379291520.0, + "grad_norm": 0.6866941632690114, + "language_loss": 0.47695303, + "learning_rate": 3.710259674588209e-06, + "loss": 0.49752012, + "num_input_tokens_seen": 195326225, + "router_z_loss_clip": 0.18457031, + "router_z_loss_mlp": 0.00823975, + "step": 6833, + "time_per_iteration": 3.018108367919922 + }, + { + "auxiliary_loss_clip": 0.01032709, + "auxiliary_loss_mlp": 0.01020584, + "balance_loss_clip": 1.0142647, + "balance_loss_mlp": 1.0196538, + "epoch": 0.19830537983866287, + "flos": 74766626951040.0, + "grad_norm": 0.6923323509654502, + "language_loss": 0.4320235, + "learning_rate": 3.7101622250546207e-06, + "loss": 0.45255643, + "num_input_tokens_seen": 195384550, + "router_z_loss_clip": 0.18457031, + "router_z_loss_mlp": 0.00927734, + "step": 6834, + "time_per_iteration": 3.1173059940338135 + }, + { + "auxiliary_loss_clip": 0.01126101, + "auxiliary_loss_mlp": 0.01047028, + "balance_loss_clip": 1.05716372, + "balance_loss_mlp": 1.03400421, + "epoch": 0.19833439730717892, + "flos": 27407885028480.0, + "grad_norm": 3.029613710500977, + "language_loss": 0.80242813, + "learning_rate": 3.710064760416187e-06, + "loss": 0.82415938, + "num_input_tokens_seen": 195397325, + "router_z_loss_clip": 0.68994141, + "router_z_loss_mlp": 0.13018799, + "step": 6835, + "time_per_iteration": 2.627810001373291 + }, + { + "auxiliary_loss_clip": 0.01126041, + "auxiliary_loss_mlp": 0.01045614, + "balance_loss_clip": 1.05250967, + "balance_loss_mlp": 1.03055787, + "epoch": 0.19836341477569497, + "flos": 48022835546880.0, + "grad_norm": 2.43547321065646, + "language_loss": 1.0402168, + "learning_rate": 3.70996728067377e-06, + "loss": 1.0619334, + "num_input_tokens_seen": 195415185, + "router_z_loss_clip": 0.734375, + "router_z_loss_mlp": 0.15057373, + "step": 6836, + "time_per_iteration": 2.689101457595825 + }, + { + "auxiliary_loss_clip": 0.01031822, + "auxiliary_loss_mlp": 0.01003343, + "balance_loss_clip": 1.01380777, + "balance_loss_mlp": 1.00247288, + "epoch": 0.19839243224421102, + "flos": 74776359536640.0, + "grad_norm": 0.6510960385932878, + "language_loss": 0.50469124, + "learning_rate": 3.7098697858282295e-06, + "loss": 0.52504289, + "num_input_tokens_seen": 195480245, + "router_z_loss_clip": 0.1796875, + "router_z_loss_mlp": 0.00872803, + "step": 6837, + "time_per_iteration": 3.179049491882324 + }, + { + "auxiliary_loss_clip": 0.01030209, + "auxiliary_loss_mlp": 0.00998804, + "balance_loss_clip": 1.01221943, + "balance_loss_mlp": 0.99796313, + "epoch": 0.19842144971272707, + "flos": 60658654769280.0, + "grad_norm": 0.620430011264525, + "language_loss": 0.4659062, + "learning_rate": 3.709772275880427e-06, + "loss": 0.48619631, + "num_input_tokens_seen": 195546840, + "router_z_loss_clip": 0.1796875, + "router_z_loss_mlp": 0.00842285, + "step": 6838, + "time_per_iteration": 3.193667411804199 + }, + { + "auxiliary_loss_clip": 0.01127345, + "auxiliary_loss_mlp": 0.01035388, + "balance_loss_clip": 1.05385542, + "balance_loss_mlp": 1.02021313, + "epoch": 0.1984504671812431, + "flos": 24472297180800.0, + "grad_norm": 2.463200896705956, + "language_loss": 0.92543513, + "learning_rate": 3.7096747508312243e-06, + "loss": 0.94706243, + "num_input_tokens_seen": 195566325, + "router_z_loss_clip": 0.73486328, + "router_z_loss_mlp": 0.15185547, + "step": 6839, + "time_per_iteration": 2.594655752182007 + }, + { + "auxiliary_loss_clip": 0.01123753, + "auxiliary_loss_mlp": 0.01039334, + "balance_loss_clip": 1.05480933, + "balance_loss_mlp": 1.02585173, + "epoch": 0.19847948464975915, + "flos": 16502317896960.0, + "grad_norm": 2.054388430835937, + "language_loss": 0.60277653, + "learning_rate": 3.709577210681482e-06, + "loss": 0.62440741, + "num_input_tokens_seen": 195580195, + "router_z_loss_clip": 0.69042969, + "router_z_loss_mlp": 0.1348877, + "step": 6840, + "time_per_iteration": 2.5394127368927 + }, + { + "auxiliary_loss_clip": 0.0102898, + "auxiliary_loss_mlp": 0.01005689, + "balance_loss_clip": 1.01086402, + "balance_loss_mlp": 1.00479472, + "epoch": 0.1985085021182752, + "flos": 74778119303040.0, + "grad_norm": 0.6875691330489542, + "language_loss": 0.48220837, + "learning_rate": 3.7094796554320624e-06, + "loss": 0.50255507, + "num_input_tokens_seen": 195642055, + "router_z_loss_clip": 0.18164062, + "router_z_loss_mlp": 0.00891113, + "step": 6841, + "time_per_iteration": 3.1694865226745605 + }, + { + "auxiliary_loss_clip": 0.01028377, + "auxiliary_loss_mlp": 0.01005168, + "balance_loss_clip": 1.01047373, + "balance_loss_mlp": 1.00429749, + "epoch": 0.19853751958679125, + "flos": 67879176779520.0, + "grad_norm": 0.6932441544373397, + "language_loss": 0.47046435, + "learning_rate": 3.7093820850838268e-06, + "loss": 0.49079978, + "num_input_tokens_seen": 195700545, + "router_z_loss_clip": 0.1796875, + "router_z_loss_mlp": 0.00872803, + "step": 6842, + "time_per_iteration": 3.082956075668335 + }, + { + "auxiliary_loss_clip": 0.011275, + "auxiliary_loss_mlp": 0.01042079, + "balance_loss_clip": 1.05152273, + "balance_loss_mlp": 1.02532971, + "epoch": 0.1985665370553073, + "flos": 26351716498560.0, + "grad_norm": 2.212468672344907, + "language_loss": 0.85159367, + "learning_rate": 3.7092844996376362e-06, + "loss": 0.87328947, + "num_input_tokens_seen": 195715330, + "router_z_loss_clip": 0.75976562, + "router_z_loss_mlp": 0.16760254, + "step": 6843, + "time_per_iteration": 2.5901520252227783 + }, + { + "auxiliary_loss_clip": 0.01117569, + "auxiliary_loss_mlp": 0.01039321, + "balance_loss_clip": 1.04873538, + "balance_loss_mlp": 1.02528405, + "epoch": 0.19859555452382335, + "flos": 46420106693760.0, + "grad_norm": 1.4301221359695817, + "language_loss": 0.75466383, + "learning_rate": 3.7091868990943544e-06, + "loss": 0.77623272, + "num_input_tokens_seen": 195741565, + "router_z_loss_clip": 0.68896484, + "router_z_loss_mlp": 0.14038086, + "step": 6844, + "time_per_iteration": 2.8512706756591797 + }, + { + "auxiliary_loss_clip": 0.01028207, + "auxiliary_loss_mlp": 0.00998892, + "balance_loss_clip": 1.0104351, + "balance_loss_mlp": 0.9980818, + "epoch": 0.19862457199233938, + "flos": 74782105712640.0, + "grad_norm": 0.5933146058017206, + "language_loss": 0.40948012, + "learning_rate": 3.7090892834548414e-06, + "loss": 0.42975113, + "num_input_tokens_seen": 195806420, + "router_z_loss_clip": 0.17773438, + "router_z_loss_mlp": 0.00811768, + "step": 6845, + "time_per_iteration": 3.1863391399383545 + }, + { + "auxiliary_loss_clip": 0.01123116, + "auxiliary_loss_mlp": 0.01046674, + "balance_loss_clip": 1.05315745, + "balance_loss_mlp": 1.03249979, + "epoch": 0.19865358946085543, + "flos": 24200272506240.0, + "grad_norm": 1.7778536132203098, + "language_loss": 0.6943531, + "learning_rate": 3.70899165271996e-06, + "loss": 0.71605092, + "num_input_tokens_seen": 195820200, + "router_z_loss_clip": 0.70019531, + "router_z_loss_mlp": 0.14172363, + "step": 6846, + "time_per_iteration": 2.5730748176574707 + }, + { + "auxiliary_loss_clip": 0.01125005, + "auxiliary_loss_mlp": 0.01058615, + "balance_loss_clip": 1.05357158, + "balance_loss_mlp": 1.04357123, + "epoch": 0.19868260692937148, + "flos": 21645264234240.0, + "grad_norm": 2.084778853799352, + "language_loss": 0.70092487, + "learning_rate": 3.7088940068905732e-06, + "loss": 0.72276103, + "num_input_tokens_seen": 195834175, + "router_z_loss_clip": 0.71533203, + "router_z_loss_mlp": 0.15039062, + "step": 6847, + "time_per_iteration": 2.5734949111938477 + }, + { + "auxiliary_loss_clip": 0.01121881, + "auxiliary_loss_mlp": 0.01046251, + "balance_loss_clip": 1.05215955, + "balance_loss_mlp": 1.03270888, + "epoch": 0.19871162439788753, + "flos": 31535960497920.0, + "grad_norm": 2.5627368037372693, + "language_loss": 0.82938087, + "learning_rate": 3.708796345967543e-06, + "loss": 0.85106218, + "num_input_tokens_seen": 195850265, + "router_z_loss_clip": 0.69677734, + "router_z_loss_mlp": 0.13543701, + "step": 6848, + "time_per_iteration": 2.6064469814300537 + }, + { + "auxiliary_loss_clip": 0.01126265, + "auxiliary_loss_mlp": 0.01052728, + "balance_loss_clip": 1.05612803, + "balance_loss_mlp": 1.03833365, + "epoch": 0.19874064186640358, + "flos": 38975860823040.0, + "grad_norm": 2.3423044629989405, + "language_loss": 0.80577743, + "learning_rate": 3.708698669951732e-06, + "loss": 0.82756734, + "num_input_tokens_seen": 195872100, + "router_z_loss_clip": 0.70166016, + "router_z_loss_mlp": 0.1439209, + "step": 6849, + "time_per_iteration": 2.749620199203491 + }, + { + "auxiliary_loss_clip": 0.01118719, + "auxiliary_loss_mlp": 0.01038346, + "balance_loss_clip": 1.05034971, + "balance_loss_mlp": 1.02532804, + "epoch": 0.19876965933491964, + "flos": 15078711191040.0, + "grad_norm": 2.4120172884688786, + "language_loss": 0.88456076, + "learning_rate": 3.7086009788440026e-06, + "loss": 0.90613139, + "num_input_tokens_seen": 195886955, + "router_z_loss_clip": 0.68505859, + "router_z_loss_mlp": 0.13006592, + "step": 6850, + "time_per_iteration": 2.4857068061828613 + }, + { + "auxiliary_loss_clip": 0.01117464, + "auxiliary_loss_mlp": 0.01041448, + "balance_loss_clip": 1.05174184, + "balance_loss_mlp": 1.02963459, + "epoch": 0.19879867680343566, + "flos": 14641283364480.0, + "grad_norm": 2.6757920580103525, + "language_loss": 0.6901114, + "learning_rate": 3.7085032726452186e-06, + "loss": 0.71170056, + "num_input_tokens_seen": 195900725, + "router_z_loss_clip": 0.65771484, + "router_z_loss_mlp": 0.11816406, + "step": 6851, + "time_per_iteration": 2.51000714302063 + }, + { + "auxiliary_loss_clip": 0.0103273, + "auxiliary_loss_mlp": 0.01031085, + "balance_loss_clip": 1.01454735, + "balance_loss_mlp": 1.03026223, + "epoch": 0.1988276942719517, + "flos": 66574291501440.0, + "grad_norm": 0.6320447411708618, + "language_loss": 0.49335063, + "learning_rate": 3.7084055513562424e-06, + "loss": 0.51398873, + "num_input_tokens_seen": 195967160, + "router_z_loss_clip": 0.18164062, + "router_z_loss_mlp": 0.00823975, + "step": 6852, + "time_per_iteration": 3.1730453968048096 + }, + { + "auxiliary_loss_clip": 0.01118018, + "auxiliary_loss_mlp": 0.0104468, + "balance_loss_clip": 1.04964638, + "balance_loss_mlp": 1.03138196, + "epoch": 0.19885671174046776, + "flos": 19748175425280.0, + "grad_norm": 2.519740778800795, + "language_loss": 0.93024278, + "learning_rate": 3.7083078149779363e-06, + "loss": 0.95186985, + "num_input_tokens_seen": 195979925, + "router_z_loss_clip": 0.68408203, + "router_z_loss_mlp": 0.13293457, + "step": 6853, + "time_per_iteration": 2.5185108184814453 + }, + { + "auxiliary_loss_clip": 0.0111959, + "auxiliary_loss_mlp": 0.01034082, + "balance_loss_clip": 1.04997039, + "balance_loss_mlp": 1.02003336, + "epoch": 0.1988857292089838, + "flos": 35147101386240.0, + "grad_norm": 2.3833226861216894, + "language_loss": 0.78605258, + "learning_rate": 3.7082100635111646e-06, + "loss": 0.80758929, + "num_input_tokens_seen": 195999745, + "router_z_loss_clip": 0.69628906, + "router_z_loss_mlp": 0.14050293, + "step": 6854, + "time_per_iteration": 2.6455795764923096 + }, + { + "auxiliary_loss_clip": 0.0113184, + "auxiliary_loss_mlp": 0.01047676, + "balance_loss_clip": 1.05528057, + "balance_loss_mlp": 1.03281617, + "epoch": 0.19891474667749987, + "flos": 16793373801600.0, + "grad_norm": 2.339205467046479, + "language_loss": 0.80605721, + "learning_rate": 3.7081122969567905e-06, + "loss": 0.82785243, + "num_input_tokens_seen": 196012065, + "router_z_loss_clip": 0.765625, + "router_z_loss_mlp": 0.14855957, + "step": 6855, + "time_per_iteration": 2.5504841804504395 + }, + { + "auxiliary_loss_clip": 0.01128836, + "auxiliary_loss_mlp": 0.01043578, + "balance_loss_clip": 1.0512023, + "balance_loss_mlp": 1.02685308, + "epoch": 0.1989437641460159, + "flos": 30802664344320.0, + "grad_norm": 4.416563193876334, + "language_loss": 0.83199745, + "learning_rate": 3.7080145153156775e-06, + "loss": 0.85372162, + "num_input_tokens_seen": 196027310, + "router_z_loss_clip": 0.77636719, + "router_z_loss_mlp": 0.16741943, + "step": 6856, + "time_per_iteration": 2.5435283184051514 + }, + { + "auxiliary_loss_clip": 0.01117217, + "auxiliary_loss_mlp": 0.01032387, + "balance_loss_clip": 1.0511862, + "balance_loss_mlp": 1.01833868, + "epoch": 0.19897278161453194, + "flos": 12525462685440.0, + "grad_norm": 2.179359409332894, + "language_loss": 0.73750108, + "learning_rate": 3.7079167185886887e-06, + "loss": 0.75899714, + "num_input_tokens_seen": 196040600, + "router_z_loss_clip": 0.66064453, + "router_z_loss_mlp": 0.14056396, + "step": 6857, + "time_per_iteration": 2.5276286602020264 + }, + { + "auxiliary_loss_clip": 0.01032126, + "auxiliary_loss_mlp": 0.01006044, + "balance_loss_clip": 1.01402283, + "balance_loss_mlp": 1.0051856, + "epoch": 0.199001799083048, + "flos": 62081363635200.0, + "grad_norm": 0.6857975091536945, + "language_loss": 0.47908416, + "learning_rate": 3.7078189067766886e-06, + "loss": 0.49946585, + "num_input_tokens_seen": 196102675, + "router_z_loss_clip": 0.18164062, + "router_z_loss_mlp": 0.00860596, + "step": 6858, + "time_per_iteration": 3.0886905193328857 + }, + { + "auxiliary_loss_clip": 0.01117575, + "auxiliary_loss_mlp": 0.0102678, + "balance_loss_clip": 1.05089223, + "balance_loss_mlp": 1.01407838, + "epoch": 0.19903081655156404, + "flos": 22527984965760.0, + "grad_norm": 1.8761565939492841, + "language_loss": 0.71197224, + "learning_rate": 3.7077210798805403e-06, + "loss": 0.73341584, + "num_input_tokens_seen": 196119545, + "router_z_loss_clip": 0.66699219, + "router_z_loss_mlp": 0.12695312, + "step": 6859, + "time_per_iteration": 2.53776478767395 + }, + { + "auxiliary_loss_clip": 0.01032019, + "auxiliary_loss_mlp": 0.01008502, + "balance_loss_clip": 1.01397896, + "balance_loss_mlp": 1.0076077, + "epoch": 0.1990598340200801, + "flos": 66377823085440.0, + "grad_norm": 0.6431150870010771, + "language_loss": 0.46671075, + "learning_rate": 3.707623237901109e-06, + "loss": 0.48711595, + "num_input_tokens_seen": 196180960, + "router_z_loss_clip": 0.1796875, + "router_z_loss_mlp": 0.00891113, + "step": 6860, + "time_per_iteration": 3.075911521911621 + }, + { + "auxiliary_loss_clip": 0.01129236, + "auxiliary_loss_mlp": 0.01040723, + "balance_loss_clip": 1.0547626, + "balance_loss_mlp": 1.02651334, + "epoch": 0.19908885148859615, + "flos": 16210831029120.0, + "grad_norm": 1.8780132703234702, + "language_loss": 0.68855965, + "learning_rate": 3.7075253808392583e-06, + "loss": 0.7102592, + "num_input_tokens_seen": 196196625, + "router_z_loss_clip": 0.74511719, + "router_z_loss_mlp": 0.14202881, + "step": 6861, + "time_per_iteration": 2.5001001358032227 + }, + { + "auxiliary_loss_clip": 0.01029481, + "auxiliary_loss_mlp": 0.01003173, + "balance_loss_clip": 1.01142311, + "balance_loss_mlp": 1.00227916, + "epoch": 0.19911786895711217, + "flos": 74784511923840.0, + "grad_norm": 0.6217657095153937, + "language_loss": 0.44574076, + "learning_rate": 3.7074275086958525e-06, + "loss": 0.46606731, + "num_input_tokens_seen": 196268375, + "router_z_loss_clip": 0.18066406, + "router_z_loss_mlp": 0.00891113, + "step": 6862, + "time_per_iteration": 3.326315402984619 + }, + { + "auxiliary_loss_clip": 0.01131956, + "auxiliary_loss_mlp": 0.01041282, + "balance_loss_clip": 1.05496895, + "balance_loss_mlp": 1.02615476, + "epoch": 0.19914688642562822, + "flos": 25113301338240.0, + "grad_norm": 6.400797639250856, + "language_loss": 0.7886802, + "learning_rate": 3.7073296214717557e-06, + "loss": 0.81041253, + "num_input_tokens_seen": 196282655, + "router_z_loss_clip": 0.77050781, + "router_z_loss_mlp": 0.15136719, + "step": 6863, + "time_per_iteration": 2.5519521236419678 + }, + { + "auxiliary_loss_clip": 0.01120407, + "auxiliary_loss_mlp": 0.01038394, + "balance_loss_clip": 1.05071902, + "balance_loss_mlp": 1.02377915, + "epoch": 0.19917590389414427, + "flos": 33213204115200.0, + "grad_norm": 2.335453887133759, + "language_loss": 0.79345858, + "learning_rate": 3.707231719167833e-06, + "loss": 0.81504661, + "num_input_tokens_seen": 196302055, + "router_z_loss_clip": 0.69775391, + "router_z_loss_mlp": 0.14624023, + "step": 6864, + "time_per_iteration": 2.632605791091919 + }, + { + "auxiliary_loss_clip": 0.01128644, + "auxiliary_loss_mlp": 0.010349, + "balance_loss_clip": 1.05275059, + "balance_loss_mlp": 1.01958728, + "epoch": 0.19920492136266033, + "flos": 21537137318400.0, + "grad_norm": 2.265414709946401, + "language_loss": 0.84801257, + "learning_rate": 3.707133801784949e-06, + "loss": 0.86964798, + "num_input_tokens_seen": 196317145, + "router_z_loss_clip": 0.75830078, + "router_z_loss_mlp": 0.15325928, + "step": 6865, + "time_per_iteration": 2.532224178314209 + }, + { + "auxiliary_loss_clip": 0.01028211, + "auxiliary_loss_mlp": 0.01006113, + "balance_loss_clip": 1.01026499, + "balance_loss_mlp": 1.00514126, + "epoch": 0.19923393883117638, + "flos": 74772013991040.0, + "grad_norm": 0.6208591640742692, + "language_loss": 0.49493703, + "learning_rate": 3.7070358693239683e-06, + "loss": 0.51528031, + "num_input_tokens_seen": 196381050, + "router_z_loss_clip": 0.1796875, + "router_z_loss_mlp": 0.00970459, + "step": 6866, + "time_per_iteration": 3.1178362369537354 + }, + { + "auxiliary_loss_clip": 0.01119711, + "auxiliary_loss_mlp": 0.0103835, + "balance_loss_clip": 1.05024827, + "balance_loss_mlp": 1.02473044, + "epoch": 0.19926295629969243, + "flos": 11869374816000.0, + "grad_norm": 2.3021557044024945, + "language_loss": 0.8915695, + "learning_rate": 3.706937921785756e-06, + "loss": 0.91315007, + "num_input_tokens_seen": 196393040, + "router_z_loss_clip": 0.69482422, + "router_z_loss_mlp": 0.13623047, + "step": 6867, + "time_per_iteration": 2.521413803100586 + }, + { + "auxiliary_loss_clip": 0.01124135, + "auxiliary_loss_mlp": 0.01042517, + "balance_loss_clip": 1.05446804, + "balance_loss_mlp": 1.02682924, + "epoch": 0.19929197376820845, + "flos": 11687702803200.0, + "grad_norm": 2.442968367171773, + "language_loss": 0.69474161, + "learning_rate": 3.7068399591711773e-06, + "loss": 0.71640813, + "num_input_tokens_seen": 196407755, + "router_z_loss_clip": 0.69677734, + "router_z_loss_mlp": 0.15679932, + "step": 6868, + "time_per_iteration": 2.524203062057495 + }, + { + "auxiliary_loss_clip": 0.01027822, + "auxiliary_loss_mlp": 0.01010845, + "balance_loss_clip": 1.00991273, + "balance_loss_mlp": 1.00991559, + "epoch": 0.1993209912367245, + "flos": 61606409074560.0, + "grad_norm": 0.6817412897133359, + "language_loss": 0.46002069, + "learning_rate": 3.706741981481097e-06, + "loss": 0.48040739, + "num_input_tokens_seen": 196468430, + "router_z_loss_clip": 0.1796875, + "router_z_loss_mlp": 0.00927734, + "step": 6869, + "time_per_iteration": 3.051684856414795 + }, + { + "auxiliary_loss_clip": 0.01120198, + "auxiliary_loss_mlp": 0.01029304, + "balance_loss_clip": 1.05069947, + "balance_loss_mlp": 1.0162921, + "epoch": 0.19935000870524056, + "flos": 12851531372160.0, + "grad_norm": 3.069842504729078, + "language_loss": 0.77934456, + "learning_rate": 3.7066439887163816e-06, + "loss": 0.80083954, + "num_input_tokens_seen": 196480225, + "router_z_loss_clip": 0.69482422, + "router_z_loss_mlp": 0.13024902, + "step": 6870, + "time_per_iteration": 2.525092840194702 + }, + { + "auxiliary_loss_clip": 0.01028143, + "auxiliary_loss_mlp": 0.0100564, + "balance_loss_clip": 1.01023436, + "balance_loss_mlp": 1.00481117, + "epoch": 0.1993790261737566, + "flos": 74770146483840.0, + "grad_norm": 0.629066218252479, + "language_loss": 0.45148537, + "learning_rate": 3.7065459808778954e-06, + "loss": 0.47182322, + "num_input_tokens_seen": 196542390, + "router_z_loss_clip": 0.1796875, + "router_z_loss_mlp": 0.00830078, + "step": 6871, + "time_per_iteration": 3.1883726119995117 + }, + { + "auxiliary_loss_clip": 0.0112593, + "auxiliary_loss_mlp": 0.01040631, + "balance_loss_clip": 1.052495, + "balance_loss_mlp": 1.02475798, + "epoch": 0.19940804364227266, + "flos": 16393149486720.0, + "grad_norm": 3.3941239523773454, + "language_loss": 0.92889845, + "learning_rate": 3.706447957966505e-06, + "loss": 0.95056403, + "num_input_tokens_seen": 196555315, + "router_z_loss_clip": 0.734375, + "router_z_loss_mlp": 0.15856934, + "step": 6872, + "time_per_iteration": 2.484644889831543 + }, + { + "auxiliary_loss_clip": 0.01027743, + "auxiliary_loss_mlp": 0.01001127, + "balance_loss_clip": 1.00985289, + "balance_loss_mlp": 1.00024462, + "epoch": 0.19943706111078868, + "flos": 74778873488640.0, + "grad_norm": 0.672110637189409, + "language_loss": 0.49353358, + "learning_rate": 3.7063499199830752e-06, + "loss": 0.51382232, + "num_input_tokens_seen": 196620950, + "router_z_loss_clip": 0.17871094, + "router_z_loss_mlp": 0.0088501, + "step": 6873, + "time_per_iteration": 3.1843409538269043 + }, + { + "auxiliary_loss_clip": 0.01116912, + "auxiliary_loss_mlp": 0.01034828, + "balance_loss_clip": 1.0486095, + "balance_loss_mlp": 1.02127945, + "epoch": 0.19946607857930473, + "flos": 16178044890240.0, + "grad_norm": 2.3131575991765865, + "language_loss": 0.87620467, + "learning_rate": 3.7062518669284727e-06, + "loss": 0.89772213, + "num_input_tokens_seen": 196633935, + "router_z_loss_clip": 0.68359375, + "router_z_loss_mlp": 0.13549805, + "step": 6874, + "time_per_iteration": 2.4979264736175537 + }, + { + "auxiliary_loss_clip": 0.01124528, + "auxiliary_loss_mlp": 0.01041275, + "balance_loss_clip": 1.05114508, + "balance_loss_mlp": 1.02485967, + "epoch": 0.19949509604782079, + "flos": 19273544087040.0, + "grad_norm": 1.945624124545716, + "language_loss": 0.6882031, + "learning_rate": 3.7061537988035633e-06, + "loss": 0.70986116, + "num_input_tokens_seen": 196647365, + "router_z_loss_clip": 0.73339844, + "router_z_loss_mlp": 0.16418457, + "step": 6875, + "time_per_iteration": 2.5488641262054443 + }, + { + "auxiliary_loss_clip": 0.01119076, + "auxiliary_loss_mlp": 0.0103796, + "balance_loss_clip": 1.05041862, + "balance_loss_mlp": 1.02388167, + "epoch": 0.19952411351633684, + "flos": 20951685544320.0, + "grad_norm": 2.3635157692215056, + "language_loss": 0.68007886, + "learning_rate": 3.7060557156092127e-06, + "loss": 0.70164925, + "num_input_tokens_seen": 196661675, + "router_z_loss_clip": 0.68603516, + "router_z_loss_mlp": 0.14086914, + "step": 6876, + "time_per_iteration": 2.474482774734497 + }, + { + "auxiliary_loss_clip": 0.01122974, + "auxiliary_loss_mlp": 0.01037187, + "balance_loss_clip": 1.05043554, + "balance_loss_mlp": 1.02215433, + "epoch": 0.1995531309848529, + "flos": 32773980608640.0, + "grad_norm": 1.9601198044346144, + "language_loss": 0.82834268, + "learning_rate": 3.7059576173462883e-06, + "loss": 0.84994435, + "num_input_tokens_seen": 196680805, + "router_z_loss_clip": 0.72509766, + "router_z_loss_mlp": 0.15020752, + "step": 6877, + "time_per_iteration": 2.60662841796875 + }, + { + "auxiliary_loss_clip": 0.01116566, + "auxiliary_loss_mlp": 0.01026583, + "balance_loss_clip": 1.04666233, + "balance_loss_mlp": 1.01323199, + "epoch": 0.19958214845336894, + "flos": 27082714181760.0, + "grad_norm": 2.4571645173214205, + "language_loss": 0.81592, + "learning_rate": 3.705859504015655e-06, + "loss": 0.83735144, + "num_input_tokens_seen": 196694730, + "router_z_loss_clip": 0.69921875, + "router_z_loss_mlp": 0.13366699, + "step": 6878, + "time_per_iteration": 2.56426739692688 + }, + { + "auxiliary_loss_clip": 0.01121646, + "auxiliary_loss_mlp": 0.01037874, + "balance_loss_clip": 1.05191147, + "balance_loss_mlp": 1.02469516, + "epoch": 0.19961116592188496, + "flos": 24784144081920.0, + "grad_norm": 2.2372637581161077, + "language_loss": 0.82262123, + "learning_rate": 3.705761375618181e-06, + "loss": 0.84421641, + "num_input_tokens_seen": 196709120, + "router_z_loss_clip": 0.69677734, + "router_z_loss_mlp": 0.13183594, + "step": 6879, + "time_per_iteration": 2.4870686531066895 + }, + { + "auxiliary_loss_clip": 0.01028846, + "auxiliary_loss_mlp": 0.01008406, + "balance_loss_clip": 1.01098514, + "balance_loss_mlp": 1.00754762, + "epoch": 0.19964018339040102, + "flos": 60535695536640.0, + "grad_norm": 0.7321102571311181, + "language_loss": 0.51281232, + "learning_rate": 3.705663232154732e-06, + "loss": 0.53318483, + "num_input_tokens_seen": 196762055, + "router_z_loss_clip": 0.17871094, + "router_z_loss_mlp": 0.00860596, + "step": 6880, + "time_per_iteration": 2.9408016204833984 + }, + { + "auxiliary_loss_clip": 0.01028052, + "auxiliary_loss_mlp": 0.01003277, + "balance_loss_clip": 1.01027119, + "balance_loss_mlp": 1.00237703, + "epoch": 0.19966920085891707, + "flos": 74785948467840.0, + "grad_norm": 0.7032553740138019, + "language_loss": 0.50722617, + "learning_rate": 3.705565073626176e-06, + "loss": 0.52753949, + "num_input_tokens_seen": 196831425, + "router_z_loss_clip": 0.17773438, + "router_z_loss_mlp": 0.00897217, + "step": 6881, + "time_per_iteration": 3.2271997928619385 + }, + { + "auxiliary_loss_clip": 0.01132765, + "auxiliary_loss_mlp": 0.0104443, + "balance_loss_clip": 1.05370259, + "balance_loss_mlp": 1.0272814, + "epoch": 0.19969821832743312, + "flos": 25403315748480.0, + "grad_norm": 2.663904323015062, + "language_loss": 0.90090853, + "learning_rate": 3.705466900033378e-06, + "loss": 0.9226805, + "num_input_tokens_seen": 196848300, + "router_z_loss_clip": 0.78955078, + "router_z_loss_mlp": 0.17163086, + "step": 6882, + "time_per_iteration": 2.5955026149749756 + }, + { + "auxiliary_loss_clip": 0.01118118, + "auxiliary_loss_mlp": 0.01034892, + "balance_loss_clip": 1.05074131, + "balance_loss_mlp": 1.02129579, + "epoch": 0.19972723579594917, + "flos": 49995301046400.0, + "grad_norm": 1.7103476413592844, + "language_loss": 0.59182167, + "learning_rate": 3.705368711377207e-06, + "loss": 0.61335182, + "num_input_tokens_seen": 196868800, + "router_z_loss_clip": 0.67382812, + "router_z_loss_mlp": 0.13586426, + "step": 6883, + "time_per_iteration": 2.674304485321045 + }, + { + "auxiliary_loss_clip": 0.01028264, + "auxiliary_loss_mlp": 0.01006322, + "balance_loss_clip": 1.01036406, + "balance_loss_mlp": 1.00545144, + "epoch": 0.1997562532644652, + "flos": 74782859898240.0, + "grad_norm": 0.6231312028973535, + "language_loss": 0.47505969, + "learning_rate": 3.7052705076585285e-06, + "loss": 0.49540555, + "num_input_tokens_seen": 196938530, + "router_z_loss_clip": 0.1796875, + "router_z_loss_mlp": 0.00872803, + "step": 6884, + "time_per_iteration": 3.202134847640991 + }, + { + "auxiliary_loss_clip": 0.01118241, + "auxiliary_loss_mlp": 0.01036911, + "balance_loss_clip": 1.04930449, + "balance_loss_mlp": 1.02239728, + "epoch": 0.19978527073298125, + "flos": 23653855837440.0, + "grad_norm": 2.070706354743774, + "language_loss": 0.80379736, + "learning_rate": 3.705172288878211e-06, + "loss": 0.82534891, + "num_input_tokens_seen": 196951335, + "router_z_loss_clip": 0.68994141, + "router_z_loss_mlp": 0.1451416, + "step": 6885, + "time_per_iteration": 2.541602849960327 + }, + { + "auxiliary_loss_clip": 0.01116014, + "auxiliary_loss_mlp": 0.01043869, + "balance_loss_clip": 1.04907346, + "balance_loss_mlp": 1.03069043, + "epoch": 0.1998142882014973, + "flos": 16028081608320.0, + "grad_norm": 2.182036678690645, + "language_loss": 0.70960855, + "learning_rate": 3.705074055037122e-06, + "loss": 0.73120737, + "num_input_tokens_seen": 196965700, + "router_z_loss_clip": 0.66943359, + "router_z_loss_mlp": 0.13183594, + "step": 6886, + "time_per_iteration": 2.5388832092285156 + }, + { + "auxiliary_loss_clip": 0.01118489, + "auxiliary_loss_mlp": 0.01036221, + "balance_loss_clip": 1.04931617, + "balance_loss_mlp": 1.0220232, + "epoch": 0.19984330567001335, + "flos": 16104356138880.0, + "grad_norm": 2.0903664523002803, + "language_loss": 0.77718467, + "learning_rate": 3.7049758061361294e-06, + "loss": 0.7987318, + "num_input_tokens_seen": 196979550, + "router_z_loss_clip": 0.69140625, + "router_z_loss_mlp": 0.14196777, + "step": 6887, + "time_per_iteration": 2.4906630516052246 + }, + { + "auxiliary_loss_clip": 0.01121739, + "auxiliary_loss_mlp": 0.01045705, + "balance_loss_clip": 1.05089486, + "balance_loss_mlp": 1.0305531, + "epoch": 0.1998723231385294, + "flos": 37298365810560.0, + "grad_norm": 1.86711771396805, + "language_loss": 1.00894332, + "learning_rate": 3.7048775421761006e-06, + "loss": 1.03061783, + "num_input_tokens_seen": 196999370, + "router_z_loss_clip": 0.70849609, + "router_z_loss_mlp": 0.15136719, + "step": 6888, + "time_per_iteration": 2.6514968872070312 + }, + { + "auxiliary_loss_clip": 0.01117522, + "auxiliary_loss_mlp": 0.0103675, + "balance_loss_clip": 1.04845989, + "balance_loss_mlp": 1.02292776, + "epoch": 0.19990134060704545, + "flos": 11614157706240.0, + "grad_norm": 2.727965577766237, + "language_loss": 0.61779892, + "learning_rate": 3.7047792631579025e-06, + "loss": 0.63934171, + "num_input_tokens_seen": 197011485, + "router_z_loss_clip": 0.68994141, + "router_z_loss_mlp": 0.13830566, + "step": 6889, + "time_per_iteration": 2.4934000968933105 + }, + { + "auxiliary_loss_clip": 0.0112181, + "auxiliary_loss_mlp": 0.010339, + "balance_loss_clip": 1.04944849, + "balance_loss_mlp": 1.01882625, + "epoch": 0.19993035807556148, + "flos": 23657878160640.0, + "grad_norm": 2.0716433381665307, + "language_loss": 0.92689133, + "learning_rate": 3.704680969082405e-06, + "loss": 0.94844842, + "num_input_tokens_seen": 197031645, + "router_z_loss_clip": 0.72363281, + "router_z_loss_mlp": 0.15063477, + "step": 6890, + "time_per_iteration": 2.585301160812378 + }, + { + "auxiliary_loss_clip": 0.01118238, + "auxiliary_loss_mlp": 0.01031237, + "balance_loss_clip": 1.04870474, + "balance_loss_mlp": 1.01756907, + "epoch": 0.19995937554407753, + "flos": 20921413357440.0, + "grad_norm": 1.9871962024089405, + "language_loss": 0.72035784, + "learning_rate": 3.704582659950475e-06, + "loss": 0.74185264, + "num_input_tokens_seen": 197045930, + "router_z_loss_clip": 0.69628906, + "router_z_loss_mlp": 0.13677979, + "step": 6891, + "time_per_iteration": 2.485797643661499 + }, + { + "auxiliary_loss_clip": 0.01114521, + "auxiliary_loss_mlp": 0.010407, + "balance_loss_clip": 1.04875481, + "balance_loss_mlp": 1.0267998, + "epoch": 0.19998839301259358, + "flos": 20952906606720.0, + "grad_norm": 2.474467183889748, + "language_loss": 0.96891773, + "learning_rate": 3.7044843357629818e-06, + "loss": 0.99046993, + "num_input_tokens_seen": 197060565, + "router_z_loss_clip": 0.65820312, + "router_z_loss_mlp": 0.13909912, + "step": 6892, + "time_per_iteration": 2.546567440032959 + } + ], + "logging_steps": 1.0, + "max_steps": 34462, + "num_input_tokens_seen": 197060565, + "num_train_epochs": 1, + "save_steps": 6892, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 8.289085393144381e+17, + "train_batch_size": 5, + "trial_name": null, + "trial_params": null +} diff --git a/sft/1M3/Full_smoe/checkpoint-6893/training_args.bin b/sft/1M3/Full_smoe/checkpoint-6893/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..215478b0c8536bd36cbea6c4bd17520030a7a4c6 --- /dev/null +++ b/sft/1M3/Full_smoe/checkpoint-6893/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00db8a51acb988c307aaf3152f0c74973b86bafd34390fe19b81d50de1db9354 +size 7992 diff --git a/sft/1M3/Full_smoe/checkpoint-6893/zero_to_fp32.py b/sft/1M3/Full_smoe/checkpoint-6893/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..24cc342e78d1a006c782b3a4cd68d9ce786d8fd8 --- /dev/null +++ b/sft/1M3/Full_smoe/checkpoint-6893/zero_to_fp32.py @@ -0,0 +1,604 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: python zero_to_fp32.py . pytorch_model.bin + +import argparse +import torch +import glob +import math +import os +import re +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + + total_files = len(files) + state_dicts = [] + for f in files: + state_dict = torch.load(f, map_location=device) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + if zero_stage <= 2: + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + elif zero_stage == 3: + # if there is more than one param group, there will be multiple flattened tensors - one + # flattened tensor per group - for simplicity merge them into a single tensor + # + # XXX: could make the script more memory efficient for when there are multiple groups - it + # will require matching the sub-lists of param_shapes for each param group flattened tensor + + fp32_flat_groups = [ + torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts)) + ] + + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = fp32_flat_groups[0].numel() * world_size + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + for name, shape in param_shapes.items(): + + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # XXX: memory usage doubles here + state_dict[name] = torch.cat( + tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)), + 0).narrow(0, 0, unpartitioned_numel).view(shape) + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + + Returns: + - pytorch ``state_dict`` + + Note: this approach may not work if your application doesn't have sufficient free CPU memory and + you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters) + print(f"Saving fp32 state dict to {output_file}") + torch.save(state_dict, output_file) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info(f"Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info(f"Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument( + "output_file", + type=str, + help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_file, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters)