diff --git a/.gitattributes b/.gitattributes index 67be2332e285f77b3356405c2bed973479f7955b..24b859fd9bb66abeac8055a091024043937e7f64 100644 --- a/.gitattributes +++ b/.gitattributes @@ -720,3 +720,12 @@ checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/artifacts/models--meta checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/artifacts/models--meta-llama--Llama-3.2-3B/blobs/4719a04514ec2f060240711b7c33ab21187cac730ecaba3040b7a0fd95a9cefb filter=lfs diff=lfs merge=lfs -text checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-900/tokenizer.json filter=lfs diff=lfs merge=lfs -text checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-100/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-100/generation_config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-100/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9c389b841a9a29ddd904e02d1eb0e08dcce82ad9 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-100/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": 128001, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.45.1" +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-100/model.safetensors.index.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-100/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..ed64de846d720b9a7859dc20575fea8e8ca51940 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-100/model.safetensors.index.json @@ -0,0 +1,262 @@ +{ + "metadata": { + "total_size": 7213504512 + }, + "weight_map": { + "lm_head.weight": "model-00002-of-00002.safetensors", + "model.embed_tokens.weight": "model-00001-of-00002.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.norm.weight": "model-00002-of-00002.safetensors" + } +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-100/special_tokens_map.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-100/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..6949c5975ee0e961ef61cf31010dce04df0a03f8 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-100/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "[PAD]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-100/training_args.bin b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-100/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..6cdf93817e8f6d23b60d79888b226cc66ed2e88b --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-100/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a50e4918f3c49370326e14a7ed2f73b48b05e521f4b3399b53438fac6abee926 +size 6011 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fb0f9a53a507ffcdb60410deb71d60ef801bf350 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/config.json @@ -0,0 +1,36 @@ +{ + "_name_or_path": "meta-llama/Llama-3.2-3B", + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "eos_token_id": 128001, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 3072, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 24, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "torch_dtype": "float16", + "transformers_version": "4.45.1", + "use_cache": true, + "vocab_size": 128256 +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/generation_config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9c389b841a9a29ddd904e02d1eb0e08dcce82ad9 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": 128001, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.45.1" +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/latest b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/latest new file mode 100644 index 0000000000000000000000000000000000000000..22cd5c3402316b70299aed2025d7943595f5d495 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/latest @@ -0,0 +1 @@ +global_step1100 \ No newline at end of file diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/model.safetensors.index.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..ed64de846d720b9a7859dc20575fea8e8ca51940 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/model.safetensors.index.json @@ -0,0 +1,262 @@ +{ + "metadata": { + "total_size": 7213504512 + }, + "weight_map": { + "lm_head.weight": "model-00002-of-00002.safetensors", + "model.embed_tokens.weight": "model-00001-of-00002.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.norm.weight": "model-00002-of-00002.safetensors" + } +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/rng_state_0.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..fad444f8143e1badd2a8d15a75e619abd485ceab --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:186b472c4073bfc086d5b4584aeaefc23f9c40f1a8199963bda08775617e1e12 +size 16567 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/rng_state_1.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..39b6620f3f8d750109fe44053f80f2e221c6e290 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8957d748b592f49a56723f1936535313c244cee56956dbc893c99d40f6ef84b +size 16567 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/rng_state_2.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..04d30c4b88bb88b6860f56816d3db2c6c7742594 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be2cd13083af0074c65198c01c90c0a3c599fbe6e0ea705a8835ca51f393d0c0 +size 16567 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/scheduler.pt b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..b0a59b3d69f7c73d7ad02040ef9ddf0af64846ba --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:500aa1e2983f7f455a3e8210712a5fb824056d0fb864457f35db87cedc4f15d7 +size 627 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/special_tokens_map.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..6949c5975ee0e961ef61cf31010dce04df0a03f8 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "[PAD]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/tokenizer.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f28ecaeab53ae07feed29ccf8624d2b0a8344df9 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19fb2e1e3cdd6f7433d89fd6d62c82042599dd4984f342efe7fec6e159e6a8f6 +size 17210734 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/tokenizer_config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8991b49e9c2a43fc527dab9e09ad8171f0cc5943 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/tokenizer_config.json @@ -0,0 +1,2086 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "[PAD]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128257": { + "content": "🅂", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "128258": { + "content": "🄿", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "[PAD]", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/trainer_state.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..d67cd44d8c5569e55d69d6a6ebc64bd2617c9820 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/trainer_state.json @@ -0,0 +1,8613 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.3690105787181084, + "eval_steps": 10, + "global_step": 1100, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0012445550715619166, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8368, + "step": 1 + }, + { + "epoch": 0.002489110143123833, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.86, + "step": 2 + }, + { + "epoch": 0.00373366521468575, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8265, + "step": 3 + }, + { + "epoch": 0.004978220286247666, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8271, + "step": 4 + }, + { + "epoch": 0.006222775357809583, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8768, + "step": 5 + }, + { + "epoch": 0.0074673304293715, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8402, + "step": 6 + }, + { + "epoch": 0.008711885500933417, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.785, + "step": 7 + }, + { + "epoch": 0.009956440572495333, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8466, + "step": 8 + }, + { + "epoch": 0.01120099564405725, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.7673, + "step": 9 + }, + { + "epoch": 0.012445550715619166, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8074, + "step": 10 + }, + { + "epoch": 0.012445550715619166, + "eval_loss": 2.8535053730010986, + "eval_runtime": 42.446, + "eval_samples_per_second": 23.559, + "eval_steps_per_second": 0.989, + "step": 10 + }, + { + "epoch": 0.013690105787181083, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8592, + "step": 11 + }, + { + "epoch": 0.014934660858743, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.804, + "step": 12 + }, + { + "epoch": 0.016179215930304917, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8076, + "step": 13 + }, + { + "epoch": 0.017423771001866834, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8219, + "step": 14 + }, + { + "epoch": 0.018668326073428748, + "grad_norm": 5.105710506439209, + "learning_rate": 2.0746887966804982e-08, + "loss": 2.7995, + "step": 15 + }, + { + "epoch": 0.019912881144990666, + "grad_norm": 5.156588077545166, + "learning_rate": 4.1493775933609963e-08, + "loss": 2.8577, + "step": 16 + }, + { + "epoch": 0.021157436216552583, + "grad_norm": 5.709373950958252, + "learning_rate": 6.224066390041494e-08, + "loss": 2.8433, + "step": 17 + }, + { + "epoch": 0.0224019912881145, + "grad_norm": 5.64658784866333, + "learning_rate": 8.298755186721993e-08, + "loss": 2.8428, + "step": 18 + }, + { + "epoch": 0.023646546359676415, + "grad_norm": 5.541937351226807, + "learning_rate": 1.037344398340249e-07, + "loss": 2.7785, + "step": 19 + }, + { + "epoch": 0.024891101431238332, + "grad_norm": 5.233374118804932, + "learning_rate": 1.2448132780082988e-07, + "loss": 2.8599, + "step": 20 + }, + { + "epoch": 0.024891101431238332, + "eval_loss": 2.8521182537078857, + "eval_runtime": 43.5664, + "eval_samples_per_second": 22.953, + "eval_steps_per_second": 0.964, + "step": 20 + }, + { + "epoch": 0.02613565650280025, + "grad_norm": 5.407598972320557, + "learning_rate": 1.4522821576763488e-07, + "loss": 2.8081, + "step": 21 + }, + { + "epoch": 0.027380211574362167, + "grad_norm": 5.0483317375183105, + "learning_rate": 1.6597510373443985e-07, + "loss": 2.8489, + "step": 22 + }, + { + "epoch": 0.02862476664592408, + "grad_norm": 4.985588550567627, + "learning_rate": 1.8672199170124483e-07, + "loss": 2.8375, + "step": 23 + }, + { + "epoch": 0.029869321717486, + "grad_norm": 5.201700687408447, + "learning_rate": 2.074688796680498e-07, + "loss": 2.8355, + "step": 24 + }, + { + "epoch": 0.031113876789047916, + "grad_norm": 5.009235382080078, + "learning_rate": 2.2821576763485478e-07, + "loss": 2.7774, + "step": 25 + }, + { + "epoch": 0.03235843186060983, + "grad_norm": 4.509210109710693, + "learning_rate": 2.4896265560165975e-07, + "loss": 2.7992, + "step": 26 + }, + { + "epoch": 0.03360298693217175, + "grad_norm": 4.529262065887451, + "learning_rate": 2.6970954356846476e-07, + "loss": 2.836, + "step": 27 + }, + { + "epoch": 0.03484754200373367, + "grad_norm": 4.526787281036377, + "learning_rate": 2.9045643153526976e-07, + "loss": 2.8128, + "step": 28 + }, + { + "epoch": 0.03609209707529558, + "grad_norm": 4.016609191894531, + "learning_rate": 3.112033195020747e-07, + "loss": 2.7947, + "step": 29 + }, + { + "epoch": 0.037336652146857496, + "grad_norm": 3.4784562587738037, + "learning_rate": 3.319502074688797e-07, + "loss": 2.7854, + "step": 30 + }, + { + "epoch": 0.037336652146857496, + "eval_loss": 2.8020100593566895, + "eval_runtime": 44.8464, + "eval_samples_per_second": 22.298, + "eval_steps_per_second": 0.937, + "step": 30 + }, + { + "epoch": 0.038581207218419414, + "grad_norm": 3.4887945652008057, + "learning_rate": 3.5269709543568466e-07, + "loss": 2.7908, + "step": 31 + }, + { + "epoch": 0.03982576228998133, + "grad_norm": 3.4533374309539795, + "learning_rate": 3.7344398340248966e-07, + "loss": 2.7358, + "step": 32 + }, + { + "epoch": 0.04107031736154325, + "grad_norm": 3.270954132080078, + "learning_rate": 3.941908713692946e-07, + "loss": 2.8013, + "step": 33 + }, + { + "epoch": 0.042314872433105166, + "grad_norm": 3.61322283744812, + "learning_rate": 4.149377593360996e-07, + "loss": 2.7521, + "step": 34 + }, + { + "epoch": 0.043559427504667084, + "grad_norm": 3.158719062805176, + "learning_rate": 4.3568464730290456e-07, + "loss": 2.7573, + "step": 35 + }, + { + "epoch": 0.044803982576229, + "grad_norm": 2.888746500015259, + "learning_rate": 4.5643153526970956e-07, + "loss": 2.7795, + "step": 36 + }, + { + "epoch": 0.04604853764779091, + "grad_norm": 3.022629499435425, + "learning_rate": 4.771784232365145e-07, + "loss": 2.7891, + "step": 37 + }, + { + "epoch": 0.04729309271935283, + "grad_norm": 3.038097381591797, + "learning_rate": 4.979253112033195e-07, + "loss": 2.7376, + "step": 38 + }, + { + "epoch": 0.04853764779091475, + "grad_norm": 2.9992239475250244, + "learning_rate": 5.186721991701245e-07, + "loss": 2.7279, + "step": 39 + }, + { + "epoch": 0.049782202862476664, + "grad_norm": 2.908842086791992, + "learning_rate": 5.394190871369295e-07, + "loss": 2.6582, + "step": 40 + }, + { + "epoch": 0.049782202862476664, + "eval_loss": 2.725968837738037, + "eval_runtime": 44.9498, + "eval_samples_per_second": 22.247, + "eval_steps_per_second": 0.934, + "step": 40 + }, + { + "epoch": 0.05102675793403858, + "grad_norm": 2.8025174140930176, + "learning_rate": 5.601659751037345e-07, + "loss": 2.6809, + "step": 41 + }, + { + "epoch": 0.0522713130056005, + "grad_norm": 2.7555699348449707, + "learning_rate": 5.809128630705395e-07, + "loss": 2.6954, + "step": 42 + }, + { + "epoch": 0.053515868077162417, + "grad_norm": 2.4732933044433594, + "learning_rate": 6.016597510373444e-07, + "loss": 2.7307, + "step": 43 + }, + { + "epoch": 0.054760423148724334, + "grad_norm": 2.451366424560547, + "learning_rate": 6.224066390041494e-07, + "loss": 2.7064, + "step": 44 + }, + { + "epoch": 0.056004978220286245, + "grad_norm": 2.5768678188323975, + "learning_rate": 6.431535269709543e-07, + "loss": 2.6466, + "step": 45 + }, + { + "epoch": 0.05724953329184816, + "grad_norm": 2.579332113265991, + "learning_rate": 6.639004149377594e-07, + "loss": 2.6127, + "step": 46 + }, + { + "epoch": 0.05849408836341008, + "grad_norm": 2.231207847595215, + "learning_rate": 6.846473029045644e-07, + "loss": 2.6429, + "step": 47 + }, + { + "epoch": 0.059738643434972, + "grad_norm": 2.134375810623169, + "learning_rate": 7.053941908713693e-07, + "loss": 2.6423, + "step": 48 + }, + { + "epoch": 0.060983198506533914, + "grad_norm": 2.0485494136810303, + "learning_rate": 7.261410788381744e-07, + "loss": 2.581, + "step": 49 + }, + { + "epoch": 0.06222775357809583, + "grad_norm": 2.02909517288208, + "learning_rate": 7.468879668049793e-07, + "loss": 2.6357, + "step": 50 + }, + { + "epoch": 0.06222775357809583, + "eval_loss": 2.632822036743164, + "eval_runtime": 47.0104, + "eval_samples_per_second": 21.272, + "eval_steps_per_second": 0.893, + "step": 50 + }, + { + "epoch": 0.06347230864965775, + "grad_norm": 1.892500877380371, + "learning_rate": 7.676348547717843e-07, + "loss": 2.5945, + "step": 51 + }, + { + "epoch": 0.06471686372121967, + "grad_norm": 2.107921600341797, + "learning_rate": 7.883817427385892e-07, + "loss": 2.5867, + "step": 52 + }, + { + "epoch": 0.06596141879278158, + "grad_norm": 1.8287527561187744, + "learning_rate": 8.091286307053943e-07, + "loss": 2.5862, + "step": 53 + }, + { + "epoch": 0.0672059738643435, + "grad_norm": 1.7342065572738647, + "learning_rate": 8.298755186721992e-07, + "loss": 2.5735, + "step": 54 + }, + { + "epoch": 0.06845052893590542, + "grad_norm": 1.7905038595199585, + "learning_rate": 8.506224066390042e-07, + "loss": 2.572, + "step": 55 + }, + { + "epoch": 0.06969508400746734, + "grad_norm": 1.9715144634246826, + "learning_rate": 8.713692946058091e-07, + "loss": 2.5797, + "step": 56 + }, + { + "epoch": 0.07093963907902924, + "grad_norm": 1.788966417312622, + "learning_rate": 8.921161825726142e-07, + "loss": 2.5567, + "step": 57 + }, + { + "epoch": 0.07218419415059116, + "grad_norm": 1.8978915214538574, + "learning_rate": 9.128630705394191e-07, + "loss": 2.5776, + "step": 58 + }, + { + "epoch": 0.07342874922215308, + "grad_norm": 1.6401689052581787, + "learning_rate": 9.336099585062241e-07, + "loss": 2.5662, + "step": 59 + }, + { + "epoch": 0.07467330429371499, + "grad_norm": 1.8200798034667969, + "learning_rate": 9.54356846473029e-07, + "loss": 2.5001, + "step": 60 + }, + { + "epoch": 0.07467330429371499, + "eval_loss": 2.569110870361328, + "eval_runtime": 47.3292, + "eval_samples_per_second": 21.129, + "eval_steps_per_second": 0.887, + "step": 60 + }, + { + "epoch": 0.07591785936527691, + "grad_norm": 1.5468674898147583, + "learning_rate": 9.751037344398341e-07, + "loss": 2.5308, + "step": 61 + }, + { + "epoch": 0.07716241443683883, + "grad_norm": 1.723684310913086, + "learning_rate": 9.95850622406639e-07, + "loss": 2.4907, + "step": 62 + }, + { + "epoch": 0.07840696950840075, + "grad_norm": 1.6346293687820435, + "learning_rate": 1.0165975103734441e-06, + "loss": 2.514, + "step": 63 + }, + { + "epoch": 0.07965152457996266, + "grad_norm": 1.8902088403701782, + "learning_rate": 1.037344398340249e-06, + "loss": 2.4859, + "step": 64 + }, + { + "epoch": 0.08089607965152458, + "grad_norm": 2.092611312866211, + "learning_rate": 1.058091286307054e-06, + "loss": 2.4888, + "step": 65 + }, + { + "epoch": 0.0821406347230865, + "grad_norm": 1.758482575416565, + "learning_rate": 1.078838174273859e-06, + "loss": 2.4537, + "step": 66 + }, + { + "epoch": 0.08338518979464841, + "grad_norm": 1.8941714763641357, + "learning_rate": 1.099585062240664e-06, + "loss": 2.5416, + "step": 67 + }, + { + "epoch": 0.08462974486621033, + "grad_norm": 1.7030234336853027, + "learning_rate": 1.120331950207469e-06, + "loss": 2.4791, + "step": 68 + }, + { + "epoch": 0.08587429993777225, + "grad_norm": 1.7602269649505615, + "learning_rate": 1.141078838174274e-06, + "loss": 2.4316, + "step": 69 + }, + { + "epoch": 0.08711885500933417, + "grad_norm": 1.7635431289672852, + "learning_rate": 1.161825726141079e-06, + "loss": 2.4674, + "step": 70 + }, + { + "epoch": 0.08711885500933417, + "eval_loss": 2.4922080039978027, + "eval_runtime": 45.3028, + "eval_samples_per_second": 22.074, + "eval_steps_per_second": 0.927, + "step": 70 + }, + { + "epoch": 0.08836341008089608, + "grad_norm": 2.0913803577423096, + "learning_rate": 1.182572614107884e-06, + "loss": 2.4341, + "step": 71 + }, + { + "epoch": 0.089607965152458, + "grad_norm": 1.8292183876037598, + "learning_rate": 1.2033195020746888e-06, + "loss": 2.4291, + "step": 72 + }, + { + "epoch": 0.09085252022401992, + "grad_norm": 2.206770181655884, + "learning_rate": 1.224066390041494e-06, + "loss": 2.3764, + "step": 73 + }, + { + "epoch": 0.09209707529558182, + "grad_norm": 1.8263559341430664, + "learning_rate": 1.2448132780082988e-06, + "loss": 2.4082, + "step": 74 + }, + { + "epoch": 0.09334163036714374, + "grad_norm": 1.9405455589294434, + "learning_rate": 1.2655601659751037e-06, + "loss": 2.4673, + "step": 75 + }, + { + "epoch": 0.09458618543870566, + "grad_norm": 2.138108015060425, + "learning_rate": 1.2863070539419086e-06, + "loss": 2.3879, + "step": 76 + }, + { + "epoch": 0.09583074051026758, + "grad_norm": 1.9024745225906372, + "learning_rate": 1.307053941908714e-06, + "loss": 2.4344, + "step": 77 + }, + { + "epoch": 0.0970752955818295, + "grad_norm": 1.9136689901351929, + "learning_rate": 1.3278008298755188e-06, + "loss": 2.474, + "step": 78 + }, + { + "epoch": 0.09831985065339141, + "grad_norm": 2.4752795696258545, + "learning_rate": 1.3485477178423237e-06, + "loss": 2.3412, + "step": 79 + }, + { + "epoch": 0.09956440572495333, + "grad_norm": 1.8720006942749023, + "learning_rate": 1.3692946058091288e-06, + "loss": 2.3438, + "step": 80 + }, + { + "epoch": 0.09956440572495333, + "eval_loss": 2.3980512619018555, + "eval_runtime": 48.2788, + "eval_samples_per_second": 20.713, + "eval_steps_per_second": 0.87, + "step": 80 + }, + { + "epoch": 0.10080896079651525, + "grad_norm": 2.671691656112671, + "learning_rate": 1.3900414937759337e-06, + "loss": 2.3336, + "step": 81 + }, + { + "epoch": 0.10205351586807716, + "grad_norm": 2.2953391075134277, + "learning_rate": 1.4107883817427386e-06, + "loss": 2.377, + "step": 82 + }, + { + "epoch": 0.10329807093963908, + "grad_norm": 3.009018898010254, + "learning_rate": 1.4315352697095435e-06, + "loss": 2.2977, + "step": 83 + }, + { + "epoch": 0.104542626011201, + "grad_norm": 2.664454936981201, + "learning_rate": 1.4522821576763488e-06, + "loss": 2.3271, + "step": 84 + }, + { + "epoch": 0.10578718108276292, + "grad_norm": 3.017303705215454, + "learning_rate": 1.4730290456431537e-06, + "loss": 2.3251, + "step": 85 + }, + { + "epoch": 0.10703173615432483, + "grad_norm": 2.634716510772705, + "learning_rate": 1.4937759336099586e-06, + "loss": 2.332, + "step": 86 + }, + { + "epoch": 0.10827629122588675, + "grad_norm": 3.059644937515259, + "learning_rate": 1.5145228215767635e-06, + "loss": 2.3478, + "step": 87 + }, + { + "epoch": 0.10952084629744867, + "grad_norm": 2.6962637901306152, + "learning_rate": 1.5352697095435686e-06, + "loss": 2.2792, + "step": 88 + }, + { + "epoch": 0.11076540136901059, + "grad_norm": 3.419729709625244, + "learning_rate": 1.5560165975103735e-06, + "loss": 2.2571, + "step": 89 + }, + { + "epoch": 0.11200995644057249, + "grad_norm": 2.740781545639038, + "learning_rate": 1.5767634854771784e-06, + "loss": 2.2875, + "step": 90 + }, + { + "epoch": 0.11200995644057249, + "eval_loss": 2.30843186378479, + "eval_runtime": 49.9264, + "eval_samples_per_second": 20.029, + "eval_steps_per_second": 0.841, + "step": 90 + }, + { + "epoch": 0.1132545115121344, + "grad_norm": 2.5608789920806885, + "learning_rate": 1.5975103734439833e-06, + "loss": 2.2592, + "step": 91 + }, + { + "epoch": 0.11449906658369632, + "grad_norm": 2.687999963760376, + "learning_rate": 1.6182572614107886e-06, + "loss": 2.2546, + "step": 92 + }, + { + "epoch": 0.11574362165525824, + "grad_norm": 2.695909023284912, + "learning_rate": 1.6390041493775935e-06, + "loss": 2.2525, + "step": 93 + }, + { + "epoch": 0.11698817672682016, + "grad_norm": 2.818357467651367, + "learning_rate": 1.6597510373443984e-06, + "loss": 2.216, + "step": 94 + }, + { + "epoch": 0.11823273179838208, + "grad_norm": 2.884119987487793, + "learning_rate": 1.6804979253112035e-06, + "loss": 2.2321, + "step": 95 + }, + { + "epoch": 0.119477286869944, + "grad_norm": 2.52104115486145, + "learning_rate": 1.7012448132780084e-06, + "loss": 2.199, + "step": 96 + }, + { + "epoch": 0.12072184194150591, + "grad_norm": 2.420313596725464, + "learning_rate": 1.7219917012448133e-06, + "loss": 2.1862, + "step": 97 + }, + { + "epoch": 0.12196639701306783, + "grad_norm": 2.8047542572021484, + "learning_rate": 1.7427385892116182e-06, + "loss": 2.1793, + "step": 98 + }, + { + "epoch": 0.12321095208462975, + "grad_norm": 2.836482286453247, + "learning_rate": 1.7634854771784235e-06, + "loss": 2.2271, + "step": 99 + }, + { + "epoch": 0.12445550715619166, + "grad_norm": 2.5282301902770996, + "learning_rate": 1.7842323651452284e-06, + "loss": 2.1768, + "step": 100 + }, + { + "epoch": 0.12445550715619166, + "eval_loss": 2.2312686443328857, + "eval_runtime": 49.2382, + "eval_samples_per_second": 20.309, + "eval_steps_per_second": 0.853, + "step": 100 + }, + { + "epoch": 0.12570006222775357, + "grad_norm": 3.3407280445098877, + "learning_rate": 1.8049792531120333e-06, + "loss": 2.1666, + "step": 101 + }, + { + "epoch": 0.1269446172993155, + "grad_norm": 2.4754133224487305, + "learning_rate": 1.8257261410788382e-06, + "loss": 2.1768, + "step": 102 + }, + { + "epoch": 0.1281891723708774, + "grad_norm": 3.430889129638672, + "learning_rate": 1.8464730290456433e-06, + "loss": 2.1953, + "step": 103 + }, + { + "epoch": 0.12943372744243933, + "grad_norm": 2.835294246673584, + "learning_rate": 1.8672199170124482e-06, + "loss": 2.146, + "step": 104 + }, + { + "epoch": 0.13067828251400124, + "grad_norm": 3.1532323360443115, + "learning_rate": 1.8879668049792531e-06, + "loss": 2.1729, + "step": 105 + }, + { + "epoch": 0.13192283758556317, + "grad_norm": 3.2278342247009277, + "learning_rate": 1.908713692946058e-06, + "loss": 2.1336, + "step": 106 + }, + { + "epoch": 0.13316739265712507, + "grad_norm": 2.7892515659332275, + "learning_rate": 1.929460580912863e-06, + "loss": 2.141, + "step": 107 + }, + { + "epoch": 0.134411947728687, + "grad_norm": 3.0179977416992188, + "learning_rate": 1.9502074688796682e-06, + "loss": 2.1255, + "step": 108 + }, + { + "epoch": 0.1356565028002489, + "grad_norm": 2.977935552597046, + "learning_rate": 1.970954356846473e-06, + "loss": 2.0986, + "step": 109 + }, + { + "epoch": 0.13690105787181084, + "grad_norm": 3.3042492866516113, + "learning_rate": 1.991701244813278e-06, + "loss": 2.0992, + "step": 110 + }, + { + "epoch": 0.13690105787181084, + "eval_loss": 2.171299695968628, + "eval_runtime": 42.9511, + "eval_samples_per_second": 23.282, + "eval_steps_per_second": 0.978, + "step": 110 + }, + { + "epoch": 0.13814561294337274, + "grad_norm": 2.9750890731811523, + "learning_rate": 2.012448132780083e-06, + "loss": 2.1116, + "step": 111 + }, + { + "epoch": 0.13939016801493467, + "grad_norm": 2.8199286460876465, + "learning_rate": 2.0331950207468883e-06, + "loss": 2.0822, + "step": 112 + }, + { + "epoch": 0.14063472308649658, + "grad_norm": 2.930532217025757, + "learning_rate": 2.053941908713693e-06, + "loss": 2.161, + "step": 113 + }, + { + "epoch": 0.14187927815805848, + "grad_norm": 3.3400321006774902, + "learning_rate": 2.074688796680498e-06, + "loss": 2.0953, + "step": 114 + }, + { + "epoch": 0.1431238332296204, + "grad_norm": 2.716564655303955, + "learning_rate": 2.095435684647303e-06, + "loss": 2.0757, + "step": 115 + }, + { + "epoch": 0.14436838830118232, + "grad_norm": 2.7586236000061035, + "learning_rate": 2.116182572614108e-06, + "loss": 2.0979, + "step": 116 + }, + { + "epoch": 0.14561294337274425, + "grad_norm": 2.622126340866089, + "learning_rate": 2.136929460580913e-06, + "loss": 2.0748, + "step": 117 + }, + { + "epoch": 0.14685749844430615, + "grad_norm": 2.646477460861206, + "learning_rate": 2.157676348547718e-06, + "loss": 2.0605, + "step": 118 + }, + { + "epoch": 0.14810205351586808, + "grad_norm": 2.8882410526275635, + "learning_rate": 2.178423236514523e-06, + "loss": 2.0501, + "step": 119 + }, + { + "epoch": 0.14934660858742999, + "grad_norm": 2.8302247524261475, + "learning_rate": 2.199170124481328e-06, + "loss": 2.0592, + "step": 120 + }, + { + "epoch": 0.14934660858742999, + "eval_loss": 2.1291966438293457, + "eval_runtime": 47.5399, + "eval_samples_per_second": 21.035, + "eval_steps_per_second": 0.883, + "step": 120 + }, + { + "epoch": 0.15059116365899192, + "grad_norm": 3.26979660987854, + "learning_rate": 2.219917012448133e-06, + "loss": 2.0422, + "step": 121 + }, + { + "epoch": 0.15183571873055382, + "grad_norm": 2.7483913898468018, + "learning_rate": 2.240663900414938e-06, + "loss": 2.0872, + "step": 122 + }, + { + "epoch": 0.15308027380211575, + "grad_norm": 3.1252024173736572, + "learning_rate": 2.2614107883817427e-06, + "loss": 2.0684, + "step": 123 + }, + { + "epoch": 0.15432482887367766, + "grad_norm": 2.9175283908843994, + "learning_rate": 2.282157676348548e-06, + "loss": 2.0522, + "step": 124 + }, + { + "epoch": 0.1555693839452396, + "grad_norm": 2.7950754165649414, + "learning_rate": 2.302904564315353e-06, + "loss": 2.065, + "step": 125 + }, + { + "epoch": 0.1568139390168015, + "grad_norm": 2.84049654006958, + "learning_rate": 2.323651452282158e-06, + "loss": 2.0781, + "step": 126 + }, + { + "epoch": 0.15805849408836342, + "grad_norm": 2.7396647930145264, + "learning_rate": 2.3443983402489627e-06, + "loss": 2.1758, + "step": 127 + }, + { + "epoch": 0.15930304915992533, + "grad_norm": 3.0765926837921143, + "learning_rate": 2.365145228215768e-06, + "loss": 2.0172, + "step": 128 + }, + { + "epoch": 0.16054760423148726, + "grad_norm": 3.217189073562622, + "learning_rate": 2.385892116182573e-06, + "loss": 2.0286, + "step": 129 + }, + { + "epoch": 0.16179215930304916, + "grad_norm": 3.141545295715332, + "learning_rate": 2.4066390041493776e-06, + "loss": 2.0583, + "step": 130 + }, + { + "epoch": 0.16179215930304916, + "eval_loss": 2.0946149826049805, + "eval_runtime": 54.92, + "eval_samples_per_second": 18.208, + "eval_steps_per_second": 0.765, + "step": 130 + }, + { + "epoch": 0.16303671437461106, + "grad_norm": 2.850052833557129, + "learning_rate": 2.4273858921161828e-06, + "loss": 2.0746, + "step": 131 + }, + { + "epoch": 0.164281269446173, + "grad_norm": 3.28913950920105, + "learning_rate": 2.448132780082988e-06, + "loss": 2.0539, + "step": 132 + }, + { + "epoch": 0.1655258245177349, + "grad_norm": 2.8819124698638916, + "learning_rate": 2.468879668049793e-06, + "loss": 1.9843, + "step": 133 + }, + { + "epoch": 0.16677037958929683, + "grad_norm": 3.6254632472991943, + "learning_rate": 2.4896265560165977e-06, + "loss": 2.0233, + "step": 134 + }, + { + "epoch": 0.16801493466085873, + "grad_norm": 2.7385146617889404, + "learning_rate": 2.5103734439834028e-06, + "loss": 1.9667, + "step": 135 + }, + { + "epoch": 0.16925948973242066, + "grad_norm": 2.9722647666931152, + "learning_rate": 2.5311203319502074e-06, + "loss": 1.9527, + "step": 136 + }, + { + "epoch": 0.17050404480398257, + "grad_norm": 3.139526605606079, + "learning_rate": 2.5518672199170125e-06, + "loss": 1.9967, + "step": 137 + }, + { + "epoch": 0.1717485998755445, + "grad_norm": 3.399920701980591, + "learning_rate": 2.5726141078838172e-06, + "loss": 1.9798, + "step": 138 + }, + { + "epoch": 0.1729931549471064, + "grad_norm": 2.8017327785491943, + "learning_rate": 2.5933609958506228e-06, + "loss": 1.983, + "step": 139 + }, + { + "epoch": 0.17423771001866833, + "grad_norm": 3.9967198371887207, + "learning_rate": 2.614107883817428e-06, + "loss": 1.977, + "step": 140 + }, + { + "epoch": 0.17423771001866833, + "eval_loss": 2.059037685394287, + "eval_runtime": 52.6232, + "eval_samples_per_second": 19.003, + "eval_steps_per_second": 0.798, + "step": 140 + }, + { + "epoch": 0.17548226509023024, + "grad_norm": 2.893092393875122, + "learning_rate": 2.6348547717842326e-06, + "loss": 1.9888, + "step": 141 + }, + { + "epoch": 0.17672682016179217, + "grad_norm": 3.634352207183838, + "learning_rate": 2.6556016597510377e-06, + "loss": 2.0726, + "step": 142 + }, + { + "epoch": 0.17797137523335407, + "grad_norm": 3.3651444911956787, + "learning_rate": 2.6763485477178423e-06, + "loss": 1.973, + "step": 143 + }, + { + "epoch": 0.179215930304916, + "grad_norm": 3.968986988067627, + "learning_rate": 2.6970954356846475e-06, + "loss": 1.9948, + "step": 144 + }, + { + "epoch": 0.1804604853764779, + "grad_norm": 3.20105242729187, + "learning_rate": 2.717842323651452e-06, + "loss": 1.9238, + "step": 145 + }, + { + "epoch": 0.18170504044803984, + "grad_norm": 3.648339033126831, + "learning_rate": 2.7385892116182577e-06, + "loss": 1.9923, + "step": 146 + }, + { + "epoch": 0.18294959551960174, + "grad_norm": 3.0856316089630127, + "learning_rate": 2.7593360995850628e-06, + "loss": 1.9749, + "step": 147 + }, + { + "epoch": 0.18419415059116365, + "grad_norm": 3.2681071758270264, + "learning_rate": 2.7800829875518675e-06, + "loss": 1.9304, + "step": 148 + }, + { + "epoch": 0.18543870566272558, + "grad_norm": 2.634958267211914, + "learning_rate": 2.8008298755186726e-06, + "loss": 1.9237, + "step": 149 + }, + { + "epoch": 0.18668326073428748, + "grad_norm": 2.769491672515869, + "learning_rate": 2.8215767634854773e-06, + "loss": 1.8963, + "step": 150 + }, + { + "epoch": 0.18668326073428748, + "eval_loss": 2.0394654273986816, + "eval_runtime": 55.2136, + "eval_samples_per_second": 18.111, + "eval_steps_per_second": 0.761, + "step": 150 + }, + { + "epoch": 0.1879278158058494, + "grad_norm": 3.4345781803131104, + "learning_rate": 2.8423236514522824e-06, + "loss": 1.9836, + "step": 151 + }, + { + "epoch": 0.18917237087741132, + "grad_norm": 3.115727424621582, + "learning_rate": 2.863070539419087e-06, + "loss": 1.9352, + "step": 152 + }, + { + "epoch": 0.19041692594897325, + "grad_norm": 3.050652503967285, + "learning_rate": 2.883817427385892e-06, + "loss": 1.9031, + "step": 153 + }, + { + "epoch": 0.19166148102053515, + "grad_norm": 2.9404428005218506, + "learning_rate": 2.9045643153526977e-06, + "loss": 1.9196, + "step": 154 + }, + { + "epoch": 0.19290603609209708, + "grad_norm": 3.080810785293579, + "learning_rate": 2.9253112033195024e-06, + "loss": 1.9405, + "step": 155 + }, + { + "epoch": 0.194150591163659, + "grad_norm": 3.058558702468872, + "learning_rate": 2.9460580912863075e-06, + "loss": 1.9052, + "step": 156 + }, + { + "epoch": 0.19539514623522092, + "grad_norm": 3.307955026626587, + "learning_rate": 2.966804979253112e-06, + "loss": 1.9245, + "step": 157 + }, + { + "epoch": 0.19663970130678282, + "grad_norm": 2.845506191253662, + "learning_rate": 2.9875518672199173e-06, + "loss": 1.9461, + "step": 158 + }, + { + "epoch": 0.19788425637834475, + "grad_norm": 2.758654832839966, + "learning_rate": 3.008298755186722e-06, + "loss": 1.8885, + "step": 159 + }, + { + "epoch": 0.19912881144990666, + "grad_norm": 3.161252737045288, + "learning_rate": 3.029045643153527e-06, + "loss": 1.8535, + "step": 160 + }, + { + "epoch": 0.19912881144990666, + "eval_loss": 2.018101453781128, + "eval_runtime": 51.0744, + "eval_samples_per_second": 19.579, + "eval_steps_per_second": 0.822, + "step": 160 + }, + { + "epoch": 0.2003733665214686, + "grad_norm": 3.196847677230835, + "learning_rate": 3.0497925311203326e-06, + "loss": 1.8853, + "step": 161 + }, + { + "epoch": 0.2016179215930305, + "grad_norm": 2.8364663124084473, + "learning_rate": 3.0705394190871373e-06, + "loss": 1.9463, + "step": 162 + }, + { + "epoch": 0.2028624766645924, + "grad_norm": 3.0874054431915283, + "learning_rate": 3.0912863070539424e-06, + "loss": 1.9254, + "step": 163 + }, + { + "epoch": 0.20410703173615433, + "grad_norm": 2.7914493083953857, + "learning_rate": 3.112033195020747e-06, + "loss": 1.9213, + "step": 164 + }, + { + "epoch": 0.20535158680771623, + "grad_norm": 3.3871428966522217, + "learning_rate": 3.132780082987552e-06, + "loss": 1.8993, + "step": 165 + }, + { + "epoch": 0.20659614187927816, + "grad_norm": 3.096653461456299, + "learning_rate": 3.153526970954357e-06, + "loss": 1.8827, + "step": 166 + }, + { + "epoch": 0.20784069695084006, + "grad_norm": 2.8276076316833496, + "learning_rate": 3.174273858921162e-06, + "loss": 1.912, + "step": 167 + }, + { + "epoch": 0.209085252022402, + "grad_norm": 3.3058435916900635, + "learning_rate": 3.1950207468879666e-06, + "loss": 1.8381, + "step": 168 + }, + { + "epoch": 0.2103298070939639, + "grad_norm": 3.5017333030700684, + "learning_rate": 3.215767634854772e-06, + "loss": 1.9544, + "step": 169 + }, + { + "epoch": 0.21157436216552583, + "grad_norm": 3.457296133041382, + "learning_rate": 3.2365145228215773e-06, + "loss": 1.922, + "step": 170 + }, + { + "epoch": 0.21157436216552583, + "eval_loss": 1.988455891609192, + "eval_runtime": 56.2099, + "eval_samples_per_second": 17.79, + "eval_steps_per_second": 0.747, + "step": 170 + }, + { + "epoch": 0.21281891723708773, + "grad_norm": 3.1964566707611084, + "learning_rate": 3.257261410788382e-06, + "loss": 1.8925, + "step": 171 + }, + { + "epoch": 0.21406347230864967, + "grad_norm": 3.234652042388916, + "learning_rate": 3.278008298755187e-06, + "loss": 1.884, + "step": 172 + }, + { + "epoch": 0.21530802738021157, + "grad_norm": 3.14414119720459, + "learning_rate": 3.2987551867219918e-06, + "loss": 1.8905, + "step": 173 + }, + { + "epoch": 0.2165525824517735, + "grad_norm": 3.606379508972168, + "learning_rate": 3.319502074688797e-06, + "loss": 1.946, + "step": 174 + }, + { + "epoch": 0.2177971375233354, + "grad_norm": 2.986646890640259, + "learning_rate": 3.3402489626556016e-06, + "loss": 1.8691, + "step": 175 + }, + { + "epoch": 0.21904169259489734, + "grad_norm": 3.7298269271850586, + "learning_rate": 3.360995850622407e-06, + "loss": 1.8717, + "step": 176 + }, + { + "epoch": 0.22028624766645924, + "grad_norm": 3.422295093536377, + "learning_rate": 3.381742738589212e-06, + "loss": 1.8568, + "step": 177 + }, + { + "epoch": 0.22153080273802117, + "grad_norm": 3.2293782234191895, + "learning_rate": 3.402489626556017e-06, + "loss": 1.8471, + "step": 178 + }, + { + "epoch": 0.22277535780958307, + "grad_norm": 3.2293782234191895, + "learning_rate": 3.402489626556017e-06, + "loss": 1.9303, + "step": 179 + }, + { + "epoch": 0.22401991288114498, + "grad_norm": 3.506223440170288, + "learning_rate": 3.423236514522822e-06, + "loss": 1.8237, + "step": 180 + }, + { + "epoch": 0.22401991288114498, + "eval_loss": 1.9734643697738647, + "eval_runtime": 52.2815, + "eval_samples_per_second": 19.127, + "eval_steps_per_second": 0.803, + "step": 180 + }, + { + "epoch": 0.2252644679527069, + "grad_norm": 3.0156619548797607, + "learning_rate": 3.4439834024896267e-06, + "loss": 1.908, + "step": 181 + }, + { + "epoch": 0.2265090230242688, + "grad_norm": 3.55517315864563, + "learning_rate": 3.4647302904564318e-06, + "loss": 1.9104, + "step": 182 + }, + { + "epoch": 0.22775357809583074, + "grad_norm": 3.144984006881714, + "learning_rate": 3.4854771784232365e-06, + "loss": 1.8198, + "step": 183 + }, + { + "epoch": 0.22899813316739265, + "grad_norm": 4.302074432373047, + "learning_rate": 3.5062240663900416e-06, + "loss": 1.8237, + "step": 184 + }, + { + "epoch": 0.23024268823895458, + "grad_norm": 3.0522446632385254, + "learning_rate": 3.526970954356847e-06, + "loss": 1.8409, + "step": 185 + }, + { + "epoch": 0.23148724331051648, + "grad_norm": 3.8607394695281982, + "learning_rate": 3.5477178423236518e-06, + "loss": 1.8807, + "step": 186 + }, + { + "epoch": 0.23273179838207841, + "grad_norm": 2.9236302375793457, + "learning_rate": 3.568464730290457e-06, + "loss": 1.7873, + "step": 187 + }, + { + "epoch": 0.23397635345364032, + "grad_norm": 4.013780117034912, + "learning_rate": 3.5892116182572616e-06, + "loss": 1.7909, + "step": 188 + }, + { + "epoch": 0.23522090852520225, + "grad_norm": 3.0933122634887695, + "learning_rate": 3.6099585062240667e-06, + "loss": 1.8469, + "step": 189 + }, + { + "epoch": 0.23646546359676415, + "grad_norm": 3.487816095352173, + "learning_rate": 3.6307053941908714e-06, + "loss": 1.8469, + "step": 190 + }, + { + "epoch": 0.23646546359676415, + "eval_loss": 1.9604240655899048, + "eval_runtime": 53.4604, + "eval_samples_per_second": 18.705, + "eval_steps_per_second": 0.786, + "step": 190 + }, + { + "epoch": 0.23771001866832608, + "grad_norm": 2.829219341278076, + "learning_rate": 3.6514522821576765e-06, + "loss": 1.8639, + "step": 191 + }, + { + "epoch": 0.238954573739888, + "grad_norm": 3.595534324645996, + "learning_rate": 3.672199170124482e-06, + "loss": 1.8988, + "step": 192 + }, + { + "epoch": 0.24019912881144992, + "grad_norm": 2.9022483825683594, + "learning_rate": 3.6929460580912867e-06, + "loss": 1.8972, + "step": 193 + }, + { + "epoch": 0.24144368388301182, + "grad_norm": 3.0667994022369385, + "learning_rate": 3.713692946058092e-06, + "loss": 1.8499, + "step": 194 + }, + { + "epoch": 0.24268823895457373, + "grad_norm": 3.0498485565185547, + "learning_rate": 3.7344398340248965e-06, + "loss": 1.8341, + "step": 195 + }, + { + "epoch": 0.24393279402613566, + "grad_norm": 4.058084964752197, + "learning_rate": 3.7551867219917016e-06, + "loss": 1.8641, + "step": 196 + }, + { + "epoch": 0.24517734909769756, + "grad_norm": 2.9995028972625732, + "learning_rate": 3.7759336099585063e-06, + "loss": 1.7885, + "step": 197 + }, + { + "epoch": 0.2464219041692595, + "grad_norm": 3.254704475402832, + "learning_rate": 3.7966804979253114e-06, + "loss": 1.7544, + "step": 198 + }, + { + "epoch": 0.2476664592408214, + "grad_norm": 2.9126744270324707, + "learning_rate": 3.817427385892116e-06, + "loss": 1.8461, + "step": 199 + }, + { + "epoch": 0.24891101431238333, + "grad_norm": 3.475247621536255, + "learning_rate": 3.838174273858922e-06, + "loss": 1.8095, + "step": 200 + }, + { + "epoch": 0.24891101431238333, + "eval_loss": 1.9373760223388672, + "eval_runtime": 51.8733, + "eval_samples_per_second": 19.278, + "eval_steps_per_second": 0.81, + "step": 200 + }, + { + "epoch": 0.25015556938394523, + "grad_norm": 3.2663731575012207, + "learning_rate": 3.858921161825726e-06, + "loss": 1.8076, + "step": 201 + }, + { + "epoch": 0.25140012445550713, + "grad_norm": 3.1151726245880127, + "learning_rate": 3.879668049792531e-06, + "loss": 1.8143, + "step": 202 + }, + { + "epoch": 0.2526446795270691, + "grad_norm": 3.102038860321045, + "learning_rate": 3.9004149377593365e-06, + "loss": 1.8279, + "step": 203 + }, + { + "epoch": 0.253889234598631, + "grad_norm": 3.370642900466919, + "learning_rate": 3.921161825726142e-06, + "loss": 1.8522, + "step": 204 + }, + { + "epoch": 0.2551337896701929, + "grad_norm": 3.116128921508789, + "learning_rate": 3.941908713692946e-06, + "loss": 1.8104, + "step": 205 + }, + { + "epoch": 0.2563783447417548, + "grad_norm": 3.12028431892395, + "learning_rate": 3.962655601659751e-06, + "loss": 1.7723, + "step": 206 + }, + { + "epoch": 0.25762289981331676, + "grad_norm": 3.1251418590545654, + "learning_rate": 3.983402489626556e-06, + "loss": 1.863, + "step": 207 + }, + { + "epoch": 0.25886745488487867, + "grad_norm": 3.073702812194824, + "learning_rate": 4.004149377593361e-06, + "loss": 1.805, + "step": 208 + }, + { + "epoch": 0.26011200995644057, + "grad_norm": 2.9772469997406006, + "learning_rate": 4.024896265560166e-06, + "loss": 1.8251, + "step": 209 + }, + { + "epoch": 0.2613565650280025, + "grad_norm": 2.9970037937164307, + "learning_rate": 4.045643153526971e-06, + "loss": 1.8386, + "step": 210 + }, + { + "epoch": 0.2613565650280025, + "eval_loss": 1.9167065620422363, + "eval_runtime": 42.3328, + "eval_samples_per_second": 23.622, + "eval_steps_per_second": 0.992, + "step": 210 + }, + { + "epoch": 0.26260112009956443, + "grad_norm": 3.1391561031341553, + "learning_rate": 4.0663900414937765e-06, + "loss": 1.8109, + "step": 211 + }, + { + "epoch": 0.26384567517112634, + "grad_norm": 3.161538600921631, + "learning_rate": 4.087136929460581e-06, + "loss": 1.8441, + "step": 212 + }, + { + "epoch": 0.26509023024268824, + "grad_norm": 3.12412166595459, + "learning_rate": 4.107883817427386e-06, + "loss": 1.7736, + "step": 213 + }, + { + "epoch": 0.26633478531425014, + "grad_norm": 3.3241145610809326, + "learning_rate": 4.128630705394191e-06, + "loss": 1.8067, + "step": 214 + }, + { + "epoch": 0.26757934038581205, + "grad_norm": 3.503307819366455, + "learning_rate": 4.149377593360996e-06, + "loss": 1.7858, + "step": 215 + }, + { + "epoch": 0.268823895457374, + "grad_norm": 3.1009578704833984, + "learning_rate": 4.170124481327801e-06, + "loss": 1.7542, + "step": 216 + }, + { + "epoch": 0.2700684505289359, + "grad_norm": 3.2236011028289795, + "learning_rate": 4.190871369294606e-06, + "loss": 1.8006, + "step": 217 + }, + { + "epoch": 0.2713130056004978, + "grad_norm": 3.048935651779175, + "learning_rate": 4.211618257261411e-06, + "loss": 1.7706, + "step": 218 + }, + { + "epoch": 0.2725575606720597, + "grad_norm": 3.3416147232055664, + "learning_rate": 4.232365145228216e-06, + "loss": 1.8108, + "step": 219 + }, + { + "epoch": 0.2738021157436217, + "grad_norm": 3.2010738849639893, + "learning_rate": 4.253112033195021e-06, + "loss": 1.8561, + "step": 220 + }, + { + "epoch": 0.2738021157436217, + "eval_loss": 1.9111930131912231, + "eval_runtime": 50.4077, + "eval_samples_per_second": 19.838, + "eval_steps_per_second": 0.833, + "step": 220 + }, + { + "epoch": 0.2750466708151836, + "grad_norm": 3.4033374786376953, + "learning_rate": 4.273858921161826e-06, + "loss": 1.8218, + "step": 221 + }, + { + "epoch": 0.2762912258867455, + "grad_norm": 3.2086002826690674, + "learning_rate": 4.294605809128631e-06, + "loss": 1.8213, + "step": 222 + }, + { + "epoch": 0.2775357809583074, + "grad_norm": 4.090956211090088, + "learning_rate": 4.315352697095436e-06, + "loss": 1.8455, + "step": 223 + }, + { + "epoch": 0.27878033602986935, + "grad_norm": 3.1142985820770264, + "learning_rate": 4.336099585062241e-06, + "loss": 1.7737, + "step": 224 + }, + { + "epoch": 0.28002489110143125, + "grad_norm": 3.369669198989868, + "learning_rate": 4.356846473029046e-06, + "loss": 1.824, + "step": 225 + }, + { + "epoch": 0.28126944617299315, + "grad_norm": 3.165672779083252, + "learning_rate": 4.3775933609958506e-06, + "loss": 1.7768, + "step": 226 + }, + { + "epoch": 0.28251400124455506, + "grad_norm": 3.3343470096588135, + "learning_rate": 4.398340248962656e-06, + "loss": 1.7278, + "step": 227 + }, + { + "epoch": 0.28375855631611696, + "grad_norm": 3.2574994564056396, + "learning_rate": 4.419087136929461e-06, + "loss": 1.8439, + "step": 228 + }, + { + "epoch": 0.2850031113876789, + "grad_norm": 3.043928623199463, + "learning_rate": 4.439834024896266e-06, + "loss": 1.818, + "step": 229 + }, + { + "epoch": 0.2862476664592408, + "grad_norm": 2.9703125953674316, + "learning_rate": 4.460580912863071e-06, + "loss": 1.773, + "step": 230 + }, + { + "epoch": 0.2862476664592408, + "eval_loss": 1.898109793663025, + "eval_runtime": 45.6023, + "eval_samples_per_second": 21.929, + "eval_steps_per_second": 0.921, + "step": 230 + }, + { + "epoch": 0.2874922215308027, + "grad_norm": 2.9997220039367676, + "learning_rate": 4.481327800829876e-06, + "loss": 1.7727, + "step": 231 + }, + { + "epoch": 0.28873677660236463, + "grad_norm": 3.1358482837677, + "learning_rate": 4.502074688796681e-06, + "loss": 1.7764, + "step": 232 + }, + { + "epoch": 0.2899813316739266, + "grad_norm": 3.395747661590576, + "learning_rate": 4.5228215767634855e-06, + "loss": 1.7826, + "step": 233 + }, + { + "epoch": 0.2912258867454885, + "grad_norm": 3.7494754791259766, + "learning_rate": 4.543568464730291e-06, + "loss": 1.7181, + "step": 234 + }, + { + "epoch": 0.2924704418170504, + "grad_norm": 3.42293381690979, + "learning_rate": 4.564315352697096e-06, + "loss": 1.7773, + "step": 235 + }, + { + "epoch": 0.2937149968886123, + "grad_norm": 3.2524514198303223, + "learning_rate": 4.585062240663901e-06, + "loss": 1.7294, + "step": 236 + }, + { + "epoch": 0.29495955196017426, + "grad_norm": 3.657869577407837, + "learning_rate": 4.605809128630706e-06, + "loss": 1.762, + "step": 237 + }, + { + "epoch": 0.29620410703173616, + "grad_norm": 3.127372980117798, + "learning_rate": 4.626556016597511e-06, + "loss": 1.7623, + "step": 238 + }, + { + "epoch": 0.29744866210329807, + "grad_norm": 4.166962146759033, + "learning_rate": 4.647302904564316e-06, + "loss": 1.6995, + "step": 239 + }, + { + "epoch": 0.29869321717485997, + "grad_norm": 3.094264030456543, + "learning_rate": 4.66804979253112e-06, + "loss": 1.7469, + "step": 240 + }, + { + "epoch": 0.29869321717485997, + "eval_loss": 1.8908178806304932, + "eval_runtime": 44.6332, + "eval_samples_per_second": 22.405, + "eval_steps_per_second": 0.941, + "step": 240 + }, + { + "epoch": 0.29993777224642193, + "grad_norm": 3.8644745349884033, + "learning_rate": 4.6887966804979255e-06, + "loss": 1.7644, + "step": 241 + }, + { + "epoch": 0.30118232731798383, + "grad_norm": 3.1488852500915527, + "learning_rate": 4.709543568464731e-06, + "loss": 1.772, + "step": 242 + }, + { + "epoch": 0.30242688238954574, + "grad_norm": 3.3179638385772705, + "learning_rate": 4.730290456431536e-06, + "loss": 1.7588, + "step": 243 + }, + { + "epoch": 0.30367143746110764, + "grad_norm": 3.328355550765991, + "learning_rate": 4.751037344398341e-06, + "loss": 1.7417, + "step": 244 + }, + { + "epoch": 0.30491599253266954, + "grad_norm": 3.2690482139587402, + "learning_rate": 4.771784232365146e-06, + "loss": 1.7562, + "step": 245 + }, + { + "epoch": 0.3061605476042315, + "grad_norm": 3.2759454250335693, + "learning_rate": 4.792531120331951e-06, + "loss": 1.7536, + "step": 246 + }, + { + "epoch": 0.3074051026757934, + "grad_norm": 2.937964916229248, + "learning_rate": 4.813278008298755e-06, + "loss": 1.7518, + "step": 247 + }, + { + "epoch": 0.3086496577473553, + "grad_norm": 3.0995302200317383, + "learning_rate": 4.83402489626556e-06, + "loss": 1.7593, + "step": 248 + }, + { + "epoch": 0.3098942128189172, + "grad_norm": 3.230459213256836, + "learning_rate": 4.8547717842323655e-06, + "loss": 1.6962, + "step": 249 + }, + { + "epoch": 0.3111387678904792, + "grad_norm": 3.242576837539673, + "learning_rate": 4.875518672199171e-06, + "loss": 1.7307, + "step": 250 + }, + { + "epoch": 0.3111387678904792, + "eval_loss": 1.8805371522903442, + "eval_runtime": 45.9886, + "eval_samples_per_second": 21.745, + "eval_steps_per_second": 0.913, + "step": 250 + }, + { + "epoch": 0.3123833229620411, + "grad_norm": 3.097045421600342, + "learning_rate": 4.896265560165976e-06, + "loss": 1.7438, + "step": 251 + }, + { + "epoch": 0.313627878033603, + "grad_norm": 3.2428948879241943, + "learning_rate": 4.91701244813278e-06, + "loss": 1.7179, + "step": 252 + }, + { + "epoch": 0.3148724331051649, + "grad_norm": 3.196274518966675, + "learning_rate": 4.937759336099586e-06, + "loss": 1.7855, + "step": 253 + }, + { + "epoch": 0.31611698817672684, + "grad_norm": 2.978203535079956, + "learning_rate": 4.95850622406639e-06, + "loss": 1.7144, + "step": 254 + }, + { + "epoch": 0.31736154324828875, + "grad_norm": 3.2641701698303223, + "learning_rate": 4.979253112033195e-06, + "loss": 1.7427, + "step": 255 + }, + { + "epoch": 0.31860609831985065, + "grad_norm": 2.7441232204437256, + "learning_rate": 5e-06, + "loss": 1.7619, + "step": 256 + }, + { + "epoch": 0.31985065339141255, + "grad_norm": 3.0723495483398438, + "learning_rate": 4.99769372693727e-06, + "loss": 1.8017, + "step": 257 + }, + { + "epoch": 0.3210952084629745, + "grad_norm": 3.104752540588379, + "learning_rate": 4.995387453874539e-06, + "loss": 1.7761, + "step": 258 + }, + { + "epoch": 0.3223397635345364, + "grad_norm": 3.138627052307129, + "learning_rate": 4.993081180811809e-06, + "loss": 1.7879, + "step": 259 + }, + { + "epoch": 0.3235843186060983, + "grad_norm": 2.8567333221435547, + "learning_rate": 4.990774907749078e-06, + "loss": 1.7561, + "step": 260 + }, + { + "epoch": 0.3235843186060983, + "eval_loss": 1.8606494665145874, + "eval_runtime": 46.453, + "eval_samples_per_second": 21.527, + "eval_steps_per_second": 0.904, + "step": 260 + }, + { + "epoch": 0.3248288736776602, + "grad_norm": 3.2696540355682373, + "learning_rate": 4.988468634686347e-06, + "loss": 1.7201, + "step": 261 + }, + { + "epoch": 0.3260734287492221, + "grad_norm": 2.6692731380462646, + "learning_rate": 4.986162361623617e-06, + "loss": 1.7264, + "step": 262 + }, + { + "epoch": 0.3273179838207841, + "grad_norm": 3.047549247741699, + "learning_rate": 4.983856088560886e-06, + "loss": 1.7362, + "step": 263 + }, + { + "epoch": 0.328562538892346, + "grad_norm": 3.0667457580566406, + "learning_rate": 4.981549815498156e-06, + "loss": 1.7321, + "step": 264 + }, + { + "epoch": 0.3298070939639079, + "grad_norm": 2.560047149658203, + "learning_rate": 4.979243542435424e-06, + "loss": 1.7508, + "step": 265 + }, + { + "epoch": 0.3310516490354698, + "grad_norm": 3.26595401763916, + "learning_rate": 4.976937269372694e-06, + "loss": 1.7248, + "step": 266 + }, + { + "epoch": 0.33229620410703176, + "grad_norm": 2.929210662841797, + "learning_rate": 4.974630996309964e-06, + "loss": 1.697, + "step": 267 + }, + { + "epoch": 0.33354075917859366, + "grad_norm": 3.1526286602020264, + "learning_rate": 4.972324723247233e-06, + "loss": 1.7385, + "step": 268 + }, + { + "epoch": 0.33478531425015556, + "grad_norm": 3.001619577407837, + "learning_rate": 4.970018450184502e-06, + "loss": 1.7072, + "step": 269 + }, + { + "epoch": 0.33602986932171747, + "grad_norm": 3.2464189529418945, + "learning_rate": 4.9677121771217715e-06, + "loss": 1.7361, + "step": 270 + }, + { + "epoch": 0.33602986932171747, + "eval_loss": 1.860226035118103, + "eval_runtime": 46.6881, + "eval_samples_per_second": 21.419, + "eval_steps_per_second": 0.9, + "step": 270 + }, + { + "epoch": 0.3372744243932794, + "grad_norm": 3.2104530334472656, + "learning_rate": 4.965405904059041e-06, + "loss": 1.7171, + "step": 271 + }, + { + "epoch": 0.33851897946484133, + "grad_norm": 3.146847724914551, + "learning_rate": 4.96309963099631e-06, + "loss": 1.7311, + "step": 272 + }, + { + "epoch": 0.33976353453640323, + "grad_norm": 3.1431286334991455, + "learning_rate": 4.96079335793358e-06, + "loss": 1.7345, + "step": 273 + }, + { + "epoch": 0.34100808960796514, + "grad_norm": 2.847163677215576, + "learning_rate": 4.958487084870849e-06, + "loss": 1.7262, + "step": 274 + }, + { + "epoch": 0.3422526446795271, + "grad_norm": 3.1987810134887695, + "learning_rate": 4.956180811808119e-06, + "loss": 1.7945, + "step": 275 + }, + { + "epoch": 0.343497199751089, + "grad_norm": 2.7475385665893555, + "learning_rate": 4.953874538745388e-06, + "loss": 1.76, + "step": 276 + }, + { + "epoch": 0.3447417548226509, + "grad_norm": 3.0427663326263428, + "learning_rate": 4.9515682656826574e-06, + "loss": 1.7324, + "step": 277 + }, + { + "epoch": 0.3459863098942128, + "grad_norm": 3.605212688446045, + "learning_rate": 4.949261992619927e-06, + "loss": 1.7712, + "step": 278 + }, + { + "epoch": 0.3472308649657747, + "grad_norm": 3.0564935207366943, + "learning_rate": 4.946955719557196e-06, + "loss": 1.6792, + "step": 279 + }, + { + "epoch": 0.34847542003733667, + "grad_norm": 3.1069741249084473, + "learning_rate": 4.944649446494466e-06, + "loss": 1.7145, + "step": 280 + }, + { + "epoch": 0.34847542003733667, + "eval_loss": 1.8537051677703857, + "eval_runtime": 47.4813, + "eval_samples_per_second": 21.061, + "eval_steps_per_second": 0.885, + "step": 280 + }, + { + "epoch": 0.3497199751088986, + "grad_norm": 2.8021512031555176, + "learning_rate": 4.942343173431734e-06, + "loss": 1.7145, + "step": 281 + }, + { + "epoch": 0.3509645301804605, + "grad_norm": 3.5549023151397705, + "learning_rate": 4.940036900369004e-06, + "loss": 1.7642, + "step": 282 + }, + { + "epoch": 0.3522090852520224, + "grad_norm": 2.7648985385894775, + "learning_rate": 4.937730627306274e-06, + "loss": 1.7255, + "step": 283 + }, + { + "epoch": 0.35345364032358434, + "grad_norm": 3.0815863609313965, + "learning_rate": 4.9354243542435426e-06, + "loss": 1.7055, + "step": 284 + }, + { + "epoch": 0.35469819539514624, + "grad_norm": 2.9009227752685547, + "learning_rate": 4.933118081180812e-06, + "loss": 1.7019, + "step": 285 + }, + { + "epoch": 0.35594275046670815, + "grad_norm": 2.618429660797119, + "learning_rate": 4.930811808118081e-06, + "loss": 1.6318, + "step": 286 + }, + { + "epoch": 0.35718730553827005, + "grad_norm": 3.368230104446411, + "learning_rate": 4.928505535055351e-06, + "loss": 1.7304, + "step": 287 + }, + { + "epoch": 0.358431860609832, + "grad_norm": 3.256889820098877, + "learning_rate": 4.92619926199262e-06, + "loss": 1.7019, + "step": 288 + }, + { + "epoch": 0.3596764156813939, + "grad_norm": 2.9366304874420166, + "learning_rate": 4.92389298892989e-06, + "loss": 1.734, + "step": 289 + }, + { + "epoch": 0.3609209707529558, + "grad_norm": 3.0393142700195312, + "learning_rate": 4.921586715867159e-06, + "loss": 1.7504, + "step": 290 + }, + { + "epoch": 0.3609209707529558, + "eval_loss": 1.8359886407852173, + "eval_runtime": 49.7622, + "eval_samples_per_second": 20.096, + "eval_steps_per_second": 0.844, + "step": 290 + }, + { + "epoch": 0.3621655258245177, + "grad_norm": 2.7288589477539062, + "learning_rate": 4.9192804428044285e-06, + "loss": 1.7004, + "step": 291 + }, + { + "epoch": 0.3634100808960797, + "grad_norm": 3.2793378829956055, + "learning_rate": 4.916974169741698e-06, + "loss": 1.7028, + "step": 292 + }, + { + "epoch": 0.3646546359676416, + "grad_norm": 2.960880756378174, + "learning_rate": 4.914667896678967e-06, + "loss": 1.6759, + "step": 293 + }, + { + "epoch": 0.3658991910392035, + "grad_norm": 2.836421012878418, + "learning_rate": 4.912361623616237e-06, + "loss": 1.6475, + "step": 294 + }, + { + "epoch": 0.3671437461107654, + "grad_norm": 2.9495465755462646, + "learning_rate": 4.910055350553506e-06, + "loss": 1.7076, + "step": 295 + }, + { + "epoch": 0.3683883011823273, + "grad_norm": 2.954730272293091, + "learning_rate": 4.907749077490776e-06, + "loss": 1.7261, + "step": 296 + }, + { + "epoch": 0.36963285625388925, + "grad_norm": 2.8167543411254883, + "learning_rate": 4.905442804428044e-06, + "loss": 1.6939, + "step": 297 + }, + { + "epoch": 0.37087741132545116, + "grad_norm": 2.857316017150879, + "learning_rate": 4.903136531365314e-06, + "loss": 1.7362, + "step": 298 + }, + { + "epoch": 0.37212196639701306, + "grad_norm": 2.8612918853759766, + "learning_rate": 4.900830258302584e-06, + "loss": 1.6489, + "step": 299 + }, + { + "epoch": 0.37336652146857496, + "grad_norm": 2.9618914127349854, + "learning_rate": 4.898523985239853e-06, + "loss": 1.718, + "step": 300 + }, + { + "epoch": 0.37336652146857496, + "eval_loss": 1.827571988105774, + "eval_runtime": 46.4552, + "eval_samples_per_second": 21.526, + "eval_steps_per_second": 0.904, + "step": 300 + }, + { + "epoch": 0.3746110765401369, + "grad_norm": 2.8706953525543213, + "learning_rate": 4.896217712177122e-06, + "loss": 1.6925, + "step": 301 + }, + { + "epoch": 0.3758556316116988, + "grad_norm": 3.2565090656280518, + "learning_rate": 4.893911439114391e-06, + "loss": 1.7369, + "step": 302 + }, + { + "epoch": 0.37710018668326073, + "grad_norm": 2.928858995437622, + "learning_rate": 4.891605166051661e-06, + "loss": 1.6882, + "step": 303 + }, + { + "epoch": 0.37834474175482263, + "grad_norm": 3.042314052581787, + "learning_rate": 4.88929889298893e-06, + "loss": 1.6466, + "step": 304 + }, + { + "epoch": 0.3795892968263846, + "grad_norm": 2.7509143352508545, + "learning_rate": 4.8869926199262e-06, + "loss": 1.685, + "step": 305 + }, + { + "epoch": 0.3808338518979465, + "grad_norm": 2.9601588249206543, + "learning_rate": 4.884686346863469e-06, + "loss": 1.6772, + "step": 306 + }, + { + "epoch": 0.3820784069695084, + "grad_norm": 2.618608236312866, + "learning_rate": 4.8823800738007384e-06, + "loss": 1.7127, + "step": 307 + }, + { + "epoch": 0.3833229620410703, + "grad_norm": 3.047581195831299, + "learning_rate": 4.880073800738008e-06, + "loss": 1.7174, + "step": 308 + }, + { + "epoch": 0.3845675171126322, + "grad_norm": 2.87735652923584, + "learning_rate": 4.877767527675277e-06, + "loss": 1.6677, + "step": 309 + }, + { + "epoch": 0.38581207218419417, + "grad_norm": 2.7169721126556396, + "learning_rate": 4.875461254612546e-06, + "loss": 1.7486, + "step": 310 + }, + { + "epoch": 0.38581207218419417, + "eval_loss": 1.8221794366836548, + "eval_runtime": 43.1681, + "eval_samples_per_second": 23.165, + "eval_steps_per_second": 0.973, + "step": 310 + }, + { + "epoch": 0.38705662725575607, + "grad_norm": 2.7094991207122803, + "learning_rate": 4.873154981549816e-06, + "loss": 1.7348, + "step": 311 + }, + { + "epoch": 0.388301182327318, + "grad_norm": 2.8989078998565674, + "learning_rate": 4.8708487084870856e-06, + "loss": 1.7352, + "step": 312 + }, + { + "epoch": 0.3895457373988799, + "grad_norm": 2.9565601348876953, + "learning_rate": 4.868542435424355e-06, + "loss": 1.7063, + "step": 313 + }, + { + "epoch": 0.39079029247044184, + "grad_norm": 2.8354082107543945, + "learning_rate": 4.8662361623616235e-06, + "loss": 1.6808, + "step": 314 + }, + { + "epoch": 0.39203484754200374, + "grad_norm": 2.8287479877471924, + "learning_rate": 4.863929889298894e-06, + "loss": 1.6731, + "step": 315 + }, + { + "epoch": 0.39327940261356564, + "grad_norm": 2.6537821292877197, + "learning_rate": 4.861623616236163e-06, + "loss": 1.695, + "step": 316 + }, + { + "epoch": 0.39452395768512755, + "grad_norm": 2.7135190963745117, + "learning_rate": 4.859317343173432e-06, + "loss": 1.6897, + "step": 317 + }, + { + "epoch": 0.3957685127566895, + "grad_norm": 3.1019129753112793, + "learning_rate": 4.857011070110701e-06, + "loss": 1.7219, + "step": 318 + }, + { + "epoch": 0.3970130678282514, + "grad_norm": 2.945737838745117, + "learning_rate": 4.854704797047971e-06, + "loss": 1.7316, + "step": 319 + }, + { + "epoch": 0.3982576228998133, + "grad_norm": 2.7668251991271973, + "learning_rate": 4.85239852398524e-06, + "loss": 1.6461, + "step": 320 + }, + { + "epoch": 0.3982576228998133, + "eval_loss": 1.8210570812225342, + "eval_runtime": 46.2552, + "eval_samples_per_second": 21.619, + "eval_steps_per_second": 0.908, + "step": 320 + }, + { + "epoch": 0.3995021779713752, + "grad_norm": 2.738109588623047, + "learning_rate": 4.8500922509225095e-06, + "loss": 1.6907, + "step": 321 + }, + { + "epoch": 0.4007467330429372, + "grad_norm": 3.0195062160491943, + "learning_rate": 4.847785977859779e-06, + "loss": 1.7099, + "step": 322 + }, + { + "epoch": 0.4019912881144991, + "grad_norm": 2.8672404289245605, + "learning_rate": 4.845479704797048e-06, + "loss": 1.6509, + "step": 323 + }, + { + "epoch": 0.403235843186061, + "grad_norm": 2.9041008949279785, + "learning_rate": 4.843173431734318e-06, + "loss": 1.5818, + "step": 324 + }, + { + "epoch": 0.4044803982576229, + "grad_norm": 2.9105985164642334, + "learning_rate": 4.840867158671587e-06, + "loss": 1.7002, + "step": 325 + }, + { + "epoch": 0.4057249533291848, + "grad_norm": 2.749009847640991, + "learning_rate": 4.838560885608857e-06, + "loss": 1.7021, + "step": 326 + }, + { + "epoch": 0.40696950840074675, + "grad_norm": 2.6998822689056396, + "learning_rate": 4.836254612546126e-06, + "loss": 1.6853, + "step": 327 + }, + { + "epoch": 0.40821406347230865, + "grad_norm": 2.678201675415039, + "learning_rate": 4.8339483394833955e-06, + "loss": 1.6934, + "step": 328 + }, + { + "epoch": 0.40945861854387056, + "grad_norm": 2.774845600128174, + "learning_rate": 4.831642066420665e-06, + "loss": 1.657, + "step": 329 + }, + { + "epoch": 0.41070317361543246, + "grad_norm": 2.6363000869750977, + "learning_rate": 4.8293357933579335e-06, + "loss": 1.6555, + "step": 330 + }, + { + "epoch": 0.41070317361543246, + "eval_loss": 1.8126581907272339, + "eval_runtime": 52.7717, + "eval_samples_per_second": 18.95, + "eval_steps_per_second": 0.796, + "step": 330 + }, + { + "epoch": 0.4119477286869944, + "grad_norm": 3.011659622192383, + "learning_rate": 4.827029520295204e-06, + "loss": 1.7202, + "step": 331 + }, + { + "epoch": 0.4131922837585563, + "grad_norm": 2.8322594165802, + "learning_rate": 4.824723247232473e-06, + "loss": 1.6736, + "step": 332 + }, + { + "epoch": 0.4144368388301182, + "grad_norm": 2.9360146522521973, + "learning_rate": 4.822416974169742e-06, + "loss": 1.7213, + "step": 333 + }, + { + "epoch": 0.41568139390168013, + "grad_norm": 2.810206413269043, + "learning_rate": 4.820110701107011e-06, + "loss": 1.6867, + "step": 334 + }, + { + "epoch": 0.4169259489732421, + "grad_norm": 2.6864097118377686, + "learning_rate": 4.817804428044281e-06, + "loss": 1.6854, + "step": 335 + }, + { + "epoch": 0.418170504044804, + "grad_norm": 2.7396061420440674, + "learning_rate": 4.81549815498155e-06, + "loss": 1.68, + "step": 336 + }, + { + "epoch": 0.4194150591163659, + "grad_norm": 2.602771520614624, + "learning_rate": 4.8131918819188194e-06, + "loss": 1.6269, + "step": 337 + }, + { + "epoch": 0.4206596141879278, + "grad_norm": 2.8174757957458496, + "learning_rate": 4.810885608856089e-06, + "loss": 1.6981, + "step": 338 + }, + { + "epoch": 0.42190416925948976, + "grad_norm": 3.2398130893707275, + "learning_rate": 4.808579335793358e-06, + "loss": 1.7061, + "step": 339 + }, + { + "epoch": 0.42314872433105166, + "grad_norm": 2.7119736671447754, + "learning_rate": 4.806273062730628e-06, + "loss": 1.6875, + "step": 340 + }, + { + "epoch": 0.42314872433105166, + "eval_loss": 1.7973068952560425, + "eval_runtime": 53.5791, + "eval_samples_per_second": 18.664, + "eval_steps_per_second": 0.784, + "step": 340 + }, + { + "epoch": 0.42439327940261357, + "grad_norm": 2.868533134460449, + "learning_rate": 4.803966789667897e-06, + "loss": 1.6967, + "step": 341 + }, + { + "epoch": 0.42563783447417547, + "grad_norm": 2.528083562850952, + "learning_rate": 4.8016605166051665e-06, + "loss": 1.6195, + "step": 342 + }, + { + "epoch": 0.4268823895457374, + "grad_norm": 2.7885115146636963, + "learning_rate": 4.799354243542436e-06, + "loss": 1.6197, + "step": 343 + }, + { + "epoch": 0.42812694461729933, + "grad_norm": 2.768247604370117, + "learning_rate": 4.797047970479705e-06, + "loss": 1.6271, + "step": 344 + }, + { + "epoch": 0.42937149968886124, + "grad_norm": 2.7710275650024414, + "learning_rate": 4.794741697416975e-06, + "loss": 1.6768, + "step": 345 + }, + { + "epoch": 0.43061605476042314, + "grad_norm": 2.968337297439575, + "learning_rate": 4.792435424354243e-06, + "loss": 1.6433, + "step": 346 + }, + { + "epoch": 0.43186060983198504, + "grad_norm": 2.5739808082580566, + "learning_rate": 4.790129151291514e-06, + "loss": 1.6327, + "step": 347 + }, + { + "epoch": 0.433105164903547, + "grad_norm": 2.7676408290863037, + "learning_rate": 4.787822878228783e-06, + "loss": 1.6178, + "step": 348 + }, + { + "epoch": 0.4343497199751089, + "grad_norm": 2.959059476852417, + "learning_rate": 4.7855166051660525e-06, + "loss": 1.6809, + "step": 349 + }, + { + "epoch": 0.4355942750466708, + "grad_norm": 2.559967279434204, + "learning_rate": 4.783210332103321e-06, + "loss": 1.6704, + "step": 350 + }, + { + "epoch": 0.4355942750466708, + "eval_loss": 1.8013949394226074, + "eval_runtime": 50.4649, + "eval_samples_per_second": 19.816, + "eval_steps_per_second": 0.832, + "step": 350 + }, + { + "epoch": 0.4368388301182327, + "grad_norm": 2.817901611328125, + "learning_rate": 4.7809040590405905e-06, + "loss": 1.6296, + "step": 351 + }, + { + "epoch": 0.43808338518979467, + "grad_norm": 2.7116055488586426, + "learning_rate": 4.778597785977861e-06, + "loss": 1.6652, + "step": 352 + }, + { + "epoch": 0.4393279402613566, + "grad_norm": 2.544750452041626, + "learning_rate": 4.776291512915129e-06, + "loss": 1.5668, + "step": 353 + }, + { + "epoch": 0.4405724953329185, + "grad_norm": 2.720534086227417, + "learning_rate": 4.773985239852399e-06, + "loss": 1.6152, + "step": 354 + }, + { + "epoch": 0.4418170504044804, + "grad_norm": 2.7270126342773438, + "learning_rate": 4.771678966789668e-06, + "loss": 1.5624, + "step": 355 + }, + { + "epoch": 0.44306160547604234, + "grad_norm": 2.7986624240875244, + "learning_rate": 4.769372693726938e-06, + "loss": 1.6621, + "step": 356 + }, + { + "epoch": 0.44430616054760425, + "grad_norm": 2.943107843399048, + "learning_rate": 4.767066420664207e-06, + "loss": 1.6713, + "step": 357 + }, + { + "epoch": 0.44555071561916615, + "grad_norm": 2.739898681640625, + "learning_rate": 4.7647601476014765e-06, + "loss": 1.6363, + "step": 358 + }, + { + "epoch": 0.44679527069072805, + "grad_norm": 2.8729329109191895, + "learning_rate": 4.762453874538746e-06, + "loss": 1.6605, + "step": 359 + }, + { + "epoch": 0.44803982576228996, + "grad_norm": 2.909029483795166, + "learning_rate": 4.760147601476015e-06, + "loss": 1.6277, + "step": 360 + }, + { + "epoch": 0.44803982576228996, + "eval_loss": 1.8110179901123047, + "eval_runtime": 47.924, + "eval_samples_per_second": 20.866, + "eval_steps_per_second": 0.876, + "step": 360 + }, + { + "epoch": 0.4492843808338519, + "grad_norm": 3.3826682567596436, + "learning_rate": 4.757841328413285e-06, + "loss": 1.7155, + "step": 361 + }, + { + "epoch": 0.4505289359054138, + "grad_norm": 3.0586729049682617, + "learning_rate": 4.755535055350554e-06, + "loss": 1.5991, + "step": 362 + }, + { + "epoch": 0.4517734909769757, + "grad_norm": 2.645576238632202, + "learning_rate": 4.753228782287823e-06, + "loss": 1.6504, + "step": 363 + }, + { + "epoch": 0.4530180460485376, + "grad_norm": 2.705967664718628, + "learning_rate": 4.750922509225093e-06, + "loss": 1.6761, + "step": 364 + }, + { + "epoch": 0.4542626011200996, + "grad_norm": 2.7651922702789307, + "learning_rate": 4.748616236162362e-06, + "loss": 1.61, + "step": 365 + }, + { + "epoch": 0.4555071561916615, + "grad_norm": 2.5654966831207275, + "learning_rate": 4.746309963099631e-06, + "loss": 1.6335, + "step": 366 + }, + { + "epoch": 0.4567517112632234, + "grad_norm": 2.6606173515319824, + "learning_rate": 4.7440036900369e-06, + "loss": 1.6207, + "step": 367 + }, + { + "epoch": 0.4579962663347853, + "grad_norm": 2.724755048751831, + "learning_rate": 4.741697416974171e-06, + "loss": 1.6329, + "step": 368 + }, + { + "epoch": 0.45924082140634725, + "grad_norm": 2.7019615173339844, + "learning_rate": 4.739391143911439e-06, + "loss": 1.6763, + "step": 369 + }, + { + "epoch": 0.46048537647790916, + "grad_norm": 2.6594254970550537, + "learning_rate": 4.737084870848709e-06, + "loss": 1.6174, + "step": 370 + }, + { + "epoch": 0.46048537647790916, + "eval_loss": 1.7924141883850098, + "eval_runtime": 53.0349, + "eval_samples_per_second": 18.855, + "eval_steps_per_second": 0.792, + "step": 370 + }, + { + "epoch": 0.46172993154947106, + "grad_norm": 2.668421745300293, + "learning_rate": 4.734778597785978e-06, + "loss": 1.6421, + "step": 371 + }, + { + "epoch": 0.46297448662103297, + "grad_norm": 2.7148995399475098, + "learning_rate": 4.7324723247232475e-06, + "loss": 1.6635, + "step": 372 + }, + { + "epoch": 0.4642190416925949, + "grad_norm": 3.156646490097046, + "learning_rate": 4.730166051660517e-06, + "loss": 1.6679, + "step": 373 + }, + { + "epoch": 0.46546359676415683, + "grad_norm": 2.8170645236968994, + "learning_rate": 4.727859778597786e-06, + "loss": 1.6492, + "step": 374 + }, + { + "epoch": 0.46670815183571873, + "grad_norm": 2.9796457290649414, + "learning_rate": 4.725553505535056e-06, + "loss": 1.6025, + "step": 375 + }, + { + "epoch": 0.46795270690728064, + "grad_norm": 2.6493406295776367, + "learning_rate": 4.723247232472325e-06, + "loss": 1.6398, + "step": 376 + }, + { + "epoch": 0.46919726197884254, + "grad_norm": 3.2643542289733887, + "learning_rate": 4.720940959409595e-06, + "loss": 1.6413, + "step": 377 + }, + { + "epoch": 0.4704418170504045, + "grad_norm": 2.7613179683685303, + "learning_rate": 4.718634686346864e-06, + "loss": 1.6251, + "step": 378 + }, + { + "epoch": 0.4716863721219664, + "grad_norm": 3.1216518878936768, + "learning_rate": 4.716328413284133e-06, + "loss": 1.6268, + "step": 379 + }, + { + "epoch": 0.4729309271935283, + "grad_norm": 2.571647882461548, + "learning_rate": 4.714022140221403e-06, + "loss": 1.6245, + "step": 380 + }, + { + "epoch": 0.4729309271935283, + "eval_loss": 1.7875893115997314, + "eval_runtime": 49.3058, + "eval_samples_per_second": 20.282, + "eval_steps_per_second": 0.852, + "step": 380 + }, + { + "epoch": 0.4741754822650902, + "grad_norm": 3.443906545639038, + "learning_rate": 4.711715867158672e-06, + "loss": 1.6799, + "step": 381 + }, + { + "epoch": 0.47542003733665217, + "grad_norm": 2.7887957096099854, + "learning_rate": 4.709409594095941e-06, + "loss": 1.6589, + "step": 382 + }, + { + "epoch": 0.47666459240821407, + "grad_norm": 3.2833151817321777, + "learning_rate": 4.70710332103321e-06, + "loss": 1.6064, + "step": 383 + }, + { + "epoch": 0.477909147479776, + "grad_norm": 2.6975057125091553, + "learning_rate": 4.704797047970481e-06, + "loss": 1.6029, + "step": 384 + }, + { + "epoch": 0.4791537025513379, + "grad_norm": 2.745246171951294, + "learning_rate": 4.702490774907749e-06, + "loss": 1.7127, + "step": 385 + }, + { + "epoch": 0.48039825762289984, + "grad_norm": 3.0522775650024414, + "learning_rate": 4.700184501845019e-06, + "loss": 1.6508, + "step": 386 + }, + { + "epoch": 0.48164281269446174, + "grad_norm": 2.689218044281006, + "learning_rate": 4.697878228782288e-06, + "loss": 1.6971, + "step": 387 + }, + { + "epoch": 0.48288736776602365, + "grad_norm": 3.060361862182617, + "learning_rate": 4.6955719557195575e-06, + "loss": 1.6464, + "step": 388 + }, + { + "epoch": 0.48413192283758555, + "grad_norm": 2.872262477874756, + "learning_rate": 4.693265682656827e-06, + "loss": 1.6293, + "step": 389 + }, + { + "epoch": 0.48537647790914745, + "grad_norm": 2.980050563812256, + "learning_rate": 4.690959409594096e-06, + "loss": 1.6435, + "step": 390 + }, + { + "epoch": 0.48537647790914745, + "eval_loss": 1.7791496515274048, + "eval_runtime": 50.0299, + "eval_samples_per_second": 19.988, + "eval_steps_per_second": 0.839, + "step": 390 + }, + { + "epoch": 0.4866210329807094, + "grad_norm": 2.764827251434326, + "learning_rate": 4.688653136531366e-06, + "loss": 1.6279, + "step": 391 + }, + { + "epoch": 0.4878655880522713, + "grad_norm": 2.6912333965301514, + "learning_rate": 4.686346863468635e-06, + "loss": 1.7276, + "step": 392 + }, + { + "epoch": 0.4891101431238332, + "grad_norm": 2.8358330726623535, + "learning_rate": 4.6840405904059046e-06, + "loss": 1.6482, + "step": 393 + }, + { + "epoch": 0.4903546981953951, + "grad_norm": 2.6581575870513916, + "learning_rate": 4.681734317343174e-06, + "loss": 1.6853, + "step": 394 + }, + { + "epoch": 0.4915992532669571, + "grad_norm": 2.941425323486328, + "learning_rate": 4.6794280442804426e-06, + "loss": 1.6075, + "step": 395 + }, + { + "epoch": 0.492843808338519, + "grad_norm": 3.2334814071655273, + "learning_rate": 4.677121771217713e-06, + "loss": 1.6157, + "step": 396 + }, + { + "epoch": 0.4940883634100809, + "grad_norm": 2.984410524368286, + "learning_rate": 4.674815498154982e-06, + "loss": 1.6257, + "step": 397 + }, + { + "epoch": 0.4953329184816428, + "grad_norm": 3.0650789737701416, + "learning_rate": 4.672509225092252e-06, + "loss": 1.6573, + "step": 398 + }, + { + "epoch": 0.49657747355320475, + "grad_norm": 2.798664093017578, + "learning_rate": 4.67020295202952e-06, + "loss": 1.638, + "step": 399 + }, + { + "epoch": 0.49782202862476665, + "grad_norm": 2.8266706466674805, + "learning_rate": 4.66789667896679e-06, + "loss": 1.6484, + "step": 400 + }, + { + "epoch": 0.49782202862476665, + "eval_loss": 1.7714862823486328, + "eval_runtime": 48.0657, + "eval_samples_per_second": 20.805, + "eval_steps_per_second": 0.874, + "step": 400 + }, + { + "epoch": 0.49906658369632856, + "grad_norm": 2.9928388595581055, + "learning_rate": 4.66559040590406e-06, + "loss": 1.6721, + "step": 401 + }, + { + "epoch": 0.5003111387678905, + "grad_norm": 2.573514461517334, + "learning_rate": 4.6632841328413285e-06, + "loss": 1.679, + "step": 402 + }, + { + "epoch": 0.5015556938394524, + "grad_norm": 3.1736996173858643, + "learning_rate": 4.660977859778598e-06, + "loss": 1.7124, + "step": 403 + }, + { + "epoch": 0.5028002489110143, + "grad_norm": 2.858849287033081, + "learning_rate": 4.658671586715867e-06, + "loss": 1.6622, + "step": 404 + }, + { + "epoch": 0.5040448039825762, + "grad_norm": 2.825698137283325, + "learning_rate": 4.656365313653137e-06, + "loss": 1.6464, + "step": 405 + }, + { + "epoch": 0.5052893590541382, + "grad_norm": 3.783891439437866, + "learning_rate": 4.654059040590406e-06, + "loss": 1.6665, + "step": 406 + }, + { + "epoch": 0.5065339141257, + "grad_norm": 2.789813756942749, + "learning_rate": 4.651752767527676e-06, + "loss": 1.6064, + "step": 407 + }, + { + "epoch": 0.507778469197262, + "grad_norm": 3.3106348514556885, + "learning_rate": 4.649446494464945e-06, + "loss": 1.6542, + "step": 408 + }, + { + "epoch": 0.5090230242688238, + "grad_norm": 2.8805112838745117, + "learning_rate": 4.6471402214022145e-06, + "loss": 1.6058, + "step": 409 + }, + { + "epoch": 0.5102675793403858, + "grad_norm": 3.30898118019104, + "learning_rate": 4.644833948339484e-06, + "loss": 1.5933, + "step": 410 + }, + { + "epoch": 0.5102675793403858, + "eval_loss": 1.7787818908691406, + "eval_runtime": 44.0876, + "eval_samples_per_second": 22.682, + "eval_steps_per_second": 0.953, + "step": 410 + }, + { + "epoch": 0.5115121344119478, + "grad_norm": 3.3563625812530518, + "learning_rate": 4.642527675276753e-06, + "loss": 1.6751, + "step": 411 + }, + { + "epoch": 0.5127566894835096, + "grad_norm": 2.968120574951172, + "learning_rate": 4.640221402214023e-06, + "loss": 1.663, + "step": 412 + }, + { + "epoch": 0.5140012445550716, + "grad_norm": 3.310248374938965, + "learning_rate": 4.637915129151292e-06, + "loss": 1.658, + "step": 413 + }, + { + "epoch": 0.5152457996266335, + "grad_norm": 2.7011170387268066, + "learning_rate": 4.635608856088562e-06, + "loss": 1.6894, + "step": 414 + }, + { + "epoch": 0.5164903546981954, + "grad_norm": 2.804901361465454, + "learning_rate": 4.63330258302583e-06, + "loss": 1.602, + "step": 415 + }, + { + "epoch": 0.5177349097697573, + "grad_norm": 3.2412750720977783, + "learning_rate": 4.6309963099631e-06, + "loss": 1.6158, + "step": 416 + }, + { + "epoch": 0.5189794648413192, + "grad_norm": 2.699909210205078, + "learning_rate": 4.62869003690037e-06, + "loss": 1.5842, + "step": 417 + }, + { + "epoch": 0.5202240199128811, + "grad_norm": 3.065929412841797, + "learning_rate": 4.6263837638376384e-06, + "loss": 1.6434, + "step": 418 + }, + { + "epoch": 0.5214685749844431, + "grad_norm": 2.895305633544922, + "learning_rate": 4.624077490774908e-06, + "loss": 1.5969, + "step": 419 + }, + { + "epoch": 0.522713130056005, + "grad_norm": 3.1590664386749268, + "learning_rate": 4.621771217712177e-06, + "loss": 1.6211, + "step": 420 + }, + { + "epoch": 0.522713130056005, + "eval_loss": 1.7674189805984497, + "eval_runtime": 53.328, + "eval_samples_per_second": 18.752, + "eval_steps_per_second": 0.788, + "step": 420 + }, + { + "epoch": 0.5239576851275669, + "grad_norm": 2.7126312255859375, + "learning_rate": 4.619464944649447e-06, + "loss": 1.6246, + "step": 421 + }, + { + "epoch": 0.5252022401991289, + "grad_norm": 2.918508768081665, + "learning_rate": 4.617158671586716e-06, + "loss": 1.6937, + "step": 422 + }, + { + "epoch": 0.5264467952706907, + "grad_norm": 3.008610725402832, + "learning_rate": 4.6148523985239856e-06, + "loss": 1.6864, + "step": 423 + }, + { + "epoch": 0.5276913503422527, + "grad_norm": 2.6612043380737305, + "learning_rate": 4.612546125461255e-06, + "loss": 1.5785, + "step": 424 + }, + { + "epoch": 0.5289359054138145, + "grad_norm": 2.850679874420166, + "learning_rate": 4.610239852398524e-06, + "loss": 1.5899, + "step": 425 + }, + { + "epoch": 0.5301804604853765, + "grad_norm": 2.9205031394958496, + "learning_rate": 4.607933579335794e-06, + "loss": 1.5867, + "step": 426 + }, + { + "epoch": 0.5314250155569384, + "grad_norm": 2.929532527923584, + "learning_rate": 4.605627306273063e-06, + "loss": 1.6182, + "step": 427 + }, + { + "epoch": 0.5326695706285003, + "grad_norm": 2.874936103820801, + "learning_rate": 4.603321033210333e-06, + "loss": 1.5789, + "step": 428 + }, + { + "epoch": 0.5339141257000622, + "grad_norm": 2.8703525066375732, + "learning_rate": 4.601014760147602e-06, + "loss": 1.5859, + "step": 429 + }, + { + "epoch": 0.5351586807716241, + "grad_norm": 2.7076902389526367, + "learning_rate": 4.5987084870848715e-06, + "loss": 1.644, + "step": 430 + }, + { + "epoch": 0.5351586807716241, + "eval_loss": 1.7592095136642456, + "eval_runtime": 47.9438, + "eval_samples_per_second": 20.858, + "eval_steps_per_second": 0.876, + "step": 430 + }, + { + "epoch": 0.536403235843186, + "grad_norm": 2.839160680770874, + "learning_rate": 4.59640221402214e-06, + "loss": 1.6357, + "step": 431 + }, + { + "epoch": 0.537647790914748, + "grad_norm": 2.674025535583496, + "learning_rate": 4.5940959409594095e-06, + "loss": 1.676, + "step": 432 + }, + { + "epoch": 0.5388923459863099, + "grad_norm": 3.0619537830352783, + "learning_rate": 4.59178966789668e-06, + "loss": 1.5617, + "step": 433 + }, + { + "epoch": 0.5401369010578718, + "grad_norm": 3.058418035507202, + "learning_rate": 4.589483394833948e-06, + "loss": 1.5951, + "step": 434 + }, + { + "epoch": 0.5413814561294338, + "grad_norm": 3.0046396255493164, + "learning_rate": 4.587177121771218e-06, + "loss": 1.6473, + "step": 435 + }, + { + "epoch": 0.5426260112009956, + "grad_norm": 2.7736752033233643, + "learning_rate": 4.584870848708487e-06, + "loss": 1.6262, + "step": 436 + }, + { + "epoch": 0.5438705662725576, + "grad_norm": 2.7804994583129883, + "learning_rate": 4.5825645756457575e-06, + "loss": 1.6886, + "step": 437 + }, + { + "epoch": 0.5451151213441194, + "grad_norm": 3.0717954635620117, + "learning_rate": 4.580258302583026e-06, + "loss": 1.6287, + "step": 438 + }, + { + "epoch": 0.5463596764156814, + "grad_norm": 2.800243854522705, + "learning_rate": 4.5779520295202955e-06, + "loss": 1.6169, + "step": 439 + }, + { + "epoch": 0.5476042314872434, + "grad_norm": 2.9581644535064697, + "learning_rate": 4.575645756457565e-06, + "loss": 1.6617, + "step": 440 + }, + { + "epoch": 0.5476042314872434, + "eval_loss": 1.7642868757247925, + "eval_runtime": 51.2646, + "eval_samples_per_second": 19.507, + "eval_steps_per_second": 0.819, + "step": 440 + }, + { + "epoch": 0.5488487865588052, + "grad_norm": 3.0067460536956787, + "learning_rate": 4.573339483394834e-06, + "loss": 1.6482, + "step": 441 + }, + { + "epoch": 0.5500933416303672, + "grad_norm": 2.9075019359588623, + "learning_rate": 4.571033210332104e-06, + "loss": 1.6595, + "step": 442 + }, + { + "epoch": 0.551337896701929, + "grad_norm": 2.7788755893707275, + "learning_rate": 4.568726937269373e-06, + "loss": 1.5733, + "step": 443 + }, + { + "epoch": 0.552582451773491, + "grad_norm": 2.7775425910949707, + "learning_rate": 4.566420664206643e-06, + "loss": 1.6925, + "step": 444 + }, + { + "epoch": 0.5538270068450529, + "grad_norm": 2.6949567794799805, + "learning_rate": 4.564114391143912e-06, + "loss": 1.5738, + "step": 445 + }, + { + "epoch": 0.5550715619166148, + "grad_norm": 2.7093007564544678, + "learning_rate": 4.5618081180811814e-06, + "loss": 1.5848, + "step": 446 + }, + { + "epoch": 0.5563161169881767, + "grad_norm": 2.6920173168182373, + "learning_rate": 4.559501845018451e-06, + "loss": 1.6092, + "step": 447 + }, + { + "epoch": 0.5575606720597387, + "grad_norm": 2.733731985092163, + "learning_rate": 4.5571955719557194e-06, + "loss": 1.6056, + "step": 448 + }, + { + "epoch": 0.5588052271313005, + "grad_norm": 2.9156274795532227, + "learning_rate": 4.55488929889299e-06, + "loss": 1.6436, + "step": 449 + }, + { + "epoch": 0.5600497822028625, + "grad_norm": 2.8802616596221924, + "learning_rate": 4.552583025830259e-06, + "loss": 1.6309, + "step": 450 + }, + { + "epoch": 0.5600497822028625, + "eval_loss": 1.7486340999603271, + "eval_runtime": 47.2219, + "eval_samples_per_second": 21.177, + "eval_steps_per_second": 0.889, + "step": 450 + }, + { + "epoch": 0.5612943372744243, + "grad_norm": 2.729743242263794, + "learning_rate": 4.550276752767528e-06, + "loss": 1.7171, + "step": 451 + }, + { + "epoch": 0.5625388923459863, + "grad_norm": 2.800049066543579, + "learning_rate": 4.547970479704797e-06, + "loss": 1.6098, + "step": 452 + }, + { + "epoch": 0.5637834474175483, + "grad_norm": 2.9659311771392822, + "learning_rate": 4.5456642066420666e-06, + "loss": 1.6646, + "step": 453 + }, + { + "epoch": 0.5650280024891101, + "grad_norm": 2.5417754650115967, + "learning_rate": 4.543357933579336e-06, + "loss": 1.5847, + "step": 454 + }, + { + "epoch": 0.5662725575606721, + "grad_norm": 2.8471338748931885, + "learning_rate": 4.541051660516605e-06, + "loss": 1.6235, + "step": 455 + }, + { + "epoch": 0.5675171126322339, + "grad_norm": 2.7039637565612793, + "learning_rate": 4.538745387453875e-06, + "loss": 1.5613, + "step": 456 + }, + { + "epoch": 0.5687616677037959, + "grad_norm": 3.0383341312408447, + "learning_rate": 4.536439114391144e-06, + "loss": 1.6485, + "step": 457 + }, + { + "epoch": 0.5700062227753578, + "grad_norm": 2.528388261795044, + "learning_rate": 4.534132841328414e-06, + "loss": 1.6117, + "step": 458 + }, + { + "epoch": 0.5712507778469197, + "grad_norm": 2.7623119354248047, + "learning_rate": 4.531826568265683e-06, + "loss": 1.6017, + "step": 459 + }, + { + "epoch": 0.5724953329184816, + "grad_norm": 2.9213945865631104, + "learning_rate": 4.5295202952029525e-06, + "loss": 1.643, + "step": 460 + }, + { + "epoch": 0.5724953329184816, + "eval_loss": 1.747575044631958, + "eval_runtime": 45.2844, + "eval_samples_per_second": 22.083, + "eval_steps_per_second": 0.927, + "step": 460 + }, + { + "epoch": 0.5737398879900436, + "grad_norm": 2.664275884628296, + "learning_rate": 4.527214022140222e-06, + "loss": 1.6154, + "step": 461 + }, + { + "epoch": 0.5749844430616055, + "grad_norm": 2.5891788005828857, + "learning_rate": 4.524907749077491e-06, + "loss": 1.5975, + "step": 462 + }, + { + "epoch": 0.5762289981331674, + "grad_norm": 2.8126487731933594, + "learning_rate": 4.522601476014761e-06, + "loss": 1.6306, + "step": 463 + }, + { + "epoch": 0.5774735532047293, + "grad_norm": 2.799391031265259, + "learning_rate": 4.520295202952029e-06, + "loss": 1.6139, + "step": 464 + }, + { + "epoch": 0.5787181082762912, + "grad_norm": 2.7323718070983887, + "learning_rate": 4.5179889298893e-06, + "loss": 1.632, + "step": 465 + }, + { + "epoch": 0.5799626633478532, + "grad_norm": 2.601386785507202, + "learning_rate": 4.515682656826569e-06, + "loss": 1.6139, + "step": 466 + }, + { + "epoch": 0.581207218419415, + "grad_norm": 2.562162399291992, + "learning_rate": 4.513376383763838e-06, + "loss": 1.5168, + "step": 467 + }, + { + "epoch": 0.582451773490977, + "grad_norm": 2.6912002563476562, + "learning_rate": 4.511070110701107e-06, + "loss": 1.6249, + "step": 468 + }, + { + "epoch": 0.583696328562539, + "grad_norm": 2.6971495151519775, + "learning_rate": 4.5087638376383765e-06, + "loss": 1.616, + "step": 469 + }, + { + "epoch": 0.5849408836341008, + "grad_norm": 2.6898910999298096, + "learning_rate": 4.506457564575646e-06, + "loss": 1.614, + "step": 470 + }, + { + "epoch": 0.5849408836341008, + "eval_loss": 1.7460769414901733, + "eval_runtime": 45.3055, + "eval_samples_per_second": 22.072, + "eval_steps_per_second": 0.927, + "step": 470 + }, + { + "epoch": 0.5861854387056628, + "grad_norm": 2.698180675506592, + "learning_rate": 4.504151291512915e-06, + "loss": 1.5993, + "step": 471 + }, + { + "epoch": 0.5874299937772246, + "grad_norm": 2.7125210762023926, + "learning_rate": 4.501845018450185e-06, + "loss": 1.6113, + "step": 472 + }, + { + "epoch": 0.5886745488487866, + "grad_norm": 2.754445791244507, + "learning_rate": 4.499538745387454e-06, + "loss": 1.6719, + "step": 473 + }, + { + "epoch": 0.5899191039203485, + "grad_norm": 2.6979362964630127, + "learning_rate": 4.497232472324724e-06, + "loss": 1.608, + "step": 474 + }, + { + "epoch": 0.5911636589919104, + "grad_norm": 2.7694528102874756, + "learning_rate": 4.494926199261993e-06, + "loss": 1.5695, + "step": 475 + }, + { + "epoch": 0.5924082140634723, + "grad_norm": 2.653353214263916, + "learning_rate": 4.4926199261992624e-06, + "loss": 1.5523, + "step": 476 + }, + { + "epoch": 0.5936527691350342, + "grad_norm": 2.6632070541381836, + "learning_rate": 4.490313653136532e-06, + "loss": 1.5675, + "step": 477 + }, + { + "epoch": 0.5948973242065961, + "grad_norm": 3.038543939590454, + "learning_rate": 4.488007380073801e-06, + "loss": 1.5692, + "step": 478 + }, + { + "epoch": 0.5961418792781581, + "grad_norm": 2.8123953342437744, + "learning_rate": 4.485701107011071e-06, + "loss": 1.6223, + "step": 479 + }, + { + "epoch": 0.5973864343497199, + "grad_norm": 2.5845773220062256, + "learning_rate": 4.483394833948339e-06, + "loss": 1.5773, + "step": 480 + }, + { + "epoch": 0.5973864343497199, + "eval_loss": 1.7419319152832031, + "eval_runtime": 44.6842, + "eval_samples_per_second": 22.379, + "eval_steps_per_second": 0.94, + "step": 480 + }, + { + "epoch": 0.5986309894212819, + "grad_norm": 3.069936752319336, + "learning_rate": 4.4810885608856096e-06, + "loss": 1.6323, + "step": 481 + }, + { + "epoch": 0.5998755444928439, + "grad_norm": 2.9085781574249268, + "learning_rate": 4.478782287822879e-06, + "loss": 1.5845, + "step": 482 + }, + { + "epoch": 0.6011200995644057, + "grad_norm": 2.8568010330200195, + "learning_rate": 4.4764760147601476e-06, + "loss": 1.5898, + "step": 483 + }, + { + "epoch": 0.6023646546359677, + "grad_norm": 3.089081048965454, + "learning_rate": 4.474169741697417e-06, + "loss": 1.6622, + "step": 484 + }, + { + "epoch": 0.6036092097075295, + "grad_norm": 2.8515470027923584, + "learning_rate": 4.471863468634686e-06, + "loss": 1.5859, + "step": 485 + }, + { + "epoch": 0.6048537647790915, + "grad_norm": 2.647765874862671, + "learning_rate": 4.469557195571957e-06, + "loss": 1.5926, + "step": 486 + }, + { + "epoch": 0.6060983198506534, + "grad_norm": 2.61676025390625, + "learning_rate": 4.467250922509225e-06, + "loss": 1.5684, + "step": 487 + }, + { + "epoch": 0.6073428749222153, + "grad_norm": 2.7127039432525635, + "learning_rate": 4.464944649446495e-06, + "loss": 1.5702, + "step": 488 + }, + { + "epoch": 0.6085874299937772, + "grad_norm": 2.9932055473327637, + "learning_rate": 4.462638376383764e-06, + "loss": 1.5976, + "step": 489 + }, + { + "epoch": 0.6098319850653391, + "grad_norm": 2.6508774757385254, + "learning_rate": 4.4603321033210335e-06, + "loss": 1.586, + "step": 490 + }, + { + "epoch": 0.6098319850653391, + "eval_loss": 1.7357326745986938, + "eval_runtime": 45.393, + "eval_samples_per_second": 22.03, + "eval_steps_per_second": 0.925, + "step": 490 + }, + { + "epoch": 0.611076540136901, + "grad_norm": 2.7626044750213623, + "learning_rate": 4.458025830258303e-06, + "loss": 1.5773, + "step": 491 + }, + { + "epoch": 0.612321095208463, + "grad_norm": 2.595003604888916, + "learning_rate": 4.455719557195572e-06, + "loss": 1.601, + "step": 492 + }, + { + "epoch": 0.6135656502800249, + "grad_norm": 2.595767021179199, + "learning_rate": 4.453413284132842e-06, + "loss": 1.6287, + "step": 493 + }, + { + "epoch": 0.6148102053515868, + "grad_norm": 2.755845308303833, + "learning_rate": 4.451107011070111e-06, + "loss": 1.593, + "step": 494 + }, + { + "epoch": 0.6160547604231488, + "grad_norm": 2.7302653789520264, + "learning_rate": 4.448800738007381e-06, + "loss": 1.5768, + "step": 495 + }, + { + "epoch": 0.6172993154947106, + "grad_norm": 2.6493024826049805, + "learning_rate": 4.446494464944649e-06, + "loss": 1.6059, + "step": 496 + }, + { + "epoch": 0.6185438705662726, + "grad_norm": 2.8633735179901123, + "learning_rate": 4.4441881918819195e-06, + "loss": 1.598, + "step": 497 + }, + { + "epoch": 0.6197884256378344, + "grad_norm": 2.65639066696167, + "learning_rate": 4.441881918819189e-06, + "loss": 1.6036, + "step": 498 + }, + { + "epoch": 0.6210329807093964, + "grad_norm": 2.6867759227752686, + "learning_rate": 4.439575645756458e-06, + "loss": 1.5676, + "step": 499 + }, + { + "epoch": 0.6222775357809583, + "grad_norm": 2.7986443042755127, + "learning_rate": 4.437269372693727e-06, + "loss": 1.524, + "step": 500 + }, + { + "epoch": 0.6222775357809583, + "eval_loss": 1.7449125051498413, + "eval_runtime": 43.9329, + "eval_samples_per_second": 22.762, + "eval_steps_per_second": 0.956, + "step": 500 + }, + { + "epoch": 0.6235220908525202, + "grad_norm": 2.665905714035034, + "learning_rate": 4.434963099630996e-06, + "loss": 1.5635, + "step": 501 + }, + { + "epoch": 0.6247666459240822, + "grad_norm": 2.902435541152954, + "learning_rate": 4.432656826568267e-06, + "loss": 1.6377, + "step": 502 + }, + { + "epoch": 0.6260112009956441, + "grad_norm": 2.6872262954711914, + "learning_rate": 4.430350553505535e-06, + "loss": 1.5553, + "step": 503 + }, + { + "epoch": 0.627255756067206, + "grad_norm": 2.676621913909912, + "learning_rate": 4.428044280442805e-06, + "loss": 1.6101, + "step": 504 + }, + { + "epoch": 0.6285003111387679, + "grad_norm": 2.756347179412842, + "learning_rate": 4.425738007380074e-06, + "loss": 1.5884, + "step": 505 + }, + { + "epoch": 0.6297448662103298, + "grad_norm": 2.856882333755493, + "learning_rate": 4.4234317343173434e-06, + "loss": 1.5635, + "step": 506 + }, + { + "epoch": 0.6309894212818917, + "grad_norm": 2.7708330154418945, + "learning_rate": 4.421125461254613e-06, + "loss": 1.6028, + "step": 507 + }, + { + "epoch": 0.6322339763534537, + "grad_norm": 2.8167600631713867, + "learning_rate": 4.418819188191882e-06, + "loss": 1.6026, + "step": 508 + }, + { + "epoch": 0.6334785314250155, + "grad_norm": 2.54194974899292, + "learning_rate": 4.416512915129152e-06, + "loss": 1.6001, + "step": 509 + }, + { + "epoch": 0.6347230864965775, + "grad_norm": 2.683037519454956, + "learning_rate": 4.414206642066421e-06, + "loss": 1.6084, + "step": 510 + }, + { + "epoch": 0.6347230864965775, + "eval_loss": 1.7376186847686768, + "eval_runtime": 42.3026, + "eval_samples_per_second": 23.639, + "eval_steps_per_second": 0.993, + "step": 510 + }, + { + "epoch": 0.6359676415681393, + "grad_norm": 2.82627010345459, + "learning_rate": 4.4119003690036905e-06, + "loss": 1.6258, + "step": 511 + }, + { + "epoch": 0.6372121966397013, + "grad_norm": 2.6139848232269287, + "learning_rate": 4.40959409594096e-06, + "loss": 1.6029, + "step": 512 + }, + { + "epoch": 0.6384567517112633, + "grad_norm": 2.646712303161621, + "learning_rate": 4.407287822878229e-06, + "loss": 1.5602, + "step": 513 + }, + { + "epoch": 0.6397013067828251, + "grad_norm": 2.910935163497925, + "learning_rate": 4.404981549815499e-06, + "loss": 1.5757, + "step": 514 + }, + { + "epoch": 0.6409458618543871, + "grad_norm": 2.6001148223876953, + "learning_rate": 4.402675276752768e-06, + "loss": 1.5857, + "step": 515 + }, + { + "epoch": 0.642190416925949, + "grad_norm": 2.8525235652923584, + "learning_rate": 4.400369003690037e-06, + "loss": 1.6092, + "step": 516 + }, + { + "epoch": 0.6434349719975109, + "grad_norm": 2.5335488319396973, + "learning_rate": 4.398062730627306e-06, + "loss": 1.5849, + "step": 517 + }, + { + "epoch": 0.6446795270690728, + "grad_norm": 2.5787103176116943, + "learning_rate": 4.3957564575645765e-06, + "loss": 1.634, + "step": 518 + }, + { + "epoch": 0.6459240821406347, + "grad_norm": 2.6188197135925293, + "learning_rate": 4.393450184501845e-06, + "loss": 1.5477, + "step": 519 + }, + { + "epoch": 0.6471686372121966, + "grad_norm": 2.6548666954040527, + "learning_rate": 4.3911439114391145e-06, + "loss": 1.6133, + "step": 520 + }, + { + "epoch": 0.6471686372121966, + "eval_loss": 1.7393039464950562, + "eval_runtime": 45.9234, + "eval_samples_per_second": 21.775, + "eval_steps_per_second": 0.915, + "step": 520 + }, + { + "epoch": 0.6484131922837586, + "grad_norm": 2.907928228378296, + "learning_rate": 4.388837638376384e-06, + "loss": 1.587, + "step": 521 + }, + { + "epoch": 0.6496577473553204, + "grad_norm": 2.7899692058563232, + "learning_rate": 4.386531365313653e-06, + "loss": 1.6126, + "step": 522 + }, + { + "epoch": 0.6509023024268824, + "grad_norm": 2.680147886276245, + "learning_rate": 4.384225092250923e-06, + "loss": 1.565, + "step": 523 + }, + { + "epoch": 0.6521468574984443, + "grad_norm": 2.6590754985809326, + "learning_rate": 4.381918819188192e-06, + "loss": 1.6127, + "step": 524 + }, + { + "epoch": 0.6533914125700062, + "grad_norm": 2.6638906002044678, + "learning_rate": 4.379612546125462e-06, + "loss": 1.5615, + "step": 525 + }, + { + "epoch": 0.6546359676415682, + "grad_norm": 2.6518194675445557, + "learning_rate": 4.377306273062731e-06, + "loss": 1.6202, + "step": 526 + }, + { + "epoch": 0.65588052271313, + "grad_norm": 2.5323619842529297, + "learning_rate": 4.3750000000000005e-06, + "loss": 1.5305, + "step": 527 + }, + { + "epoch": 0.657125077784692, + "grad_norm": 2.6871442794799805, + "learning_rate": 4.37269372693727e-06, + "loss": 1.6043, + "step": 528 + }, + { + "epoch": 0.6583696328562539, + "grad_norm": 2.760746717453003, + "learning_rate": 4.370387453874539e-06, + "loss": 1.6059, + "step": 529 + }, + { + "epoch": 0.6596141879278158, + "grad_norm": 2.780482769012451, + "learning_rate": 4.368081180811809e-06, + "loss": 1.6082, + "step": 530 + }, + { + "epoch": 0.6596141879278158, + "eval_loss": 1.7219713926315308, + "eval_runtime": 51.8478, + "eval_samples_per_second": 19.287, + "eval_steps_per_second": 0.81, + "step": 530 + }, + { + "epoch": 0.6608587429993777, + "grad_norm": 2.605890989303589, + "learning_rate": 4.365774907749078e-06, + "loss": 1.5554, + "step": 531 + }, + { + "epoch": 0.6621032980709396, + "grad_norm": 2.731555461883545, + "learning_rate": 4.363468634686347e-06, + "loss": 1.606, + "step": 532 + }, + { + "epoch": 0.6633478531425016, + "grad_norm": 2.8356943130493164, + "learning_rate": 4.361162361623616e-06, + "loss": 1.5601, + "step": 533 + }, + { + "epoch": 0.6645924082140635, + "grad_norm": 2.7196593284606934, + "learning_rate": 4.3588560885608864e-06, + "loss": 1.5722, + "step": 534 + }, + { + "epoch": 0.6658369632856254, + "grad_norm": 2.621371269226074, + "learning_rate": 4.356549815498156e-06, + "loss": 1.6194, + "step": 535 + }, + { + "epoch": 0.6670815183571873, + "grad_norm": 2.653916120529175, + "learning_rate": 4.354243542435424e-06, + "loss": 1.5592, + "step": 536 + }, + { + "epoch": 0.6683260734287492, + "grad_norm": 2.88431453704834, + "learning_rate": 4.351937269372694e-06, + "loss": 1.6296, + "step": 537 + }, + { + "epoch": 0.6695706285003111, + "grad_norm": 2.667130470275879, + "learning_rate": 4.349630996309963e-06, + "loss": 1.5624, + "step": 538 + }, + { + "epoch": 0.6708151835718731, + "grad_norm": 2.6453566551208496, + "learning_rate": 4.347324723247233e-06, + "loss": 1.6025, + "step": 539 + }, + { + "epoch": 0.6720597386434349, + "grad_norm": 3.032271146774292, + "learning_rate": 4.345018450184502e-06, + "loss": 1.6771, + "step": 540 + }, + { + "epoch": 0.6720597386434349, + "eval_loss": 1.721895456314087, + "eval_runtime": 50.62, + "eval_samples_per_second": 19.755, + "eval_steps_per_second": 0.83, + "step": 540 + }, + { + "epoch": 0.6733042937149969, + "grad_norm": 2.8476362228393555, + "learning_rate": 4.3427121771217715e-06, + "loss": 1.5663, + "step": 541 + }, + { + "epoch": 0.6745488487865589, + "grad_norm": 2.615602970123291, + "learning_rate": 4.340405904059041e-06, + "loss": 1.5815, + "step": 542 + }, + { + "epoch": 0.6757934038581207, + "grad_norm": 3.221571683883667, + "learning_rate": 4.33809963099631e-06, + "loss": 1.5955, + "step": 543 + }, + { + "epoch": 0.6770379589296827, + "grad_norm": 2.602994203567505, + "learning_rate": 4.33579335793358e-06, + "loss": 1.5667, + "step": 544 + }, + { + "epoch": 0.6782825140012445, + "grad_norm": 2.9380433559417725, + "learning_rate": 4.333487084870848e-06, + "loss": 1.5651, + "step": 545 + }, + { + "epoch": 0.6795270690728065, + "grad_norm": 3.0689425468444824, + "learning_rate": 4.331180811808119e-06, + "loss": 1.6003, + "step": 546 + }, + { + "epoch": 0.6807716241443684, + "grad_norm": 2.770317316055298, + "learning_rate": 4.328874538745388e-06, + "loss": 1.5576, + "step": 547 + }, + { + "epoch": 0.6820161792159303, + "grad_norm": 2.8631815910339355, + "learning_rate": 4.3265682656826575e-06, + "loss": 1.5308, + "step": 548 + }, + { + "epoch": 0.6832607342874922, + "grad_norm": 2.785576820373535, + "learning_rate": 4.324261992619926e-06, + "loss": 1.6113, + "step": 549 + }, + { + "epoch": 0.6845052893590542, + "grad_norm": 2.723919630050659, + "learning_rate": 4.321955719557196e-06, + "loss": 1.5606, + "step": 550 + }, + { + "epoch": 0.6845052893590542, + "eval_loss": 1.721803069114685, + "eval_runtime": 51.6036, + "eval_samples_per_second": 19.378, + "eval_steps_per_second": 0.814, + "step": 550 + }, + { + "epoch": 0.685749844430616, + "grad_norm": 3.081198215484619, + "learning_rate": 4.319649446494466e-06, + "loss": 1.5643, + "step": 551 + }, + { + "epoch": 0.686994399502178, + "grad_norm": 2.757392168045044, + "learning_rate": 4.317343173431734e-06, + "loss": 1.5909, + "step": 552 + }, + { + "epoch": 0.6882389545737398, + "grad_norm": 3.038753032684326, + "learning_rate": 4.315036900369004e-06, + "loss": 1.6023, + "step": 553 + }, + { + "epoch": 0.6894835096453018, + "grad_norm": 2.8246452808380127, + "learning_rate": 4.312730627306273e-06, + "loss": 1.5548, + "step": 554 + }, + { + "epoch": 0.6907280647168638, + "grad_norm": 2.589320182800293, + "learning_rate": 4.310424354243543e-06, + "loss": 1.5424, + "step": 555 + }, + { + "epoch": 0.6919726197884256, + "grad_norm": 2.6413373947143555, + "learning_rate": 4.308118081180812e-06, + "loss": 1.6243, + "step": 556 + }, + { + "epoch": 0.6932171748599876, + "grad_norm": 2.764784336090088, + "learning_rate": 4.3058118081180815e-06, + "loss": 1.589, + "step": 557 + }, + { + "epoch": 0.6944617299315494, + "grad_norm": 2.806821823120117, + "learning_rate": 4.303505535055351e-06, + "loss": 1.621, + "step": 558 + }, + { + "epoch": 0.6957062850031114, + "grad_norm": 2.718017578125, + "learning_rate": 4.30119926199262e-06, + "loss": 1.5773, + "step": 559 + }, + { + "epoch": 0.6969508400746733, + "grad_norm": 2.7620160579681396, + "learning_rate": 4.29889298892989e-06, + "loss": 1.5868, + "step": 560 + }, + { + "epoch": 0.6969508400746733, + "eval_loss": 1.7152249813079834, + "eval_runtime": 52.0463, + "eval_samples_per_second": 19.214, + "eval_steps_per_second": 0.807, + "step": 560 + }, + { + "epoch": 0.6981953951462352, + "grad_norm": 2.66686749458313, + "learning_rate": 4.296586715867159e-06, + "loss": 1.534, + "step": 561 + }, + { + "epoch": 0.6994399502177971, + "grad_norm": 2.812577247619629, + "learning_rate": 4.2942804428044286e-06, + "loss": 1.5501, + "step": 562 + }, + { + "epoch": 0.7006845052893591, + "grad_norm": 2.578508138656616, + "learning_rate": 4.291974169741698e-06, + "loss": 1.6286, + "step": 563 + }, + { + "epoch": 0.701929060360921, + "grad_norm": 2.524249315261841, + "learning_rate": 4.289667896678967e-06, + "loss": 1.5719, + "step": 564 + }, + { + "epoch": 0.7031736154324829, + "grad_norm": 2.827235460281372, + "learning_rate": 4.287361623616236e-06, + "loss": 1.5333, + "step": 565 + }, + { + "epoch": 0.7044181705040448, + "grad_norm": 2.6359963417053223, + "learning_rate": 4.285055350553506e-06, + "loss": 1.5849, + "step": 566 + }, + { + "epoch": 0.7056627255756067, + "grad_norm": 2.930530071258545, + "learning_rate": 4.282749077490776e-06, + "loss": 1.5672, + "step": 567 + }, + { + "epoch": 0.7069072806471687, + "grad_norm": 2.750102996826172, + "learning_rate": 4.280442804428044e-06, + "loss": 1.5696, + "step": 568 + }, + { + "epoch": 0.7081518357187305, + "grad_norm": 2.869690418243408, + "learning_rate": 4.278136531365314e-06, + "loss": 1.5689, + "step": 569 + }, + { + "epoch": 0.7093963907902925, + "grad_norm": 2.954852819442749, + "learning_rate": 4.275830258302583e-06, + "loss": 1.5931, + "step": 570 + }, + { + "epoch": 0.7093963907902925, + "eval_loss": 1.7143300771713257, + "eval_runtime": 52.7293, + "eval_samples_per_second": 18.965, + "eval_steps_per_second": 0.797, + "step": 570 + }, + { + "epoch": 0.7106409458618543, + "grad_norm": 2.702223539352417, + "learning_rate": 4.273523985239853e-06, + "loss": 1.5635, + "step": 571 + }, + { + "epoch": 0.7118855009334163, + "grad_norm": 2.689995050430298, + "learning_rate": 4.271217712177122e-06, + "loss": 1.5545, + "step": 572 + }, + { + "epoch": 0.7131300560049783, + "grad_norm": 2.68979811668396, + "learning_rate": 4.268911439114391e-06, + "loss": 1.5404, + "step": 573 + }, + { + "epoch": 0.7143746110765401, + "grad_norm": 2.7477986812591553, + "learning_rate": 4.266605166051661e-06, + "loss": 1.5719, + "step": 574 + }, + { + "epoch": 0.7156191661481021, + "grad_norm": 2.975778341293335, + "learning_rate": 4.26429889298893e-06, + "loss": 1.5587, + "step": 575 + }, + { + "epoch": 0.716863721219664, + "grad_norm": 2.658170700073242, + "learning_rate": 4.2619926199262e-06, + "loss": 1.5746, + "step": 576 + }, + { + "epoch": 0.7181082762912259, + "grad_norm": 2.8825011253356934, + "learning_rate": 4.259686346863469e-06, + "loss": 1.5246, + "step": 577 + }, + { + "epoch": 0.7193528313627878, + "grad_norm": 2.845280170440674, + "learning_rate": 4.2573800738007385e-06, + "loss": 1.5703, + "step": 578 + }, + { + "epoch": 0.7205973864343497, + "grad_norm": 2.660616636276245, + "learning_rate": 4.255073800738008e-06, + "loss": 1.5373, + "step": 579 + }, + { + "epoch": 0.7218419415059116, + "grad_norm": 2.749447822570801, + "learning_rate": 4.252767527675277e-06, + "loss": 1.59, + "step": 580 + }, + { + "epoch": 0.7218419415059116, + "eval_loss": 1.7149444818496704, + "eval_runtime": 52.2099, + "eval_samples_per_second": 19.153, + "eval_steps_per_second": 0.804, + "step": 580 + }, + { + "epoch": 0.7230864965774736, + "grad_norm": 2.813328742980957, + "learning_rate": 4.250461254612546e-06, + "loss": 1.5694, + "step": 581 + }, + { + "epoch": 0.7243310516490354, + "grad_norm": 2.823866844177246, + "learning_rate": 4.248154981549816e-06, + "loss": 1.5143, + "step": 582 + }, + { + "epoch": 0.7255756067205974, + "grad_norm": 2.71337890625, + "learning_rate": 4.245848708487086e-06, + "loss": 1.5915, + "step": 583 + }, + { + "epoch": 0.7268201617921594, + "grad_norm": 2.718085765838623, + "learning_rate": 4.243542435424355e-06, + "loss": 1.604, + "step": 584 + }, + { + "epoch": 0.7280647168637212, + "grad_norm": 2.5741796493530273, + "learning_rate": 4.241236162361624e-06, + "loss": 1.5398, + "step": 585 + }, + { + "epoch": 0.7293092719352832, + "grad_norm": 2.727114200592041, + "learning_rate": 4.238929889298893e-06, + "loss": 1.5886, + "step": 586 + }, + { + "epoch": 0.730553827006845, + "grad_norm": 2.6703338623046875, + "learning_rate": 4.236623616236163e-06, + "loss": 1.5959, + "step": 587 + }, + { + "epoch": 0.731798382078407, + "grad_norm": 2.750326633453369, + "learning_rate": 4.234317343173432e-06, + "loss": 1.5916, + "step": 588 + }, + { + "epoch": 0.7330429371499689, + "grad_norm": 2.6680715084075928, + "learning_rate": 4.232011070110701e-06, + "loss": 1.562, + "step": 589 + }, + { + "epoch": 0.7342874922215308, + "grad_norm": 2.6603832244873047, + "learning_rate": 4.229704797047971e-06, + "loss": 1.6121, + "step": 590 + }, + { + "epoch": 0.7342874922215308, + "eval_loss": 1.7144734859466553, + "eval_runtime": 50.5317, + "eval_samples_per_second": 19.79, + "eval_steps_per_second": 0.831, + "step": 590 + }, + { + "epoch": 0.7355320472930927, + "grad_norm": 2.7868523597717285, + "learning_rate": 4.22739852398524e-06, + "loss": 1.5488, + "step": 591 + }, + { + "epoch": 0.7367766023646546, + "grad_norm": 2.755270004272461, + "learning_rate": 4.2250922509225096e-06, + "loss": 1.5801, + "step": 592 + }, + { + "epoch": 0.7380211574362165, + "grad_norm": 2.7614126205444336, + "learning_rate": 4.222785977859779e-06, + "loss": 1.5507, + "step": 593 + }, + { + "epoch": 0.7392657125077785, + "grad_norm": 2.8674862384796143, + "learning_rate": 4.220479704797048e-06, + "loss": 1.5807, + "step": 594 + }, + { + "epoch": 0.7405102675793404, + "grad_norm": 2.7366859912872314, + "learning_rate": 4.218173431734318e-06, + "loss": 1.6594, + "step": 595 + }, + { + "epoch": 0.7417548226509023, + "grad_norm": 2.6231045722961426, + "learning_rate": 4.215867158671587e-06, + "loss": 1.5609, + "step": 596 + }, + { + "epoch": 0.7429993777224643, + "grad_norm": 2.7694146633148193, + "learning_rate": 4.213560885608857e-06, + "loss": 1.5889, + "step": 597 + }, + { + "epoch": 0.7442439327940261, + "grad_norm": 2.6275386810302734, + "learning_rate": 4.211254612546125e-06, + "loss": 1.575, + "step": 598 + }, + { + "epoch": 0.7454884878655881, + "grad_norm": 2.9413866996765137, + "learning_rate": 4.2089483394833955e-06, + "loss": 1.589, + "step": 599 + }, + { + "epoch": 0.7467330429371499, + "grad_norm": 2.768606185913086, + "learning_rate": 4.206642066420665e-06, + "loss": 1.5738, + "step": 600 + }, + { + "epoch": 0.7467330429371499, + "eval_loss": 1.7148027420043945, + "eval_runtime": 54.7031, + "eval_samples_per_second": 18.28, + "eval_steps_per_second": 0.768, + "step": 600 + }, + { + "epoch": 0.7479775980087119, + "grad_norm": 2.7917706966400146, + "learning_rate": 4.2043357933579335e-06, + "loss": 1.499, + "step": 601 + }, + { + "epoch": 0.7492221530802738, + "grad_norm": 2.5982208251953125, + "learning_rate": 4.202029520295203e-06, + "loss": 1.5113, + "step": 602 + }, + { + "epoch": 0.7504667081518357, + "grad_norm": 2.692032814025879, + "learning_rate": 4.199723247232473e-06, + "loss": 1.4721, + "step": 603 + }, + { + "epoch": 0.7517112632233977, + "grad_norm": 2.975860834121704, + "learning_rate": 4.197416974169742e-06, + "loss": 1.5147, + "step": 604 + }, + { + "epoch": 0.7529558182949595, + "grad_norm": 2.758610248565674, + "learning_rate": 4.195110701107011e-06, + "loss": 1.5674, + "step": 605 + }, + { + "epoch": 0.7542003733665215, + "grad_norm": 2.7620184421539307, + "learning_rate": 4.192804428044281e-06, + "loss": 1.5669, + "step": 606 + }, + { + "epoch": 0.7554449284380834, + "grad_norm": 3.043940305709839, + "learning_rate": 4.19049815498155e-06, + "loss": 1.587, + "step": 607 + }, + { + "epoch": 0.7566894835096453, + "grad_norm": 2.7482151985168457, + "learning_rate": 4.1881918819188195e-06, + "loss": 1.5451, + "step": 608 + }, + { + "epoch": 0.7579340385812072, + "grad_norm": 2.7259294986724854, + "learning_rate": 4.185885608856089e-06, + "loss": 1.5295, + "step": 609 + }, + { + "epoch": 0.7591785936527692, + "grad_norm": 2.6340832710266113, + "learning_rate": 4.183579335793358e-06, + "loss": 1.5335, + "step": 610 + }, + { + "epoch": 0.7591785936527692, + "eval_loss": 1.7038393020629883, + "eval_runtime": 44.0792, + "eval_samples_per_second": 22.686, + "eval_steps_per_second": 0.953, + "step": 610 + }, + { + "epoch": 0.760423148724331, + "grad_norm": 2.6144909858703613, + "learning_rate": 4.181273062730628e-06, + "loss": 1.5907, + "step": 611 + }, + { + "epoch": 0.761667703795893, + "grad_norm": 2.7754175662994385, + "learning_rate": 4.178966789667897e-06, + "loss": 1.5773, + "step": 612 + }, + { + "epoch": 0.7629122588674548, + "grad_norm": 2.6313252449035645, + "learning_rate": 4.176660516605167e-06, + "loss": 1.5675, + "step": 613 + }, + { + "epoch": 0.7641568139390168, + "grad_norm": 2.549074649810791, + "learning_rate": 4.174354243542435e-06, + "loss": 1.595, + "step": 614 + }, + { + "epoch": 0.7654013690105788, + "grad_norm": 2.611804246902466, + "learning_rate": 4.1720479704797054e-06, + "loss": 1.5416, + "step": 615 + }, + { + "epoch": 0.7666459240821406, + "grad_norm": 2.6322927474975586, + "learning_rate": 4.169741697416975e-06, + "loss": 1.604, + "step": 616 + }, + { + "epoch": 0.7678904791537026, + "grad_norm": 2.5792219638824463, + "learning_rate": 4.1674354243542434e-06, + "loss": 1.5976, + "step": 617 + }, + { + "epoch": 0.7691350342252644, + "grad_norm": 2.845416307449341, + "learning_rate": 4.165129151291513e-06, + "loss": 1.5945, + "step": 618 + }, + { + "epoch": 0.7703795892968264, + "grad_norm": 2.8647871017456055, + "learning_rate": 4.162822878228783e-06, + "loss": 1.5637, + "step": 619 + }, + { + "epoch": 0.7716241443683883, + "grad_norm": 2.590719699859619, + "learning_rate": 4.1605166051660526e-06, + "loss": 1.556, + "step": 620 + }, + { + "epoch": 0.7716241443683883, + "eval_loss": 1.7115222215652466, + "eval_runtime": 50.3905, + "eval_samples_per_second": 19.845, + "eval_steps_per_second": 0.833, + "step": 620 + }, + { + "epoch": 0.7728686994399502, + "grad_norm": 2.626723527908325, + "learning_rate": 4.158210332103321e-06, + "loss": 1.5486, + "step": 621 + }, + { + "epoch": 0.7741132545115121, + "grad_norm": 2.713069438934326, + "learning_rate": 4.1559040590405906e-06, + "loss": 1.5474, + "step": 622 + }, + { + "epoch": 0.7753578095830741, + "grad_norm": 2.685563087463379, + "learning_rate": 4.15359778597786e-06, + "loss": 1.5529, + "step": 623 + }, + { + "epoch": 0.776602364654636, + "grad_norm": 2.690586805343628, + "learning_rate": 4.151291512915129e-06, + "loss": 1.5293, + "step": 624 + }, + { + "epoch": 0.7778469197261979, + "grad_norm": 2.865345001220703, + "learning_rate": 4.148985239852399e-06, + "loss": 1.5095, + "step": 625 + }, + { + "epoch": 0.7790914747977598, + "grad_norm": 2.92393159866333, + "learning_rate": 4.146678966789668e-06, + "loss": 1.5561, + "step": 626 + }, + { + "epoch": 0.7803360298693217, + "grad_norm": 2.554480791091919, + "learning_rate": 4.144372693726938e-06, + "loss": 1.4876, + "step": 627 + }, + { + "epoch": 0.7815805849408837, + "grad_norm": 2.6128833293914795, + "learning_rate": 4.142066420664207e-06, + "loss": 1.5378, + "step": 628 + }, + { + "epoch": 0.7828251400124455, + "grad_norm": 3.0110888481140137, + "learning_rate": 4.1397601476014765e-06, + "loss": 1.5859, + "step": 629 + }, + { + "epoch": 0.7840696950840075, + "grad_norm": 2.8152706623077393, + "learning_rate": 4.137453874538745e-06, + "loss": 1.5841, + "step": 630 + }, + { + "epoch": 0.7840696950840075, + "eval_loss": 1.702181339263916, + "eval_runtime": 53.6926, + "eval_samples_per_second": 18.625, + "eval_steps_per_second": 0.782, + "step": 630 + }, + { + "epoch": 0.7853142501555694, + "grad_norm": 2.8634233474731445, + "learning_rate": 4.135147601476015e-06, + "loss": 1.564, + "step": 631 + }, + { + "epoch": 0.7865588052271313, + "grad_norm": 2.8939666748046875, + "learning_rate": 4.132841328413285e-06, + "loss": 1.5859, + "step": 632 + }, + { + "epoch": 0.7878033602986932, + "grad_norm": 2.6560444831848145, + "learning_rate": 4.130535055350554e-06, + "loss": 1.5199, + "step": 633 + }, + { + "epoch": 0.7890479153702551, + "grad_norm": 2.964721202850342, + "learning_rate": 4.128228782287823e-06, + "loss": 1.5888, + "step": 634 + }, + { + "epoch": 0.790292470441817, + "grad_norm": 2.74668288230896, + "learning_rate": 4.125922509225092e-06, + "loss": 1.5847, + "step": 635 + }, + { + "epoch": 0.791537025513379, + "grad_norm": 2.723123550415039, + "learning_rate": 4.1236162361623625e-06, + "loss": 1.5166, + "step": 636 + }, + { + "epoch": 0.7927815805849409, + "grad_norm": 2.7265713214874268, + "learning_rate": 4.121309963099631e-06, + "loss": 1.5954, + "step": 637 + }, + { + "epoch": 0.7940261356565028, + "grad_norm": 2.979126214981079, + "learning_rate": 4.1190036900369005e-06, + "loss": 1.5583, + "step": 638 + }, + { + "epoch": 0.7952706907280647, + "grad_norm": 2.844376802444458, + "learning_rate": 4.11669741697417e-06, + "loss": 1.5564, + "step": 639 + }, + { + "epoch": 0.7965152457996266, + "grad_norm": 2.7583703994750977, + "learning_rate": 4.114391143911439e-06, + "loss": 1.5094, + "step": 640 + }, + { + "epoch": 0.7965152457996266, + "eval_loss": 1.6986565589904785, + "eval_runtime": 47.3549, + "eval_samples_per_second": 21.117, + "eval_steps_per_second": 0.887, + "step": 640 + }, + { + "epoch": 0.7977598008711886, + "grad_norm": 2.8942224979400635, + "learning_rate": 4.112084870848709e-06, + "loss": 1.5504, + "step": 641 + }, + { + "epoch": 0.7990043559427504, + "grad_norm": 2.6590495109558105, + "learning_rate": 4.109778597785978e-06, + "loss": 1.5227, + "step": 642 + }, + { + "epoch": 0.8002489110143124, + "grad_norm": 2.5988378524780273, + "learning_rate": 4.107472324723248e-06, + "loss": 1.5054, + "step": 643 + }, + { + "epoch": 0.8014934660858744, + "grad_norm": 2.787335157394409, + "learning_rate": 4.105166051660517e-06, + "loss": 1.5844, + "step": 644 + }, + { + "epoch": 0.8027380211574362, + "grad_norm": 2.8202896118164062, + "learning_rate": 4.1028597785977864e-06, + "loss": 1.5596, + "step": 645 + }, + { + "epoch": 0.8039825762289982, + "grad_norm": 2.65376615524292, + "learning_rate": 4.100553505535056e-06, + "loss": 1.5893, + "step": 646 + }, + { + "epoch": 0.80522713130056, + "grad_norm": 2.5933308601379395, + "learning_rate": 4.098247232472325e-06, + "loss": 1.5403, + "step": 647 + }, + { + "epoch": 0.806471686372122, + "grad_norm": 2.777070999145508, + "learning_rate": 4.095940959409595e-06, + "loss": 1.5231, + "step": 648 + }, + { + "epoch": 0.8077162414436839, + "grad_norm": 2.6427664756774902, + "learning_rate": 4.093634686346864e-06, + "loss": 1.5937, + "step": 649 + }, + { + "epoch": 0.8089607965152458, + "grad_norm": 2.651561737060547, + "learning_rate": 4.091328413284133e-06, + "loss": 1.4847, + "step": 650 + }, + { + "epoch": 0.8089607965152458, + "eval_loss": 1.7089996337890625, + "eval_runtime": 47.4346, + "eval_samples_per_second": 21.082, + "eval_steps_per_second": 0.885, + "step": 650 + }, + { + "epoch": 0.8102053515868077, + "grad_norm": 2.844067335128784, + "learning_rate": 4.089022140221402e-06, + "loss": 1.488, + "step": 651 + }, + { + "epoch": 0.8114499066583696, + "grad_norm": 2.603480577468872, + "learning_rate": 4.086715867158672e-06, + "loss": 1.5248, + "step": 652 + }, + { + "epoch": 0.8126944617299315, + "grad_norm": 2.789322853088379, + "learning_rate": 4.084409594095941e-06, + "loss": 1.5119, + "step": 653 + }, + { + "epoch": 0.8139390168014935, + "grad_norm": 2.7183773517608643, + "learning_rate": 4.08210332103321e-06, + "loss": 1.5123, + "step": 654 + }, + { + "epoch": 0.8151835718730553, + "grad_norm": 2.80613374710083, + "learning_rate": 4.07979704797048e-06, + "loss": 1.5526, + "step": 655 + }, + { + "epoch": 0.8164281269446173, + "grad_norm": 2.641035795211792, + "learning_rate": 4.077490774907749e-06, + "loss": 1.6127, + "step": 656 + }, + { + "epoch": 0.8176726820161793, + "grad_norm": 2.773684024810791, + "learning_rate": 4.075184501845019e-06, + "loss": 1.5404, + "step": 657 + }, + { + "epoch": 0.8189172370877411, + "grad_norm": 2.6951382160186768, + "learning_rate": 4.072878228782288e-06, + "loss": 1.4928, + "step": 658 + }, + { + "epoch": 0.8201617921593031, + "grad_norm": 2.931771755218506, + "learning_rate": 4.0705719557195575e-06, + "loss": 1.5949, + "step": 659 + }, + { + "epoch": 0.8214063472308649, + "grad_norm": 2.8786075115203857, + "learning_rate": 4.068265682656827e-06, + "loss": 1.5582, + "step": 660 + }, + { + "epoch": 0.8214063472308649, + "eval_loss": 1.698564052581787, + "eval_runtime": 46.6299, + "eval_samples_per_second": 21.445, + "eval_steps_per_second": 0.901, + "step": 660 + }, + { + "epoch": 0.8226509023024269, + "grad_norm": 2.7402451038360596, + "learning_rate": 4.065959409594096e-06, + "loss": 1.5562, + "step": 661 + }, + { + "epoch": 0.8238954573739888, + "grad_norm": 2.7968692779541016, + "learning_rate": 4.063653136531366e-06, + "loss": 1.556, + "step": 662 + }, + { + "epoch": 0.8251400124455507, + "grad_norm": 2.8233132362365723, + "learning_rate": 4.061346863468635e-06, + "loss": 1.5527, + "step": 663 + }, + { + "epoch": 0.8263845675171126, + "grad_norm": 2.713390827178955, + "learning_rate": 4.059040590405905e-06, + "loss": 1.6012, + "step": 664 + }, + { + "epoch": 0.8276291225886746, + "grad_norm": 2.7783989906311035, + "learning_rate": 4.056734317343174e-06, + "loss": 1.5247, + "step": 665 + }, + { + "epoch": 0.8288736776602365, + "grad_norm": 2.808554172515869, + "learning_rate": 4.054428044280443e-06, + "loss": 1.5412, + "step": 666 + }, + { + "epoch": 0.8301182327317984, + "grad_norm": 2.8476831912994385, + "learning_rate": 4.052121771217712e-06, + "loss": 1.5307, + "step": 667 + }, + { + "epoch": 0.8313627878033603, + "grad_norm": 2.8287601470947266, + "learning_rate": 4.049815498154982e-06, + "loss": 1.6021, + "step": 668 + }, + { + "epoch": 0.8326073428749222, + "grad_norm": 2.975510835647583, + "learning_rate": 4.047509225092252e-06, + "loss": 1.5474, + "step": 669 + }, + { + "epoch": 0.8338518979464842, + "grad_norm": 2.723236560821533, + "learning_rate": 4.04520295202952e-06, + "loss": 1.5242, + "step": 670 + }, + { + "epoch": 0.8338518979464842, + "eval_loss": 1.7005239725112915, + "eval_runtime": 49.4504, + "eval_samples_per_second": 20.222, + "eval_steps_per_second": 0.849, + "step": 670 + }, + { + "epoch": 0.835096453018046, + "grad_norm": 2.7521748542785645, + "learning_rate": 4.04289667896679e-06, + "loss": 1.5467, + "step": 671 + }, + { + "epoch": 0.836341008089608, + "grad_norm": 2.7770044803619385, + "learning_rate": 4.04059040590406e-06, + "loss": 1.5407, + "step": 672 + }, + { + "epoch": 0.8375855631611698, + "grad_norm": 2.744323492050171, + "learning_rate": 4.038284132841329e-06, + "loss": 1.5422, + "step": 673 + }, + { + "epoch": 0.8388301182327318, + "grad_norm": 2.6699817180633545, + "learning_rate": 4.035977859778598e-06, + "loss": 1.5508, + "step": 674 + }, + { + "epoch": 0.8400746733042938, + "grad_norm": 2.705273151397705, + "learning_rate": 4.0336715867158674e-06, + "loss": 1.5099, + "step": 675 + }, + { + "epoch": 0.8413192283758556, + "grad_norm": 2.9485747814178467, + "learning_rate": 4.031365313653137e-06, + "loss": 1.5529, + "step": 676 + }, + { + "epoch": 0.8425637834474176, + "grad_norm": 3.1750423908233643, + "learning_rate": 4.029059040590406e-06, + "loss": 1.5894, + "step": 677 + }, + { + "epoch": 0.8438083385189795, + "grad_norm": 2.76448655128479, + "learning_rate": 4.026752767527676e-06, + "loss": 1.5752, + "step": 678 + }, + { + "epoch": 0.8450528935905414, + "grad_norm": 2.676708698272705, + "learning_rate": 4.024446494464945e-06, + "loss": 1.5487, + "step": 679 + }, + { + "epoch": 0.8462974486621033, + "grad_norm": 2.6832051277160645, + "learning_rate": 4.0221402214022145e-06, + "loss": 1.5055, + "step": 680 + }, + { + "epoch": 0.8462974486621033, + "eval_loss": 1.6945017576217651, + "eval_runtime": 43.9369, + "eval_samples_per_second": 22.76, + "eval_steps_per_second": 0.956, + "step": 680 + }, + { + "epoch": 0.8475420037336652, + "grad_norm": 2.6812336444854736, + "learning_rate": 4.019833948339484e-06, + "loss": 1.5315, + "step": 681 + }, + { + "epoch": 0.8487865588052271, + "grad_norm": 2.9394326210021973, + "learning_rate": 4.017527675276753e-06, + "loss": 1.5636, + "step": 682 + }, + { + "epoch": 0.8500311138767891, + "grad_norm": 2.944952964782715, + "learning_rate": 4.015221402214022e-06, + "loss": 1.56, + "step": 683 + }, + { + "epoch": 0.8512756689483509, + "grad_norm": 2.7475314140319824, + "learning_rate": 4.012915129151292e-06, + "loss": 1.5442, + "step": 684 + }, + { + "epoch": 0.8525202240199129, + "grad_norm": 3.0572879314422607, + "learning_rate": 4.010608856088562e-06, + "loss": 1.5023, + "step": 685 + }, + { + "epoch": 0.8537647790914747, + "grad_norm": 2.75365948677063, + "learning_rate": 4.00830258302583e-06, + "loss": 1.5414, + "step": 686 + }, + { + "epoch": 0.8550093341630367, + "grad_norm": 2.5714449882507324, + "learning_rate": 4.0059963099631e-06, + "loss": 1.5027, + "step": 687 + }, + { + "epoch": 0.8562538892345987, + "grad_norm": 2.755167007446289, + "learning_rate": 4.003690036900369e-06, + "loss": 1.5788, + "step": 688 + }, + { + "epoch": 0.8574984443061605, + "grad_norm": 2.798967123031616, + "learning_rate": 4.0013837638376385e-06, + "loss": 1.5162, + "step": 689 + }, + { + "epoch": 0.8587429993777225, + "grad_norm": 2.803614854812622, + "learning_rate": 3.999077490774908e-06, + "loss": 1.5079, + "step": 690 + }, + { + "epoch": 0.8587429993777225, + "eval_loss": 1.6924962997436523, + "eval_runtime": 44.47, + "eval_samples_per_second": 22.487, + "eval_steps_per_second": 0.944, + "step": 690 + }, + { + "epoch": 0.8599875544492844, + "grad_norm": 2.6898066997528076, + "learning_rate": 3.996771217712177e-06, + "loss": 1.5478, + "step": 691 + }, + { + "epoch": 0.8612321095208463, + "grad_norm": 2.6376144886016846, + "learning_rate": 3.994464944649447e-06, + "loss": 1.5362, + "step": 692 + }, + { + "epoch": 0.8624766645924082, + "grad_norm": 2.7240405082702637, + "learning_rate": 3.992158671586716e-06, + "loss": 1.5502, + "step": 693 + }, + { + "epoch": 0.8637212196639701, + "grad_norm": 2.929445266723633, + "learning_rate": 3.989852398523986e-06, + "loss": 1.5799, + "step": 694 + }, + { + "epoch": 0.864965774735532, + "grad_norm": 2.593223810195923, + "learning_rate": 3.987546125461255e-06, + "loss": 1.5352, + "step": 695 + }, + { + "epoch": 0.866210329807094, + "grad_norm": 2.7710154056549072, + "learning_rate": 3.9852398523985245e-06, + "loss": 1.5048, + "step": 696 + }, + { + "epoch": 0.8674548848786559, + "grad_norm": 2.7370848655700684, + "learning_rate": 3.982933579335794e-06, + "loss": 1.5233, + "step": 697 + }, + { + "epoch": 0.8686994399502178, + "grad_norm": 2.8632168769836426, + "learning_rate": 3.980627306273063e-06, + "loss": 1.5435, + "step": 698 + }, + { + "epoch": 0.8699439950217797, + "grad_norm": 2.752298593521118, + "learning_rate": 3.978321033210332e-06, + "loss": 1.5728, + "step": 699 + }, + { + "epoch": 0.8711885500933416, + "grad_norm": 2.8345625400543213, + "learning_rate": 3.976014760147602e-06, + "loss": 1.5727, + "step": 700 + }, + { + "epoch": 0.8711885500933416, + "eval_loss": 1.6855305433273315, + "eval_runtime": 43.3683, + "eval_samples_per_second": 23.058, + "eval_steps_per_second": 0.968, + "step": 700 + }, + { + "epoch": 0.8724331051649036, + "grad_norm": 2.8135335445404053, + "learning_rate": 3.973708487084872e-06, + "loss": 1.5086, + "step": 701 + }, + { + "epoch": 0.8736776602364654, + "grad_norm": 2.7416603565216064, + "learning_rate": 3.97140221402214e-06, + "loss": 1.545, + "step": 702 + }, + { + "epoch": 0.8749222153080274, + "grad_norm": 2.6902289390563965, + "learning_rate": 3.96909594095941e-06, + "loss": 1.5509, + "step": 703 + }, + { + "epoch": 0.8761667703795893, + "grad_norm": 2.6537084579467773, + "learning_rate": 3.966789667896679e-06, + "loss": 1.562, + "step": 704 + }, + { + "epoch": 0.8774113254511512, + "grad_norm": 2.6979284286499023, + "learning_rate": 3.964483394833948e-06, + "loss": 1.4829, + "step": 705 + }, + { + "epoch": 0.8786558805227132, + "grad_norm": 2.5795764923095703, + "learning_rate": 3.962177121771218e-06, + "loss": 1.4677, + "step": 706 + }, + { + "epoch": 0.879900435594275, + "grad_norm": 3.0185086727142334, + "learning_rate": 3.959870848708487e-06, + "loss": 1.5608, + "step": 707 + }, + { + "epoch": 0.881144990665837, + "grad_norm": 2.6458113193511963, + "learning_rate": 3.957564575645757e-06, + "loss": 1.5342, + "step": 708 + }, + { + "epoch": 0.8823895457373989, + "grad_norm": 2.7985665798187256, + "learning_rate": 3.955258302583026e-06, + "loss": 1.527, + "step": 709 + }, + { + "epoch": 0.8836341008089608, + "grad_norm": 2.9069082736968994, + "learning_rate": 3.9529520295202955e-06, + "loss": 1.5156, + "step": 710 + }, + { + "epoch": 0.8836341008089608, + "eval_loss": 1.6898822784423828, + "eval_runtime": 42.8749, + "eval_samples_per_second": 23.324, + "eval_steps_per_second": 0.98, + "step": 710 + }, + { + "epoch": 0.8848786558805227, + "grad_norm": 2.760143995285034, + "learning_rate": 3.950645756457565e-06, + "loss": 1.5124, + "step": 711 + }, + { + "epoch": 0.8861232109520847, + "grad_norm": 2.8085529804229736, + "learning_rate": 3.948339483394834e-06, + "loss": 1.5862, + "step": 712 + }, + { + "epoch": 0.8873677660236465, + "grad_norm": 2.909905195236206, + "learning_rate": 3.946033210332104e-06, + "loss": 1.5557, + "step": 713 + }, + { + "epoch": 0.8886123210952085, + "grad_norm": 2.826899290084839, + "learning_rate": 3.943726937269373e-06, + "loss": 1.5277, + "step": 714 + }, + { + "epoch": 0.8898568761667703, + "grad_norm": 2.6269052028656006, + "learning_rate": 3.941420664206642e-06, + "loss": 1.4819, + "step": 715 + }, + { + "epoch": 0.8911014312383323, + "grad_norm": 2.6687698364257812, + "learning_rate": 3.939114391143912e-06, + "loss": 1.5156, + "step": 716 + }, + { + "epoch": 0.8923459863098943, + "grad_norm": 2.9605629444122314, + "learning_rate": 3.9368081180811815e-06, + "loss": 1.5358, + "step": 717 + }, + { + "epoch": 0.8935905413814561, + "grad_norm": 3.0367166996002197, + "learning_rate": 3.934501845018451e-06, + "loss": 1.5847, + "step": 718 + }, + { + "epoch": 0.8948350964530181, + "grad_norm": 2.528796434402466, + "learning_rate": 3.9321955719557195e-06, + "loss": 1.4851, + "step": 719 + }, + { + "epoch": 0.8960796515245799, + "grad_norm": 2.7116613388061523, + "learning_rate": 3.929889298892989e-06, + "loss": 1.4706, + "step": 720 + }, + { + "epoch": 0.8960796515245799, + "eval_loss": 1.6845883131027222, + "eval_runtime": 44.4734, + "eval_samples_per_second": 22.485, + "eval_steps_per_second": 0.944, + "step": 720 + }, + { + "epoch": 0.8973242065961419, + "grad_norm": 2.771864414215088, + "learning_rate": 3.927583025830259e-06, + "loss": 1.5444, + "step": 721 + }, + { + "epoch": 0.8985687616677038, + "grad_norm": 2.773746967315674, + "learning_rate": 3.925276752767528e-06, + "loss": 1.564, + "step": 722 + }, + { + "epoch": 0.8998133167392657, + "grad_norm": 2.7069854736328125, + "learning_rate": 3.922970479704797e-06, + "loss": 1.5601, + "step": 723 + }, + { + "epoch": 0.9010578718108276, + "grad_norm": 2.840261697769165, + "learning_rate": 3.920664206642067e-06, + "loss": 1.5733, + "step": 724 + }, + { + "epoch": 0.9023024268823896, + "grad_norm": 2.709897041320801, + "learning_rate": 3.918357933579336e-06, + "loss": 1.528, + "step": 725 + }, + { + "epoch": 0.9035469819539514, + "grad_norm": 2.662367343902588, + "learning_rate": 3.9160516605166055e-06, + "loss": 1.5532, + "step": 726 + }, + { + "epoch": 0.9047915370255134, + "grad_norm": 2.6775717735290527, + "learning_rate": 3.913745387453875e-06, + "loss": 1.5464, + "step": 727 + }, + { + "epoch": 0.9060360920970753, + "grad_norm": 2.617842674255371, + "learning_rate": 3.911439114391144e-06, + "loss": 1.4539, + "step": 728 + }, + { + "epoch": 0.9072806471686372, + "grad_norm": 2.668649435043335, + "learning_rate": 3.909132841328414e-06, + "loss": 1.5627, + "step": 729 + }, + { + "epoch": 0.9085252022401992, + "grad_norm": 2.5331950187683105, + "learning_rate": 3.906826568265683e-06, + "loss": 1.5165, + "step": 730 + }, + { + "epoch": 0.9085252022401992, + "eval_loss": 1.6860331296920776, + "eval_runtime": 46.094, + "eval_samples_per_second": 21.695, + "eval_steps_per_second": 0.911, + "step": 730 + }, + { + "epoch": 0.909769757311761, + "grad_norm": 2.602492332458496, + "learning_rate": 3.9045202952029526e-06, + "loss": 1.5145, + "step": 731 + }, + { + "epoch": 0.911014312383323, + "grad_norm": 2.632782459259033, + "learning_rate": 3.902214022140222e-06, + "loss": 1.4413, + "step": 732 + }, + { + "epoch": 0.9122588674548848, + "grad_norm": 2.6025912761688232, + "learning_rate": 3.899907749077491e-06, + "loss": 1.5263, + "step": 733 + }, + { + "epoch": 0.9135034225264468, + "grad_norm": 2.770116090774536, + "learning_rate": 3.897601476014761e-06, + "loss": 1.5514, + "step": 734 + }, + { + "epoch": 0.9147479775980087, + "grad_norm": 2.7822961807250977, + "learning_rate": 3.895295202952029e-06, + "loss": 1.5296, + "step": 735 + }, + { + "epoch": 0.9159925326695706, + "grad_norm": 2.6689720153808594, + "learning_rate": 3.892988929889299e-06, + "loss": 1.4552, + "step": 736 + }, + { + "epoch": 0.9172370877411326, + "grad_norm": 2.8021798133850098, + "learning_rate": 3.890682656826569e-06, + "loss": 1.5308, + "step": 737 + }, + { + "epoch": 0.9184816428126945, + "grad_norm": 2.575313091278076, + "learning_rate": 3.888376383763838e-06, + "loss": 1.4807, + "step": 738 + }, + { + "epoch": 0.9197261978842564, + "grad_norm": 2.5955779552459717, + "learning_rate": 3.886070110701107e-06, + "loss": 1.4222, + "step": 739 + }, + { + "epoch": 0.9209707529558183, + "grad_norm": 2.647939920425415, + "learning_rate": 3.8837638376383765e-06, + "loss": 1.5285, + "step": 740 + }, + { + "epoch": 0.9209707529558183, + "eval_loss": 1.6907480955123901, + "eval_runtime": 45.7678, + "eval_samples_per_second": 21.849, + "eval_steps_per_second": 0.918, + "step": 740 + }, + { + "epoch": 0.9222153080273802, + "grad_norm": 2.7664523124694824, + "learning_rate": 3.881457564575646e-06, + "loss": 1.5349, + "step": 741 + }, + { + "epoch": 0.9234598630989421, + "grad_norm": 2.5770998001098633, + "learning_rate": 3.879151291512915e-06, + "loss": 1.5071, + "step": 742 + }, + { + "epoch": 0.9247044181705041, + "grad_norm": 2.502567768096924, + "learning_rate": 3.876845018450185e-06, + "loss": 1.492, + "step": 743 + }, + { + "epoch": 0.9259489732420659, + "grad_norm": 2.6531119346618652, + "learning_rate": 3.874538745387454e-06, + "loss": 1.5045, + "step": 744 + }, + { + "epoch": 0.9271935283136279, + "grad_norm": 2.7343876361846924, + "learning_rate": 3.872232472324724e-06, + "loss": 1.5194, + "step": 745 + }, + { + "epoch": 0.9284380833851898, + "grad_norm": 2.842313528060913, + "learning_rate": 3.869926199261993e-06, + "loss": 1.5588, + "step": 746 + }, + { + "epoch": 0.9296826384567517, + "grad_norm": 2.718245267868042, + "learning_rate": 3.8676199261992625e-06, + "loss": 1.5252, + "step": 747 + }, + { + "epoch": 0.9309271935283137, + "grad_norm": 2.695392370223999, + "learning_rate": 3.865313653136532e-06, + "loss": 1.5596, + "step": 748 + }, + { + "epoch": 0.9321717485998755, + "grad_norm": 2.7859959602355957, + "learning_rate": 3.863007380073801e-06, + "loss": 1.5151, + "step": 749 + }, + { + "epoch": 0.9334163036714375, + "grad_norm": 2.5199174880981445, + "learning_rate": 3.860701107011071e-06, + "loss": 1.4948, + "step": 750 + }, + { + "epoch": 0.9334163036714375, + "eval_loss": 1.6868674755096436, + "eval_runtime": 46.4115, + "eval_samples_per_second": 21.546, + "eval_steps_per_second": 0.905, + "step": 750 + }, + { + "epoch": 0.9346608587429994, + "grad_norm": 2.8194046020507812, + "learning_rate": 3.858394833948339e-06, + "loss": 1.5144, + "step": 751 + }, + { + "epoch": 0.9359054138145613, + "grad_norm": 2.810380697250366, + "learning_rate": 3.856088560885609e-06, + "loss": 1.4998, + "step": 752 + }, + { + "epoch": 0.9371499688861232, + "grad_norm": 2.6906750202178955, + "learning_rate": 3.853782287822879e-06, + "loss": 1.4748, + "step": 753 + }, + { + "epoch": 0.9383945239576851, + "grad_norm": 2.9030277729034424, + "learning_rate": 3.851476014760148e-06, + "loss": 1.615, + "step": 754 + }, + { + "epoch": 0.939639079029247, + "grad_norm": 2.8690176010131836, + "learning_rate": 3.849169741697417e-06, + "loss": 1.5083, + "step": 755 + }, + { + "epoch": 0.940883634100809, + "grad_norm": 2.669646978378296, + "learning_rate": 3.8468634686346865e-06, + "loss": 1.5572, + "step": 756 + }, + { + "epoch": 0.9421281891723708, + "grad_norm": 2.6517696380615234, + "learning_rate": 3.844557195571956e-06, + "loss": 1.5142, + "step": 757 + }, + { + "epoch": 0.9433727442439328, + "grad_norm": 2.5524444580078125, + "learning_rate": 3.842250922509225e-06, + "loss": 1.4924, + "step": 758 + }, + { + "epoch": 0.9446172993154948, + "grad_norm": 2.6532633304595947, + "learning_rate": 3.839944649446495e-06, + "loss": 1.484, + "step": 759 + }, + { + "epoch": 0.9458618543870566, + "grad_norm": 2.7779057025909424, + "learning_rate": 3.837638376383764e-06, + "loss": 1.5106, + "step": 760 + }, + { + "epoch": 0.9458618543870566, + "eval_loss": 1.6845752000808716, + "eval_runtime": 50.3415, + "eval_samples_per_second": 19.864, + "eval_steps_per_second": 0.834, + "step": 760 + }, + { + "epoch": 0.9471064094586186, + "grad_norm": 2.6192541122436523, + "learning_rate": 3.8353321033210336e-06, + "loss": 1.5454, + "step": 761 + }, + { + "epoch": 0.9483509645301804, + "grad_norm": 2.561861991882324, + "learning_rate": 3.833025830258303e-06, + "loss": 1.5141, + "step": 762 + }, + { + "epoch": 0.9495955196017424, + "grad_norm": 2.661829948425293, + "learning_rate": 3.830719557195572e-06, + "loss": 1.4997, + "step": 763 + }, + { + "epoch": 0.9508400746733043, + "grad_norm": 2.725275754928589, + "learning_rate": 3.828413284132842e-06, + "loss": 1.5224, + "step": 764 + }, + { + "epoch": 0.9520846297448662, + "grad_norm": 2.5664193630218506, + "learning_rate": 3.826107011070111e-06, + "loss": 1.5073, + "step": 765 + }, + { + "epoch": 0.9533291848164281, + "grad_norm": 2.595189332962036, + "learning_rate": 3.823800738007381e-06, + "loss": 1.5379, + "step": 766 + }, + { + "epoch": 0.95457373988799, + "grad_norm": 2.738060235977173, + "learning_rate": 3.821494464944649e-06, + "loss": 1.4814, + "step": 767 + }, + { + "epoch": 0.955818294959552, + "grad_norm": 2.601071357727051, + "learning_rate": 3.819188191881919e-06, + "loss": 1.4812, + "step": 768 + }, + { + "epoch": 0.9570628500311139, + "grad_norm": 2.657212257385254, + "learning_rate": 3.816881918819189e-06, + "loss": 1.4748, + "step": 769 + }, + { + "epoch": 0.9583074051026758, + "grad_norm": 2.7673351764678955, + "learning_rate": 3.814575645756458e-06, + "loss": 1.5535, + "step": 770 + }, + { + "epoch": 0.9583074051026758, + "eval_loss": 1.677243947982788, + "eval_runtime": 51.5537, + "eval_samples_per_second": 19.397, + "eval_steps_per_second": 0.815, + "step": 770 + }, + { + "epoch": 0.9595519601742377, + "grad_norm": 2.760890007019043, + "learning_rate": 3.812269372693727e-06, + "loss": 1.5654, + "step": 771 + }, + { + "epoch": 0.9607965152457997, + "grad_norm": 2.6934309005737305, + "learning_rate": 3.809963099630997e-06, + "loss": 1.5149, + "step": 772 + }, + { + "epoch": 0.9620410703173615, + "grad_norm": 2.729950428009033, + "learning_rate": 3.8076568265682662e-06, + "loss": 1.5579, + "step": 773 + }, + { + "epoch": 0.9632856253889235, + "grad_norm": 2.732926607131958, + "learning_rate": 3.8053505535055352e-06, + "loss": 1.5179, + "step": 774 + }, + { + "epoch": 0.9645301804604853, + "grad_norm": 2.745391368865967, + "learning_rate": 3.8030442804428046e-06, + "loss": 1.5124, + "step": 775 + }, + { + "epoch": 0.9657747355320473, + "grad_norm": 2.8050146102905273, + "learning_rate": 3.800738007380074e-06, + "loss": 1.519, + "step": 776 + }, + { + "epoch": 0.9670192906036092, + "grad_norm": 2.697171449661255, + "learning_rate": 3.798431734317343e-06, + "loss": 1.5219, + "step": 777 + }, + { + "epoch": 0.9682638456751711, + "grad_norm": 2.6375980377197266, + "learning_rate": 3.796125461254613e-06, + "loss": 1.5345, + "step": 778 + }, + { + "epoch": 0.9695084007467331, + "grad_norm": 2.586636781692505, + "learning_rate": 3.7938191881918823e-06, + "loss": 1.5182, + "step": 779 + }, + { + "epoch": 0.9707529558182949, + "grad_norm": 2.626453399658203, + "learning_rate": 3.7915129151291518e-06, + "loss": 1.4817, + "step": 780 + }, + { + "epoch": 0.9707529558182949, + "eval_loss": 1.6734713315963745, + "eval_runtime": 48.1436, + "eval_samples_per_second": 20.771, + "eval_steps_per_second": 0.872, + "step": 780 + }, + { + "epoch": 0.9719975108898569, + "grad_norm": 2.579970121383667, + "learning_rate": 3.7892066420664208e-06, + "loss": 1.4981, + "step": 781 + }, + { + "epoch": 0.9732420659614188, + "grad_norm": 2.6834843158721924, + "learning_rate": 3.7869003690036906e-06, + "loss": 1.5044, + "step": 782 + }, + { + "epoch": 0.9744866210329807, + "grad_norm": 2.586982488632202, + "learning_rate": 3.78459409594096e-06, + "loss": 1.5079, + "step": 783 + }, + { + "epoch": 0.9757311761045426, + "grad_norm": 2.910027503967285, + "learning_rate": 3.782287822878229e-06, + "loss": 1.5695, + "step": 784 + }, + { + "epoch": 0.9769757311761046, + "grad_norm": 2.7088494300842285, + "learning_rate": 3.7799815498154984e-06, + "loss": 1.4986, + "step": 785 + }, + { + "epoch": 0.9782202862476664, + "grad_norm": 2.581325054168701, + "learning_rate": 3.777675276752768e-06, + "loss": 1.5137, + "step": 786 + }, + { + "epoch": 0.9794648413192284, + "grad_norm": 2.700709819793701, + "learning_rate": 3.775369003690037e-06, + "loss": 1.4718, + "step": 787 + }, + { + "epoch": 0.9807093963907902, + "grad_norm": 2.6712708473205566, + "learning_rate": 3.7730627306273067e-06, + "loss": 1.5208, + "step": 788 + }, + { + "epoch": 0.9819539514623522, + "grad_norm": 2.6799817085266113, + "learning_rate": 3.770756457564576e-06, + "loss": 1.5374, + "step": 789 + }, + { + "epoch": 0.9831985065339142, + "grad_norm": 2.618988037109375, + "learning_rate": 3.768450184501845e-06, + "loss": 1.4997, + "step": 790 + }, + { + "epoch": 0.9831985065339142, + "eval_loss": 1.6860820055007935, + "eval_runtime": 51.7654, + "eval_samples_per_second": 19.318, + "eval_steps_per_second": 0.811, + "step": 790 + }, + { + "epoch": 0.984443061605476, + "grad_norm": 2.6899526119232178, + "learning_rate": 3.7661439114391146e-06, + "loss": 1.5216, + "step": 791 + }, + { + "epoch": 0.985687616677038, + "grad_norm": 2.8003487586975098, + "learning_rate": 3.763837638376384e-06, + "loss": 1.557, + "step": 792 + }, + { + "epoch": 0.9869321717485999, + "grad_norm": 2.744536876678467, + "learning_rate": 3.761531365313654e-06, + "loss": 1.5355, + "step": 793 + }, + { + "epoch": 0.9881767268201618, + "grad_norm": 2.587250232696533, + "learning_rate": 3.759225092250923e-06, + "loss": 1.5259, + "step": 794 + }, + { + "epoch": 0.9894212818917237, + "grad_norm": 2.616291046142578, + "learning_rate": 3.7569188191881922e-06, + "loss": 1.5176, + "step": 795 + }, + { + "epoch": 0.9906658369632856, + "grad_norm": 2.6410577297210693, + "learning_rate": 3.7546125461254617e-06, + "loss": 1.511, + "step": 796 + }, + { + "epoch": 0.9919103920348475, + "grad_norm": 2.577373504638672, + "learning_rate": 3.7523062730627307e-06, + "loss": 1.4715, + "step": 797 + }, + { + "epoch": 0.9931549471064095, + "grad_norm": 2.67305326461792, + "learning_rate": 3.7500000000000005e-06, + "loss": 1.4828, + "step": 798 + }, + { + "epoch": 0.9943995021779714, + "grad_norm": 3.1631500720977783, + "learning_rate": 3.74769372693727e-06, + "loss": 1.5602, + "step": 799 + }, + { + "epoch": 0.9956440572495333, + "grad_norm": 2.9222350120544434, + "learning_rate": 3.745387453874539e-06, + "loss": 1.4814, + "step": 800 + }, + { + "epoch": 0.9956440572495333, + "eval_loss": 1.6810544729232788, + "eval_runtime": 51.1262, + "eval_samples_per_second": 19.559, + "eval_steps_per_second": 0.821, + "step": 800 + }, + { + "epoch": 0.9968886123210952, + "grad_norm": 2.884312629699707, + "learning_rate": 3.7430811808118084e-06, + "loss": 1.4553, + "step": 801 + }, + { + "epoch": 0.9981331673926571, + "grad_norm": 2.712655782699585, + "learning_rate": 3.7407749077490778e-06, + "loss": 1.5275, + "step": 802 + }, + { + "epoch": 0.9993777224642191, + "grad_norm": 2.8700571060180664, + "learning_rate": 3.7384686346863468e-06, + "loss": 1.4923, + "step": 803 + }, + { + "epoch": 1.000622277535781, + "grad_norm": 2.7779245376586914, + "learning_rate": 3.7361623616236166e-06, + "loss": 1.5224, + "step": 804 + }, + { + "epoch": 1.0018668326073428, + "grad_norm": 2.7679810523986816, + "learning_rate": 3.733856088560886e-06, + "loss": 1.4977, + "step": 805 + }, + { + "epoch": 1.0031113876789048, + "grad_norm": 2.797023296356201, + "learning_rate": 3.7315498154981555e-06, + "loss": 1.4396, + "step": 806 + }, + { + "epoch": 1.0043559427504667, + "grad_norm": 3.007962465286255, + "learning_rate": 3.7292435424354245e-06, + "loss": 1.5015, + "step": 807 + }, + { + "epoch": 1.0056004978220285, + "grad_norm": 3.127639055252075, + "learning_rate": 3.726937269372694e-06, + "loss": 1.5176, + "step": 808 + }, + { + "epoch": 1.0068450528935906, + "grad_norm": 2.9001357555389404, + "learning_rate": 3.7246309963099637e-06, + "loss": 1.4735, + "step": 809 + }, + { + "epoch": 1.0080896079651525, + "grad_norm": 2.8245413303375244, + "learning_rate": 3.7223247232472327e-06, + "loss": 1.486, + "step": 810 + }, + { + "epoch": 1.0080896079651525, + "eval_loss": 1.6835130453109741, + "eval_runtime": 44.5682, + "eval_samples_per_second": 22.438, + "eval_steps_per_second": 0.942, + "step": 810 + }, + { + "epoch": 1.0093341630367143, + "grad_norm": 2.780172824859619, + "learning_rate": 3.720018450184502e-06, + "loss": 1.4437, + "step": 811 + }, + { + "epoch": 1.0105787181082764, + "grad_norm": 2.6013996601104736, + "learning_rate": 3.7177121771217716e-06, + "loss": 1.4529, + "step": 812 + }, + { + "epoch": 1.0118232731798382, + "grad_norm": 2.831015110015869, + "learning_rate": 3.7154059040590406e-06, + "loss": 1.4825, + "step": 813 + }, + { + "epoch": 1.0130678282514, + "grad_norm": 2.8341829776763916, + "learning_rate": 3.71309963099631e-06, + "loss": 1.4736, + "step": 814 + }, + { + "epoch": 1.0143123833229621, + "grad_norm": 2.769540309906006, + "learning_rate": 3.71079335793358e-06, + "loss": 1.4724, + "step": 815 + }, + { + "epoch": 1.015556938394524, + "grad_norm": 3.0083069801330566, + "learning_rate": 3.708487084870849e-06, + "loss": 1.4985, + "step": 816 + }, + { + "epoch": 1.0168014934660858, + "grad_norm": 3.052168607711792, + "learning_rate": 3.7061808118081183e-06, + "loss": 1.5193, + "step": 817 + }, + { + "epoch": 1.0180460485376477, + "grad_norm": 2.924877405166626, + "learning_rate": 3.7038745387453877e-06, + "loss": 1.5219, + "step": 818 + }, + { + "epoch": 1.0192906036092098, + "grad_norm": 3.150123357772827, + "learning_rate": 3.7015682656826576e-06, + "loss": 1.5026, + "step": 819 + }, + { + "epoch": 1.0205351586807716, + "grad_norm": 2.881655216217041, + "learning_rate": 3.6992619926199266e-06, + "loss": 1.5011, + "step": 820 + }, + { + "epoch": 1.0205351586807716, + "eval_loss": 1.6745303869247437, + "eval_runtime": 50.6791, + "eval_samples_per_second": 19.732, + "eval_steps_per_second": 0.829, + "step": 820 + }, + { + "epoch": 1.0217797137523335, + "grad_norm": 2.824249505996704, + "learning_rate": 3.696955719557196e-06, + "loss": 1.4642, + "step": 821 + }, + { + "epoch": 1.0230242688238955, + "grad_norm": 2.859144926071167, + "learning_rate": 3.6946494464944654e-06, + "loss": 1.4681, + "step": 822 + }, + { + "epoch": 1.0242688238954574, + "grad_norm": 2.90312123298645, + "learning_rate": 3.6923431734317344e-06, + "loss": 1.4926, + "step": 823 + }, + { + "epoch": 1.0255133789670192, + "grad_norm": 2.8340659141540527, + "learning_rate": 3.690036900369004e-06, + "loss": 1.4834, + "step": 824 + }, + { + "epoch": 1.0267579340385813, + "grad_norm": 2.9743151664733887, + "learning_rate": 3.6877306273062737e-06, + "loss": 1.4574, + "step": 825 + }, + { + "epoch": 1.0280024891101431, + "grad_norm": 3.032179594039917, + "learning_rate": 3.6854243542435427e-06, + "loss": 1.5571, + "step": 826 + }, + { + "epoch": 1.029247044181705, + "grad_norm": 2.816826343536377, + "learning_rate": 3.683118081180812e-06, + "loss": 1.455, + "step": 827 + }, + { + "epoch": 1.030491599253267, + "grad_norm": 3.046696186065674, + "learning_rate": 3.6808118081180815e-06, + "loss": 1.4871, + "step": 828 + }, + { + "epoch": 1.031736154324829, + "grad_norm": 2.8517391681671143, + "learning_rate": 3.678505535055351e-06, + "loss": 1.5292, + "step": 829 + }, + { + "epoch": 1.0329807093963908, + "grad_norm": 2.752811908721924, + "learning_rate": 3.67619926199262e-06, + "loss": 1.5091, + "step": 830 + }, + { + "epoch": 1.0329807093963908, + "eval_loss": 1.6692287921905518, + "eval_runtime": 50.8347, + "eval_samples_per_second": 19.672, + "eval_steps_per_second": 0.826, + "step": 830 + }, + { + "epoch": 1.0342252644679526, + "grad_norm": 2.9220356941223145, + "learning_rate": 3.6738929889298898e-06, + "loss": 1.4247, + "step": 831 + }, + { + "epoch": 1.0354698195395147, + "grad_norm": 2.8903002738952637, + "learning_rate": 3.671586715867159e-06, + "loss": 1.5595, + "step": 832 + }, + { + "epoch": 1.0367143746110765, + "grad_norm": 2.6495542526245117, + "learning_rate": 3.669280442804428e-06, + "loss": 1.4076, + "step": 833 + }, + { + "epoch": 1.0379589296826384, + "grad_norm": 2.880809783935547, + "learning_rate": 3.6669741697416976e-06, + "loss": 1.5285, + "step": 834 + }, + { + "epoch": 1.0392034847542004, + "grad_norm": 2.919142007827759, + "learning_rate": 3.6646678966789675e-06, + "loss": 1.5165, + "step": 835 + }, + { + "epoch": 1.0404480398257623, + "grad_norm": 3.1180522441864014, + "learning_rate": 3.6623616236162365e-06, + "loss": 1.5224, + "step": 836 + }, + { + "epoch": 1.0416925948973241, + "grad_norm": 2.767709255218506, + "learning_rate": 3.660055350553506e-06, + "loss": 1.4859, + "step": 837 + }, + { + "epoch": 1.0429371499688862, + "grad_norm": 2.708828926086426, + "learning_rate": 3.6577490774907753e-06, + "loss": 1.4803, + "step": 838 + }, + { + "epoch": 1.044181705040448, + "grad_norm": 2.688359260559082, + "learning_rate": 3.6554428044280443e-06, + "loss": 1.4571, + "step": 839 + }, + { + "epoch": 1.04542626011201, + "grad_norm": 2.691812753677368, + "learning_rate": 3.6531365313653137e-06, + "loss": 1.5043, + "step": 840 + }, + { + "epoch": 1.04542626011201, + "eval_loss": 1.6769559383392334, + "eval_runtime": 49.6233, + "eval_samples_per_second": 20.152, + "eval_steps_per_second": 0.846, + "step": 840 + }, + { + "epoch": 1.046670815183572, + "grad_norm": 2.686372995376587, + "learning_rate": 3.6508302583025836e-06, + "loss": 1.5221, + "step": 841 + }, + { + "epoch": 1.0479153702551338, + "grad_norm": 2.742859125137329, + "learning_rate": 3.648523985239853e-06, + "loss": 1.4685, + "step": 842 + }, + { + "epoch": 1.0491599253266957, + "grad_norm": 2.874018907546997, + "learning_rate": 3.646217712177122e-06, + "loss": 1.4813, + "step": 843 + }, + { + "epoch": 1.0504044803982575, + "grad_norm": 2.8159990310668945, + "learning_rate": 3.6439114391143914e-06, + "loss": 1.4421, + "step": 844 + }, + { + "epoch": 1.0516490354698196, + "grad_norm": 2.714787006378174, + "learning_rate": 3.641605166051661e-06, + "loss": 1.4879, + "step": 845 + }, + { + "epoch": 1.0528935905413814, + "grad_norm": 2.885148525238037, + "learning_rate": 3.63929889298893e-06, + "loss": 1.4725, + "step": 846 + }, + { + "epoch": 1.0541381456129433, + "grad_norm": 2.7272610664367676, + "learning_rate": 3.6369926199261997e-06, + "loss": 1.4276, + "step": 847 + }, + { + "epoch": 1.0553827006845053, + "grad_norm": 2.9114978313446045, + "learning_rate": 3.634686346863469e-06, + "loss": 1.4194, + "step": 848 + }, + { + "epoch": 1.0566272557560672, + "grad_norm": 3.019313335418701, + "learning_rate": 3.632380073800738e-06, + "loss": 1.5266, + "step": 849 + }, + { + "epoch": 1.057871810827629, + "grad_norm": 2.714751958847046, + "learning_rate": 3.6300738007380075e-06, + "loss": 1.4593, + "step": 850 + }, + { + "epoch": 1.057871810827629, + "eval_loss": 1.6760298013687134, + "eval_runtime": 52.1681, + "eval_samples_per_second": 19.169, + "eval_steps_per_second": 0.805, + "step": 850 + }, + { + "epoch": 1.0591163658991911, + "grad_norm": 3.0507936477661133, + "learning_rate": 3.627767527675277e-06, + "loss": 1.5064, + "step": 851 + }, + { + "epoch": 1.060360920970753, + "grad_norm": 2.8116295337677, + "learning_rate": 3.625461254612546e-06, + "loss": 1.5218, + "step": 852 + }, + { + "epoch": 1.0616054760423148, + "grad_norm": 2.8801465034484863, + "learning_rate": 3.623154981549816e-06, + "loss": 1.4641, + "step": 853 + }, + { + "epoch": 1.0628500311138769, + "grad_norm": 3.0581862926483154, + "learning_rate": 3.6208487084870852e-06, + "loss": 1.483, + "step": 854 + }, + { + "epoch": 1.0640945861854387, + "grad_norm": 2.898780584335327, + "learning_rate": 3.6185424354243547e-06, + "loss": 1.4623, + "step": 855 + }, + { + "epoch": 1.0653391412570006, + "grad_norm": 2.971820116043091, + "learning_rate": 3.6162361623616237e-06, + "loss": 1.4455, + "step": 856 + }, + { + "epoch": 1.0665836963285624, + "grad_norm": 3.0021913051605225, + "learning_rate": 3.6139298892988935e-06, + "loss": 1.4878, + "step": 857 + }, + { + "epoch": 1.0678282514001245, + "grad_norm": 3.0697457790374756, + "learning_rate": 3.611623616236163e-06, + "loss": 1.5008, + "step": 858 + }, + { + "epoch": 1.0690728064716863, + "grad_norm": 3.0787923336029053, + "learning_rate": 3.609317343173432e-06, + "loss": 1.4612, + "step": 859 + }, + { + "epoch": 1.0703173615432482, + "grad_norm": 2.789339780807495, + "learning_rate": 3.6070110701107014e-06, + "loss": 1.4287, + "step": 860 + }, + { + "epoch": 1.0703173615432482, + "eval_loss": 1.6747506856918335, + "eval_runtime": 51.4912, + "eval_samples_per_second": 19.421, + "eval_steps_per_second": 0.816, + "step": 860 + }, + { + "epoch": 1.0715619166148103, + "grad_norm": 2.9622788429260254, + "learning_rate": 3.6047047970479708e-06, + "loss": 1.4905, + "step": 861 + }, + { + "epoch": 1.072806471686372, + "grad_norm": 3.0675268173217773, + "learning_rate": 3.6023985239852398e-06, + "loss": 1.461, + "step": 862 + }, + { + "epoch": 1.074051026757934, + "grad_norm": 2.820996046066284, + "learning_rate": 3.6000922509225096e-06, + "loss": 1.4948, + "step": 863 + }, + { + "epoch": 1.075295581829496, + "grad_norm": 2.833761215209961, + "learning_rate": 3.597785977859779e-06, + "loss": 1.4683, + "step": 864 + }, + { + "epoch": 1.0765401369010579, + "grad_norm": 2.917313814163208, + "learning_rate": 3.595479704797048e-06, + "loss": 1.5191, + "step": 865 + }, + { + "epoch": 1.0777846919726197, + "grad_norm": 2.876166820526123, + "learning_rate": 3.5931734317343175e-06, + "loss": 1.4854, + "step": 866 + }, + { + "epoch": 1.0790292470441818, + "grad_norm": 3.139049768447876, + "learning_rate": 3.590867158671587e-06, + "loss": 1.5134, + "step": 867 + }, + { + "epoch": 1.0802738021157436, + "grad_norm": 2.9956753253936768, + "learning_rate": 3.5885608856088567e-06, + "loss": 1.5003, + "step": 868 + }, + { + "epoch": 1.0815183571873055, + "grad_norm": 2.957181930541992, + "learning_rate": 3.5862546125461257e-06, + "loss": 1.4588, + "step": 869 + }, + { + "epoch": 1.0827629122588673, + "grad_norm": 2.8576438426971436, + "learning_rate": 3.583948339483395e-06, + "loss": 1.48, + "step": 870 + }, + { + "epoch": 1.0827629122588673, + "eval_loss": 1.6724848747253418, + "eval_runtime": 50.9669, + "eval_samples_per_second": 19.621, + "eval_steps_per_second": 0.824, + "step": 870 + }, + { + "epoch": 1.0840074673304294, + "grad_norm": 2.8183560371398926, + "learning_rate": 3.5816420664206646e-06, + "loss": 1.4818, + "step": 871 + }, + { + "epoch": 1.0852520224019913, + "grad_norm": 2.7531797885894775, + "learning_rate": 3.5793357933579336e-06, + "loss": 1.4712, + "step": 872 + }, + { + "epoch": 1.086496577473553, + "grad_norm": 2.923962354660034, + "learning_rate": 3.5770295202952034e-06, + "loss": 1.4864, + "step": 873 + }, + { + "epoch": 1.0877411325451152, + "grad_norm": 2.851724147796631, + "learning_rate": 3.574723247232473e-06, + "loss": 1.4771, + "step": 874 + }, + { + "epoch": 1.088985687616677, + "grad_norm": 3.024634838104248, + "learning_rate": 3.572416974169742e-06, + "loss": 1.4643, + "step": 875 + }, + { + "epoch": 1.0902302426882389, + "grad_norm": 2.7100844383239746, + "learning_rate": 3.5701107011070113e-06, + "loss": 1.4773, + "step": 876 + }, + { + "epoch": 1.091474797759801, + "grad_norm": 2.730502128601074, + "learning_rate": 3.5678044280442807e-06, + "loss": 1.5051, + "step": 877 + }, + { + "epoch": 1.0927193528313628, + "grad_norm": 2.747082233428955, + "learning_rate": 3.5654981549815497e-06, + "loss": 1.5207, + "step": 878 + }, + { + "epoch": 1.0939639079029246, + "grad_norm": 2.718358039855957, + "learning_rate": 3.5631918819188195e-06, + "loss": 1.5067, + "step": 879 + }, + { + "epoch": 1.0952084629744867, + "grad_norm": 2.7794573307037354, + "learning_rate": 3.560885608856089e-06, + "loss": 1.4616, + "step": 880 + }, + { + "epoch": 1.0952084629744867, + "eval_loss": 1.6676132678985596, + "eval_runtime": 51.6157, + "eval_samples_per_second": 19.374, + "eval_steps_per_second": 0.814, + "step": 880 + }, + { + "epoch": 1.0964530180460486, + "grad_norm": 2.727416753768921, + "learning_rate": 3.5585793357933584e-06, + "loss": 1.4043, + "step": 881 + }, + { + "epoch": 1.0976975731176104, + "grad_norm": 2.856900930404663, + "learning_rate": 3.5562730627306274e-06, + "loss": 1.5198, + "step": 882 + }, + { + "epoch": 1.0989421281891725, + "grad_norm": 2.872823715209961, + "learning_rate": 3.553966789667897e-06, + "loss": 1.4981, + "step": 883 + }, + { + "epoch": 1.1001866832607343, + "grad_norm": 2.7724361419677734, + "learning_rate": 3.5516605166051667e-06, + "loss": 1.4819, + "step": 884 + }, + { + "epoch": 1.1014312383322962, + "grad_norm": 2.884434461593628, + "learning_rate": 3.5493542435424357e-06, + "loss": 1.433, + "step": 885 + }, + { + "epoch": 1.102675793403858, + "grad_norm": 2.887873411178589, + "learning_rate": 3.547047970479705e-06, + "loss": 1.4677, + "step": 886 + }, + { + "epoch": 1.10392034847542, + "grad_norm": 2.946845054626465, + "learning_rate": 3.5447416974169745e-06, + "loss": 1.4983, + "step": 887 + }, + { + "epoch": 1.105164903546982, + "grad_norm": 2.752552032470703, + "learning_rate": 3.5424354243542435e-06, + "loss": 1.4766, + "step": 888 + }, + { + "epoch": 1.1064094586185438, + "grad_norm": 2.788634777069092, + "learning_rate": 3.5401291512915133e-06, + "loss": 1.4747, + "step": 889 + }, + { + "epoch": 1.1076540136901059, + "grad_norm": 2.782792568206787, + "learning_rate": 3.5378228782287828e-06, + "loss": 1.5503, + "step": 890 + }, + { + "epoch": 1.1076540136901059, + "eval_loss": 1.6698333024978638, + "eval_runtime": 49.1373, + "eval_samples_per_second": 20.351, + "eval_steps_per_second": 0.855, + "step": 890 + }, + { + "epoch": 1.1088985687616677, + "grad_norm": 2.8351166248321533, + "learning_rate": 3.535516605166052e-06, + "loss": 1.4589, + "step": 891 + }, + { + "epoch": 1.1101431238332296, + "grad_norm": 2.798872709274292, + "learning_rate": 3.533210332103321e-06, + "loss": 1.4496, + "step": 892 + }, + { + "epoch": 1.1113876789047916, + "grad_norm": 3.0863239765167236, + "learning_rate": 3.5309040590405906e-06, + "loss": 1.4698, + "step": 893 + }, + { + "epoch": 1.1126322339763535, + "grad_norm": 2.853862762451172, + "learning_rate": 3.5285977859778605e-06, + "loss": 1.4429, + "step": 894 + }, + { + "epoch": 1.1138767890479153, + "grad_norm": 2.8946170806884766, + "learning_rate": 3.5262915129151295e-06, + "loss": 1.4697, + "step": 895 + }, + { + "epoch": 1.1151213441194774, + "grad_norm": 3.043823003768921, + "learning_rate": 3.523985239852399e-06, + "loss": 1.4666, + "step": 896 + }, + { + "epoch": 1.1163658991910392, + "grad_norm": 2.7822980880737305, + "learning_rate": 3.5216789667896683e-06, + "loss": 1.4523, + "step": 897 + }, + { + "epoch": 1.117610454262601, + "grad_norm": 2.9454171657562256, + "learning_rate": 3.5193726937269373e-06, + "loss": 1.5009, + "step": 898 + }, + { + "epoch": 1.1188550093341632, + "grad_norm": 2.8533740043640137, + "learning_rate": 3.5170664206642067e-06, + "loss": 1.4265, + "step": 899 + }, + { + "epoch": 1.120099564405725, + "grad_norm": 2.894937038421631, + "learning_rate": 3.5147601476014766e-06, + "loss": 1.4895, + "step": 900 + }, + { + "epoch": 1.120099564405725, + "eval_loss": 1.6701407432556152, + "eval_runtime": 45.477, + "eval_samples_per_second": 21.989, + "eval_steps_per_second": 0.924, + "step": 900 + }, + { + "epoch": 1.1213441194772868, + "grad_norm": 2.933821201324463, + "learning_rate": 3.5124538745387456e-06, + "loss": 1.468, + "step": 901 + }, + { + "epoch": 1.1225886745488487, + "grad_norm": 2.834265947341919, + "learning_rate": 3.510147601476015e-06, + "loss": 1.4621, + "step": 902 + }, + { + "epoch": 1.1238332296204108, + "grad_norm": 2.9064722061157227, + "learning_rate": 3.5078413284132844e-06, + "loss": 1.504, + "step": 903 + }, + { + "epoch": 1.1250777846919726, + "grad_norm": 2.8753886222839355, + "learning_rate": 3.505535055350554e-06, + "loss": 1.4643, + "step": 904 + }, + { + "epoch": 1.1263223397635345, + "grad_norm": 2.894659996032715, + "learning_rate": 3.503228782287823e-06, + "loss": 1.4909, + "step": 905 + }, + { + "epoch": 1.1275668948350965, + "grad_norm": 3.085977792739868, + "learning_rate": 3.5009225092250927e-06, + "loss": 1.459, + "step": 906 + }, + { + "epoch": 1.1288114499066584, + "grad_norm": 2.9958064556121826, + "learning_rate": 3.498616236162362e-06, + "loss": 1.4152, + "step": 907 + }, + { + "epoch": 1.1300560049782202, + "grad_norm": 2.9136762619018555, + "learning_rate": 3.496309963099631e-06, + "loss": 1.473, + "step": 908 + }, + { + "epoch": 1.1313005600497823, + "grad_norm": 2.956437826156616, + "learning_rate": 3.4940036900369005e-06, + "loss": 1.5167, + "step": 909 + }, + { + "epoch": 1.1325451151213441, + "grad_norm": 2.7354865074157715, + "learning_rate": 3.4916974169741704e-06, + "loss": 1.4413, + "step": 910 + }, + { + "epoch": 1.1325451151213441, + "eval_loss": 1.6629912853240967, + "eval_runtime": 41.5478, + "eval_samples_per_second": 24.069, + "eval_steps_per_second": 1.011, + "step": 910 + }, + { + "epoch": 1.133789670192906, + "grad_norm": 2.7443008422851562, + "learning_rate": 3.4893911439114394e-06, + "loss": 1.4475, + "step": 911 + }, + { + "epoch": 1.135034225264468, + "grad_norm": 2.7811238765716553, + "learning_rate": 3.487084870848709e-06, + "loss": 1.4257, + "step": 912 + }, + { + "epoch": 1.13627878033603, + "grad_norm": 2.7916479110717773, + "learning_rate": 3.4847785977859782e-06, + "loss": 1.4048, + "step": 913 + }, + { + "epoch": 1.1375233354075918, + "grad_norm": 2.7835633754730225, + "learning_rate": 3.4824723247232472e-06, + "loss": 1.3996, + "step": 914 + }, + { + "epoch": 1.1387678904791536, + "grad_norm": 2.8188705444335938, + "learning_rate": 3.4801660516605166e-06, + "loss": 1.4541, + "step": 915 + }, + { + "epoch": 1.1400124455507157, + "grad_norm": 3.0134100914001465, + "learning_rate": 3.4778597785977865e-06, + "loss": 1.4353, + "step": 916 + }, + { + "epoch": 1.1412570006222775, + "grad_norm": 2.6965606212615967, + "learning_rate": 3.475553505535056e-06, + "loss": 1.4308, + "step": 917 + }, + { + "epoch": 1.1425015556938394, + "grad_norm": 2.8179919719696045, + "learning_rate": 3.473247232472325e-06, + "loss": 1.5204, + "step": 918 + }, + { + "epoch": 1.1437461107654014, + "grad_norm": 2.7213902473449707, + "learning_rate": 3.4709409594095943e-06, + "loss": 1.4344, + "step": 919 + }, + { + "epoch": 1.1449906658369633, + "grad_norm": 2.850193738937378, + "learning_rate": 3.4686346863468638e-06, + "loss": 1.473, + "step": 920 + }, + { + "epoch": 1.1449906658369633, + "eval_loss": 1.663345217704773, + "eval_runtime": 43.8397, + "eval_samples_per_second": 22.81, + "eval_steps_per_second": 0.958, + "step": 920 + }, + { + "epoch": 1.1462352209085251, + "grad_norm": 2.780198574066162, + "learning_rate": 3.4663284132841328e-06, + "loss": 1.4472, + "step": 921 + }, + { + "epoch": 1.1474797759800872, + "grad_norm": 2.9373831748962402, + "learning_rate": 3.4640221402214026e-06, + "loss": 1.426, + "step": 922 + }, + { + "epoch": 1.148724331051649, + "grad_norm": 2.6986873149871826, + "learning_rate": 3.461715867158672e-06, + "loss": 1.3953, + "step": 923 + }, + { + "epoch": 1.149968886123211, + "grad_norm": 2.927441120147705, + "learning_rate": 3.459409594095941e-06, + "loss": 1.4578, + "step": 924 + }, + { + "epoch": 1.151213441194773, + "grad_norm": 2.8469648361206055, + "learning_rate": 3.4571033210332105e-06, + "loss": 1.4313, + "step": 925 + }, + { + "epoch": 1.1524579962663348, + "grad_norm": 2.8114013671875, + "learning_rate": 3.4547970479704803e-06, + "loss": 1.4267, + "step": 926 + }, + { + "epoch": 1.1537025513378967, + "grad_norm": 2.977605104446411, + "learning_rate": 3.4524907749077493e-06, + "loss": 1.4747, + "step": 927 + }, + { + "epoch": 1.1549471064094585, + "grad_norm": 2.9327852725982666, + "learning_rate": 3.4501845018450187e-06, + "loss": 1.4219, + "step": 928 + }, + { + "epoch": 1.1561916614810206, + "grad_norm": 2.844038963317871, + "learning_rate": 3.447878228782288e-06, + "loss": 1.5146, + "step": 929 + }, + { + "epoch": 1.1574362165525824, + "grad_norm": 2.9402201175689697, + "learning_rate": 3.4455719557195576e-06, + "loss": 1.45, + "step": 930 + }, + { + "epoch": 1.1574362165525824, + "eval_loss": 1.6620122194290161, + "eval_runtime": 46.1748, + "eval_samples_per_second": 21.657, + "eval_steps_per_second": 0.91, + "step": 930 + }, + { + "epoch": 1.1586807716241443, + "grad_norm": 2.842651128768921, + "learning_rate": 3.4432656826568266e-06, + "loss": 1.4415, + "step": 931 + }, + { + "epoch": 1.1599253266957064, + "grad_norm": 2.8909685611724854, + "learning_rate": 3.4409594095940964e-06, + "loss": 1.4529, + "step": 932 + }, + { + "epoch": 1.1611698817672682, + "grad_norm": 2.8025808334350586, + "learning_rate": 3.438653136531366e-06, + "loss": 1.4552, + "step": 933 + }, + { + "epoch": 1.16241443683883, + "grad_norm": 2.6995794773101807, + "learning_rate": 3.436346863468635e-06, + "loss": 1.4435, + "step": 934 + }, + { + "epoch": 1.1636589919103921, + "grad_norm": 2.7307121753692627, + "learning_rate": 3.4340405904059043e-06, + "loss": 1.4447, + "step": 935 + }, + { + "epoch": 1.164903546981954, + "grad_norm": 2.9532501697540283, + "learning_rate": 3.4317343173431737e-06, + "loss": 1.4747, + "step": 936 + }, + { + "epoch": 1.1661481020535158, + "grad_norm": 3.0135788917541504, + "learning_rate": 3.4294280442804427e-06, + "loss": 1.525, + "step": 937 + }, + { + "epoch": 1.167392657125078, + "grad_norm": 2.796355962753296, + "learning_rate": 3.4271217712177125e-06, + "loss": 1.4539, + "step": 938 + }, + { + "epoch": 1.1686372121966397, + "grad_norm": 2.9422309398651123, + "learning_rate": 3.424815498154982e-06, + "loss": 1.4664, + "step": 939 + }, + { + "epoch": 1.1698817672682016, + "grad_norm": 2.7680039405822754, + "learning_rate": 3.4225092250922514e-06, + "loss": 1.4837, + "step": 940 + }, + { + "epoch": 1.1698817672682016, + "eval_loss": 1.660908818244934, + "eval_runtime": 45.6998, + "eval_samples_per_second": 21.882, + "eval_steps_per_second": 0.919, + "step": 940 + }, + { + "epoch": 1.1711263223397634, + "grad_norm": 2.8888845443725586, + "learning_rate": 3.4202029520295204e-06, + "loss": 1.5315, + "step": 941 + }, + { + "epoch": 1.1723708774113255, + "grad_norm": 2.7124130725860596, + "learning_rate": 3.41789667896679e-06, + "loss": 1.4188, + "step": 942 + }, + { + "epoch": 1.1736154324828874, + "grad_norm": 2.75203013420105, + "learning_rate": 3.4155904059040596e-06, + "loss": 1.4533, + "step": 943 + }, + { + "epoch": 1.1748599875544492, + "grad_norm": 2.865480422973633, + "learning_rate": 3.4132841328413286e-06, + "loss": 1.4532, + "step": 944 + }, + { + "epoch": 1.1761045426260113, + "grad_norm": 2.8618321418762207, + "learning_rate": 3.410977859778598e-06, + "loss": 1.4386, + "step": 945 + }, + { + "epoch": 1.1773490976975731, + "grad_norm": 2.7656145095825195, + "learning_rate": 3.4086715867158675e-06, + "loss": 1.3996, + "step": 946 + }, + { + "epoch": 1.178593652769135, + "grad_norm": 2.76213002204895, + "learning_rate": 3.4063653136531365e-06, + "loss": 1.4979, + "step": 947 + }, + { + "epoch": 1.179838207840697, + "grad_norm": 2.8581957817077637, + "learning_rate": 3.4040590405904063e-06, + "loss": 1.4673, + "step": 948 + }, + { + "epoch": 1.1810827629122589, + "grad_norm": 2.7825586795806885, + "learning_rate": 3.4017527675276758e-06, + "loss": 1.4453, + "step": 949 + }, + { + "epoch": 1.1823273179838207, + "grad_norm": 2.9226131439208984, + "learning_rate": 3.3994464944649448e-06, + "loss": 1.4879, + "step": 950 + }, + { + "epoch": 1.1823273179838207, + "eval_loss": 1.6621263027191162, + "eval_runtime": 45.1222, + "eval_samples_per_second": 22.162, + "eval_steps_per_second": 0.931, + "step": 950 + }, + { + "epoch": 1.1835718730553828, + "grad_norm": 2.9997975826263428, + "learning_rate": 3.397140221402214e-06, + "loss": 1.505, + "step": 951 + }, + { + "epoch": 1.1848164281269447, + "grad_norm": 2.7225453853607178, + "learning_rate": 3.3948339483394836e-06, + "loss": 1.4695, + "step": 952 + }, + { + "epoch": 1.1860609831985065, + "grad_norm": 2.916473150253296, + "learning_rate": 3.3925276752767534e-06, + "loss": 1.4901, + "step": 953 + }, + { + "epoch": 1.1873055382700684, + "grad_norm": 2.8371644020080566, + "learning_rate": 3.3902214022140224e-06, + "loss": 1.4944, + "step": 954 + }, + { + "epoch": 1.1885500933416304, + "grad_norm": 2.813662528991699, + "learning_rate": 3.387915129151292e-06, + "loss": 1.4603, + "step": 955 + }, + { + "epoch": 1.1897946484131923, + "grad_norm": 2.991931676864624, + "learning_rate": 3.3856088560885613e-06, + "loss": 1.5496, + "step": 956 + }, + { + "epoch": 1.1910392034847541, + "grad_norm": 2.6252501010894775, + "learning_rate": 3.3833025830258303e-06, + "loss": 1.4405, + "step": 957 + }, + { + "epoch": 1.1922837585563162, + "grad_norm": 2.747725009918213, + "learning_rate": 3.3809963099630997e-06, + "loss": 1.4689, + "step": 958 + }, + { + "epoch": 1.193528313627878, + "grad_norm": 2.887763261795044, + "learning_rate": 3.3786900369003696e-06, + "loss": 1.4501, + "step": 959 + }, + { + "epoch": 1.1947728686994399, + "grad_norm": 2.9954354763031006, + "learning_rate": 3.3763837638376386e-06, + "loss": 1.4978, + "step": 960 + }, + { + "epoch": 1.1947728686994399, + "eval_loss": 1.664442539215088, + "eval_runtime": 49.1266, + "eval_samples_per_second": 20.356, + "eval_steps_per_second": 0.855, + "step": 960 + }, + { + "epoch": 1.196017423771002, + "grad_norm": 3.0174152851104736, + "learning_rate": 3.374077490774908e-06, + "loss": 1.4353, + "step": 961 + }, + { + "epoch": 1.1972619788425638, + "grad_norm": 2.9082882404327393, + "learning_rate": 3.3717712177121774e-06, + "loss": 1.4878, + "step": 962 + }, + { + "epoch": 1.1985065339141256, + "grad_norm": 2.897993326187134, + "learning_rate": 3.3694649446494464e-06, + "loss": 1.4975, + "step": 963 + }, + { + "epoch": 1.1997510889856877, + "grad_norm": 2.739470958709717, + "learning_rate": 3.3671586715867163e-06, + "loss": 1.4202, + "step": 964 + }, + { + "epoch": 1.2009956440572496, + "grad_norm": 2.817786693572998, + "learning_rate": 3.3648523985239857e-06, + "loss": 1.5195, + "step": 965 + }, + { + "epoch": 1.2022401991288114, + "grad_norm": 2.8476386070251465, + "learning_rate": 3.362546125461255e-06, + "loss": 1.444, + "step": 966 + }, + { + "epoch": 1.2034847542003733, + "grad_norm": 2.804047107696533, + "learning_rate": 3.360239852398524e-06, + "loss": 1.4442, + "step": 967 + }, + { + "epoch": 1.2047293092719353, + "grad_norm": 2.9285430908203125, + "learning_rate": 3.3579335793357935e-06, + "loss": 1.4877, + "step": 968 + }, + { + "epoch": 1.2059738643434972, + "grad_norm": 2.8814878463745117, + "learning_rate": 3.3556273062730634e-06, + "loss": 1.5078, + "step": 969 + }, + { + "epoch": 1.207218419415059, + "grad_norm": 2.970728635787964, + "learning_rate": 3.3533210332103324e-06, + "loss": 1.4166, + "step": 970 + }, + { + "epoch": 1.207218419415059, + "eval_loss": 1.6635315418243408, + "eval_runtime": 47.2321, + "eval_samples_per_second": 21.172, + "eval_steps_per_second": 0.889, + "step": 970 + }, + { + "epoch": 1.208462974486621, + "grad_norm": 2.9836292266845703, + "learning_rate": 3.3510147601476018e-06, + "loss": 1.4777, + "step": 971 + }, + { + "epoch": 1.209707529558183, + "grad_norm": 2.8495216369628906, + "learning_rate": 3.348708487084871e-06, + "loss": 1.4829, + "step": 972 + }, + { + "epoch": 1.2109520846297448, + "grad_norm": 2.7709312438964844, + "learning_rate": 3.34640221402214e-06, + "loss": 1.4242, + "step": 973 + }, + { + "epoch": 1.2121966397013069, + "grad_norm": 2.999462366104126, + "learning_rate": 3.3440959409594096e-06, + "loss": 1.4148, + "step": 974 + }, + { + "epoch": 1.2134411947728687, + "grad_norm": 2.8719074726104736, + "learning_rate": 3.3417896678966795e-06, + "loss": 1.4196, + "step": 975 + }, + { + "epoch": 1.2146857498444306, + "grad_norm": 2.8469338417053223, + "learning_rate": 3.3394833948339485e-06, + "loss": 1.4941, + "step": 976 + }, + { + "epoch": 1.2159303049159926, + "grad_norm": 2.9780938625335693, + "learning_rate": 3.337177121771218e-06, + "loss": 1.513, + "step": 977 + }, + { + "epoch": 1.2171748599875545, + "grad_norm": 2.814760684967041, + "learning_rate": 3.3348708487084873e-06, + "loss": 1.4204, + "step": 978 + }, + { + "epoch": 1.2184194150591163, + "grad_norm": 3.016261339187622, + "learning_rate": 3.332564575645757e-06, + "loss": 1.4389, + "step": 979 + }, + { + "epoch": 1.2196639701306782, + "grad_norm": 2.9385855197906494, + "learning_rate": 3.330258302583026e-06, + "loss": 1.4885, + "step": 980 + }, + { + "epoch": 1.2196639701306782, + "eval_loss": 1.6688873767852783, + "eval_runtime": 46.2942, + "eval_samples_per_second": 21.601, + "eval_steps_per_second": 0.907, + "step": 980 + }, + { + "epoch": 1.2209085252022402, + "grad_norm": 3.1434757709503174, + "learning_rate": 3.3279520295202956e-06, + "loss": 1.5157, + "step": 981 + }, + { + "epoch": 1.222153080273802, + "grad_norm": 3.0104827880859375, + "learning_rate": 3.325645756457565e-06, + "loss": 1.49, + "step": 982 + }, + { + "epoch": 1.223397635345364, + "grad_norm": 3.1913645267486572, + "learning_rate": 3.323339483394834e-06, + "loss": 1.4937, + "step": 983 + }, + { + "epoch": 1.224642190416926, + "grad_norm": 3.19496750831604, + "learning_rate": 3.3210332103321034e-06, + "loss": 1.5197, + "step": 984 + }, + { + "epoch": 1.2258867454884879, + "grad_norm": 3.046232223510742, + "learning_rate": 3.3187269372693733e-06, + "loss": 1.4497, + "step": 985 + }, + { + "epoch": 1.2271313005600497, + "grad_norm": 2.850675344467163, + "learning_rate": 3.3164206642066423e-06, + "loss": 1.4595, + "step": 986 + }, + { + "epoch": 1.2283758556316118, + "grad_norm": 2.7744576930999756, + "learning_rate": 3.3141143911439117e-06, + "loss": 1.4726, + "step": 987 + }, + { + "epoch": 1.2296204107031736, + "grad_norm": 3.017153024673462, + "learning_rate": 3.311808118081181e-06, + "loss": 1.493, + "step": 988 + }, + { + "epoch": 1.2308649657747355, + "grad_norm": 2.8821120262145996, + "learning_rate": 3.3095018450184506e-06, + "loss": 1.4734, + "step": 989 + }, + { + "epoch": 1.2321095208462975, + "grad_norm": 2.8805840015411377, + "learning_rate": 3.3071955719557196e-06, + "loss": 1.4562, + "step": 990 + }, + { + "epoch": 1.2321095208462975, + "eval_loss": 1.669345736503601, + "eval_runtime": 47.0893, + "eval_samples_per_second": 21.236, + "eval_steps_per_second": 0.892, + "step": 990 + }, + { + "epoch": 1.2333540759178594, + "grad_norm": 2.7599008083343506, + "learning_rate": 3.3048892988929894e-06, + "loss": 1.4218, + "step": 991 + }, + { + "epoch": 1.2345986309894212, + "grad_norm": 2.9835240840911865, + "learning_rate": 3.302583025830259e-06, + "loss": 1.4596, + "step": 992 + }, + { + "epoch": 1.235843186060983, + "grad_norm": 2.843358039855957, + "learning_rate": 3.300276752767528e-06, + "loss": 1.4561, + "step": 993 + }, + { + "epoch": 1.2370877411325452, + "grad_norm": 2.7386093139648438, + "learning_rate": 3.2979704797047972e-06, + "loss": 1.4475, + "step": 994 + }, + { + "epoch": 1.238332296204107, + "grad_norm": 2.668161392211914, + "learning_rate": 3.2956642066420667e-06, + "loss": 1.4661, + "step": 995 + }, + { + "epoch": 1.2395768512756689, + "grad_norm": 2.947235107421875, + "learning_rate": 3.2933579335793357e-06, + "loss": 1.4358, + "step": 996 + }, + { + "epoch": 1.240821406347231, + "grad_norm": 2.8159878253936768, + "learning_rate": 3.2910516605166055e-06, + "loss": 1.4432, + "step": 997 + }, + { + "epoch": 1.2420659614187928, + "grad_norm": 2.896597385406494, + "learning_rate": 3.288745387453875e-06, + "loss": 1.5585, + "step": 998 + }, + { + "epoch": 1.2433105164903546, + "grad_norm": 2.80241322517395, + "learning_rate": 3.286439114391144e-06, + "loss": 1.5076, + "step": 999 + }, + { + "epoch": 1.2445550715619167, + "grad_norm": 3.008939027786255, + "learning_rate": 3.2841328413284134e-06, + "loss": 1.4234, + "step": 1000 + }, + { + "epoch": 1.2445550715619167, + "eval_loss": 1.6621588468551636, + "eval_runtime": 50.4926, + "eval_samples_per_second": 19.805, + "eval_steps_per_second": 0.832, + "step": 1000 + }, + { + "epoch": 1.2457996266334785, + "grad_norm": 2.9872381687164307, + "learning_rate": 3.281826568265683e-06, + "loss": 1.453, + "step": 1001 + }, + { + "epoch": 1.2470441817050404, + "grad_norm": 2.704676628112793, + "learning_rate": 3.2795202952029526e-06, + "loss": 1.4711, + "step": 1002 + }, + { + "epoch": 1.2482887367766025, + "grad_norm": 2.8100757598876953, + "learning_rate": 3.2772140221402216e-06, + "loss": 1.3884, + "step": 1003 + }, + { + "epoch": 1.2495332918481643, + "grad_norm": 2.902953624725342, + "learning_rate": 3.274907749077491e-06, + "loss": 1.4986, + "step": 1004 + }, + { + "epoch": 1.2507778469197262, + "grad_norm": 2.980163097381592, + "learning_rate": 3.2726014760147605e-06, + "loss": 1.5067, + "step": 1005 + }, + { + "epoch": 1.252022401991288, + "grad_norm": 2.8820345401763916, + "learning_rate": 3.2702952029520295e-06, + "loss": 1.4995, + "step": 1006 + }, + { + "epoch": 1.25326695706285, + "grad_norm": 2.8219635486602783, + "learning_rate": 3.2679889298892993e-06, + "loss": 1.4881, + "step": 1007 + }, + { + "epoch": 1.254511512134412, + "grad_norm": 2.8593199253082275, + "learning_rate": 3.2656826568265687e-06, + "loss": 1.4469, + "step": 1008 + }, + { + "epoch": 1.255756067205974, + "grad_norm": 2.888357639312744, + "learning_rate": 3.2633763837638377e-06, + "loss": 1.4869, + "step": 1009 + }, + { + "epoch": 1.2570006222775358, + "grad_norm": 2.9985013008117676, + "learning_rate": 3.261070110701107e-06, + "loss": 1.4724, + "step": 1010 + }, + { + "epoch": 1.2570006222775358, + "eval_loss": 1.6492141485214233, + "eval_runtime": 45.2197, + "eval_samples_per_second": 22.114, + "eval_steps_per_second": 0.929, + "step": 1010 + }, + { + "epoch": 1.2582451773490977, + "grad_norm": 2.860780715942383, + "learning_rate": 3.2587638376383766e-06, + "loss": 1.4618, + "step": 1011 + }, + { + "epoch": 1.2594897324206595, + "grad_norm": 2.766479730606079, + "learning_rate": 3.2564575645756456e-06, + "loss": 1.4642, + "step": 1012 + }, + { + "epoch": 1.2607342874922216, + "grad_norm": 2.711759328842163, + "learning_rate": 3.2541512915129154e-06, + "loss": 1.4283, + "step": 1013 + }, + { + "epoch": 1.2619788425637835, + "grad_norm": 2.8365769386291504, + "learning_rate": 3.251845018450185e-06, + "loss": 1.4248, + "step": 1014 + }, + { + "epoch": 1.2632233976353453, + "grad_norm": 2.957620143890381, + "learning_rate": 3.2495387453874543e-06, + "loss": 1.4662, + "step": 1015 + }, + { + "epoch": 1.2644679527069074, + "grad_norm": 2.8066208362579346, + "learning_rate": 3.2472324723247233e-06, + "loss": 1.4447, + "step": 1016 + }, + { + "epoch": 1.2657125077784692, + "grad_norm": 2.902040958404541, + "learning_rate": 3.244926199261993e-06, + "loss": 1.4743, + "step": 1017 + }, + { + "epoch": 1.266957062850031, + "grad_norm": 2.7636687755584717, + "learning_rate": 3.2426199261992625e-06, + "loss": 1.4465, + "step": 1018 + }, + { + "epoch": 1.268201617921593, + "grad_norm": 2.95261549949646, + "learning_rate": 3.2403136531365315e-06, + "loss": 1.4769, + "step": 1019 + }, + { + "epoch": 1.269446172993155, + "grad_norm": 2.911128520965576, + "learning_rate": 3.238007380073801e-06, + "loss": 1.3843, + "step": 1020 + }, + { + "epoch": 1.269446172993155, + "eval_loss": 1.6526964902877808, + "eval_runtime": 48.2126, + "eval_samples_per_second": 20.741, + "eval_steps_per_second": 0.871, + "step": 1020 + }, + { + "epoch": 1.2706907280647168, + "grad_norm": 2.769951820373535, + "learning_rate": 3.2357011070110704e-06, + "loss": 1.4847, + "step": 1021 + }, + { + "epoch": 1.271935283136279, + "grad_norm": 3.0451674461364746, + "learning_rate": 3.2333948339483394e-06, + "loss": 1.4453, + "step": 1022 + }, + { + "epoch": 1.2731798382078408, + "grad_norm": 2.9543521404266357, + "learning_rate": 3.2310885608856092e-06, + "loss": 1.4789, + "step": 1023 + }, + { + "epoch": 1.2744243932794026, + "grad_norm": 2.9174394607543945, + "learning_rate": 3.2287822878228787e-06, + "loss": 1.4807, + "step": 1024 + }, + { + "epoch": 1.2756689483509644, + "grad_norm": 2.888258934020996, + "learning_rate": 3.2264760147601477e-06, + "loss": 1.4367, + "step": 1025 + }, + { + "epoch": 1.2769135034225265, + "grad_norm": 2.8346047401428223, + "learning_rate": 3.224169741697417e-06, + "loss": 1.4313, + "step": 1026 + }, + { + "epoch": 1.2781580584940884, + "grad_norm": 2.7531332969665527, + "learning_rate": 3.2218634686346865e-06, + "loss": 1.4018, + "step": 1027 + }, + { + "epoch": 1.2794026135656502, + "grad_norm": 3.136946439743042, + "learning_rate": 3.2195571955719564e-06, + "loss": 1.4417, + "step": 1028 + }, + { + "epoch": 1.2806471686372123, + "grad_norm": 2.9618165493011475, + "learning_rate": 3.2172509225092254e-06, + "loss": 1.5117, + "step": 1029 + }, + { + "epoch": 1.2818917237087741, + "grad_norm": 3.0740747451782227, + "learning_rate": 3.2149446494464948e-06, + "loss": 1.4641, + "step": 1030 + }, + { + "epoch": 1.2818917237087741, + "eval_loss": 1.6572761535644531, + "eval_runtime": 51.6696, + "eval_samples_per_second": 19.354, + "eval_steps_per_second": 0.813, + "step": 1030 + }, + { + "epoch": 1.283136278780336, + "grad_norm": 2.9225106239318848, + "learning_rate": 3.212638376383764e-06, + "loss": 1.4421, + "step": 1031 + }, + { + "epoch": 1.2843808338518978, + "grad_norm": 2.9530649185180664, + "learning_rate": 3.210332103321033e-06, + "loss": 1.5138, + "step": 1032 + }, + { + "epoch": 1.28562538892346, + "grad_norm": 2.8513238430023193, + "learning_rate": 3.208025830258303e-06, + "loss": 1.411, + "step": 1033 + }, + { + "epoch": 1.2868699439950217, + "grad_norm": 2.7440171241760254, + "learning_rate": 3.2057195571955725e-06, + "loss": 1.4182, + "step": 1034 + }, + { + "epoch": 1.2881144990665838, + "grad_norm": 2.8126630783081055, + "learning_rate": 3.2034132841328415e-06, + "loss": 1.4333, + "step": 1035 + }, + { + "epoch": 1.2893590541381457, + "grad_norm": 2.8598744869232178, + "learning_rate": 3.201107011070111e-06, + "loss": 1.439, + "step": 1036 + }, + { + "epoch": 1.2906036092097075, + "grad_norm": 2.901622772216797, + "learning_rate": 3.1988007380073803e-06, + "loss": 1.4461, + "step": 1037 + }, + { + "epoch": 1.2918481642812694, + "grad_norm": 2.7670180797576904, + "learning_rate": 3.1964944649446493e-06, + "loss": 1.4772, + "step": 1038 + }, + { + "epoch": 1.2930927193528314, + "grad_norm": 2.7860350608825684, + "learning_rate": 3.194188191881919e-06, + "loss": 1.4414, + "step": 1039 + }, + { + "epoch": 1.2943372744243933, + "grad_norm": 2.822052478790283, + "learning_rate": 3.1918819188191886e-06, + "loss": 1.4192, + "step": 1040 + }, + { + "epoch": 1.2943372744243933, + "eval_loss": 1.663284420967102, + "eval_runtime": 47.2822, + "eval_samples_per_second": 21.15, + "eval_steps_per_second": 0.888, + "step": 1040 + }, + { + "epoch": 1.2955818294959551, + "grad_norm": 2.909109115600586, + "learning_rate": 3.189575645756458e-06, + "loss": 1.4795, + "step": 1041 + }, + { + "epoch": 1.2968263845675172, + "grad_norm": 2.933673143386841, + "learning_rate": 3.187269372693727e-06, + "loss": 1.5038, + "step": 1042 + }, + { + "epoch": 1.298070939639079, + "grad_norm": 3.032518148422241, + "learning_rate": 3.1849630996309964e-06, + "loss": 1.5133, + "step": 1043 + }, + { + "epoch": 1.299315494710641, + "grad_norm": 2.826040506362915, + "learning_rate": 3.1826568265682663e-06, + "loss": 1.4395, + "step": 1044 + }, + { + "epoch": 1.3005600497822027, + "grad_norm": 2.8152658939361572, + "learning_rate": 3.1803505535055353e-06, + "loss": 1.4787, + "step": 1045 + }, + { + "epoch": 1.3018046048537648, + "grad_norm": 2.9410431385040283, + "learning_rate": 3.1780442804428047e-06, + "loss": 1.4375, + "step": 1046 + }, + { + "epoch": 1.3030491599253267, + "grad_norm": 2.948636531829834, + "learning_rate": 3.175738007380074e-06, + "loss": 1.4847, + "step": 1047 + }, + { + "epoch": 1.3042937149968887, + "grad_norm": 2.6287615299224854, + "learning_rate": 3.173431734317343e-06, + "loss": 1.4186, + "step": 1048 + }, + { + "epoch": 1.3055382700684506, + "grad_norm": 2.836029052734375, + "learning_rate": 3.1711254612546125e-06, + "loss": 1.4251, + "step": 1049 + }, + { + "epoch": 1.3067828251400124, + "grad_norm": 2.982081651687622, + "learning_rate": 3.1688191881918824e-06, + "loss": 1.4951, + "step": 1050 + }, + { + "epoch": 1.3067828251400124, + "eval_loss": 1.6578067541122437, + "eval_runtime": 47.5847, + "eval_samples_per_second": 21.015, + "eval_steps_per_second": 0.883, + "step": 1050 + }, + { + "epoch": 1.3080273802115743, + "grad_norm": 2.9297118186950684, + "learning_rate": 3.166512915129152e-06, + "loss": 1.4699, + "step": 1051 + }, + { + "epoch": 1.3092719352831363, + "grad_norm": 2.906435251235962, + "learning_rate": 3.164206642066421e-06, + "loss": 1.4562, + "step": 1052 + }, + { + "epoch": 1.3105164903546982, + "grad_norm": 2.8655896186828613, + "learning_rate": 3.1619003690036902e-06, + "loss": 1.4351, + "step": 1053 + }, + { + "epoch": 1.31176104542626, + "grad_norm": 2.779244899749756, + "learning_rate": 3.15959409594096e-06, + "loss": 1.4479, + "step": 1054 + }, + { + "epoch": 1.3130056004978221, + "grad_norm": 2.9863739013671875, + "learning_rate": 3.157287822878229e-06, + "loss": 1.4964, + "step": 1055 + }, + { + "epoch": 1.314250155569384, + "grad_norm": 2.821871042251587, + "learning_rate": 3.1549815498154985e-06, + "loss": 1.4173, + "step": 1056 + }, + { + "epoch": 1.3154947106409458, + "grad_norm": 2.8573882579803467, + "learning_rate": 3.152675276752768e-06, + "loss": 1.4877, + "step": 1057 + }, + { + "epoch": 1.3167392657125077, + "grad_norm": 2.9824137687683105, + "learning_rate": 3.150369003690037e-06, + "loss": 1.5124, + "step": 1058 + }, + { + "epoch": 1.3179838207840697, + "grad_norm": 2.881098747253418, + "learning_rate": 3.1480627306273063e-06, + "loss": 1.4346, + "step": 1059 + }, + { + "epoch": 1.3192283758556316, + "grad_norm": 2.918782949447632, + "learning_rate": 3.145756457564576e-06, + "loss": 1.4797, + "step": 1060 + }, + { + "epoch": 1.3192283758556316, + "eval_loss": 1.6510179042816162, + "eval_runtime": 53.6356, + "eval_samples_per_second": 18.644, + "eval_steps_per_second": 0.783, + "step": 1060 + }, + { + "epoch": 1.3204729309271936, + "grad_norm": 3.045776128768921, + "learning_rate": 3.143450184501845e-06, + "loss": 1.5039, + "step": 1061 + }, + { + "epoch": 1.3217174859987555, + "grad_norm": 3.041501760482788, + "learning_rate": 3.1411439114391146e-06, + "loss": 1.4878, + "step": 1062 + }, + { + "epoch": 1.3229620410703173, + "grad_norm": 2.917837142944336, + "learning_rate": 3.138837638376384e-06, + "loss": 1.4202, + "step": 1063 + }, + { + "epoch": 1.3242065961418792, + "grad_norm": 2.708070755004883, + "learning_rate": 3.1365313653136535e-06, + "loss": 1.4071, + "step": 1064 + }, + { + "epoch": 1.3254511512134413, + "grad_norm": 2.9146156311035156, + "learning_rate": 3.1342250922509225e-06, + "loss": 1.4428, + "step": 1065 + }, + { + "epoch": 1.326695706285003, + "grad_norm": 3.220919132232666, + "learning_rate": 3.1319188191881923e-06, + "loss": 1.4733, + "step": 1066 + }, + { + "epoch": 1.327940261356565, + "grad_norm": 2.9127554893493652, + "learning_rate": 3.1296125461254617e-06, + "loss": 1.4554, + "step": 1067 + }, + { + "epoch": 1.329184816428127, + "grad_norm": 3.053191661834717, + "learning_rate": 3.1273062730627307e-06, + "loss": 1.4044, + "step": 1068 + }, + { + "epoch": 1.3304293714996889, + "grad_norm": 3.2937021255493164, + "learning_rate": 3.125e-06, + "loss": 1.4949, + "step": 1069 + }, + { + "epoch": 1.3316739265712507, + "grad_norm": 2.976717948913574, + "learning_rate": 3.12269372693727e-06, + "loss": 1.446, + "step": 1070 + }, + { + "epoch": 1.3316739265712507, + "eval_loss": 1.6503105163574219, + "eval_runtime": 50.7579, + "eval_samples_per_second": 19.701, + "eval_steps_per_second": 0.827, + "step": 1070 + }, + { + "epoch": 1.3329184816428126, + "grad_norm": 3.0860936641693115, + "learning_rate": 3.120387453874539e-06, + "loss": 1.4055, + "step": 1071 + }, + { + "epoch": 1.3341630367143746, + "grad_norm": 3.0401217937469482, + "learning_rate": 3.1180811808118084e-06, + "loss": 1.4688, + "step": 1072 + }, + { + "epoch": 1.3354075917859365, + "grad_norm": 2.953253984451294, + "learning_rate": 3.115774907749078e-06, + "loss": 1.5001, + "step": 1073 + }, + { + "epoch": 1.3366521468574986, + "grad_norm": 3.0665059089660645, + "learning_rate": 3.113468634686347e-06, + "loss": 1.4217, + "step": 1074 + }, + { + "epoch": 1.3378967019290604, + "grad_norm": 3.121600389480591, + "learning_rate": 3.1111623616236163e-06, + "loss": 1.4508, + "step": 1075 + }, + { + "epoch": 1.3391412570006223, + "grad_norm": 2.8057339191436768, + "learning_rate": 3.108856088560886e-06, + "loss": 1.4241, + "step": 1076 + }, + { + "epoch": 1.340385812072184, + "grad_norm": 3.0147411823272705, + "learning_rate": 3.1065498154981555e-06, + "loss": 1.479, + "step": 1077 + }, + { + "epoch": 1.3416303671437462, + "grad_norm": 3.030010223388672, + "learning_rate": 3.1042435424354245e-06, + "loss": 1.4861, + "step": 1078 + }, + { + "epoch": 1.342874922215308, + "grad_norm": 3.027998208999634, + "learning_rate": 3.101937269372694e-06, + "loss": 1.4916, + "step": 1079 + }, + { + "epoch": 1.3441194772868699, + "grad_norm": 2.9252536296844482, + "learning_rate": 3.0996309963099634e-06, + "loss": 1.418, + "step": 1080 + }, + { + "epoch": 1.3441194772868699, + "eval_loss": 1.6490333080291748, + "eval_runtime": 47.4026, + "eval_samples_per_second": 21.096, + "eval_steps_per_second": 0.886, + "step": 1080 + }, + { + "epoch": 1.345364032358432, + "grad_norm": 3.110847234725952, + "learning_rate": 3.0973247232472324e-06, + "loss": 1.4127, + "step": 1081 + }, + { + "epoch": 1.3466085874299938, + "grad_norm": 2.846491575241089, + "learning_rate": 3.0950184501845022e-06, + "loss": 1.444, + "step": 1082 + }, + { + "epoch": 1.3478531425015556, + "grad_norm": 2.950533151626587, + "learning_rate": 3.0927121771217716e-06, + "loss": 1.4456, + "step": 1083 + }, + { + "epoch": 1.3490976975731175, + "grad_norm": 2.875964403152466, + "learning_rate": 3.0904059040590406e-06, + "loss": 1.4485, + "step": 1084 + }, + { + "epoch": 1.3503422526446796, + "grad_norm": 2.9612326622009277, + "learning_rate": 3.08809963099631e-06, + "loss": 1.4446, + "step": 1085 + }, + { + "epoch": 1.3515868077162414, + "grad_norm": 2.925297975540161, + "learning_rate": 3.0857933579335795e-06, + "loss": 1.4847, + "step": 1086 + }, + { + "epoch": 1.3528313627878035, + "grad_norm": 2.991616725921631, + "learning_rate": 3.0834870848708485e-06, + "loss": 1.5151, + "step": 1087 + }, + { + "epoch": 1.3540759178593653, + "grad_norm": 2.96329927444458, + "learning_rate": 3.0811808118081183e-06, + "loss": 1.4574, + "step": 1088 + }, + { + "epoch": 1.3553204729309272, + "grad_norm": 2.948702573776245, + "learning_rate": 3.0788745387453878e-06, + "loss": 1.4585, + "step": 1089 + }, + { + "epoch": 1.356565028002489, + "grad_norm": 3.218463659286499, + "learning_rate": 3.076568265682657e-06, + "loss": 1.4439, + "step": 1090 + }, + { + "epoch": 1.356565028002489, + "eval_loss": 1.645838975906372, + "eval_runtime": 47.048, + "eval_samples_per_second": 21.255, + "eval_steps_per_second": 0.893, + "step": 1090 + }, + { + "epoch": 1.357809583074051, + "grad_norm": 2.9799578189849854, + "learning_rate": 3.074261992619926e-06, + "loss": 1.4883, + "step": 1091 + }, + { + "epoch": 1.359054138145613, + "grad_norm": 2.7660505771636963, + "learning_rate": 3.071955719557196e-06, + "loss": 1.4349, + "step": 1092 + }, + { + "epoch": 1.3602986932171748, + "grad_norm": 3.0201916694641113, + "learning_rate": 3.0696494464944655e-06, + "loss": 1.418, + "step": 1093 + }, + { + "epoch": 1.3615432482887369, + "grad_norm": 2.844207763671875, + "learning_rate": 3.0673431734317345e-06, + "loss": 1.4769, + "step": 1094 + }, + { + "epoch": 1.3627878033602987, + "grad_norm": 2.8306753635406494, + "learning_rate": 3.065036900369004e-06, + "loss": 1.4653, + "step": 1095 + }, + { + "epoch": 1.3640323584318605, + "grad_norm": 2.805023431777954, + "learning_rate": 3.0627306273062733e-06, + "loss": 1.4771, + "step": 1096 + }, + { + "epoch": 1.3652769135034224, + "grad_norm": 3.020883083343506, + "learning_rate": 3.0604243542435423e-06, + "loss": 1.4237, + "step": 1097 + }, + { + "epoch": 1.3665214685749845, + "grad_norm": 2.8395731449127197, + "learning_rate": 3.058118081180812e-06, + "loss": 1.4647, + "step": 1098 + }, + { + "epoch": 1.3677660236465463, + "grad_norm": 2.871156692504883, + "learning_rate": 3.0558118081180816e-06, + "loss": 1.4196, + "step": 1099 + }, + { + "epoch": 1.3690105787181084, + "grad_norm": 2.974905014038086, + "learning_rate": 3.053505535055351e-06, + "loss": 1.4779, + "step": 1100 + }, + { + "epoch": 1.3690105787181084, + "eval_loss": 1.651503562927246, + "eval_runtime": 54.2713, + "eval_samples_per_second": 18.426, + "eval_steps_per_second": 0.774, + "step": 1100 + } + ], + "logging_steps": 1, + "max_steps": 2409, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 100, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3.429040350298112e+17, + "train_batch_size": 3, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/training_args.bin b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..6cdf93817e8f6d23b60d79888b226cc66ed2e88b --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a50e4918f3c49370326e14a7ed2f73b48b05e521f4b3399b53438fac6abee926 +size 6011 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/zero_to_fp32.py b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..24cc342e78d1a006c782b3a4cd68d9ce786d8fd8 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/zero_to_fp32.py @@ -0,0 +1,604 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: python zero_to_fp32.py . pytorch_model.bin + +import argparse +import torch +import glob +import math +import os +import re +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + + total_files = len(files) + state_dicts = [] + for f in files: + state_dict = torch.load(f, map_location=device) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + if zero_stage <= 2: + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + elif zero_stage == 3: + # if there is more than one param group, there will be multiple flattened tensors - one + # flattened tensor per group - for simplicity merge them into a single tensor + # + # XXX: could make the script more memory efficient for when there are multiple groups - it + # will require matching the sub-lists of param_shapes for each param group flattened tensor + + fp32_flat_groups = [ + torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts)) + ] + + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = fp32_flat_groups[0].numel() * world_size + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + for name, shape in param_shapes.items(): + + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # XXX: memory usage doubles here + state_dict[name] = torch.cat( + tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)), + 0).narrow(0, 0, unpartitioned_numel).view(shape) + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + + Returns: + - pytorch ``state_dict`` + + Note: this approach may not work if your application doesn't have sufficient free CPU memory and + you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters) + print(f"Saving fp32 state dict to {output_file}") + torch.save(state_dict, output_file) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info(f"Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info(f"Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument( + "output_file", + type=str, + help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_file, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fb0f9a53a507ffcdb60410deb71d60ef801bf350 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/config.json @@ -0,0 +1,36 @@ +{ + "_name_or_path": "meta-llama/Llama-3.2-3B", + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "eos_token_id": 128001, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 3072, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 24, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "torch_dtype": "float16", + "transformers_version": "4.45.1", + "use_cache": true, + "vocab_size": 128256 +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/generation_config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9c389b841a9a29ddd904e02d1eb0e08dcce82ad9 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": 128001, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.45.1" +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/latest b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/latest new file mode 100644 index 0000000000000000000000000000000000000000..873b08355ae55e303606c474ab5fba25f9e8262a --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/latest @@ -0,0 +1 @@ +global_step1200 \ No newline at end of file diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/model.safetensors.index.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..ed64de846d720b9a7859dc20575fea8e8ca51940 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/model.safetensors.index.json @@ -0,0 +1,262 @@ +{ + "metadata": { + "total_size": 7213504512 + }, + "weight_map": { + "lm_head.weight": "model-00002-of-00002.safetensors", + "model.embed_tokens.weight": "model-00001-of-00002.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.norm.weight": "model-00002-of-00002.safetensors" + } +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/rng_state_0.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..3259c6605b4fec3d2abdf15c7c3ccbdef85f5d96 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b6aefb2eb9cf046f058c1e4bd20c70066eedd4535bf2e9f2b3f178ddbfbfd9e +size 16567 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/rng_state_1.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..9f4723485ff30930b59224e8afe0cb0b2b3c51bc --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f34c8972483cae48cde438f54c14179a5393e31298d634393f2e166c6a9bfb3e +size 16567 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/rng_state_2.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..6f245457df6ae96d9dae4e4269752c2f036e6ee3 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d0068d515416f5010564895d620c337c347a04bf8d5382015b5833708d52850 +size 16567 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/scheduler.pt b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..ca7b87736ad96984e06c88dc58b2e5efcdb20f2d --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5d60eb151191f050cb4b3590ac23ee7ff247807e742892633ea85518fbb160a +size 627 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/special_tokens_map.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..6949c5975ee0e961ef61cf31010dce04df0a03f8 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "[PAD]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/tokenizer.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f28ecaeab53ae07feed29ccf8624d2b0a8344df9 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19fb2e1e3cdd6f7433d89fd6d62c82042599dd4984f342efe7fec6e159e6a8f6 +size 17210734 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/tokenizer_config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8991b49e9c2a43fc527dab9e09ad8171f0cc5943 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/tokenizer_config.json @@ -0,0 +1,2086 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "[PAD]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128257": { + "content": "🅂", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "128258": { + "content": "🄿", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "[PAD]", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/trainer_state.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..889f1fbfac8838c0f487ac60d00ed057472813a6 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/trainer_state.json @@ -0,0 +1,9393 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.4934660858742999, + "eval_steps": 10, + "global_step": 1200, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0012445550715619166, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8368, + "step": 1 + }, + { + "epoch": 0.002489110143123833, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.86, + "step": 2 + }, + { + "epoch": 0.00373366521468575, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8265, + "step": 3 + }, + { + "epoch": 0.004978220286247666, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8271, + "step": 4 + }, + { + "epoch": 0.006222775357809583, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8768, + "step": 5 + }, + { + "epoch": 0.0074673304293715, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8402, + "step": 6 + }, + { + "epoch": 0.008711885500933417, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.785, + "step": 7 + }, + { + "epoch": 0.009956440572495333, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8466, + "step": 8 + }, + { + "epoch": 0.01120099564405725, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.7673, + "step": 9 + }, + { + "epoch": 0.012445550715619166, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8074, + "step": 10 + }, + { + "epoch": 0.012445550715619166, + "eval_loss": 2.8535053730010986, + "eval_runtime": 42.446, + "eval_samples_per_second": 23.559, + "eval_steps_per_second": 0.989, + "step": 10 + }, + { + "epoch": 0.013690105787181083, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8592, + "step": 11 + }, + { + "epoch": 0.014934660858743, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.804, + "step": 12 + }, + { + "epoch": 0.016179215930304917, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8076, + "step": 13 + }, + { + "epoch": 0.017423771001866834, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8219, + "step": 14 + }, + { + "epoch": 0.018668326073428748, + "grad_norm": 5.105710506439209, + "learning_rate": 2.0746887966804982e-08, + "loss": 2.7995, + "step": 15 + }, + { + "epoch": 0.019912881144990666, + "grad_norm": 5.156588077545166, + "learning_rate": 4.1493775933609963e-08, + "loss": 2.8577, + "step": 16 + }, + { + "epoch": 0.021157436216552583, + "grad_norm": 5.709373950958252, + "learning_rate": 6.224066390041494e-08, + "loss": 2.8433, + "step": 17 + }, + { + "epoch": 0.0224019912881145, + "grad_norm": 5.64658784866333, + "learning_rate": 8.298755186721993e-08, + "loss": 2.8428, + "step": 18 + }, + { + "epoch": 0.023646546359676415, + "grad_norm": 5.541937351226807, + "learning_rate": 1.037344398340249e-07, + "loss": 2.7785, + "step": 19 + }, + { + "epoch": 0.024891101431238332, + "grad_norm": 5.233374118804932, + "learning_rate": 1.2448132780082988e-07, + "loss": 2.8599, + "step": 20 + }, + { + "epoch": 0.024891101431238332, + "eval_loss": 2.8521182537078857, + "eval_runtime": 43.5664, + "eval_samples_per_second": 22.953, + "eval_steps_per_second": 0.964, + "step": 20 + }, + { + "epoch": 0.02613565650280025, + "grad_norm": 5.407598972320557, + "learning_rate": 1.4522821576763488e-07, + "loss": 2.8081, + "step": 21 + }, + { + "epoch": 0.027380211574362167, + "grad_norm": 5.0483317375183105, + "learning_rate": 1.6597510373443985e-07, + "loss": 2.8489, + "step": 22 + }, + { + "epoch": 0.02862476664592408, + "grad_norm": 4.985588550567627, + "learning_rate": 1.8672199170124483e-07, + "loss": 2.8375, + "step": 23 + }, + { + "epoch": 0.029869321717486, + "grad_norm": 5.201700687408447, + "learning_rate": 2.074688796680498e-07, + "loss": 2.8355, + "step": 24 + }, + { + "epoch": 0.031113876789047916, + "grad_norm": 5.009235382080078, + "learning_rate": 2.2821576763485478e-07, + "loss": 2.7774, + "step": 25 + }, + { + "epoch": 0.03235843186060983, + "grad_norm": 4.509210109710693, + "learning_rate": 2.4896265560165975e-07, + "loss": 2.7992, + "step": 26 + }, + { + "epoch": 0.03360298693217175, + "grad_norm": 4.529262065887451, + "learning_rate": 2.6970954356846476e-07, + "loss": 2.836, + "step": 27 + }, + { + "epoch": 0.03484754200373367, + "grad_norm": 4.526787281036377, + "learning_rate": 2.9045643153526976e-07, + "loss": 2.8128, + "step": 28 + }, + { + "epoch": 0.03609209707529558, + "grad_norm": 4.016609191894531, + "learning_rate": 3.112033195020747e-07, + "loss": 2.7947, + "step": 29 + }, + { + "epoch": 0.037336652146857496, + "grad_norm": 3.4784562587738037, + "learning_rate": 3.319502074688797e-07, + "loss": 2.7854, + "step": 30 + }, + { + "epoch": 0.037336652146857496, + "eval_loss": 2.8020100593566895, + "eval_runtime": 44.8464, + "eval_samples_per_second": 22.298, + "eval_steps_per_second": 0.937, + "step": 30 + }, + { + "epoch": 0.038581207218419414, + "grad_norm": 3.4887945652008057, + "learning_rate": 3.5269709543568466e-07, + "loss": 2.7908, + "step": 31 + }, + { + "epoch": 0.03982576228998133, + "grad_norm": 3.4533374309539795, + "learning_rate": 3.7344398340248966e-07, + "loss": 2.7358, + "step": 32 + }, + { + "epoch": 0.04107031736154325, + "grad_norm": 3.270954132080078, + "learning_rate": 3.941908713692946e-07, + "loss": 2.8013, + "step": 33 + }, + { + "epoch": 0.042314872433105166, + "grad_norm": 3.61322283744812, + "learning_rate": 4.149377593360996e-07, + "loss": 2.7521, + "step": 34 + }, + { + "epoch": 0.043559427504667084, + "grad_norm": 3.158719062805176, + "learning_rate": 4.3568464730290456e-07, + "loss": 2.7573, + "step": 35 + }, + { + "epoch": 0.044803982576229, + "grad_norm": 2.888746500015259, + "learning_rate": 4.5643153526970956e-07, + "loss": 2.7795, + "step": 36 + }, + { + "epoch": 0.04604853764779091, + "grad_norm": 3.022629499435425, + "learning_rate": 4.771784232365145e-07, + "loss": 2.7891, + "step": 37 + }, + { + "epoch": 0.04729309271935283, + "grad_norm": 3.038097381591797, + "learning_rate": 4.979253112033195e-07, + "loss": 2.7376, + "step": 38 + }, + { + "epoch": 0.04853764779091475, + "grad_norm": 2.9992239475250244, + "learning_rate": 5.186721991701245e-07, + "loss": 2.7279, + "step": 39 + }, + { + "epoch": 0.049782202862476664, + "grad_norm": 2.908842086791992, + "learning_rate": 5.394190871369295e-07, + "loss": 2.6582, + "step": 40 + }, + { + "epoch": 0.049782202862476664, + "eval_loss": 2.725968837738037, + "eval_runtime": 44.9498, + "eval_samples_per_second": 22.247, + "eval_steps_per_second": 0.934, + "step": 40 + }, + { + "epoch": 0.05102675793403858, + "grad_norm": 2.8025174140930176, + "learning_rate": 5.601659751037345e-07, + "loss": 2.6809, + "step": 41 + }, + { + "epoch": 0.0522713130056005, + "grad_norm": 2.7555699348449707, + "learning_rate": 5.809128630705395e-07, + "loss": 2.6954, + "step": 42 + }, + { + "epoch": 0.053515868077162417, + "grad_norm": 2.4732933044433594, + "learning_rate": 6.016597510373444e-07, + "loss": 2.7307, + "step": 43 + }, + { + "epoch": 0.054760423148724334, + "grad_norm": 2.451366424560547, + "learning_rate": 6.224066390041494e-07, + "loss": 2.7064, + "step": 44 + }, + { + "epoch": 0.056004978220286245, + "grad_norm": 2.5768678188323975, + "learning_rate": 6.431535269709543e-07, + "loss": 2.6466, + "step": 45 + }, + { + "epoch": 0.05724953329184816, + "grad_norm": 2.579332113265991, + "learning_rate": 6.639004149377594e-07, + "loss": 2.6127, + "step": 46 + }, + { + "epoch": 0.05849408836341008, + "grad_norm": 2.231207847595215, + "learning_rate": 6.846473029045644e-07, + "loss": 2.6429, + "step": 47 + }, + { + "epoch": 0.059738643434972, + "grad_norm": 2.134375810623169, + "learning_rate": 7.053941908713693e-07, + "loss": 2.6423, + "step": 48 + }, + { + "epoch": 0.060983198506533914, + "grad_norm": 2.0485494136810303, + "learning_rate": 7.261410788381744e-07, + "loss": 2.581, + "step": 49 + }, + { + "epoch": 0.06222775357809583, + "grad_norm": 2.02909517288208, + "learning_rate": 7.468879668049793e-07, + "loss": 2.6357, + "step": 50 + }, + { + "epoch": 0.06222775357809583, + "eval_loss": 2.632822036743164, + "eval_runtime": 47.0104, + "eval_samples_per_second": 21.272, + "eval_steps_per_second": 0.893, + "step": 50 + }, + { + "epoch": 0.06347230864965775, + "grad_norm": 1.892500877380371, + "learning_rate": 7.676348547717843e-07, + "loss": 2.5945, + "step": 51 + }, + { + "epoch": 0.06471686372121967, + "grad_norm": 2.107921600341797, + "learning_rate": 7.883817427385892e-07, + "loss": 2.5867, + "step": 52 + }, + { + "epoch": 0.06596141879278158, + "grad_norm": 1.8287527561187744, + "learning_rate": 8.091286307053943e-07, + "loss": 2.5862, + "step": 53 + }, + { + "epoch": 0.0672059738643435, + "grad_norm": 1.7342065572738647, + "learning_rate": 8.298755186721992e-07, + "loss": 2.5735, + "step": 54 + }, + { + "epoch": 0.06845052893590542, + "grad_norm": 1.7905038595199585, + "learning_rate": 8.506224066390042e-07, + "loss": 2.572, + "step": 55 + }, + { + "epoch": 0.06969508400746734, + "grad_norm": 1.9715144634246826, + "learning_rate": 8.713692946058091e-07, + "loss": 2.5797, + "step": 56 + }, + { + "epoch": 0.07093963907902924, + "grad_norm": 1.788966417312622, + "learning_rate": 8.921161825726142e-07, + "loss": 2.5567, + "step": 57 + }, + { + "epoch": 0.07218419415059116, + "grad_norm": 1.8978915214538574, + "learning_rate": 9.128630705394191e-07, + "loss": 2.5776, + "step": 58 + }, + { + "epoch": 0.07342874922215308, + "grad_norm": 1.6401689052581787, + "learning_rate": 9.336099585062241e-07, + "loss": 2.5662, + "step": 59 + }, + { + "epoch": 0.07467330429371499, + "grad_norm": 1.8200798034667969, + "learning_rate": 9.54356846473029e-07, + "loss": 2.5001, + "step": 60 + }, + { + "epoch": 0.07467330429371499, + "eval_loss": 2.569110870361328, + "eval_runtime": 47.3292, + "eval_samples_per_second": 21.129, + "eval_steps_per_second": 0.887, + "step": 60 + }, + { + "epoch": 0.07591785936527691, + "grad_norm": 1.5468674898147583, + "learning_rate": 9.751037344398341e-07, + "loss": 2.5308, + "step": 61 + }, + { + "epoch": 0.07716241443683883, + "grad_norm": 1.723684310913086, + "learning_rate": 9.95850622406639e-07, + "loss": 2.4907, + "step": 62 + }, + { + "epoch": 0.07840696950840075, + "grad_norm": 1.6346293687820435, + "learning_rate": 1.0165975103734441e-06, + "loss": 2.514, + "step": 63 + }, + { + "epoch": 0.07965152457996266, + "grad_norm": 1.8902088403701782, + "learning_rate": 1.037344398340249e-06, + "loss": 2.4859, + "step": 64 + }, + { + "epoch": 0.08089607965152458, + "grad_norm": 2.092611312866211, + "learning_rate": 1.058091286307054e-06, + "loss": 2.4888, + "step": 65 + }, + { + "epoch": 0.0821406347230865, + "grad_norm": 1.758482575416565, + "learning_rate": 1.078838174273859e-06, + "loss": 2.4537, + "step": 66 + }, + { + "epoch": 0.08338518979464841, + "grad_norm": 1.8941714763641357, + "learning_rate": 1.099585062240664e-06, + "loss": 2.5416, + "step": 67 + }, + { + "epoch": 0.08462974486621033, + "grad_norm": 1.7030234336853027, + "learning_rate": 1.120331950207469e-06, + "loss": 2.4791, + "step": 68 + }, + { + "epoch": 0.08587429993777225, + "grad_norm": 1.7602269649505615, + "learning_rate": 1.141078838174274e-06, + "loss": 2.4316, + "step": 69 + }, + { + "epoch": 0.08711885500933417, + "grad_norm": 1.7635431289672852, + "learning_rate": 1.161825726141079e-06, + "loss": 2.4674, + "step": 70 + }, + { + "epoch": 0.08711885500933417, + "eval_loss": 2.4922080039978027, + "eval_runtime": 45.3028, + "eval_samples_per_second": 22.074, + "eval_steps_per_second": 0.927, + "step": 70 + }, + { + "epoch": 0.08836341008089608, + "grad_norm": 2.0913803577423096, + "learning_rate": 1.182572614107884e-06, + "loss": 2.4341, + "step": 71 + }, + { + "epoch": 0.089607965152458, + "grad_norm": 1.8292183876037598, + "learning_rate": 1.2033195020746888e-06, + "loss": 2.4291, + "step": 72 + }, + { + "epoch": 0.09085252022401992, + "grad_norm": 2.206770181655884, + "learning_rate": 1.224066390041494e-06, + "loss": 2.3764, + "step": 73 + }, + { + "epoch": 0.09209707529558182, + "grad_norm": 1.8263559341430664, + "learning_rate": 1.2448132780082988e-06, + "loss": 2.4082, + "step": 74 + }, + { + "epoch": 0.09334163036714374, + "grad_norm": 1.9405455589294434, + "learning_rate": 1.2655601659751037e-06, + "loss": 2.4673, + "step": 75 + }, + { + "epoch": 0.09458618543870566, + "grad_norm": 2.138108015060425, + "learning_rate": 1.2863070539419086e-06, + "loss": 2.3879, + "step": 76 + }, + { + "epoch": 0.09583074051026758, + "grad_norm": 1.9024745225906372, + "learning_rate": 1.307053941908714e-06, + "loss": 2.4344, + "step": 77 + }, + { + "epoch": 0.0970752955818295, + "grad_norm": 1.9136689901351929, + "learning_rate": 1.3278008298755188e-06, + "loss": 2.474, + "step": 78 + }, + { + "epoch": 0.09831985065339141, + "grad_norm": 2.4752795696258545, + "learning_rate": 1.3485477178423237e-06, + "loss": 2.3412, + "step": 79 + }, + { + "epoch": 0.09956440572495333, + "grad_norm": 1.8720006942749023, + "learning_rate": 1.3692946058091288e-06, + "loss": 2.3438, + "step": 80 + }, + { + "epoch": 0.09956440572495333, + "eval_loss": 2.3980512619018555, + "eval_runtime": 48.2788, + "eval_samples_per_second": 20.713, + "eval_steps_per_second": 0.87, + "step": 80 + }, + { + "epoch": 0.10080896079651525, + "grad_norm": 2.671691656112671, + "learning_rate": 1.3900414937759337e-06, + "loss": 2.3336, + "step": 81 + }, + { + "epoch": 0.10205351586807716, + "grad_norm": 2.2953391075134277, + "learning_rate": 1.4107883817427386e-06, + "loss": 2.377, + "step": 82 + }, + { + "epoch": 0.10329807093963908, + "grad_norm": 3.009018898010254, + "learning_rate": 1.4315352697095435e-06, + "loss": 2.2977, + "step": 83 + }, + { + "epoch": 0.104542626011201, + "grad_norm": 2.664454936981201, + "learning_rate": 1.4522821576763488e-06, + "loss": 2.3271, + "step": 84 + }, + { + "epoch": 0.10578718108276292, + "grad_norm": 3.017303705215454, + "learning_rate": 1.4730290456431537e-06, + "loss": 2.3251, + "step": 85 + }, + { + "epoch": 0.10703173615432483, + "grad_norm": 2.634716510772705, + "learning_rate": 1.4937759336099586e-06, + "loss": 2.332, + "step": 86 + }, + { + "epoch": 0.10827629122588675, + "grad_norm": 3.059644937515259, + "learning_rate": 1.5145228215767635e-06, + "loss": 2.3478, + "step": 87 + }, + { + "epoch": 0.10952084629744867, + "grad_norm": 2.6962637901306152, + "learning_rate": 1.5352697095435686e-06, + "loss": 2.2792, + "step": 88 + }, + { + "epoch": 0.11076540136901059, + "grad_norm": 3.419729709625244, + "learning_rate": 1.5560165975103735e-06, + "loss": 2.2571, + "step": 89 + }, + { + "epoch": 0.11200995644057249, + "grad_norm": 2.740781545639038, + "learning_rate": 1.5767634854771784e-06, + "loss": 2.2875, + "step": 90 + }, + { + "epoch": 0.11200995644057249, + "eval_loss": 2.30843186378479, + "eval_runtime": 49.9264, + "eval_samples_per_second": 20.029, + "eval_steps_per_second": 0.841, + "step": 90 + }, + { + "epoch": 0.1132545115121344, + "grad_norm": 2.5608789920806885, + "learning_rate": 1.5975103734439833e-06, + "loss": 2.2592, + "step": 91 + }, + { + "epoch": 0.11449906658369632, + "grad_norm": 2.687999963760376, + "learning_rate": 1.6182572614107886e-06, + "loss": 2.2546, + "step": 92 + }, + { + "epoch": 0.11574362165525824, + "grad_norm": 2.695909023284912, + "learning_rate": 1.6390041493775935e-06, + "loss": 2.2525, + "step": 93 + }, + { + "epoch": 0.11698817672682016, + "grad_norm": 2.818357467651367, + "learning_rate": 1.6597510373443984e-06, + "loss": 2.216, + "step": 94 + }, + { + "epoch": 0.11823273179838208, + "grad_norm": 2.884119987487793, + "learning_rate": 1.6804979253112035e-06, + "loss": 2.2321, + "step": 95 + }, + { + "epoch": 0.119477286869944, + "grad_norm": 2.52104115486145, + "learning_rate": 1.7012448132780084e-06, + "loss": 2.199, + "step": 96 + }, + { + "epoch": 0.12072184194150591, + "grad_norm": 2.420313596725464, + "learning_rate": 1.7219917012448133e-06, + "loss": 2.1862, + "step": 97 + }, + { + "epoch": 0.12196639701306783, + "grad_norm": 2.8047542572021484, + "learning_rate": 1.7427385892116182e-06, + "loss": 2.1793, + "step": 98 + }, + { + "epoch": 0.12321095208462975, + "grad_norm": 2.836482286453247, + "learning_rate": 1.7634854771784235e-06, + "loss": 2.2271, + "step": 99 + }, + { + "epoch": 0.12445550715619166, + "grad_norm": 2.5282301902770996, + "learning_rate": 1.7842323651452284e-06, + "loss": 2.1768, + "step": 100 + }, + { + "epoch": 0.12445550715619166, + "eval_loss": 2.2312686443328857, + "eval_runtime": 49.2382, + "eval_samples_per_second": 20.309, + "eval_steps_per_second": 0.853, + "step": 100 + }, + { + "epoch": 0.12570006222775357, + "grad_norm": 3.3407280445098877, + "learning_rate": 1.8049792531120333e-06, + "loss": 2.1666, + "step": 101 + }, + { + "epoch": 0.1269446172993155, + "grad_norm": 2.4754133224487305, + "learning_rate": 1.8257261410788382e-06, + "loss": 2.1768, + "step": 102 + }, + { + "epoch": 0.1281891723708774, + "grad_norm": 3.430889129638672, + "learning_rate": 1.8464730290456433e-06, + "loss": 2.1953, + "step": 103 + }, + { + "epoch": 0.12943372744243933, + "grad_norm": 2.835294246673584, + "learning_rate": 1.8672199170124482e-06, + "loss": 2.146, + "step": 104 + }, + { + "epoch": 0.13067828251400124, + "grad_norm": 3.1532323360443115, + "learning_rate": 1.8879668049792531e-06, + "loss": 2.1729, + "step": 105 + }, + { + "epoch": 0.13192283758556317, + "grad_norm": 3.2278342247009277, + "learning_rate": 1.908713692946058e-06, + "loss": 2.1336, + "step": 106 + }, + { + "epoch": 0.13316739265712507, + "grad_norm": 2.7892515659332275, + "learning_rate": 1.929460580912863e-06, + "loss": 2.141, + "step": 107 + }, + { + "epoch": 0.134411947728687, + "grad_norm": 3.0179977416992188, + "learning_rate": 1.9502074688796682e-06, + "loss": 2.1255, + "step": 108 + }, + { + "epoch": 0.1356565028002489, + "grad_norm": 2.977935552597046, + "learning_rate": 1.970954356846473e-06, + "loss": 2.0986, + "step": 109 + }, + { + "epoch": 0.13690105787181084, + "grad_norm": 3.3042492866516113, + "learning_rate": 1.991701244813278e-06, + "loss": 2.0992, + "step": 110 + }, + { + "epoch": 0.13690105787181084, + "eval_loss": 2.171299695968628, + "eval_runtime": 42.9511, + "eval_samples_per_second": 23.282, + "eval_steps_per_second": 0.978, + "step": 110 + }, + { + "epoch": 0.13814561294337274, + "grad_norm": 2.9750890731811523, + "learning_rate": 2.012448132780083e-06, + "loss": 2.1116, + "step": 111 + }, + { + "epoch": 0.13939016801493467, + "grad_norm": 2.8199286460876465, + "learning_rate": 2.0331950207468883e-06, + "loss": 2.0822, + "step": 112 + }, + { + "epoch": 0.14063472308649658, + "grad_norm": 2.930532217025757, + "learning_rate": 2.053941908713693e-06, + "loss": 2.161, + "step": 113 + }, + { + "epoch": 0.14187927815805848, + "grad_norm": 3.3400321006774902, + "learning_rate": 2.074688796680498e-06, + "loss": 2.0953, + "step": 114 + }, + { + "epoch": 0.1431238332296204, + "grad_norm": 2.716564655303955, + "learning_rate": 2.095435684647303e-06, + "loss": 2.0757, + "step": 115 + }, + { + "epoch": 0.14436838830118232, + "grad_norm": 2.7586236000061035, + "learning_rate": 2.116182572614108e-06, + "loss": 2.0979, + "step": 116 + }, + { + "epoch": 0.14561294337274425, + "grad_norm": 2.622126340866089, + "learning_rate": 2.136929460580913e-06, + "loss": 2.0748, + "step": 117 + }, + { + "epoch": 0.14685749844430615, + "grad_norm": 2.646477460861206, + "learning_rate": 2.157676348547718e-06, + "loss": 2.0605, + "step": 118 + }, + { + "epoch": 0.14810205351586808, + "grad_norm": 2.8882410526275635, + "learning_rate": 2.178423236514523e-06, + "loss": 2.0501, + "step": 119 + }, + { + "epoch": 0.14934660858742999, + "grad_norm": 2.8302247524261475, + "learning_rate": 2.199170124481328e-06, + "loss": 2.0592, + "step": 120 + }, + { + "epoch": 0.14934660858742999, + "eval_loss": 2.1291966438293457, + "eval_runtime": 47.5399, + "eval_samples_per_second": 21.035, + "eval_steps_per_second": 0.883, + "step": 120 + }, + { + "epoch": 0.15059116365899192, + "grad_norm": 3.26979660987854, + "learning_rate": 2.219917012448133e-06, + "loss": 2.0422, + "step": 121 + }, + { + "epoch": 0.15183571873055382, + "grad_norm": 2.7483913898468018, + "learning_rate": 2.240663900414938e-06, + "loss": 2.0872, + "step": 122 + }, + { + "epoch": 0.15308027380211575, + "grad_norm": 3.1252024173736572, + "learning_rate": 2.2614107883817427e-06, + "loss": 2.0684, + "step": 123 + }, + { + "epoch": 0.15432482887367766, + "grad_norm": 2.9175283908843994, + "learning_rate": 2.282157676348548e-06, + "loss": 2.0522, + "step": 124 + }, + { + "epoch": 0.1555693839452396, + "grad_norm": 2.7950754165649414, + "learning_rate": 2.302904564315353e-06, + "loss": 2.065, + "step": 125 + }, + { + "epoch": 0.1568139390168015, + "grad_norm": 2.84049654006958, + "learning_rate": 2.323651452282158e-06, + "loss": 2.0781, + "step": 126 + }, + { + "epoch": 0.15805849408836342, + "grad_norm": 2.7396647930145264, + "learning_rate": 2.3443983402489627e-06, + "loss": 2.1758, + "step": 127 + }, + { + "epoch": 0.15930304915992533, + "grad_norm": 3.0765926837921143, + "learning_rate": 2.365145228215768e-06, + "loss": 2.0172, + "step": 128 + }, + { + "epoch": 0.16054760423148726, + "grad_norm": 3.217189073562622, + "learning_rate": 2.385892116182573e-06, + "loss": 2.0286, + "step": 129 + }, + { + "epoch": 0.16179215930304916, + "grad_norm": 3.141545295715332, + "learning_rate": 2.4066390041493776e-06, + "loss": 2.0583, + "step": 130 + }, + { + "epoch": 0.16179215930304916, + "eval_loss": 2.0946149826049805, + "eval_runtime": 54.92, + "eval_samples_per_second": 18.208, + "eval_steps_per_second": 0.765, + "step": 130 + }, + { + "epoch": 0.16303671437461106, + "grad_norm": 2.850052833557129, + "learning_rate": 2.4273858921161828e-06, + "loss": 2.0746, + "step": 131 + }, + { + "epoch": 0.164281269446173, + "grad_norm": 3.28913950920105, + "learning_rate": 2.448132780082988e-06, + "loss": 2.0539, + "step": 132 + }, + { + "epoch": 0.1655258245177349, + "grad_norm": 2.8819124698638916, + "learning_rate": 2.468879668049793e-06, + "loss": 1.9843, + "step": 133 + }, + { + "epoch": 0.16677037958929683, + "grad_norm": 3.6254632472991943, + "learning_rate": 2.4896265560165977e-06, + "loss": 2.0233, + "step": 134 + }, + { + "epoch": 0.16801493466085873, + "grad_norm": 2.7385146617889404, + "learning_rate": 2.5103734439834028e-06, + "loss": 1.9667, + "step": 135 + }, + { + "epoch": 0.16925948973242066, + "grad_norm": 2.9722647666931152, + "learning_rate": 2.5311203319502074e-06, + "loss": 1.9527, + "step": 136 + }, + { + "epoch": 0.17050404480398257, + "grad_norm": 3.139526605606079, + "learning_rate": 2.5518672199170125e-06, + "loss": 1.9967, + "step": 137 + }, + { + "epoch": 0.1717485998755445, + "grad_norm": 3.399920701980591, + "learning_rate": 2.5726141078838172e-06, + "loss": 1.9798, + "step": 138 + }, + { + "epoch": 0.1729931549471064, + "grad_norm": 2.8017327785491943, + "learning_rate": 2.5933609958506228e-06, + "loss": 1.983, + "step": 139 + }, + { + "epoch": 0.17423771001866833, + "grad_norm": 3.9967198371887207, + "learning_rate": 2.614107883817428e-06, + "loss": 1.977, + "step": 140 + }, + { + "epoch": 0.17423771001866833, + "eval_loss": 2.059037685394287, + "eval_runtime": 52.6232, + "eval_samples_per_second": 19.003, + "eval_steps_per_second": 0.798, + "step": 140 + }, + { + "epoch": 0.17548226509023024, + "grad_norm": 2.893092393875122, + "learning_rate": 2.6348547717842326e-06, + "loss": 1.9888, + "step": 141 + }, + { + "epoch": 0.17672682016179217, + "grad_norm": 3.634352207183838, + "learning_rate": 2.6556016597510377e-06, + "loss": 2.0726, + "step": 142 + }, + { + "epoch": 0.17797137523335407, + "grad_norm": 3.3651444911956787, + "learning_rate": 2.6763485477178423e-06, + "loss": 1.973, + "step": 143 + }, + { + "epoch": 0.179215930304916, + "grad_norm": 3.968986988067627, + "learning_rate": 2.6970954356846475e-06, + "loss": 1.9948, + "step": 144 + }, + { + "epoch": 0.1804604853764779, + "grad_norm": 3.20105242729187, + "learning_rate": 2.717842323651452e-06, + "loss": 1.9238, + "step": 145 + }, + { + "epoch": 0.18170504044803984, + "grad_norm": 3.648339033126831, + "learning_rate": 2.7385892116182577e-06, + "loss": 1.9923, + "step": 146 + }, + { + "epoch": 0.18294959551960174, + "grad_norm": 3.0856316089630127, + "learning_rate": 2.7593360995850628e-06, + "loss": 1.9749, + "step": 147 + }, + { + "epoch": 0.18419415059116365, + "grad_norm": 3.2681071758270264, + "learning_rate": 2.7800829875518675e-06, + "loss": 1.9304, + "step": 148 + }, + { + "epoch": 0.18543870566272558, + "grad_norm": 2.634958267211914, + "learning_rate": 2.8008298755186726e-06, + "loss": 1.9237, + "step": 149 + }, + { + "epoch": 0.18668326073428748, + "grad_norm": 2.769491672515869, + "learning_rate": 2.8215767634854773e-06, + "loss": 1.8963, + "step": 150 + }, + { + "epoch": 0.18668326073428748, + "eval_loss": 2.0394654273986816, + "eval_runtime": 55.2136, + "eval_samples_per_second": 18.111, + "eval_steps_per_second": 0.761, + "step": 150 + }, + { + "epoch": 0.1879278158058494, + "grad_norm": 3.4345781803131104, + "learning_rate": 2.8423236514522824e-06, + "loss": 1.9836, + "step": 151 + }, + { + "epoch": 0.18917237087741132, + "grad_norm": 3.115727424621582, + "learning_rate": 2.863070539419087e-06, + "loss": 1.9352, + "step": 152 + }, + { + "epoch": 0.19041692594897325, + "grad_norm": 3.050652503967285, + "learning_rate": 2.883817427385892e-06, + "loss": 1.9031, + "step": 153 + }, + { + "epoch": 0.19166148102053515, + "grad_norm": 2.9404428005218506, + "learning_rate": 2.9045643153526977e-06, + "loss": 1.9196, + "step": 154 + }, + { + "epoch": 0.19290603609209708, + "grad_norm": 3.080810785293579, + "learning_rate": 2.9253112033195024e-06, + "loss": 1.9405, + "step": 155 + }, + { + "epoch": 0.194150591163659, + "grad_norm": 3.058558702468872, + "learning_rate": 2.9460580912863075e-06, + "loss": 1.9052, + "step": 156 + }, + { + "epoch": 0.19539514623522092, + "grad_norm": 3.307955026626587, + "learning_rate": 2.966804979253112e-06, + "loss": 1.9245, + "step": 157 + }, + { + "epoch": 0.19663970130678282, + "grad_norm": 2.845506191253662, + "learning_rate": 2.9875518672199173e-06, + "loss": 1.9461, + "step": 158 + }, + { + "epoch": 0.19788425637834475, + "grad_norm": 2.758654832839966, + "learning_rate": 3.008298755186722e-06, + "loss": 1.8885, + "step": 159 + }, + { + "epoch": 0.19912881144990666, + "grad_norm": 3.161252737045288, + "learning_rate": 3.029045643153527e-06, + "loss": 1.8535, + "step": 160 + }, + { + "epoch": 0.19912881144990666, + "eval_loss": 2.018101453781128, + "eval_runtime": 51.0744, + "eval_samples_per_second": 19.579, + "eval_steps_per_second": 0.822, + "step": 160 + }, + { + "epoch": 0.2003733665214686, + "grad_norm": 3.196847677230835, + "learning_rate": 3.0497925311203326e-06, + "loss": 1.8853, + "step": 161 + }, + { + "epoch": 0.2016179215930305, + "grad_norm": 2.8364663124084473, + "learning_rate": 3.0705394190871373e-06, + "loss": 1.9463, + "step": 162 + }, + { + "epoch": 0.2028624766645924, + "grad_norm": 3.0874054431915283, + "learning_rate": 3.0912863070539424e-06, + "loss": 1.9254, + "step": 163 + }, + { + "epoch": 0.20410703173615433, + "grad_norm": 2.7914493083953857, + "learning_rate": 3.112033195020747e-06, + "loss": 1.9213, + "step": 164 + }, + { + "epoch": 0.20535158680771623, + "grad_norm": 3.3871428966522217, + "learning_rate": 3.132780082987552e-06, + "loss": 1.8993, + "step": 165 + }, + { + "epoch": 0.20659614187927816, + "grad_norm": 3.096653461456299, + "learning_rate": 3.153526970954357e-06, + "loss": 1.8827, + "step": 166 + }, + { + "epoch": 0.20784069695084006, + "grad_norm": 2.8276076316833496, + "learning_rate": 3.174273858921162e-06, + "loss": 1.912, + "step": 167 + }, + { + "epoch": 0.209085252022402, + "grad_norm": 3.3058435916900635, + "learning_rate": 3.1950207468879666e-06, + "loss": 1.8381, + "step": 168 + }, + { + "epoch": 0.2103298070939639, + "grad_norm": 3.5017333030700684, + "learning_rate": 3.215767634854772e-06, + "loss": 1.9544, + "step": 169 + }, + { + "epoch": 0.21157436216552583, + "grad_norm": 3.457296133041382, + "learning_rate": 3.2365145228215773e-06, + "loss": 1.922, + "step": 170 + }, + { + "epoch": 0.21157436216552583, + "eval_loss": 1.988455891609192, + "eval_runtime": 56.2099, + "eval_samples_per_second": 17.79, + "eval_steps_per_second": 0.747, + "step": 170 + }, + { + "epoch": 0.21281891723708773, + "grad_norm": 3.1964566707611084, + "learning_rate": 3.257261410788382e-06, + "loss": 1.8925, + "step": 171 + }, + { + "epoch": 0.21406347230864967, + "grad_norm": 3.234652042388916, + "learning_rate": 3.278008298755187e-06, + "loss": 1.884, + "step": 172 + }, + { + "epoch": 0.21530802738021157, + "grad_norm": 3.14414119720459, + "learning_rate": 3.2987551867219918e-06, + "loss": 1.8905, + "step": 173 + }, + { + "epoch": 0.2165525824517735, + "grad_norm": 3.606379508972168, + "learning_rate": 3.319502074688797e-06, + "loss": 1.946, + "step": 174 + }, + { + "epoch": 0.2177971375233354, + "grad_norm": 2.986646890640259, + "learning_rate": 3.3402489626556016e-06, + "loss": 1.8691, + "step": 175 + }, + { + "epoch": 0.21904169259489734, + "grad_norm": 3.7298269271850586, + "learning_rate": 3.360995850622407e-06, + "loss": 1.8717, + "step": 176 + }, + { + "epoch": 0.22028624766645924, + "grad_norm": 3.422295093536377, + "learning_rate": 3.381742738589212e-06, + "loss": 1.8568, + "step": 177 + }, + { + "epoch": 0.22153080273802117, + "grad_norm": 3.2293782234191895, + "learning_rate": 3.402489626556017e-06, + "loss": 1.8471, + "step": 178 + }, + { + "epoch": 0.22277535780958307, + "grad_norm": 3.2293782234191895, + "learning_rate": 3.402489626556017e-06, + "loss": 1.9303, + "step": 179 + }, + { + "epoch": 0.22401991288114498, + "grad_norm": 3.506223440170288, + "learning_rate": 3.423236514522822e-06, + "loss": 1.8237, + "step": 180 + }, + { + "epoch": 0.22401991288114498, + "eval_loss": 1.9734643697738647, + "eval_runtime": 52.2815, + "eval_samples_per_second": 19.127, + "eval_steps_per_second": 0.803, + "step": 180 + }, + { + "epoch": 0.2252644679527069, + "grad_norm": 3.0156619548797607, + "learning_rate": 3.4439834024896267e-06, + "loss": 1.908, + "step": 181 + }, + { + "epoch": 0.2265090230242688, + "grad_norm": 3.55517315864563, + "learning_rate": 3.4647302904564318e-06, + "loss": 1.9104, + "step": 182 + }, + { + "epoch": 0.22775357809583074, + "grad_norm": 3.144984006881714, + "learning_rate": 3.4854771784232365e-06, + "loss": 1.8198, + "step": 183 + }, + { + "epoch": 0.22899813316739265, + "grad_norm": 4.302074432373047, + "learning_rate": 3.5062240663900416e-06, + "loss": 1.8237, + "step": 184 + }, + { + "epoch": 0.23024268823895458, + "grad_norm": 3.0522446632385254, + "learning_rate": 3.526970954356847e-06, + "loss": 1.8409, + "step": 185 + }, + { + "epoch": 0.23148724331051648, + "grad_norm": 3.8607394695281982, + "learning_rate": 3.5477178423236518e-06, + "loss": 1.8807, + "step": 186 + }, + { + "epoch": 0.23273179838207841, + "grad_norm": 2.9236302375793457, + "learning_rate": 3.568464730290457e-06, + "loss": 1.7873, + "step": 187 + }, + { + "epoch": 0.23397635345364032, + "grad_norm": 4.013780117034912, + "learning_rate": 3.5892116182572616e-06, + "loss": 1.7909, + "step": 188 + }, + { + "epoch": 0.23522090852520225, + "grad_norm": 3.0933122634887695, + "learning_rate": 3.6099585062240667e-06, + "loss": 1.8469, + "step": 189 + }, + { + "epoch": 0.23646546359676415, + "grad_norm": 3.487816095352173, + "learning_rate": 3.6307053941908714e-06, + "loss": 1.8469, + "step": 190 + }, + { + "epoch": 0.23646546359676415, + "eval_loss": 1.9604240655899048, + "eval_runtime": 53.4604, + "eval_samples_per_second": 18.705, + "eval_steps_per_second": 0.786, + "step": 190 + }, + { + "epoch": 0.23771001866832608, + "grad_norm": 2.829219341278076, + "learning_rate": 3.6514522821576765e-06, + "loss": 1.8639, + "step": 191 + }, + { + "epoch": 0.238954573739888, + "grad_norm": 3.595534324645996, + "learning_rate": 3.672199170124482e-06, + "loss": 1.8988, + "step": 192 + }, + { + "epoch": 0.24019912881144992, + "grad_norm": 2.9022483825683594, + "learning_rate": 3.6929460580912867e-06, + "loss": 1.8972, + "step": 193 + }, + { + "epoch": 0.24144368388301182, + "grad_norm": 3.0667994022369385, + "learning_rate": 3.713692946058092e-06, + "loss": 1.8499, + "step": 194 + }, + { + "epoch": 0.24268823895457373, + "grad_norm": 3.0498485565185547, + "learning_rate": 3.7344398340248965e-06, + "loss": 1.8341, + "step": 195 + }, + { + "epoch": 0.24393279402613566, + "grad_norm": 4.058084964752197, + "learning_rate": 3.7551867219917016e-06, + "loss": 1.8641, + "step": 196 + }, + { + "epoch": 0.24517734909769756, + "grad_norm": 2.9995028972625732, + "learning_rate": 3.7759336099585063e-06, + "loss": 1.7885, + "step": 197 + }, + { + "epoch": 0.2464219041692595, + "grad_norm": 3.254704475402832, + "learning_rate": 3.7966804979253114e-06, + "loss": 1.7544, + "step": 198 + }, + { + "epoch": 0.2476664592408214, + "grad_norm": 2.9126744270324707, + "learning_rate": 3.817427385892116e-06, + "loss": 1.8461, + "step": 199 + }, + { + "epoch": 0.24891101431238333, + "grad_norm": 3.475247621536255, + "learning_rate": 3.838174273858922e-06, + "loss": 1.8095, + "step": 200 + }, + { + "epoch": 0.24891101431238333, + "eval_loss": 1.9373760223388672, + "eval_runtime": 51.8733, + "eval_samples_per_second": 19.278, + "eval_steps_per_second": 0.81, + "step": 200 + }, + { + "epoch": 0.25015556938394523, + "grad_norm": 3.2663731575012207, + "learning_rate": 3.858921161825726e-06, + "loss": 1.8076, + "step": 201 + }, + { + "epoch": 0.25140012445550713, + "grad_norm": 3.1151726245880127, + "learning_rate": 3.879668049792531e-06, + "loss": 1.8143, + "step": 202 + }, + { + "epoch": 0.2526446795270691, + "grad_norm": 3.102038860321045, + "learning_rate": 3.9004149377593365e-06, + "loss": 1.8279, + "step": 203 + }, + { + "epoch": 0.253889234598631, + "grad_norm": 3.370642900466919, + "learning_rate": 3.921161825726142e-06, + "loss": 1.8522, + "step": 204 + }, + { + "epoch": 0.2551337896701929, + "grad_norm": 3.116128921508789, + "learning_rate": 3.941908713692946e-06, + "loss": 1.8104, + "step": 205 + }, + { + "epoch": 0.2563783447417548, + "grad_norm": 3.12028431892395, + "learning_rate": 3.962655601659751e-06, + "loss": 1.7723, + "step": 206 + }, + { + "epoch": 0.25762289981331676, + "grad_norm": 3.1251418590545654, + "learning_rate": 3.983402489626556e-06, + "loss": 1.863, + "step": 207 + }, + { + "epoch": 0.25886745488487867, + "grad_norm": 3.073702812194824, + "learning_rate": 4.004149377593361e-06, + "loss": 1.805, + "step": 208 + }, + { + "epoch": 0.26011200995644057, + "grad_norm": 2.9772469997406006, + "learning_rate": 4.024896265560166e-06, + "loss": 1.8251, + "step": 209 + }, + { + "epoch": 0.2613565650280025, + "grad_norm": 2.9970037937164307, + "learning_rate": 4.045643153526971e-06, + "loss": 1.8386, + "step": 210 + }, + { + "epoch": 0.2613565650280025, + "eval_loss": 1.9167065620422363, + "eval_runtime": 42.3328, + "eval_samples_per_second": 23.622, + "eval_steps_per_second": 0.992, + "step": 210 + }, + { + "epoch": 0.26260112009956443, + "grad_norm": 3.1391561031341553, + "learning_rate": 4.0663900414937765e-06, + "loss": 1.8109, + "step": 211 + }, + { + "epoch": 0.26384567517112634, + "grad_norm": 3.161538600921631, + "learning_rate": 4.087136929460581e-06, + "loss": 1.8441, + "step": 212 + }, + { + "epoch": 0.26509023024268824, + "grad_norm": 3.12412166595459, + "learning_rate": 4.107883817427386e-06, + "loss": 1.7736, + "step": 213 + }, + { + "epoch": 0.26633478531425014, + "grad_norm": 3.3241145610809326, + "learning_rate": 4.128630705394191e-06, + "loss": 1.8067, + "step": 214 + }, + { + "epoch": 0.26757934038581205, + "grad_norm": 3.503307819366455, + "learning_rate": 4.149377593360996e-06, + "loss": 1.7858, + "step": 215 + }, + { + "epoch": 0.268823895457374, + "grad_norm": 3.1009578704833984, + "learning_rate": 4.170124481327801e-06, + "loss": 1.7542, + "step": 216 + }, + { + "epoch": 0.2700684505289359, + "grad_norm": 3.2236011028289795, + "learning_rate": 4.190871369294606e-06, + "loss": 1.8006, + "step": 217 + }, + { + "epoch": 0.2713130056004978, + "grad_norm": 3.048935651779175, + "learning_rate": 4.211618257261411e-06, + "loss": 1.7706, + "step": 218 + }, + { + "epoch": 0.2725575606720597, + "grad_norm": 3.3416147232055664, + "learning_rate": 4.232365145228216e-06, + "loss": 1.8108, + "step": 219 + }, + { + "epoch": 0.2738021157436217, + "grad_norm": 3.2010738849639893, + "learning_rate": 4.253112033195021e-06, + "loss": 1.8561, + "step": 220 + }, + { + "epoch": 0.2738021157436217, + "eval_loss": 1.9111930131912231, + "eval_runtime": 50.4077, + "eval_samples_per_second": 19.838, + "eval_steps_per_second": 0.833, + "step": 220 + }, + { + "epoch": 0.2750466708151836, + "grad_norm": 3.4033374786376953, + "learning_rate": 4.273858921161826e-06, + "loss": 1.8218, + "step": 221 + }, + { + "epoch": 0.2762912258867455, + "grad_norm": 3.2086002826690674, + "learning_rate": 4.294605809128631e-06, + "loss": 1.8213, + "step": 222 + }, + { + "epoch": 0.2775357809583074, + "grad_norm": 4.090956211090088, + "learning_rate": 4.315352697095436e-06, + "loss": 1.8455, + "step": 223 + }, + { + "epoch": 0.27878033602986935, + "grad_norm": 3.1142985820770264, + "learning_rate": 4.336099585062241e-06, + "loss": 1.7737, + "step": 224 + }, + { + "epoch": 0.28002489110143125, + "grad_norm": 3.369669198989868, + "learning_rate": 4.356846473029046e-06, + "loss": 1.824, + "step": 225 + }, + { + "epoch": 0.28126944617299315, + "grad_norm": 3.165672779083252, + "learning_rate": 4.3775933609958506e-06, + "loss": 1.7768, + "step": 226 + }, + { + "epoch": 0.28251400124455506, + "grad_norm": 3.3343470096588135, + "learning_rate": 4.398340248962656e-06, + "loss": 1.7278, + "step": 227 + }, + { + "epoch": 0.28375855631611696, + "grad_norm": 3.2574994564056396, + "learning_rate": 4.419087136929461e-06, + "loss": 1.8439, + "step": 228 + }, + { + "epoch": 0.2850031113876789, + "grad_norm": 3.043928623199463, + "learning_rate": 4.439834024896266e-06, + "loss": 1.818, + "step": 229 + }, + { + "epoch": 0.2862476664592408, + "grad_norm": 2.9703125953674316, + "learning_rate": 4.460580912863071e-06, + "loss": 1.773, + "step": 230 + }, + { + "epoch": 0.2862476664592408, + "eval_loss": 1.898109793663025, + "eval_runtime": 45.6023, + "eval_samples_per_second": 21.929, + "eval_steps_per_second": 0.921, + "step": 230 + }, + { + "epoch": 0.2874922215308027, + "grad_norm": 2.9997220039367676, + "learning_rate": 4.481327800829876e-06, + "loss": 1.7727, + "step": 231 + }, + { + "epoch": 0.28873677660236463, + "grad_norm": 3.1358482837677, + "learning_rate": 4.502074688796681e-06, + "loss": 1.7764, + "step": 232 + }, + { + "epoch": 0.2899813316739266, + "grad_norm": 3.395747661590576, + "learning_rate": 4.5228215767634855e-06, + "loss": 1.7826, + "step": 233 + }, + { + "epoch": 0.2912258867454885, + "grad_norm": 3.7494754791259766, + "learning_rate": 4.543568464730291e-06, + "loss": 1.7181, + "step": 234 + }, + { + "epoch": 0.2924704418170504, + "grad_norm": 3.42293381690979, + "learning_rate": 4.564315352697096e-06, + "loss": 1.7773, + "step": 235 + }, + { + "epoch": 0.2937149968886123, + "grad_norm": 3.2524514198303223, + "learning_rate": 4.585062240663901e-06, + "loss": 1.7294, + "step": 236 + }, + { + "epoch": 0.29495955196017426, + "grad_norm": 3.657869577407837, + "learning_rate": 4.605809128630706e-06, + "loss": 1.762, + "step": 237 + }, + { + "epoch": 0.29620410703173616, + "grad_norm": 3.127372980117798, + "learning_rate": 4.626556016597511e-06, + "loss": 1.7623, + "step": 238 + }, + { + "epoch": 0.29744866210329807, + "grad_norm": 4.166962146759033, + "learning_rate": 4.647302904564316e-06, + "loss": 1.6995, + "step": 239 + }, + { + "epoch": 0.29869321717485997, + "grad_norm": 3.094264030456543, + "learning_rate": 4.66804979253112e-06, + "loss": 1.7469, + "step": 240 + }, + { + "epoch": 0.29869321717485997, + "eval_loss": 1.8908178806304932, + "eval_runtime": 44.6332, + "eval_samples_per_second": 22.405, + "eval_steps_per_second": 0.941, + "step": 240 + }, + { + "epoch": 0.29993777224642193, + "grad_norm": 3.8644745349884033, + "learning_rate": 4.6887966804979255e-06, + "loss": 1.7644, + "step": 241 + }, + { + "epoch": 0.30118232731798383, + "grad_norm": 3.1488852500915527, + "learning_rate": 4.709543568464731e-06, + "loss": 1.772, + "step": 242 + }, + { + "epoch": 0.30242688238954574, + "grad_norm": 3.3179638385772705, + "learning_rate": 4.730290456431536e-06, + "loss": 1.7588, + "step": 243 + }, + { + "epoch": 0.30367143746110764, + "grad_norm": 3.328355550765991, + "learning_rate": 4.751037344398341e-06, + "loss": 1.7417, + "step": 244 + }, + { + "epoch": 0.30491599253266954, + "grad_norm": 3.2690482139587402, + "learning_rate": 4.771784232365146e-06, + "loss": 1.7562, + "step": 245 + }, + { + "epoch": 0.3061605476042315, + "grad_norm": 3.2759454250335693, + "learning_rate": 4.792531120331951e-06, + "loss": 1.7536, + "step": 246 + }, + { + "epoch": 0.3074051026757934, + "grad_norm": 2.937964916229248, + "learning_rate": 4.813278008298755e-06, + "loss": 1.7518, + "step": 247 + }, + { + "epoch": 0.3086496577473553, + "grad_norm": 3.0995302200317383, + "learning_rate": 4.83402489626556e-06, + "loss": 1.7593, + "step": 248 + }, + { + "epoch": 0.3098942128189172, + "grad_norm": 3.230459213256836, + "learning_rate": 4.8547717842323655e-06, + "loss": 1.6962, + "step": 249 + }, + { + "epoch": 0.3111387678904792, + "grad_norm": 3.242576837539673, + "learning_rate": 4.875518672199171e-06, + "loss": 1.7307, + "step": 250 + }, + { + "epoch": 0.3111387678904792, + "eval_loss": 1.8805371522903442, + "eval_runtime": 45.9886, + "eval_samples_per_second": 21.745, + "eval_steps_per_second": 0.913, + "step": 250 + }, + { + "epoch": 0.3123833229620411, + "grad_norm": 3.097045421600342, + "learning_rate": 4.896265560165976e-06, + "loss": 1.7438, + "step": 251 + }, + { + "epoch": 0.313627878033603, + "grad_norm": 3.2428948879241943, + "learning_rate": 4.91701244813278e-06, + "loss": 1.7179, + "step": 252 + }, + { + "epoch": 0.3148724331051649, + "grad_norm": 3.196274518966675, + "learning_rate": 4.937759336099586e-06, + "loss": 1.7855, + "step": 253 + }, + { + "epoch": 0.31611698817672684, + "grad_norm": 2.978203535079956, + "learning_rate": 4.95850622406639e-06, + "loss": 1.7144, + "step": 254 + }, + { + "epoch": 0.31736154324828875, + "grad_norm": 3.2641701698303223, + "learning_rate": 4.979253112033195e-06, + "loss": 1.7427, + "step": 255 + }, + { + "epoch": 0.31860609831985065, + "grad_norm": 2.7441232204437256, + "learning_rate": 5e-06, + "loss": 1.7619, + "step": 256 + }, + { + "epoch": 0.31985065339141255, + "grad_norm": 3.0723495483398438, + "learning_rate": 4.99769372693727e-06, + "loss": 1.8017, + "step": 257 + }, + { + "epoch": 0.3210952084629745, + "grad_norm": 3.104752540588379, + "learning_rate": 4.995387453874539e-06, + "loss": 1.7761, + "step": 258 + }, + { + "epoch": 0.3223397635345364, + "grad_norm": 3.138627052307129, + "learning_rate": 4.993081180811809e-06, + "loss": 1.7879, + "step": 259 + }, + { + "epoch": 0.3235843186060983, + "grad_norm": 2.8567333221435547, + "learning_rate": 4.990774907749078e-06, + "loss": 1.7561, + "step": 260 + }, + { + "epoch": 0.3235843186060983, + "eval_loss": 1.8606494665145874, + "eval_runtime": 46.453, + "eval_samples_per_second": 21.527, + "eval_steps_per_second": 0.904, + "step": 260 + }, + { + "epoch": 0.3248288736776602, + "grad_norm": 3.2696540355682373, + "learning_rate": 4.988468634686347e-06, + "loss": 1.7201, + "step": 261 + }, + { + "epoch": 0.3260734287492221, + "grad_norm": 2.6692731380462646, + "learning_rate": 4.986162361623617e-06, + "loss": 1.7264, + "step": 262 + }, + { + "epoch": 0.3273179838207841, + "grad_norm": 3.047549247741699, + "learning_rate": 4.983856088560886e-06, + "loss": 1.7362, + "step": 263 + }, + { + "epoch": 0.328562538892346, + "grad_norm": 3.0667457580566406, + "learning_rate": 4.981549815498156e-06, + "loss": 1.7321, + "step": 264 + }, + { + "epoch": 0.3298070939639079, + "grad_norm": 2.560047149658203, + "learning_rate": 4.979243542435424e-06, + "loss": 1.7508, + "step": 265 + }, + { + "epoch": 0.3310516490354698, + "grad_norm": 3.26595401763916, + "learning_rate": 4.976937269372694e-06, + "loss": 1.7248, + "step": 266 + }, + { + "epoch": 0.33229620410703176, + "grad_norm": 2.929210662841797, + "learning_rate": 4.974630996309964e-06, + "loss": 1.697, + "step": 267 + }, + { + "epoch": 0.33354075917859366, + "grad_norm": 3.1526286602020264, + "learning_rate": 4.972324723247233e-06, + "loss": 1.7385, + "step": 268 + }, + { + "epoch": 0.33478531425015556, + "grad_norm": 3.001619577407837, + "learning_rate": 4.970018450184502e-06, + "loss": 1.7072, + "step": 269 + }, + { + "epoch": 0.33602986932171747, + "grad_norm": 3.2464189529418945, + "learning_rate": 4.9677121771217715e-06, + "loss": 1.7361, + "step": 270 + }, + { + "epoch": 0.33602986932171747, + "eval_loss": 1.860226035118103, + "eval_runtime": 46.6881, + "eval_samples_per_second": 21.419, + "eval_steps_per_second": 0.9, + "step": 270 + }, + { + "epoch": 0.3372744243932794, + "grad_norm": 3.2104530334472656, + "learning_rate": 4.965405904059041e-06, + "loss": 1.7171, + "step": 271 + }, + { + "epoch": 0.33851897946484133, + "grad_norm": 3.146847724914551, + "learning_rate": 4.96309963099631e-06, + "loss": 1.7311, + "step": 272 + }, + { + "epoch": 0.33976353453640323, + "grad_norm": 3.1431286334991455, + "learning_rate": 4.96079335793358e-06, + "loss": 1.7345, + "step": 273 + }, + { + "epoch": 0.34100808960796514, + "grad_norm": 2.847163677215576, + "learning_rate": 4.958487084870849e-06, + "loss": 1.7262, + "step": 274 + }, + { + "epoch": 0.3422526446795271, + "grad_norm": 3.1987810134887695, + "learning_rate": 4.956180811808119e-06, + "loss": 1.7945, + "step": 275 + }, + { + "epoch": 0.343497199751089, + "grad_norm": 2.7475385665893555, + "learning_rate": 4.953874538745388e-06, + "loss": 1.76, + "step": 276 + }, + { + "epoch": 0.3447417548226509, + "grad_norm": 3.0427663326263428, + "learning_rate": 4.9515682656826574e-06, + "loss": 1.7324, + "step": 277 + }, + { + "epoch": 0.3459863098942128, + "grad_norm": 3.605212688446045, + "learning_rate": 4.949261992619927e-06, + "loss": 1.7712, + "step": 278 + }, + { + "epoch": 0.3472308649657747, + "grad_norm": 3.0564935207366943, + "learning_rate": 4.946955719557196e-06, + "loss": 1.6792, + "step": 279 + }, + { + "epoch": 0.34847542003733667, + "grad_norm": 3.1069741249084473, + "learning_rate": 4.944649446494466e-06, + "loss": 1.7145, + "step": 280 + }, + { + "epoch": 0.34847542003733667, + "eval_loss": 1.8537051677703857, + "eval_runtime": 47.4813, + "eval_samples_per_second": 21.061, + "eval_steps_per_second": 0.885, + "step": 280 + }, + { + "epoch": 0.3497199751088986, + "grad_norm": 2.8021512031555176, + "learning_rate": 4.942343173431734e-06, + "loss": 1.7145, + "step": 281 + }, + { + "epoch": 0.3509645301804605, + "grad_norm": 3.5549023151397705, + "learning_rate": 4.940036900369004e-06, + "loss": 1.7642, + "step": 282 + }, + { + "epoch": 0.3522090852520224, + "grad_norm": 2.7648985385894775, + "learning_rate": 4.937730627306274e-06, + "loss": 1.7255, + "step": 283 + }, + { + "epoch": 0.35345364032358434, + "grad_norm": 3.0815863609313965, + "learning_rate": 4.9354243542435426e-06, + "loss": 1.7055, + "step": 284 + }, + { + "epoch": 0.35469819539514624, + "grad_norm": 2.9009227752685547, + "learning_rate": 4.933118081180812e-06, + "loss": 1.7019, + "step": 285 + }, + { + "epoch": 0.35594275046670815, + "grad_norm": 2.618429660797119, + "learning_rate": 4.930811808118081e-06, + "loss": 1.6318, + "step": 286 + }, + { + "epoch": 0.35718730553827005, + "grad_norm": 3.368230104446411, + "learning_rate": 4.928505535055351e-06, + "loss": 1.7304, + "step": 287 + }, + { + "epoch": 0.358431860609832, + "grad_norm": 3.256889820098877, + "learning_rate": 4.92619926199262e-06, + "loss": 1.7019, + "step": 288 + }, + { + "epoch": 0.3596764156813939, + "grad_norm": 2.9366304874420166, + "learning_rate": 4.92389298892989e-06, + "loss": 1.734, + "step": 289 + }, + { + "epoch": 0.3609209707529558, + "grad_norm": 3.0393142700195312, + "learning_rate": 4.921586715867159e-06, + "loss": 1.7504, + "step": 290 + }, + { + "epoch": 0.3609209707529558, + "eval_loss": 1.8359886407852173, + "eval_runtime": 49.7622, + "eval_samples_per_second": 20.096, + "eval_steps_per_second": 0.844, + "step": 290 + }, + { + "epoch": 0.3621655258245177, + "grad_norm": 2.7288589477539062, + "learning_rate": 4.9192804428044285e-06, + "loss": 1.7004, + "step": 291 + }, + { + "epoch": 0.3634100808960797, + "grad_norm": 3.2793378829956055, + "learning_rate": 4.916974169741698e-06, + "loss": 1.7028, + "step": 292 + }, + { + "epoch": 0.3646546359676416, + "grad_norm": 2.960880756378174, + "learning_rate": 4.914667896678967e-06, + "loss": 1.6759, + "step": 293 + }, + { + "epoch": 0.3658991910392035, + "grad_norm": 2.836421012878418, + "learning_rate": 4.912361623616237e-06, + "loss": 1.6475, + "step": 294 + }, + { + "epoch": 0.3671437461107654, + "grad_norm": 2.9495465755462646, + "learning_rate": 4.910055350553506e-06, + "loss": 1.7076, + "step": 295 + }, + { + "epoch": 0.3683883011823273, + "grad_norm": 2.954730272293091, + "learning_rate": 4.907749077490776e-06, + "loss": 1.7261, + "step": 296 + }, + { + "epoch": 0.36963285625388925, + "grad_norm": 2.8167543411254883, + "learning_rate": 4.905442804428044e-06, + "loss": 1.6939, + "step": 297 + }, + { + "epoch": 0.37087741132545116, + "grad_norm": 2.857316017150879, + "learning_rate": 4.903136531365314e-06, + "loss": 1.7362, + "step": 298 + }, + { + "epoch": 0.37212196639701306, + "grad_norm": 2.8612918853759766, + "learning_rate": 4.900830258302584e-06, + "loss": 1.6489, + "step": 299 + }, + { + "epoch": 0.37336652146857496, + "grad_norm": 2.9618914127349854, + "learning_rate": 4.898523985239853e-06, + "loss": 1.718, + "step": 300 + }, + { + "epoch": 0.37336652146857496, + "eval_loss": 1.827571988105774, + "eval_runtime": 46.4552, + "eval_samples_per_second": 21.526, + "eval_steps_per_second": 0.904, + "step": 300 + }, + { + "epoch": 0.3746110765401369, + "grad_norm": 2.8706953525543213, + "learning_rate": 4.896217712177122e-06, + "loss": 1.6925, + "step": 301 + }, + { + "epoch": 0.3758556316116988, + "grad_norm": 3.2565090656280518, + "learning_rate": 4.893911439114391e-06, + "loss": 1.7369, + "step": 302 + }, + { + "epoch": 0.37710018668326073, + "grad_norm": 2.928858995437622, + "learning_rate": 4.891605166051661e-06, + "loss": 1.6882, + "step": 303 + }, + { + "epoch": 0.37834474175482263, + "grad_norm": 3.042314052581787, + "learning_rate": 4.88929889298893e-06, + "loss": 1.6466, + "step": 304 + }, + { + "epoch": 0.3795892968263846, + "grad_norm": 2.7509143352508545, + "learning_rate": 4.8869926199262e-06, + "loss": 1.685, + "step": 305 + }, + { + "epoch": 0.3808338518979465, + "grad_norm": 2.9601588249206543, + "learning_rate": 4.884686346863469e-06, + "loss": 1.6772, + "step": 306 + }, + { + "epoch": 0.3820784069695084, + "grad_norm": 2.618608236312866, + "learning_rate": 4.8823800738007384e-06, + "loss": 1.7127, + "step": 307 + }, + { + "epoch": 0.3833229620410703, + "grad_norm": 3.047581195831299, + "learning_rate": 4.880073800738008e-06, + "loss": 1.7174, + "step": 308 + }, + { + "epoch": 0.3845675171126322, + "grad_norm": 2.87735652923584, + "learning_rate": 4.877767527675277e-06, + "loss": 1.6677, + "step": 309 + }, + { + "epoch": 0.38581207218419417, + "grad_norm": 2.7169721126556396, + "learning_rate": 4.875461254612546e-06, + "loss": 1.7486, + "step": 310 + }, + { + "epoch": 0.38581207218419417, + "eval_loss": 1.8221794366836548, + "eval_runtime": 43.1681, + "eval_samples_per_second": 23.165, + "eval_steps_per_second": 0.973, + "step": 310 + }, + { + "epoch": 0.38705662725575607, + "grad_norm": 2.7094991207122803, + "learning_rate": 4.873154981549816e-06, + "loss": 1.7348, + "step": 311 + }, + { + "epoch": 0.388301182327318, + "grad_norm": 2.8989078998565674, + "learning_rate": 4.8708487084870856e-06, + "loss": 1.7352, + "step": 312 + }, + { + "epoch": 0.3895457373988799, + "grad_norm": 2.9565601348876953, + "learning_rate": 4.868542435424355e-06, + "loss": 1.7063, + "step": 313 + }, + { + "epoch": 0.39079029247044184, + "grad_norm": 2.8354082107543945, + "learning_rate": 4.8662361623616235e-06, + "loss": 1.6808, + "step": 314 + }, + { + "epoch": 0.39203484754200374, + "grad_norm": 2.8287479877471924, + "learning_rate": 4.863929889298894e-06, + "loss": 1.6731, + "step": 315 + }, + { + "epoch": 0.39327940261356564, + "grad_norm": 2.6537821292877197, + "learning_rate": 4.861623616236163e-06, + "loss": 1.695, + "step": 316 + }, + { + "epoch": 0.39452395768512755, + "grad_norm": 2.7135190963745117, + "learning_rate": 4.859317343173432e-06, + "loss": 1.6897, + "step": 317 + }, + { + "epoch": 0.3957685127566895, + "grad_norm": 3.1019129753112793, + "learning_rate": 4.857011070110701e-06, + "loss": 1.7219, + "step": 318 + }, + { + "epoch": 0.3970130678282514, + "grad_norm": 2.945737838745117, + "learning_rate": 4.854704797047971e-06, + "loss": 1.7316, + "step": 319 + }, + { + "epoch": 0.3982576228998133, + "grad_norm": 2.7668251991271973, + "learning_rate": 4.85239852398524e-06, + "loss": 1.6461, + "step": 320 + }, + { + "epoch": 0.3982576228998133, + "eval_loss": 1.8210570812225342, + "eval_runtime": 46.2552, + "eval_samples_per_second": 21.619, + "eval_steps_per_second": 0.908, + "step": 320 + }, + { + "epoch": 0.3995021779713752, + "grad_norm": 2.738109588623047, + "learning_rate": 4.8500922509225095e-06, + "loss": 1.6907, + "step": 321 + }, + { + "epoch": 0.4007467330429372, + "grad_norm": 3.0195062160491943, + "learning_rate": 4.847785977859779e-06, + "loss": 1.7099, + "step": 322 + }, + { + "epoch": 0.4019912881144991, + "grad_norm": 2.8672404289245605, + "learning_rate": 4.845479704797048e-06, + "loss": 1.6509, + "step": 323 + }, + { + "epoch": 0.403235843186061, + "grad_norm": 2.9041008949279785, + "learning_rate": 4.843173431734318e-06, + "loss": 1.5818, + "step": 324 + }, + { + "epoch": 0.4044803982576229, + "grad_norm": 2.9105985164642334, + "learning_rate": 4.840867158671587e-06, + "loss": 1.7002, + "step": 325 + }, + { + "epoch": 0.4057249533291848, + "grad_norm": 2.749009847640991, + "learning_rate": 4.838560885608857e-06, + "loss": 1.7021, + "step": 326 + }, + { + "epoch": 0.40696950840074675, + "grad_norm": 2.6998822689056396, + "learning_rate": 4.836254612546126e-06, + "loss": 1.6853, + "step": 327 + }, + { + "epoch": 0.40821406347230865, + "grad_norm": 2.678201675415039, + "learning_rate": 4.8339483394833955e-06, + "loss": 1.6934, + "step": 328 + }, + { + "epoch": 0.40945861854387056, + "grad_norm": 2.774845600128174, + "learning_rate": 4.831642066420665e-06, + "loss": 1.657, + "step": 329 + }, + { + "epoch": 0.41070317361543246, + "grad_norm": 2.6363000869750977, + "learning_rate": 4.8293357933579335e-06, + "loss": 1.6555, + "step": 330 + }, + { + "epoch": 0.41070317361543246, + "eval_loss": 1.8126581907272339, + "eval_runtime": 52.7717, + "eval_samples_per_second": 18.95, + "eval_steps_per_second": 0.796, + "step": 330 + }, + { + "epoch": 0.4119477286869944, + "grad_norm": 3.011659622192383, + "learning_rate": 4.827029520295204e-06, + "loss": 1.7202, + "step": 331 + }, + { + "epoch": 0.4131922837585563, + "grad_norm": 2.8322594165802, + "learning_rate": 4.824723247232473e-06, + "loss": 1.6736, + "step": 332 + }, + { + "epoch": 0.4144368388301182, + "grad_norm": 2.9360146522521973, + "learning_rate": 4.822416974169742e-06, + "loss": 1.7213, + "step": 333 + }, + { + "epoch": 0.41568139390168013, + "grad_norm": 2.810206413269043, + "learning_rate": 4.820110701107011e-06, + "loss": 1.6867, + "step": 334 + }, + { + "epoch": 0.4169259489732421, + "grad_norm": 2.6864097118377686, + "learning_rate": 4.817804428044281e-06, + "loss": 1.6854, + "step": 335 + }, + { + "epoch": 0.418170504044804, + "grad_norm": 2.7396061420440674, + "learning_rate": 4.81549815498155e-06, + "loss": 1.68, + "step": 336 + }, + { + "epoch": 0.4194150591163659, + "grad_norm": 2.602771520614624, + "learning_rate": 4.8131918819188194e-06, + "loss": 1.6269, + "step": 337 + }, + { + "epoch": 0.4206596141879278, + "grad_norm": 2.8174757957458496, + "learning_rate": 4.810885608856089e-06, + "loss": 1.6981, + "step": 338 + }, + { + "epoch": 0.42190416925948976, + "grad_norm": 3.2398130893707275, + "learning_rate": 4.808579335793358e-06, + "loss": 1.7061, + "step": 339 + }, + { + "epoch": 0.42314872433105166, + "grad_norm": 2.7119736671447754, + "learning_rate": 4.806273062730628e-06, + "loss": 1.6875, + "step": 340 + }, + { + "epoch": 0.42314872433105166, + "eval_loss": 1.7973068952560425, + "eval_runtime": 53.5791, + "eval_samples_per_second": 18.664, + "eval_steps_per_second": 0.784, + "step": 340 + }, + { + "epoch": 0.42439327940261357, + "grad_norm": 2.868533134460449, + "learning_rate": 4.803966789667897e-06, + "loss": 1.6967, + "step": 341 + }, + { + "epoch": 0.42563783447417547, + "grad_norm": 2.528083562850952, + "learning_rate": 4.8016605166051665e-06, + "loss": 1.6195, + "step": 342 + }, + { + "epoch": 0.4268823895457374, + "grad_norm": 2.7885115146636963, + "learning_rate": 4.799354243542436e-06, + "loss": 1.6197, + "step": 343 + }, + { + "epoch": 0.42812694461729933, + "grad_norm": 2.768247604370117, + "learning_rate": 4.797047970479705e-06, + "loss": 1.6271, + "step": 344 + }, + { + "epoch": 0.42937149968886124, + "grad_norm": 2.7710275650024414, + "learning_rate": 4.794741697416975e-06, + "loss": 1.6768, + "step": 345 + }, + { + "epoch": 0.43061605476042314, + "grad_norm": 2.968337297439575, + "learning_rate": 4.792435424354243e-06, + "loss": 1.6433, + "step": 346 + }, + { + "epoch": 0.43186060983198504, + "grad_norm": 2.5739808082580566, + "learning_rate": 4.790129151291514e-06, + "loss": 1.6327, + "step": 347 + }, + { + "epoch": 0.433105164903547, + "grad_norm": 2.7676408290863037, + "learning_rate": 4.787822878228783e-06, + "loss": 1.6178, + "step": 348 + }, + { + "epoch": 0.4343497199751089, + "grad_norm": 2.959059476852417, + "learning_rate": 4.7855166051660525e-06, + "loss": 1.6809, + "step": 349 + }, + { + "epoch": 0.4355942750466708, + "grad_norm": 2.559967279434204, + "learning_rate": 4.783210332103321e-06, + "loss": 1.6704, + "step": 350 + }, + { + "epoch": 0.4355942750466708, + "eval_loss": 1.8013949394226074, + "eval_runtime": 50.4649, + "eval_samples_per_second": 19.816, + "eval_steps_per_second": 0.832, + "step": 350 + }, + { + "epoch": 0.4368388301182327, + "grad_norm": 2.817901611328125, + "learning_rate": 4.7809040590405905e-06, + "loss": 1.6296, + "step": 351 + }, + { + "epoch": 0.43808338518979467, + "grad_norm": 2.7116055488586426, + "learning_rate": 4.778597785977861e-06, + "loss": 1.6652, + "step": 352 + }, + { + "epoch": 0.4393279402613566, + "grad_norm": 2.544750452041626, + "learning_rate": 4.776291512915129e-06, + "loss": 1.5668, + "step": 353 + }, + { + "epoch": 0.4405724953329185, + "grad_norm": 2.720534086227417, + "learning_rate": 4.773985239852399e-06, + "loss": 1.6152, + "step": 354 + }, + { + "epoch": 0.4418170504044804, + "grad_norm": 2.7270126342773438, + "learning_rate": 4.771678966789668e-06, + "loss": 1.5624, + "step": 355 + }, + { + "epoch": 0.44306160547604234, + "grad_norm": 2.7986624240875244, + "learning_rate": 4.769372693726938e-06, + "loss": 1.6621, + "step": 356 + }, + { + "epoch": 0.44430616054760425, + "grad_norm": 2.943107843399048, + "learning_rate": 4.767066420664207e-06, + "loss": 1.6713, + "step": 357 + }, + { + "epoch": 0.44555071561916615, + "grad_norm": 2.739898681640625, + "learning_rate": 4.7647601476014765e-06, + "loss": 1.6363, + "step": 358 + }, + { + "epoch": 0.44679527069072805, + "grad_norm": 2.8729329109191895, + "learning_rate": 4.762453874538746e-06, + "loss": 1.6605, + "step": 359 + }, + { + "epoch": 0.44803982576228996, + "grad_norm": 2.909029483795166, + "learning_rate": 4.760147601476015e-06, + "loss": 1.6277, + "step": 360 + }, + { + "epoch": 0.44803982576228996, + "eval_loss": 1.8110179901123047, + "eval_runtime": 47.924, + "eval_samples_per_second": 20.866, + "eval_steps_per_second": 0.876, + "step": 360 + }, + { + "epoch": 0.4492843808338519, + "grad_norm": 3.3826682567596436, + "learning_rate": 4.757841328413285e-06, + "loss": 1.7155, + "step": 361 + }, + { + "epoch": 0.4505289359054138, + "grad_norm": 3.0586729049682617, + "learning_rate": 4.755535055350554e-06, + "loss": 1.5991, + "step": 362 + }, + { + "epoch": 0.4517734909769757, + "grad_norm": 2.645576238632202, + "learning_rate": 4.753228782287823e-06, + "loss": 1.6504, + "step": 363 + }, + { + "epoch": 0.4530180460485376, + "grad_norm": 2.705967664718628, + "learning_rate": 4.750922509225093e-06, + "loss": 1.6761, + "step": 364 + }, + { + "epoch": 0.4542626011200996, + "grad_norm": 2.7651922702789307, + "learning_rate": 4.748616236162362e-06, + "loss": 1.61, + "step": 365 + }, + { + "epoch": 0.4555071561916615, + "grad_norm": 2.5654966831207275, + "learning_rate": 4.746309963099631e-06, + "loss": 1.6335, + "step": 366 + }, + { + "epoch": 0.4567517112632234, + "grad_norm": 2.6606173515319824, + "learning_rate": 4.7440036900369e-06, + "loss": 1.6207, + "step": 367 + }, + { + "epoch": 0.4579962663347853, + "grad_norm": 2.724755048751831, + "learning_rate": 4.741697416974171e-06, + "loss": 1.6329, + "step": 368 + }, + { + "epoch": 0.45924082140634725, + "grad_norm": 2.7019615173339844, + "learning_rate": 4.739391143911439e-06, + "loss": 1.6763, + "step": 369 + }, + { + "epoch": 0.46048537647790916, + "grad_norm": 2.6594254970550537, + "learning_rate": 4.737084870848709e-06, + "loss": 1.6174, + "step": 370 + }, + { + "epoch": 0.46048537647790916, + "eval_loss": 1.7924141883850098, + "eval_runtime": 53.0349, + "eval_samples_per_second": 18.855, + "eval_steps_per_second": 0.792, + "step": 370 + }, + { + "epoch": 0.46172993154947106, + "grad_norm": 2.668421745300293, + "learning_rate": 4.734778597785978e-06, + "loss": 1.6421, + "step": 371 + }, + { + "epoch": 0.46297448662103297, + "grad_norm": 2.7148995399475098, + "learning_rate": 4.7324723247232475e-06, + "loss": 1.6635, + "step": 372 + }, + { + "epoch": 0.4642190416925949, + "grad_norm": 3.156646490097046, + "learning_rate": 4.730166051660517e-06, + "loss": 1.6679, + "step": 373 + }, + { + "epoch": 0.46546359676415683, + "grad_norm": 2.8170645236968994, + "learning_rate": 4.727859778597786e-06, + "loss": 1.6492, + "step": 374 + }, + { + "epoch": 0.46670815183571873, + "grad_norm": 2.9796457290649414, + "learning_rate": 4.725553505535056e-06, + "loss": 1.6025, + "step": 375 + }, + { + "epoch": 0.46795270690728064, + "grad_norm": 2.6493406295776367, + "learning_rate": 4.723247232472325e-06, + "loss": 1.6398, + "step": 376 + }, + { + "epoch": 0.46919726197884254, + "grad_norm": 3.2643542289733887, + "learning_rate": 4.720940959409595e-06, + "loss": 1.6413, + "step": 377 + }, + { + "epoch": 0.4704418170504045, + "grad_norm": 2.7613179683685303, + "learning_rate": 4.718634686346864e-06, + "loss": 1.6251, + "step": 378 + }, + { + "epoch": 0.4716863721219664, + "grad_norm": 3.1216518878936768, + "learning_rate": 4.716328413284133e-06, + "loss": 1.6268, + "step": 379 + }, + { + "epoch": 0.4729309271935283, + "grad_norm": 2.571647882461548, + "learning_rate": 4.714022140221403e-06, + "loss": 1.6245, + "step": 380 + }, + { + "epoch": 0.4729309271935283, + "eval_loss": 1.7875893115997314, + "eval_runtime": 49.3058, + "eval_samples_per_second": 20.282, + "eval_steps_per_second": 0.852, + "step": 380 + }, + { + "epoch": 0.4741754822650902, + "grad_norm": 3.443906545639038, + "learning_rate": 4.711715867158672e-06, + "loss": 1.6799, + "step": 381 + }, + { + "epoch": 0.47542003733665217, + "grad_norm": 2.7887957096099854, + "learning_rate": 4.709409594095941e-06, + "loss": 1.6589, + "step": 382 + }, + { + "epoch": 0.47666459240821407, + "grad_norm": 3.2833151817321777, + "learning_rate": 4.70710332103321e-06, + "loss": 1.6064, + "step": 383 + }, + { + "epoch": 0.477909147479776, + "grad_norm": 2.6975057125091553, + "learning_rate": 4.704797047970481e-06, + "loss": 1.6029, + "step": 384 + }, + { + "epoch": 0.4791537025513379, + "grad_norm": 2.745246171951294, + "learning_rate": 4.702490774907749e-06, + "loss": 1.7127, + "step": 385 + }, + { + "epoch": 0.48039825762289984, + "grad_norm": 3.0522775650024414, + "learning_rate": 4.700184501845019e-06, + "loss": 1.6508, + "step": 386 + }, + { + "epoch": 0.48164281269446174, + "grad_norm": 2.689218044281006, + "learning_rate": 4.697878228782288e-06, + "loss": 1.6971, + "step": 387 + }, + { + "epoch": 0.48288736776602365, + "grad_norm": 3.060361862182617, + "learning_rate": 4.6955719557195575e-06, + "loss": 1.6464, + "step": 388 + }, + { + "epoch": 0.48413192283758555, + "grad_norm": 2.872262477874756, + "learning_rate": 4.693265682656827e-06, + "loss": 1.6293, + "step": 389 + }, + { + "epoch": 0.48537647790914745, + "grad_norm": 2.980050563812256, + "learning_rate": 4.690959409594096e-06, + "loss": 1.6435, + "step": 390 + }, + { + "epoch": 0.48537647790914745, + "eval_loss": 1.7791496515274048, + "eval_runtime": 50.0299, + "eval_samples_per_second": 19.988, + "eval_steps_per_second": 0.839, + "step": 390 + }, + { + "epoch": 0.4866210329807094, + "grad_norm": 2.764827251434326, + "learning_rate": 4.688653136531366e-06, + "loss": 1.6279, + "step": 391 + }, + { + "epoch": 0.4878655880522713, + "grad_norm": 2.6912333965301514, + "learning_rate": 4.686346863468635e-06, + "loss": 1.7276, + "step": 392 + }, + { + "epoch": 0.4891101431238332, + "grad_norm": 2.8358330726623535, + "learning_rate": 4.6840405904059046e-06, + "loss": 1.6482, + "step": 393 + }, + { + "epoch": 0.4903546981953951, + "grad_norm": 2.6581575870513916, + "learning_rate": 4.681734317343174e-06, + "loss": 1.6853, + "step": 394 + }, + { + "epoch": 0.4915992532669571, + "grad_norm": 2.941425323486328, + "learning_rate": 4.6794280442804426e-06, + "loss": 1.6075, + "step": 395 + }, + { + "epoch": 0.492843808338519, + "grad_norm": 3.2334814071655273, + "learning_rate": 4.677121771217713e-06, + "loss": 1.6157, + "step": 396 + }, + { + "epoch": 0.4940883634100809, + "grad_norm": 2.984410524368286, + "learning_rate": 4.674815498154982e-06, + "loss": 1.6257, + "step": 397 + }, + { + "epoch": 0.4953329184816428, + "grad_norm": 3.0650789737701416, + "learning_rate": 4.672509225092252e-06, + "loss": 1.6573, + "step": 398 + }, + { + "epoch": 0.49657747355320475, + "grad_norm": 2.798664093017578, + "learning_rate": 4.67020295202952e-06, + "loss": 1.638, + "step": 399 + }, + { + "epoch": 0.49782202862476665, + "grad_norm": 2.8266706466674805, + "learning_rate": 4.66789667896679e-06, + "loss": 1.6484, + "step": 400 + }, + { + "epoch": 0.49782202862476665, + "eval_loss": 1.7714862823486328, + "eval_runtime": 48.0657, + "eval_samples_per_second": 20.805, + "eval_steps_per_second": 0.874, + "step": 400 + }, + { + "epoch": 0.49906658369632856, + "grad_norm": 2.9928388595581055, + "learning_rate": 4.66559040590406e-06, + "loss": 1.6721, + "step": 401 + }, + { + "epoch": 0.5003111387678905, + "grad_norm": 2.573514461517334, + "learning_rate": 4.6632841328413285e-06, + "loss": 1.679, + "step": 402 + }, + { + "epoch": 0.5015556938394524, + "grad_norm": 3.1736996173858643, + "learning_rate": 4.660977859778598e-06, + "loss": 1.7124, + "step": 403 + }, + { + "epoch": 0.5028002489110143, + "grad_norm": 2.858849287033081, + "learning_rate": 4.658671586715867e-06, + "loss": 1.6622, + "step": 404 + }, + { + "epoch": 0.5040448039825762, + "grad_norm": 2.825698137283325, + "learning_rate": 4.656365313653137e-06, + "loss": 1.6464, + "step": 405 + }, + { + "epoch": 0.5052893590541382, + "grad_norm": 3.783891439437866, + "learning_rate": 4.654059040590406e-06, + "loss": 1.6665, + "step": 406 + }, + { + "epoch": 0.5065339141257, + "grad_norm": 2.789813756942749, + "learning_rate": 4.651752767527676e-06, + "loss": 1.6064, + "step": 407 + }, + { + "epoch": 0.507778469197262, + "grad_norm": 3.3106348514556885, + "learning_rate": 4.649446494464945e-06, + "loss": 1.6542, + "step": 408 + }, + { + "epoch": 0.5090230242688238, + "grad_norm": 2.8805112838745117, + "learning_rate": 4.6471402214022145e-06, + "loss": 1.6058, + "step": 409 + }, + { + "epoch": 0.5102675793403858, + "grad_norm": 3.30898118019104, + "learning_rate": 4.644833948339484e-06, + "loss": 1.5933, + "step": 410 + }, + { + "epoch": 0.5102675793403858, + "eval_loss": 1.7787818908691406, + "eval_runtime": 44.0876, + "eval_samples_per_second": 22.682, + "eval_steps_per_second": 0.953, + "step": 410 + }, + { + "epoch": 0.5115121344119478, + "grad_norm": 3.3563625812530518, + "learning_rate": 4.642527675276753e-06, + "loss": 1.6751, + "step": 411 + }, + { + "epoch": 0.5127566894835096, + "grad_norm": 2.968120574951172, + "learning_rate": 4.640221402214023e-06, + "loss": 1.663, + "step": 412 + }, + { + "epoch": 0.5140012445550716, + "grad_norm": 3.310248374938965, + "learning_rate": 4.637915129151292e-06, + "loss": 1.658, + "step": 413 + }, + { + "epoch": 0.5152457996266335, + "grad_norm": 2.7011170387268066, + "learning_rate": 4.635608856088562e-06, + "loss": 1.6894, + "step": 414 + }, + { + "epoch": 0.5164903546981954, + "grad_norm": 2.804901361465454, + "learning_rate": 4.63330258302583e-06, + "loss": 1.602, + "step": 415 + }, + { + "epoch": 0.5177349097697573, + "grad_norm": 3.2412750720977783, + "learning_rate": 4.6309963099631e-06, + "loss": 1.6158, + "step": 416 + }, + { + "epoch": 0.5189794648413192, + "grad_norm": 2.699909210205078, + "learning_rate": 4.62869003690037e-06, + "loss": 1.5842, + "step": 417 + }, + { + "epoch": 0.5202240199128811, + "grad_norm": 3.065929412841797, + "learning_rate": 4.6263837638376384e-06, + "loss": 1.6434, + "step": 418 + }, + { + "epoch": 0.5214685749844431, + "grad_norm": 2.895305633544922, + "learning_rate": 4.624077490774908e-06, + "loss": 1.5969, + "step": 419 + }, + { + "epoch": 0.522713130056005, + "grad_norm": 3.1590664386749268, + "learning_rate": 4.621771217712177e-06, + "loss": 1.6211, + "step": 420 + }, + { + "epoch": 0.522713130056005, + "eval_loss": 1.7674189805984497, + "eval_runtime": 53.328, + "eval_samples_per_second": 18.752, + "eval_steps_per_second": 0.788, + "step": 420 + }, + { + "epoch": 0.5239576851275669, + "grad_norm": 2.7126312255859375, + "learning_rate": 4.619464944649447e-06, + "loss": 1.6246, + "step": 421 + }, + { + "epoch": 0.5252022401991289, + "grad_norm": 2.918508768081665, + "learning_rate": 4.617158671586716e-06, + "loss": 1.6937, + "step": 422 + }, + { + "epoch": 0.5264467952706907, + "grad_norm": 3.008610725402832, + "learning_rate": 4.6148523985239856e-06, + "loss": 1.6864, + "step": 423 + }, + { + "epoch": 0.5276913503422527, + "grad_norm": 2.6612043380737305, + "learning_rate": 4.612546125461255e-06, + "loss": 1.5785, + "step": 424 + }, + { + "epoch": 0.5289359054138145, + "grad_norm": 2.850679874420166, + "learning_rate": 4.610239852398524e-06, + "loss": 1.5899, + "step": 425 + }, + { + "epoch": 0.5301804604853765, + "grad_norm": 2.9205031394958496, + "learning_rate": 4.607933579335794e-06, + "loss": 1.5867, + "step": 426 + }, + { + "epoch": 0.5314250155569384, + "grad_norm": 2.929532527923584, + "learning_rate": 4.605627306273063e-06, + "loss": 1.6182, + "step": 427 + }, + { + "epoch": 0.5326695706285003, + "grad_norm": 2.874936103820801, + "learning_rate": 4.603321033210333e-06, + "loss": 1.5789, + "step": 428 + }, + { + "epoch": 0.5339141257000622, + "grad_norm": 2.8703525066375732, + "learning_rate": 4.601014760147602e-06, + "loss": 1.5859, + "step": 429 + }, + { + "epoch": 0.5351586807716241, + "grad_norm": 2.7076902389526367, + "learning_rate": 4.5987084870848715e-06, + "loss": 1.644, + "step": 430 + }, + { + "epoch": 0.5351586807716241, + "eval_loss": 1.7592095136642456, + "eval_runtime": 47.9438, + "eval_samples_per_second": 20.858, + "eval_steps_per_second": 0.876, + "step": 430 + }, + { + "epoch": 0.536403235843186, + "grad_norm": 2.839160680770874, + "learning_rate": 4.59640221402214e-06, + "loss": 1.6357, + "step": 431 + }, + { + "epoch": 0.537647790914748, + "grad_norm": 2.674025535583496, + "learning_rate": 4.5940959409594095e-06, + "loss": 1.676, + "step": 432 + }, + { + "epoch": 0.5388923459863099, + "grad_norm": 3.0619537830352783, + "learning_rate": 4.59178966789668e-06, + "loss": 1.5617, + "step": 433 + }, + { + "epoch": 0.5401369010578718, + "grad_norm": 3.058418035507202, + "learning_rate": 4.589483394833948e-06, + "loss": 1.5951, + "step": 434 + }, + { + "epoch": 0.5413814561294338, + "grad_norm": 3.0046396255493164, + "learning_rate": 4.587177121771218e-06, + "loss": 1.6473, + "step": 435 + }, + { + "epoch": 0.5426260112009956, + "grad_norm": 2.7736752033233643, + "learning_rate": 4.584870848708487e-06, + "loss": 1.6262, + "step": 436 + }, + { + "epoch": 0.5438705662725576, + "grad_norm": 2.7804994583129883, + "learning_rate": 4.5825645756457575e-06, + "loss": 1.6886, + "step": 437 + }, + { + "epoch": 0.5451151213441194, + "grad_norm": 3.0717954635620117, + "learning_rate": 4.580258302583026e-06, + "loss": 1.6287, + "step": 438 + }, + { + "epoch": 0.5463596764156814, + "grad_norm": 2.800243854522705, + "learning_rate": 4.5779520295202955e-06, + "loss": 1.6169, + "step": 439 + }, + { + "epoch": 0.5476042314872434, + "grad_norm": 2.9581644535064697, + "learning_rate": 4.575645756457565e-06, + "loss": 1.6617, + "step": 440 + }, + { + "epoch": 0.5476042314872434, + "eval_loss": 1.7642868757247925, + "eval_runtime": 51.2646, + "eval_samples_per_second": 19.507, + "eval_steps_per_second": 0.819, + "step": 440 + }, + { + "epoch": 0.5488487865588052, + "grad_norm": 3.0067460536956787, + "learning_rate": 4.573339483394834e-06, + "loss": 1.6482, + "step": 441 + }, + { + "epoch": 0.5500933416303672, + "grad_norm": 2.9075019359588623, + "learning_rate": 4.571033210332104e-06, + "loss": 1.6595, + "step": 442 + }, + { + "epoch": 0.551337896701929, + "grad_norm": 2.7788755893707275, + "learning_rate": 4.568726937269373e-06, + "loss": 1.5733, + "step": 443 + }, + { + "epoch": 0.552582451773491, + "grad_norm": 2.7775425910949707, + "learning_rate": 4.566420664206643e-06, + "loss": 1.6925, + "step": 444 + }, + { + "epoch": 0.5538270068450529, + "grad_norm": 2.6949567794799805, + "learning_rate": 4.564114391143912e-06, + "loss": 1.5738, + "step": 445 + }, + { + "epoch": 0.5550715619166148, + "grad_norm": 2.7093007564544678, + "learning_rate": 4.5618081180811814e-06, + "loss": 1.5848, + "step": 446 + }, + { + "epoch": 0.5563161169881767, + "grad_norm": 2.6920173168182373, + "learning_rate": 4.559501845018451e-06, + "loss": 1.6092, + "step": 447 + }, + { + "epoch": 0.5575606720597387, + "grad_norm": 2.733731985092163, + "learning_rate": 4.5571955719557194e-06, + "loss": 1.6056, + "step": 448 + }, + { + "epoch": 0.5588052271313005, + "grad_norm": 2.9156274795532227, + "learning_rate": 4.55488929889299e-06, + "loss": 1.6436, + "step": 449 + }, + { + "epoch": 0.5600497822028625, + "grad_norm": 2.8802616596221924, + "learning_rate": 4.552583025830259e-06, + "loss": 1.6309, + "step": 450 + }, + { + "epoch": 0.5600497822028625, + "eval_loss": 1.7486340999603271, + "eval_runtime": 47.2219, + "eval_samples_per_second": 21.177, + "eval_steps_per_second": 0.889, + "step": 450 + }, + { + "epoch": 0.5612943372744243, + "grad_norm": 2.729743242263794, + "learning_rate": 4.550276752767528e-06, + "loss": 1.7171, + "step": 451 + }, + { + "epoch": 0.5625388923459863, + "grad_norm": 2.800049066543579, + "learning_rate": 4.547970479704797e-06, + "loss": 1.6098, + "step": 452 + }, + { + "epoch": 0.5637834474175483, + "grad_norm": 2.9659311771392822, + "learning_rate": 4.5456642066420666e-06, + "loss": 1.6646, + "step": 453 + }, + { + "epoch": 0.5650280024891101, + "grad_norm": 2.5417754650115967, + "learning_rate": 4.543357933579336e-06, + "loss": 1.5847, + "step": 454 + }, + { + "epoch": 0.5662725575606721, + "grad_norm": 2.8471338748931885, + "learning_rate": 4.541051660516605e-06, + "loss": 1.6235, + "step": 455 + }, + { + "epoch": 0.5675171126322339, + "grad_norm": 2.7039637565612793, + "learning_rate": 4.538745387453875e-06, + "loss": 1.5613, + "step": 456 + }, + { + "epoch": 0.5687616677037959, + "grad_norm": 3.0383341312408447, + "learning_rate": 4.536439114391144e-06, + "loss": 1.6485, + "step": 457 + }, + { + "epoch": 0.5700062227753578, + "grad_norm": 2.528388261795044, + "learning_rate": 4.534132841328414e-06, + "loss": 1.6117, + "step": 458 + }, + { + "epoch": 0.5712507778469197, + "grad_norm": 2.7623119354248047, + "learning_rate": 4.531826568265683e-06, + "loss": 1.6017, + "step": 459 + }, + { + "epoch": 0.5724953329184816, + "grad_norm": 2.9213945865631104, + "learning_rate": 4.5295202952029525e-06, + "loss": 1.643, + "step": 460 + }, + { + "epoch": 0.5724953329184816, + "eval_loss": 1.747575044631958, + "eval_runtime": 45.2844, + "eval_samples_per_second": 22.083, + "eval_steps_per_second": 0.927, + "step": 460 + }, + { + "epoch": 0.5737398879900436, + "grad_norm": 2.664275884628296, + "learning_rate": 4.527214022140222e-06, + "loss": 1.6154, + "step": 461 + }, + { + "epoch": 0.5749844430616055, + "grad_norm": 2.5891788005828857, + "learning_rate": 4.524907749077491e-06, + "loss": 1.5975, + "step": 462 + }, + { + "epoch": 0.5762289981331674, + "grad_norm": 2.8126487731933594, + "learning_rate": 4.522601476014761e-06, + "loss": 1.6306, + "step": 463 + }, + { + "epoch": 0.5774735532047293, + "grad_norm": 2.799391031265259, + "learning_rate": 4.520295202952029e-06, + "loss": 1.6139, + "step": 464 + }, + { + "epoch": 0.5787181082762912, + "grad_norm": 2.7323718070983887, + "learning_rate": 4.5179889298893e-06, + "loss": 1.632, + "step": 465 + }, + { + "epoch": 0.5799626633478532, + "grad_norm": 2.601386785507202, + "learning_rate": 4.515682656826569e-06, + "loss": 1.6139, + "step": 466 + }, + { + "epoch": 0.581207218419415, + "grad_norm": 2.562162399291992, + "learning_rate": 4.513376383763838e-06, + "loss": 1.5168, + "step": 467 + }, + { + "epoch": 0.582451773490977, + "grad_norm": 2.6912002563476562, + "learning_rate": 4.511070110701107e-06, + "loss": 1.6249, + "step": 468 + }, + { + "epoch": 0.583696328562539, + "grad_norm": 2.6971495151519775, + "learning_rate": 4.5087638376383765e-06, + "loss": 1.616, + "step": 469 + }, + { + "epoch": 0.5849408836341008, + "grad_norm": 2.6898910999298096, + "learning_rate": 4.506457564575646e-06, + "loss": 1.614, + "step": 470 + }, + { + "epoch": 0.5849408836341008, + "eval_loss": 1.7460769414901733, + "eval_runtime": 45.3055, + "eval_samples_per_second": 22.072, + "eval_steps_per_second": 0.927, + "step": 470 + }, + { + "epoch": 0.5861854387056628, + "grad_norm": 2.698180675506592, + "learning_rate": 4.504151291512915e-06, + "loss": 1.5993, + "step": 471 + }, + { + "epoch": 0.5874299937772246, + "grad_norm": 2.7125210762023926, + "learning_rate": 4.501845018450185e-06, + "loss": 1.6113, + "step": 472 + }, + { + "epoch": 0.5886745488487866, + "grad_norm": 2.754445791244507, + "learning_rate": 4.499538745387454e-06, + "loss": 1.6719, + "step": 473 + }, + { + "epoch": 0.5899191039203485, + "grad_norm": 2.6979362964630127, + "learning_rate": 4.497232472324724e-06, + "loss": 1.608, + "step": 474 + }, + { + "epoch": 0.5911636589919104, + "grad_norm": 2.7694528102874756, + "learning_rate": 4.494926199261993e-06, + "loss": 1.5695, + "step": 475 + }, + { + "epoch": 0.5924082140634723, + "grad_norm": 2.653353214263916, + "learning_rate": 4.4926199261992624e-06, + "loss": 1.5523, + "step": 476 + }, + { + "epoch": 0.5936527691350342, + "grad_norm": 2.6632070541381836, + "learning_rate": 4.490313653136532e-06, + "loss": 1.5675, + "step": 477 + }, + { + "epoch": 0.5948973242065961, + "grad_norm": 3.038543939590454, + "learning_rate": 4.488007380073801e-06, + "loss": 1.5692, + "step": 478 + }, + { + "epoch": 0.5961418792781581, + "grad_norm": 2.8123953342437744, + "learning_rate": 4.485701107011071e-06, + "loss": 1.6223, + "step": 479 + }, + { + "epoch": 0.5973864343497199, + "grad_norm": 2.5845773220062256, + "learning_rate": 4.483394833948339e-06, + "loss": 1.5773, + "step": 480 + }, + { + "epoch": 0.5973864343497199, + "eval_loss": 1.7419319152832031, + "eval_runtime": 44.6842, + "eval_samples_per_second": 22.379, + "eval_steps_per_second": 0.94, + "step": 480 + }, + { + "epoch": 0.5986309894212819, + "grad_norm": 3.069936752319336, + "learning_rate": 4.4810885608856096e-06, + "loss": 1.6323, + "step": 481 + }, + { + "epoch": 0.5998755444928439, + "grad_norm": 2.9085781574249268, + "learning_rate": 4.478782287822879e-06, + "loss": 1.5845, + "step": 482 + }, + { + "epoch": 0.6011200995644057, + "grad_norm": 2.8568010330200195, + "learning_rate": 4.4764760147601476e-06, + "loss": 1.5898, + "step": 483 + }, + { + "epoch": 0.6023646546359677, + "grad_norm": 3.089081048965454, + "learning_rate": 4.474169741697417e-06, + "loss": 1.6622, + "step": 484 + }, + { + "epoch": 0.6036092097075295, + "grad_norm": 2.8515470027923584, + "learning_rate": 4.471863468634686e-06, + "loss": 1.5859, + "step": 485 + }, + { + "epoch": 0.6048537647790915, + "grad_norm": 2.647765874862671, + "learning_rate": 4.469557195571957e-06, + "loss": 1.5926, + "step": 486 + }, + { + "epoch": 0.6060983198506534, + "grad_norm": 2.61676025390625, + "learning_rate": 4.467250922509225e-06, + "loss": 1.5684, + "step": 487 + }, + { + "epoch": 0.6073428749222153, + "grad_norm": 2.7127039432525635, + "learning_rate": 4.464944649446495e-06, + "loss": 1.5702, + "step": 488 + }, + { + "epoch": 0.6085874299937772, + "grad_norm": 2.9932055473327637, + "learning_rate": 4.462638376383764e-06, + "loss": 1.5976, + "step": 489 + }, + { + "epoch": 0.6098319850653391, + "grad_norm": 2.6508774757385254, + "learning_rate": 4.4603321033210335e-06, + "loss": 1.586, + "step": 490 + }, + { + "epoch": 0.6098319850653391, + "eval_loss": 1.7357326745986938, + "eval_runtime": 45.393, + "eval_samples_per_second": 22.03, + "eval_steps_per_second": 0.925, + "step": 490 + }, + { + "epoch": 0.611076540136901, + "grad_norm": 2.7626044750213623, + "learning_rate": 4.458025830258303e-06, + "loss": 1.5773, + "step": 491 + }, + { + "epoch": 0.612321095208463, + "grad_norm": 2.595003604888916, + "learning_rate": 4.455719557195572e-06, + "loss": 1.601, + "step": 492 + }, + { + "epoch": 0.6135656502800249, + "grad_norm": 2.595767021179199, + "learning_rate": 4.453413284132842e-06, + "loss": 1.6287, + "step": 493 + }, + { + "epoch": 0.6148102053515868, + "grad_norm": 2.755845308303833, + "learning_rate": 4.451107011070111e-06, + "loss": 1.593, + "step": 494 + }, + { + "epoch": 0.6160547604231488, + "grad_norm": 2.7302653789520264, + "learning_rate": 4.448800738007381e-06, + "loss": 1.5768, + "step": 495 + }, + { + "epoch": 0.6172993154947106, + "grad_norm": 2.6493024826049805, + "learning_rate": 4.446494464944649e-06, + "loss": 1.6059, + "step": 496 + }, + { + "epoch": 0.6185438705662726, + "grad_norm": 2.8633735179901123, + "learning_rate": 4.4441881918819195e-06, + "loss": 1.598, + "step": 497 + }, + { + "epoch": 0.6197884256378344, + "grad_norm": 2.65639066696167, + "learning_rate": 4.441881918819189e-06, + "loss": 1.6036, + "step": 498 + }, + { + "epoch": 0.6210329807093964, + "grad_norm": 2.6867759227752686, + "learning_rate": 4.439575645756458e-06, + "loss": 1.5676, + "step": 499 + }, + { + "epoch": 0.6222775357809583, + "grad_norm": 2.7986443042755127, + "learning_rate": 4.437269372693727e-06, + "loss": 1.524, + "step": 500 + }, + { + "epoch": 0.6222775357809583, + "eval_loss": 1.7449125051498413, + "eval_runtime": 43.9329, + "eval_samples_per_second": 22.762, + "eval_steps_per_second": 0.956, + "step": 500 + }, + { + "epoch": 0.6235220908525202, + "grad_norm": 2.665905714035034, + "learning_rate": 4.434963099630996e-06, + "loss": 1.5635, + "step": 501 + }, + { + "epoch": 0.6247666459240822, + "grad_norm": 2.902435541152954, + "learning_rate": 4.432656826568267e-06, + "loss": 1.6377, + "step": 502 + }, + { + "epoch": 0.6260112009956441, + "grad_norm": 2.6872262954711914, + "learning_rate": 4.430350553505535e-06, + "loss": 1.5553, + "step": 503 + }, + { + "epoch": 0.627255756067206, + "grad_norm": 2.676621913909912, + "learning_rate": 4.428044280442805e-06, + "loss": 1.6101, + "step": 504 + }, + { + "epoch": 0.6285003111387679, + "grad_norm": 2.756347179412842, + "learning_rate": 4.425738007380074e-06, + "loss": 1.5884, + "step": 505 + }, + { + "epoch": 0.6297448662103298, + "grad_norm": 2.856882333755493, + "learning_rate": 4.4234317343173434e-06, + "loss": 1.5635, + "step": 506 + }, + { + "epoch": 0.6309894212818917, + "grad_norm": 2.7708330154418945, + "learning_rate": 4.421125461254613e-06, + "loss": 1.6028, + "step": 507 + }, + { + "epoch": 0.6322339763534537, + "grad_norm": 2.8167600631713867, + "learning_rate": 4.418819188191882e-06, + "loss": 1.6026, + "step": 508 + }, + { + "epoch": 0.6334785314250155, + "grad_norm": 2.54194974899292, + "learning_rate": 4.416512915129152e-06, + "loss": 1.6001, + "step": 509 + }, + { + "epoch": 0.6347230864965775, + "grad_norm": 2.683037519454956, + "learning_rate": 4.414206642066421e-06, + "loss": 1.6084, + "step": 510 + }, + { + "epoch": 0.6347230864965775, + "eval_loss": 1.7376186847686768, + "eval_runtime": 42.3026, + "eval_samples_per_second": 23.639, + "eval_steps_per_second": 0.993, + "step": 510 + }, + { + "epoch": 0.6359676415681393, + "grad_norm": 2.82627010345459, + "learning_rate": 4.4119003690036905e-06, + "loss": 1.6258, + "step": 511 + }, + { + "epoch": 0.6372121966397013, + "grad_norm": 2.6139848232269287, + "learning_rate": 4.40959409594096e-06, + "loss": 1.6029, + "step": 512 + }, + { + "epoch": 0.6384567517112633, + "grad_norm": 2.646712303161621, + "learning_rate": 4.407287822878229e-06, + "loss": 1.5602, + "step": 513 + }, + { + "epoch": 0.6397013067828251, + "grad_norm": 2.910935163497925, + "learning_rate": 4.404981549815499e-06, + "loss": 1.5757, + "step": 514 + }, + { + "epoch": 0.6409458618543871, + "grad_norm": 2.6001148223876953, + "learning_rate": 4.402675276752768e-06, + "loss": 1.5857, + "step": 515 + }, + { + "epoch": 0.642190416925949, + "grad_norm": 2.8525235652923584, + "learning_rate": 4.400369003690037e-06, + "loss": 1.6092, + "step": 516 + }, + { + "epoch": 0.6434349719975109, + "grad_norm": 2.5335488319396973, + "learning_rate": 4.398062730627306e-06, + "loss": 1.5849, + "step": 517 + }, + { + "epoch": 0.6446795270690728, + "grad_norm": 2.5787103176116943, + "learning_rate": 4.3957564575645765e-06, + "loss": 1.634, + "step": 518 + }, + { + "epoch": 0.6459240821406347, + "grad_norm": 2.6188197135925293, + "learning_rate": 4.393450184501845e-06, + "loss": 1.5477, + "step": 519 + }, + { + "epoch": 0.6471686372121966, + "grad_norm": 2.6548666954040527, + "learning_rate": 4.3911439114391145e-06, + "loss": 1.6133, + "step": 520 + }, + { + "epoch": 0.6471686372121966, + "eval_loss": 1.7393039464950562, + "eval_runtime": 45.9234, + "eval_samples_per_second": 21.775, + "eval_steps_per_second": 0.915, + "step": 520 + }, + { + "epoch": 0.6484131922837586, + "grad_norm": 2.907928228378296, + "learning_rate": 4.388837638376384e-06, + "loss": 1.587, + "step": 521 + }, + { + "epoch": 0.6496577473553204, + "grad_norm": 2.7899692058563232, + "learning_rate": 4.386531365313653e-06, + "loss": 1.6126, + "step": 522 + }, + { + "epoch": 0.6509023024268824, + "grad_norm": 2.680147886276245, + "learning_rate": 4.384225092250923e-06, + "loss": 1.565, + "step": 523 + }, + { + "epoch": 0.6521468574984443, + "grad_norm": 2.6590754985809326, + "learning_rate": 4.381918819188192e-06, + "loss": 1.6127, + "step": 524 + }, + { + "epoch": 0.6533914125700062, + "grad_norm": 2.6638906002044678, + "learning_rate": 4.379612546125462e-06, + "loss": 1.5615, + "step": 525 + }, + { + "epoch": 0.6546359676415682, + "grad_norm": 2.6518194675445557, + "learning_rate": 4.377306273062731e-06, + "loss": 1.6202, + "step": 526 + }, + { + "epoch": 0.65588052271313, + "grad_norm": 2.5323619842529297, + "learning_rate": 4.3750000000000005e-06, + "loss": 1.5305, + "step": 527 + }, + { + "epoch": 0.657125077784692, + "grad_norm": 2.6871442794799805, + "learning_rate": 4.37269372693727e-06, + "loss": 1.6043, + "step": 528 + }, + { + "epoch": 0.6583696328562539, + "grad_norm": 2.760746717453003, + "learning_rate": 4.370387453874539e-06, + "loss": 1.6059, + "step": 529 + }, + { + "epoch": 0.6596141879278158, + "grad_norm": 2.780482769012451, + "learning_rate": 4.368081180811809e-06, + "loss": 1.6082, + "step": 530 + }, + { + "epoch": 0.6596141879278158, + "eval_loss": 1.7219713926315308, + "eval_runtime": 51.8478, + "eval_samples_per_second": 19.287, + "eval_steps_per_second": 0.81, + "step": 530 + }, + { + "epoch": 0.6608587429993777, + "grad_norm": 2.605890989303589, + "learning_rate": 4.365774907749078e-06, + "loss": 1.5554, + "step": 531 + }, + { + "epoch": 0.6621032980709396, + "grad_norm": 2.731555461883545, + "learning_rate": 4.363468634686347e-06, + "loss": 1.606, + "step": 532 + }, + { + "epoch": 0.6633478531425016, + "grad_norm": 2.8356943130493164, + "learning_rate": 4.361162361623616e-06, + "loss": 1.5601, + "step": 533 + }, + { + "epoch": 0.6645924082140635, + "grad_norm": 2.7196593284606934, + "learning_rate": 4.3588560885608864e-06, + "loss": 1.5722, + "step": 534 + }, + { + "epoch": 0.6658369632856254, + "grad_norm": 2.621371269226074, + "learning_rate": 4.356549815498156e-06, + "loss": 1.6194, + "step": 535 + }, + { + "epoch": 0.6670815183571873, + "grad_norm": 2.653916120529175, + "learning_rate": 4.354243542435424e-06, + "loss": 1.5592, + "step": 536 + }, + { + "epoch": 0.6683260734287492, + "grad_norm": 2.88431453704834, + "learning_rate": 4.351937269372694e-06, + "loss": 1.6296, + "step": 537 + }, + { + "epoch": 0.6695706285003111, + "grad_norm": 2.667130470275879, + "learning_rate": 4.349630996309963e-06, + "loss": 1.5624, + "step": 538 + }, + { + "epoch": 0.6708151835718731, + "grad_norm": 2.6453566551208496, + "learning_rate": 4.347324723247233e-06, + "loss": 1.6025, + "step": 539 + }, + { + "epoch": 0.6720597386434349, + "grad_norm": 3.032271146774292, + "learning_rate": 4.345018450184502e-06, + "loss": 1.6771, + "step": 540 + }, + { + "epoch": 0.6720597386434349, + "eval_loss": 1.721895456314087, + "eval_runtime": 50.62, + "eval_samples_per_second": 19.755, + "eval_steps_per_second": 0.83, + "step": 540 + }, + { + "epoch": 0.6733042937149969, + "grad_norm": 2.8476362228393555, + "learning_rate": 4.3427121771217715e-06, + "loss": 1.5663, + "step": 541 + }, + { + "epoch": 0.6745488487865589, + "grad_norm": 2.615602970123291, + "learning_rate": 4.340405904059041e-06, + "loss": 1.5815, + "step": 542 + }, + { + "epoch": 0.6757934038581207, + "grad_norm": 3.221571683883667, + "learning_rate": 4.33809963099631e-06, + "loss": 1.5955, + "step": 543 + }, + { + "epoch": 0.6770379589296827, + "grad_norm": 2.602994203567505, + "learning_rate": 4.33579335793358e-06, + "loss": 1.5667, + "step": 544 + }, + { + "epoch": 0.6782825140012445, + "grad_norm": 2.9380433559417725, + "learning_rate": 4.333487084870848e-06, + "loss": 1.5651, + "step": 545 + }, + { + "epoch": 0.6795270690728065, + "grad_norm": 3.0689425468444824, + "learning_rate": 4.331180811808119e-06, + "loss": 1.6003, + "step": 546 + }, + { + "epoch": 0.6807716241443684, + "grad_norm": 2.770317316055298, + "learning_rate": 4.328874538745388e-06, + "loss": 1.5576, + "step": 547 + }, + { + "epoch": 0.6820161792159303, + "grad_norm": 2.8631815910339355, + "learning_rate": 4.3265682656826575e-06, + "loss": 1.5308, + "step": 548 + }, + { + "epoch": 0.6832607342874922, + "grad_norm": 2.785576820373535, + "learning_rate": 4.324261992619926e-06, + "loss": 1.6113, + "step": 549 + }, + { + "epoch": 0.6845052893590542, + "grad_norm": 2.723919630050659, + "learning_rate": 4.321955719557196e-06, + "loss": 1.5606, + "step": 550 + }, + { + "epoch": 0.6845052893590542, + "eval_loss": 1.721803069114685, + "eval_runtime": 51.6036, + "eval_samples_per_second": 19.378, + "eval_steps_per_second": 0.814, + "step": 550 + }, + { + "epoch": 0.685749844430616, + "grad_norm": 3.081198215484619, + "learning_rate": 4.319649446494466e-06, + "loss": 1.5643, + "step": 551 + }, + { + "epoch": 0.686994399502178, + "grad_norm": 2.757392168045044, + "learning_rate": 4.317343173431734e-06, + "loss": 1.5909, + "step": 552 + }, + { + "epoch": 0.6882389545737398, + "grad_norm": 3.038753032684326, + "learning_rate": 4.315036900369004e-06, + "loss": 1.6023, + "step": 553 + }, + { + "epoch": 0.6894835096453018, + "grad_norm": 2.8246452808380127, + "learning_rate": 4.312730627306273e-06, + "loss": 1.5548, + "step": 554 + }, + { + "epoch": 0.6907280647168638, + "grad_norm": 2.589320182800293, + "learning_rate": 4.310424354243543e-06, + "loss": 1.5424, + "step": 555 + }, + { + "epoch": 0.6919726197884256, + "grad_norm": 2.6413373947143555, + "learning_rate": 4.308118081180812e-06, + "loss": 1.6243, + "step": 556 + }, + { + "epoch": 0.6932171748599876, + "grad_norm": 2.764784336090088, + "learning_rate": 4.3058118081180815e-06, + "loss": 1.589, + "step": 557 + }, + { + "epoch": 0.6944617299315494, + "grad_norm": 2.806821823120117, + "learning_rate": 4.303505535055351e-06, + "loss": 1.621, + "step": 558 + }, + { + "epoch": 0.6957062850031114, + "grad_norm": 2.718017578125, + "learning_rate": 4.30119926199262e-06, + "loss": 1.5773, + "step": 559 + }, + { + "epoch": 0.6969508400746733, + "grad_norm": 2.7620160579681396, + "learning_rate": 4.29889298892989e-06, + "loss": 1.5868, + "step": 560 + }, + { + "epoch": 0.6969508400746733, + "eval_loss": 1.7152249813079834, + "eval_runtime": 52.0463, + "eval_samples_per_second": 19.214, + "eval_steps_per_second": 0.807, + "step": 560 + }, + { + "epoch": 0.6981953951462352, + "grad_norm": 2.66686749458313, + "learning_rate": 4.296586715867159e-06, + "loss": 1.534, + "step": 561 + }, + { + "epoch": 0.6994399502177971, + "grad_norm": 2.812577247619629, + "learning_rate": 4.2942804428044286e-06, + "loss": 1.5501, + "step": 562 + }, + { + "epoch": 0.7006845052893591, + "grad_norm": 2.578508138656616, + "learning_rate": 4.291974169741698e-06, + "loss": 1.6286, + "step": 563 + }, + { + "epoch": 0.701929060360921, + "grad_norm": 2.524249315261841, + "learning_rate": 4.289667896678967e-06, + "loss": 1.5719, + "step": 564 + }, + { + "epoch": 0.7031736154324829, + "grad_norm": 2.827235460281372, + "learning_rate": 4.287361623616236e-06, + "loss": 1.5333, + "step": 565 + }, + { + "epoch": 0.7044181705040448, + "grad_norm": 2.6359963417053223, + "learning_rate": 4.285055350553506e-06, + "loss": 1.5849, + "step": 566 + }, + { + "epoch": 0.7056627255756067, + "grad_norm": 2.930530071258545, + "learning_rate": 4.282749077490776e-06, + "loss": 1.5672, + "step": 567 + }, + { + "epoch": 0.7069072806471687, + "grad_norm": 2.750102996826172, + "learning_rate": 4.280442804428044e-06, + "loss": 1.5696, + "step": 568 + }, + { + "epoch": 0.7081518357187305, + "grad_norm": 2.869690418243408, + "learning_rate": 4.278136531365314e-06, + "loss": 1.5689, + "step": 569 + }, + { + "epoch": 0.7093963907902925, + "grad_norm": 2.954852819442749, + "learning_rate": 4.275830258302583e-06, + "loss": 1.5931, + "step": 570 + }, + { + "epoch": 0.7093963907902925, + "eval_loss": 1.7143300771713257, + "eval_runtime": 52.7293, + "eval_samples_per_second": 18.965, + "eval_steps_per_second": 0.797, + "step": 570 + }, + { + "epoch": 0.7106409458618543, + "grad_norm": 2.702223539352417, + "learning_rate": 4.273523985239853e-06, + "loss": 1.5635, + "step": 571 + }, + { + "epoch": 0.7118855009334163, + "grad_norm": 2.689995050430298, + "learning_rate": 4.271217712177122e-06, + "loss": 1.5545, + "step": 572 + }, + { + "epoch": 0.7131300560049783, + "grad_norm": 2.68979811668396, + "learning_rate": 4.268911439114391e-06, + "loss": 1.5404, + "step": 573 + }, + { + "epoch": 0.7143746110765401, + "grad_norm": 2.7477986812591553, + "learning_rate": 4.266605166051661e-06, + "loss": 1.5719, + "step": 574 + }, + { + "epoch": 0.7156191661481021, + "grad_norm": 2.975778341293335, + "learning_rate": 4.26429889298893e-06, + "loss": 1.5587, + "step": 575 + }, + { + "epoch": 0.716863721219664, + "grad_norm": 2.658170700073242, + "learning_rate": 4.2619926199262e-06, + "loss": 1.5746, + "step": 576 + }, + { + "epoch": 0.7181082762912259, + "grad_norm": 2.8825011253356934, + "learning_rate": 4.259686346863469e-06, + "loss": 1.5246, + "step": 577 + }, + { + "epoch": 0.7193528313627878, + "grad_norm": 2.845280170440674, + "learning_rate": 4.2573800738007385e-06, + "loss": 1.5703, + "step": 578 + }, + { + "epoch": 0.7205973864343497, + "grad_norm": 2.660616636276245, + "learning_rate": 4.255073800738008e-06, + "loss": 1.5373, + "step": 579 + }, + { + "epoch": 0.7218419415059116, + "grad_norm": 2.749447822570801, + "learning_rate": 4.252767527675277e-06, + "loss": 1.59, + "step": 580 + }, + { + "epoch": 0.7218419415059116, + "eval_loss": 1.7149444818496704, + "eval_runtime": 52.2099, + "eval_samples_per_second": 19.153, + "eval_steps_per_second": 0.804, + "step": 580 + }, + { + "epoch": 0.7230864965774736, + "grad_norm": 2.813328742980957, + "learning_rate": 4.250461254612546e-06, + "loss": 1.5694, + "step": 581 + }, + { + "epoch": 0.7243310516490354, + "grad_norm": 2.823866844177246, + "learning_rate": 4.248154981549816e-06, + "loss": 1.5143, + "step": 582 + }, + { + "epoch": 0.7255756067205974, + "grad_norm": 2.71337890625, + "learning_rate": 4.245848708487086e-06, + "loss": 1.5915, + "step": 583 + }, + { + "epoch": 0.7268201617921594, + "grad_norm": 2.718085765838623, + "learning_rate": 4.243542435424355e-06, + "loss": 1.604, + "step": 584 + }, + { + "epoch": 0.7280647168637212, + "grad_norm": 2.5741796493530273, + "learning_rate": 4.241236162361624e-06, + "loss": 1.5398, + "step": 585 + }, + { + "epoch": 0.7293092719352832, + "grad_norm": 2.727114200592041, + "learning_rate": 4.238929889298893e-06, + "loss": 1.5886, + "step": 586 + }, + { + "epoch": 0.730553827006845, + "grad_norm": 2.6703338623046875, + "learning_rate": 4.236623616236163e-06, + "loss": 1.5959, + "step": 587 + }, + { + "epoch": 0.731798382078407, + "grad_norm": 2.750326633453369, + "learning_rate": 4.234317343173432e-06, + "loss": 1.5916, + "step": 588 + }, + { + "epoch": 0.7330429371499689, + "grad_norm": 2.6680715084075928, + "learning_rate": 4.232011070110701e-06, + "loss": 1.562, + "step": 589 + }, + { + "epoch": 0.7342874922215308, + "grad_norm": 2.6603832244873047, + "learning_rate": 4.229704797047971e-06, + "loss": 1.6121, + "step": 590 + }, + { + "epoch": 0.7342874922215308, + "eval_loss": 1.7144734859466553, + "eval_runtime": 50.5317, + "eval_samples_per_second": 19.79, + "eval_steps_per_second": 0.831, + "step": 590 + }, + { + "epoch": 0.7355320472930927, + "grad_norm": 2.7868523597717285, + "learning_rate": 4.22739852398524e-06, + "loss": 1.5488, + "step": 591 + }, + { + "epoch": 0.7367766023646546, + "grad_norm": 2.755270004272461, + "learning_rate": 4.2250922509225096e-06, + "loss": 1.5801, + "step": 592 + }, + { + "epoch": 0.7380211574362165, + "grad_norm": 2.7614126205444336, + "learning_rate": 4.222785977859779e-06, + "loss": 1.5507, + "step": 593 + }, + { + "epoch": 0.7392657125077785, + "grad_norm": 2.8674862384796143, + "learning_rate": 4.220479704797048e-06, + "loss": 1.5807, + "step": 594 + }, + { + "epoch": 0.7405102675793404, + "grad_norm": 2.7366859912872314, + "learning_rate": 4.218173431734318e-06, + "loss": 1.6594, + "step": 595 + }, + { + "epoch": 0.7417548226509023, + "grad_norm": 2.6231045722961426, + "learning_rate": 4.215867158671587e-06, + "loss": 1.5609, + "step": 596 + }, + { + "epoch": 0.7429993777224643, + "grad_norm": 2.7694146633148193, + "learning_rate": 4.213560885608857e-06, + "loss": 1.5889, + "step": 597 + }, + { + "epoch": 0.7442439327940261, + "grad_norm": 2.6275386810302734, + "learning_rate": 4.211254612546125e-06, + "loss": 1.575, + "step": 598 + }, + { + "epoch": 0.7454884878655881, + "grad_norm": 2.9413866996765137, + "learning_rate": 4.2089483394833955e-06, + "loss": 1.589, + "step": 599 + }, + { + "epoch": 0.7467330429371499, + "grad_norm": 2.768606185913086, + "learning_rate": 4.206642066420665e-06, + "loss": 1.5738, + "step": 600 + }, + { + "epoch": 0.7467330429371499, + "eval_loss": 1.7148027420043945, + "eval_runtime": 54.7031, + "eval_samples_per_second": 18.28, + "eval_steps_per_second": 0.768, + "step": 600 + }, + { + "epoch": 0.7479775980087119, + "grad_norm": 2.7917706966400146, + "learning_rate": 4.2043357933579335e-06, + "loss": 1.499, + "step": 601 + }, + { + "epoch": 0.7492221530802738, + "grad_norm": 2.5982208251953125, + "learning_rate": 4.202029520295203e-06, + "loss": 1.5113, + "step": 602 + }, + { + "epoch": 0.7504667081518357, + "grad_norm": 2.692032814025879, + "learning_rate": 4.199723247232473e-06, + "loss": 1.4721, + "step": 603 + }, + { + "epoch": 0.7517112632233977, + "grad_norm": 2.975860834121704, + "learning_rate": 4.197416974169742e-06, + "loss": 1.5147, + "step": 604 + }, + { + "epoch": 0.7529558182949595, + "grad_norm": 2.758610248565674, + "learning_rate": 4.195110701107011e-06, + "loss": 1.5674, + "step": 605 + }, + { + "epoch": 0.7542003733665215, + "grad_norm": 2.7620184421539307, + "learning_rate": 4.192804428044281e-06, + "loss": 1.5669, + "step": 606 + }, + { + "epoch": 0.7554449284380834, + "grad_norm": 3.043940305709839, + "learning_rate": 4.19049815498155e-06, + "loss": 1.587, + "step": 607 + }, + { + "epoch": 0.7566894835096453, + "grad_norm": 2.7482151985168457, + "learning_rate": 4.1881918819188195e-06, + "loss": 1.5451, + "step": 608 + }, + { + "epoch": 0.7579340385812072, + "grad_norm": 2.7259294986724854, + "learning_rate": 4.185885608856089e-06, + "loss": 1.5295, + "step": 609 + }, + { + "epoch": 0.7591785936527692, + "grad_norm": 2.6340832710266113, + "learning_rate": 4.183579335793358e-06, + "loss": 1.5335, + "step": 610 + }, + { + "epoch": 0.7591785936527692, + "eval_loss": 1.7038393020629883, + "eval_runtime": 44.0792, + "eval_samples_per_second": 22.686, + "eval_steps_per_second": 0.953, + "step": 610 + }, + { + "epoch": 0.760423148724331, + "grad_norm": 2.6144909858703613, + "learning_rate": 4.181273062730628e-06, + "loss": 1.5907, + "step": 611 + }, + { + "epoch": 0.761667703795893, + "grad_norm": 2.7754175662994385, + "learning_rate": 4.178966789667897e-06, + "loss": 1.5773, + "step": 612 + }, + { + "epoch": 0.7629122588674548, + "grad_norm": 2.6313252449035645, + "learning_rate": 4.176660516605167e-06, + "loss": 1.5675, + "step": 613 + }, + { + "epoch": 0.7641568139390168, + "grad_norm": 2.549074649810791, + "learning_rate": 4.174354243542435e-06, + "loss": 1.595, + "step": 614 + }, + { + "epoch": 0.7654013690105788, + "grad_norm": 2.611804246902466, + "learning_rate": 4.1720479704797054e-06, + "loss": 1.5416, + "step": 615 + }, + { + "epoch": 0.7666459240821406, + "grad_norm": 2.6322927474975586, + "learning_rate": 4.169741697416975e-06, + "loss": 1.604, + "step": 616 + }, + { + "epoch": 0.7678904791537026, + "grad_norm": 2.5792219638824463, + "learning_rate": 4.1674354243542434e-06, + "loss": 1.5976, + "step": 617 + }, + { + "epoch": 0.7691350342252644, + "grad_norm": 2.845416307449341, + "learning_rate": 4.165129151291513e-06, + "loss": 1.5945, + "step": 618 + }, + { + "epoch": 0.7703795892968264, + "grad_norm": 2.8647871017456055, + "learning_rate": 4.162822878228783e-06, + "loss": 1.5637, + "step": 619 + }, + { + "epoch": 0.7716241443683883, + "grad_norm": 2.590719699859619, + "learning_rate": 4.1605166051660526e-06, + "loss": 1.556, + "step": 620 + }, + { + "epoch": 0.7716241443683883, + "eval_loss": 1.7115222215652466, + "eval_runtime": 50.3905, + "eval_samples_per_second": 19.845, + "eval_steps_per_second": 0.833, + "step": 620 + }, + { + "epoch": 0.7728686994399502, + "grad_norm": 2.626723527908325, + "learning_rate": 4.158210332103321e-06, + "loss": 1.5486, + "step": 621 + }, + { + "epoch": 0.7741132545115121, + "grad_norm": 2.713069438934326, + "learning_rate": 4.1559040590405906e-06, + "loss": 1.5474, + "step": 622 + }, + { + "epoch": 0.7753578095830741, + "grad_norm": 2.685563087463379, + "learning_rate": 4.15359778597786e-06, + "loss": 1.5529, + "step": 623 + }, + { + "epoch": 0.776602364654636, + "grad_norm": 2.690586805343628, + "learning_rate": 4.151291512915129e-06, + "loss": 1.5293, + "step": 624 + }, + { + "epoch": 0.7778469197261979, + "grad_norm": 2.865345001220703, + "learning_rate": 4.148985239852399e-06, + "loss": 1.5095, + "step": 625 + }, + { + "epoch": 0.7790914747977598, + "grad_norm": 2.92393159866333, + "learning_rate": 4.146678966789668e-06, + "loss": 1.5561, + "step": 626 + }, + { + "epoch": 0.7803360298693217, + "grad_norm": 2.554480791091919, + "learning_rate": 4.144372693726938e-06, + "loss": 1.4876, + "step": 627 + }, + { + "epoch": 0.7815805849408837, + "grad_norm": 2.6128833293914795, + "learning_rate": 4.142066420664207e-06, + "loss": 1.5378, + "step": 628 + }, + { + "epoch": 0.7828251400124455, + "grad_norm": 3.0110888481140137, + "learning_rate": 4.1397601476014765e-06, + "loss": 1.5859, + "step": 629 + }, + { + "epoch": 0.7840696950840075, + "grad_norm": 2.8152706623077393, + "learning_rate": 4.137453874538745e-06, + "loss": 1.5841, + "step": 630 + }, + { + "epoch": 0.7840696950840075, + "eval_loss": 1.702181339263916, + "eval_runtime": 53.6926, + "eval_samples_per_second": 18.625, + "eval_steps_per_second": 0.782, + "step": 630 + }, + { + "epoch": 0.7853142501555694, + "grad_norm": 2.8634233474731445, + "learning_rate": 4.135147601476015e-06, + "loss": 1.564, + "step": 631 + }, + { + "epoch": 0.7865588052271313, + "grad_norm": 2.8939666748046875, + "learning_rate": 4.132841328413285e-06, + "loss": 1.5859, + "step": 632 + }, + { + "epoch": 0.7878033602986932, + "grad_norm": 2.6560444831848145, + "learning_rate": 4.130535055350554e-06, + "loss": 1.5199, + "step": 633 + }, + { + "epoch": 0.7890479153702551, + "grad_norm": 2.964721202850342, + "learning_rate": 4.128228782287823e-06, + "loss": 1.5888, + "step": 634 + }, + { + "epoch": 0.790292470441817, + "grad_norm": 2.74668288230896, + "learning_rate": 4.125922509225092e-06, + "loss": 1.5847, + "step": 635 + }, + { + "epoch": 0.791537025513379, + "grad_norm": 2.723123550415039, + "learning_rate": 4.1236162361623625e-06, + "loss": 1.5166, + "step": 636 + }, + { + "epoch": 0.7927815805849409, + "grad_norm": 2.7265713214874268, + "learning_rate": 4.121309963099631e-06, + "loss": 1.5954, + "step": 637 + }, + { + "epoch": 0.7940261356565028, + "grad_norm": 2.979126214981079, + "learning_rate": 4.1190036900369005e-06, + "loss": 1.5583, + "step": 638 + }, + { + "epoch": 0.7952706907280647, + "grad_norm": 2.844376802444458, + "learning_rate": 4.11669741697417e-06, + "loss": 1.5564, + "step": 639 + }, + { + "epoch": 0.7965152457996266, + "grad_norm": 2.7583703994750977, + "learning_rate": 4.114391143911439e-06, + "loss": 1.5094, + "step": 640 + }, + { + "epoch": 0.7965152457996266, + "eval_loss": 1.6986565589904785, + "eval_runtime": 47.3549, + "eval_samples_per_second": 21.117, + "eval_steps_per_second": 0.887, + "step": 640 + }, + { + "epoch": 0.7977598008711886, + "grad_norm": 2.8942224979400635, + "learning_rate": 4.112084870848709e-06, + "loss": 1.5504, + "step": 641 + }, + { + "epoch": 0.7990043559427504, + "grad_norm": 2.6590495109558105, + "learning_rate": 4.109778597785978e-06, + "loss": 1.5227, + "step": 642 + }, + { + "epoch": 0.8002489110143124, + "grad_norm": 2.5988378524780273, + "learning_rate": 4.107472324723248e-06, + "loss": 1.5054, + "step": 643 + }, + { + "epoch": 0.8014934660858744, + "grad_norm": 2.787335157394409, + "learning_rate": 4.105166051660517e-06, + "loss": 1.5844, + "step": 644 + }, + { + "epoch": 0.8027380211574362, + "grad_norm": 2.8202896118164062, + "learning_rate": 4.1028597785977864e-06, + "loss": 1.5596, + "step": 645 + }, + { + "epoch": 0.8039825762289982, + "grad_norm": 2.65376615524292, + "learning_rate": 4.100553505535056e-06, + "loss": 1.5893, + "step": 646 + }, + { + "epoch": 0.80522713130056, + "grad_norm": 2.5933308601379395, + "learning_rate": 4.098247232472325e-06, + "loss": 1.5403, + "step": 647 + }, + { + "epoch": 0.806471686372122, + "grad_norm": 2.777070999145508, + "learning_rate": 4.095940959409595e-06, + "loss": 1.5231, + "step": 648 + }, + { + "epoch": 0.8077162414436839, + "grad_norm": 2.6427664756774902, + "learning_rate": 4.093634686346864e-06, + "loss": 1.5937, + "step": 649 + }, + { + "epoch": 0.8089607965152458, + "grad_norm": 2.651561737060547, + "learning_rate": 4.091328413284133e-06, + "loss": 1.4847, + "step": 650 + }, + { + "epoch": 0.8089607965152458, + "eval_loss": 1.7089996337890625, + "eval_runtime": 47.4346, + "eval_samples_per_second": 21.082, + "eval_steps_per_second": 0.885, + "step": 650 + }, + { + "epoch": 0.8102053515868077, + "grad_norm": 2.844067335128784, + "learning_rate": 4.089022140221402e-06, + "loss": 1.488, + "step": 651 + }, + { + "epoch": 0.8114499066583696, + "grad_norm": 2.603480577468872, + "learning_rate": 4.086715867158672e-06, + "loss": 1.5248, + "step": 652 + }, + { + "epoch": 0.8126944617299315, + "grad_norm": 2.789322853088379, + "learning_rate": 4.084409594095941e-06, + "loss": 1.5119, + "step": 653 + }, + { + "epoch": 0.8139390168014935, + "grad_norm": 2.7183773517608643, + "learning_rate": 4.08210332103321e-06, + "loss": 1.5123, + "step": 654 + }, + { + "epoch": 0.8151835718730553, + "grad_norm": 2.80613374710083, + "learning_rate": 4.07979704797048e-06, + "loss": 1.5526, + "step": 655 + }, + { + "epoch": 0.8164281269446173, + "grad_norm": 2.641035795211792, + "learning_rate": 4.077490774907749e-06, + "loss": 1.6127, + "step": 656 + }, + { + "epoch": 0.8176726820161793, + "grad_norm": 2.773684024810791, + "learning_rate": 4.075184501845019e-06, + "loss": 1.5404, + "step": 657 + }, + { + "epoch": 0.8189172370877411, + "grad_norm": 2.6951382160186768, + "learning_rate": 4.072878228782288e-06, + "loss": 1.4928, + "step": 658 + }, + { + "epoch": 0.8201617921593031, + "grad_norm": 2.931771755218506, + "learning_rate": 4.0705719557195575e-06, + "loss": 1.5949, + "step": 659 + }, + { + "epoch": 0.8214063472308649, + "grad_norm": 2.8786075115203857, + "learning_rate": 4.068265682656827e-06, + "loss": 1.5582, + "step": 660 + }, + { + "epoch": 0.8214063472308649, + "eval_loss": 1.698564052581787, + "eval_runtime": 46.6299, + "eval_samples_per_second": 21.445, + "eval_steps_per_second": 0.901, + "step": 660 + }, + { + "epoch": 0.8226509023024269, + "grad_norm": 2.7402451038360596, + "learning_rate": 4.065959409594096e-06, + "loss": 1.5562, + "step": 661 + }, + { + "epoch": 0.8238954573739888, + "grad_norm": 2.7968692779541016, + "learning_rate": 4.063653136531366e-06, + "loss": 1.556, + "step": 662 + }, + { + "epoch": 0.8251400124455507, + "grad_norm": 2.8233132362365723, + "learning_rate": 4.061346863468635e-06, + "loss": 1.5527, + "step": 663 + }, + { + "epoch": 0.8263845675171126, + "grad_norm": 2.713390827178955, + "learning_rate": 4.059040590405905e-06, + "loss": 1.6012, + "step": 664 + }, + { + "epoch": 0.8276291225886746, + "grad_norm": 2.7783989906311035, + "learning_rate": 4.056734317343174e-06, + "loss": 1.5247, + "step": 665 + }, + { + "epoch": 0.8288736776602365, + "grad_norm": 2.808554172515869, + "learning_rate": 4.054428044280443e-06, + "loss": 1.5412, + "step": 666 + }, + { + "epoch": 0.8301182327317984, + "grad_norm": 2.8476831912994385, + "learning_rate": 4.052121771217712e-06, + "loss": 1.5307, + "step": 667 + }, + { + "epoch": 0.8313627878033603, + "grad_norm": 2.8287601470947266, + "learning_rate": 4.049815498154982e-06, + "loss": 1.6021, + "step": 668 + }, + { + "epoch": 0.8326073428749222, + "grad_norm": 2.975510835647583, + "learning_rate": 4.047509225092252e-06, + "loss": 1.5474, + "step": 669 + }, + { + "epoch": 0.8338518979464842, + "grad_norm": 2.723236560821533, + "learning_rate": 4.04520295202952e-06, + "loss": 1.5242, + "step": 670 + }, + { + "epoch": 0.8338518979464842, + "eval_loss": 1.7005239725112915, + "eval_runtime": 49.4504, + "eval_samples_per_second": 20.222, + "eval_steps_per_second": 0.849, + "step": 670 + }, + { + "epoch": 0.835096453018046, + "grad_norm": 2.7521748542785645, + "learning_rate": 4.04289667896679e-06, + "loss": 1.5467, + "step": 671 + }, + { + "epoch": 0.836341008089608, + "grad_norm": 2.7770044803619385, + "learning_rate": 4.04059040590406e-06, + "loss": 1.5407, + "step": 672 + }, + { + "epoch": 0.8375855631611698, + "grad_norm": 2.744323492050171, + "learning_rate": 4.038284132841329e-06, + "loss": 1.5422, + "step": 673 + }, + { + "epoch": 0.8388301182327318, + "grad_norm": 2.6699817180633545, + "learning_rate": 4.035977859778598e-06, + "loss": 1.5508, + "step": 674 + }, + { + "epoch": 0.8400746733042938, + "grad_norm": 2.705273151397705, + "learning_rate": 4.0336715867158674e-06, + "loss": 1.5099, + "step": 675 + }, + { + "epoch": 0.8413192283758556, + "grad_norm": 2.9485747814178467, + "learning_rate": 4.031365313653137e-06, + "loss": 1.5529, + "step": 676 + }, + { + "epoch": 0.8425637834474176, + "grad_norm": 3.1750423908233643, + "learning_rate": 4.029059040590406e-06, + "loss": 1.5894, + "step": 677 + }, + { + "epoch": 0.8438083385189795, + "grad_norm": 2.76448655128479, + "learning_rate": 4.026752767527676e-06, + "loss": 1.5752, + "step": 678 + }, + { + "epoch": 0.8450528935905414, + "grad_norm": 2.676708698272705, + "learning_rate": 4.024446494464945e-06, + "loss": 1.5487, + "step": 679 + }, + { + "epoch": 0.8462974486621033, + "grad_norm": 2.6832051277160645, + "learning_rate": 4.0221402214022145e-06, + "loss": 1.5055, + "step": 680 + }, + { + "epoch": 0.8462974486621033, + "eval_loss": 1.6945017576217651, + "eval_runtime": 43.9369, + "eval_samples_per_second": 22.76, + "eval_steps_per_second": 0.956, + "step": 680 + }, + { + "epoch": 0.8475420037336652, + "grad_norm": 2.6812336444854736, + "learning_rate": 4.019833948339484e-06, + "loss": 1.5315, + "step": 681 + }, + { + "epoch": 0.8487865588052271, + "grad_norm": 2.9394326210021973, + "learning_rate": 4.017527675276753e-06, + "loss": 1.5636, + "step": 682 + }, + { + "epoch": 0.8500311138767891, + "grad_norm": 2.944952964782715, + "learning_rate": 4.015221402214022e-06, + "loss": 1.56, + "step": 683 + }, + { + "epoch": 0.8512756689483509, + "grad_norm": 2.7475314140319824, + "learning_rate": 4.012915129151292e-06, + "loss": 1.5442, + "step": 684 + }, + { + "epoch": 0.8525202240199129, + "grad_norm": 3.0572879314422607, + "learning_rate": 4.010608856088562e-06, + "loss": 1.5023, + "step": 685 + }, + { + "epoch": 0.8537647790914747, + "grad_norm": 2.75365948677063, + "learning_rate": 4.00830258302583e-06, + "loss": 1.5414, + "step": 686 + }, + { + "epoch": 0.8550093341630367, + "grad_norm": 2.5714449882507324, + "learning_rate": 4.0059963099631e-06, + "loss": 1.5027, + "step": 687 + }, + { + "epoch": 0.8562538892345987, + "grad_norm": 2.755167007446289, + "learning_rate": 4.003690036900369e-06, + "loss": 1.5788, + "step": 688 + }, + { + "epoch": 0.8574984443061605, + "grad_norm": 2.798967123031616, + "learning_rate": 4.0013837638376385e-06, + "loss": 1.5162, + "step": 689 + }, + { + "epoch": 0.8587429993777225, + "grad_norm": 2.803614854812622, + "learning_rate": 3.999077490774908e-06, + "loss": 1.5079, + "step": 690 + }, + { + "epoch": 0.8587429993777225, + "eval_loss": 1.6924962997436523, + "eval_runtime": 44.47, + "eval_samples_per_second": 22.487, + "eval_steps_per_second": 0.944, + "step": 690 + }, + { + "epoch": 0.8599875544492844, + "grad_norm": 2.6898066997528076, + "learning_rate": 3.996771217712177e-06, + "loss": 1.5478, + "step": 691 + }, + { + "epoch": 0.8612321095208463, + "grad_norm": 2.6376144886016846, + "learning_rate": 3.994464944649447e-06, + "loss": 1.5362, + "step": 692 + }, + { + "epoch": 0.8624766645924082, + "grad_norm": 2.7240405082702637, + "learning_rate": 3.992158671586716e-06, + "loss": 1.5502, + "step": 693 + }, + { + "epoch": 0.8637212196639701, + "grad_norm": 2.929445266723633, + "learning_rate": 3.989852398523986e-06, + "loss": 1.5799, + "step": 694 + }, + { + "epoch": 0.864965774735532, + "grad_norm": 2.593223810195923, + "learning_rate": 3.987546125461255e-06, + "loss": 1.5352, + "step": 695 + }, + { + "epoch": 0.866210329807094, + "grad_norm": 2.7710154056549072, + "learning_rate": 3.9852398523985245e-06, + "loss": 1.5048, + "step": 696 + }, + { + "epoch": 0.8674548848786559, + "grad_norm": 2.7370848655700684, + "learning_rate": 3.982933579335794e-06, + "loss": 1.5233, + "step": 697 + }, + { + "epoch": 0.8686994399502178, + "grad_norm": 2.8632168769836426, + "learning_rate": 3.980627306273063e-06, + "loss": 1.5435, + "step": 698 + }, + { + "epoch": 0.8699439950217797, + "grad_norm": 2.752298593521118, + "learning_rate": 3.978321033210332e-06, + "loss": 1.5728, + "step": 699 + }, + { + "epoch": 0.8711885500933416, + "grad_norm": 2.8345625400543213, + "learning_rate": 3.976014760147602e-06, + "loss": 1.5727, + "step": 700 + }, + { + "epoch": 0.8711885500933416, + "eval_loss": 1.6855305433273315, + "eval_runtime": 43.3683, + "eval_samples_per_second": 23.058, + "eval_steps_per_second": 0.968, + "step": 700 + }, + { + "epoch": 0.8724331051649036, + "grad_norm": 2.8135335445404053, + "learning_rate": 3.973708487084872e-06, + "loss": 1.5086, + "step": 701 + }, + { + "epoch": 0.8736776602364654, + "grad_norm": 2.7416603565216064, + "learning_rate": 3.97140221402214e-06, + "loss": 1.545, + "step": 702 + }, + { + "epoch": 0.8749222153080274, + "grad_norm": 2.6902289390563965, + "learning_rate": 3.96909594095941e-06, + "loss": 1.5509, + "step": 703 + }, + { + "epoch": 0.8761667703795893, + "grad_norm": 2.6537084579467773, + "learning_rate": 3.966789667896679e-06, + "loss": 1.562, + "step": 704 + }, + { + "epoch": 0.8774113254511512, + "grad_norm": 2.6979284286499023, + "learning_rate": 3.964483394833948e-06, + "loss": 1.4829, + "step": 705 + }, + { + "epoch": 0.8786558805227132, + "grad_norm": 2.5795764923095703, + "learning_rate": 3.962177121771218e-06, + "loss": 1.4677, + "step": 706 + }, + { + "epoch": 0.879900435594275, + "grad_norm": 3.0185086727142334, + "learning_rate": 3.959870848708487e-06, + "loss": 1.5608, + "step": 707 + }, + { + "epoch": 0.881144990665837, + "grad_norm": 2.6458113193511963, + "learning_rate": 3.957564575645757e-06, + "loss": 1.5342, + "step": 708 + }, + { + "epoch": 0.8823895457373989, + "grad_norm": 2.7985665798187256, + "learning_rate": 3.955258302583026e-06, + "loss": 1.527, + "step": 709 + }, + { + "epoch": 0.8836341008089608, + "grad_norm": 2.9069082736968994, + "learning_rate": 3.9529520295202955e-06, + "loss": 1.5156, + "step": 710 + }, + { + "epoch": 0.8836341008089608, + "eval_loss": 1.6898822784423828, + "eval_runtime": 42.8749, + "eval_samples_per_second": 23.324, + "eval_steps_per_second": 0.98, + "step": 710 + }, + { + "epoch": 0.8848786558805227, + "grad_norm": 2.760143995285034, + "learning_rate": 3.950645756457565e-06, + "loss": 1.5124, + "step": 711 + }, + { + "epoch": 0.8861232109520847, + "grad_norm": 2.8085529804229736, + "learning_rate": 3.948339483394834e-06, + "loss": 1.5862, + "step": 712 + }, + { + "epoch": 0.8873677660236465, + "grad_norm": 2.909905195236206, + "learning_rate": 3.946033210332104e-06, + "loss": 1.5557, + "step": 713 + }, + { + "epoch": 0.8886123210952085, + "grad_norm": 2.826899290084839, + "learning_rate": 3.943726937269373e-06, + "loss": 1.5277, + "step": 714 + }, + { + "epoch": 0.8898568761667703, + "grad_norm": 2.6269052028656006, + "learning_rate": 3.941420664206642e-06, + "loss": 1.4819, + "step": 715 + }, + { + "epoch": 0.8911014312383323, + "grad_norm": 2.6687698364257812, + "learning_rate": 3.939114391143912e-06, + "loss": 1.5156, + "step": 716 + }, + { + "epoch": 0.8923459863098943, + "grad_norm": 2.9605629444122314, + "learning_rate": 3.9368081180811815e-06, + "loss": 1.5358, + "step": 717 + }, + { + "epoch": 0.8935905413814561, + "grad_norm": 3.0367166996002197, + "learning_rate": 3.934501845018451e-06, + "loss": 1.5847, + "step": 718 + }, + { + "epoch": 0.8948350964530181, + "grad_norm": 2.528796434402466, + "learning_rate": 3.9321955719557195e-06, + "loss": 1.4851, + "step": 719 + }, + { + "epoch": 0.8960796515245799, + "grad_norm": 2.7116613388061523, + "learning_rate": 3.929889298892989e-06, + "loss": 1.4706, + "step": 720 + }, + { + "epoch": 0.8960796515245799, + "eval_loss": 1.6845883131027222, + "eval_runtime": 44.4734, + "eval_samples_per_second": 22.485, + "eval_steps_per_second": 0.944, + "step": 720 + }, + { + "epoch": 0.8973242065961419, + "grad_norm": 2.771864414215088, + "learning_rate": 3.927583025830259e-06, + "loss": 1.5444, + "step": 721 + }, + { + "epoch": 0.8985687616677038, + "grad_norm": 2.773746967315674, + "learning_rate": 3.925276752767528e-06, + "loss": 1.564, + "step": 722 + }, + { + "epoch": 0.8998133167392657, + "grad_norm": 2.7069854736328125, + "learning_rate": 3.922970479704797e-06, + "loss": 1.5601, + "step": 723 + }, + { + "epoch": 0.9010578718108276, + "grad_norm": 2.840261697769165, + "learning_rate": 3.920664206642067e-06, + "loss": 1.5733, + "step": 724 + }, + { + "epoch": 0.9023024268823896, + "grad_norm": 2.709897041320801, + "learning_rate": 3.918357933579336e-06, + "loss": 1.528, + "step": 725 + }, + { + "epoch": 0.9035469819539514, + "grad_norm": 2.662367343902588, + "learning_rate": 3.9160516605166055e-06, + "loss": 1.5532, + "step": 726 + }, + { + "epoch": 0.9047915370255134, + "grad_norm": 2.6775717735290527, + "learning_rate": 3.913745387453875e-06, + "loss": 1.5464, + "step": 727 + }, + { + "epoch": 0.9060360920970753, + "grad_norm": 2.617842674255371, + "learning_rate": 3.911439114391144e-06, + "loss": 1.4539, + "step": 728 + }, + { + "epoch": 0.9072806471686372, + "grad_norm": 2.668649435043335, + "learning_rate": 3.909132841328414e-06, + "loss": 1.5627, + "step": 729 + }, + { + "epoch": 0.9085252022401992, + "grad_norm": 2.5331950187683105, + "learning_rate": 3.906826568265683e-06, + "loss": 1.5165, + "step": 730 + }, + { + "epoch": 0.9085252022401992, + "eval_loss": 1.6860331296920776, + "eval_runtime": 46.094, + "eval_samples_per_second": 21.695, + "eval_steps_per_second": 0.911, + "step": 730 + }, + { + "epoch": 0.909769757311761, + "grad_norm": 2.602492332458496, + "learning_rate": 3.9045202952029526e-06, + "loss": 1.5145, + "step": 731 + }, + { + "epoch": 0.911014312383323, + "grad_norm": 2.632782459259033, + "learning_rate": 3.902214022140222e-06, + "loss": 1.4413, + "step": 732 + }, + { + "epoch": 0.9122588674548848, + "grad_norm": 2.6025912761688232, + "learning_rate": 3.899907749077491e-06, + "loss": 1.5263, + "step": 733 + }, + { + "epoch": 0.9135034225264468, + "grad_norm": 2.770116090774536, + "learning_rate": 3.897601476014761e-06, + "loss": 1.5514, + "step": 734 + }, + { + "epoch": 0.9147479775980087, + "grad_norm": 2.7822961807250977, + "learning_rate": 3.895295202952029e-06, + "loss": 1.5296, + "step": 735 + }, + { + "epoch": 0.9159925326695706, + "grad_norm": 2.6689720153808594, + "learning_rate": 3.892988929889299e-06, + "loss": 1.4552, + "step": 736 + }, + { + "epoch": 0.9172370877411326, + "grad_norm": 2.8021798133850098, + "learning_rate": 3.890682656826569e-06, + "loss": 1.5308, + "step": 737 + }, + { + "epoch": 0.9184816428126945, + "grad_norm": 2.575313091278076, + "learning_rate": 3.888376383763838e-06, + "loss": 1.4807, + "step": 738 + }, + { + "epoch": 0.9197261978842564, + "grad_norm": 2.5955779552459717, + "learning_rate": 3.886070110701107e-06, + "loss": 1.4222, + "step": 739 + }, + { + "epoch": 0.9209707529558183, + "grad_norm": 2.647939920425415, + "learning_rate": 3.8837638376383765e-06, + "loss": 1.5285, + "step": 740 + }, + { + "epoch": 0.9209707529558183, + "eval_loss": 1.6907480955123901, + "eval_runtime": 45.7678, + "eval_samples_per_second": 21.849, + "eval_steps_per_second": 0.918, + "step": 740 + }, + { + "epoch": 0.9222153080273802, + "grad_norm": 2.7664523124694824, + "learning_rate": 3.881457564575646e-06, + "loss": 1.5349, + "step": 741 + }, + { + "epoch": 0.9234598630989421, + "grad_norm": 2.5770998001098633, + "learning_rate": 3.879151291512915e-06, + "loss": 1.5071, + "step": 742 + }, + { + "epoch": 0.9247044181705041, + "grad_norm": 2.502567768096924, + "learning_rate": 3.876845018450185e-06, + "loss": 1.492, + "step": 743 + }, + { + "epoch": 0.9259489732420659, + "grad_norm": 2.6531119346618652, + "learning_rate": 3.874538745387454e-06, + "loss": 1.5045, + "step": 744 + }, + { + "epoch": 0.9271935283136279, + "grad_norm": 2.7343876361846924, + "learning_rate": 3.872232472324724e-06, + "loss": 1.5194, + "step": 745 + }, + { + "epoch": 0.9284380833851898, + "grad_norm": 2.842313528060913, + "learning_rate": 3.869926199261993e-06, + "loss": 1.5588, + "step": 746 + }, + { + "epoch": 0.9296826384567517, + "grad_norm": 2.718245267868042, + "learning_rate": 3.8676199261992625e-06, + "loss": 1.5252, + "step": 747 + }, + { + "epoch": 0.9309271935283137, + "grad_norm": 2.695392370223999, + "learning_rate": 3.865313653136532e-06, + "loss": 1.5596, + "step": 748 + }, + { + "epoch": 0.9321717485998755, + "grad_norm": 2.7859959602355957, + "learning_rate": 3.863007380073801e-06, + "loss": 1.5151, + "step": 749 + }, + { + "epoch": 0.9334163036714375, + "grad_norm": 2.5199174880981445, + "learning_rate": 3.860701107011071e-06, + "loss": 1.4948, + "step": 750 + }, + { + "epoch": 0.9334163036714375, + "eval_loss": 1.6868674755096436, + "eval_runtime": 46.4115, + "eval_samples_per_second": 21.546, + "eval_steps_per_second": 0.905, + "step": 750 + }, + { + "epoch": 0.9346608587429994, + "grad_norm": 2.8194046020507812, + "learning_rate": 3.858394833948339e-06, + "loss": 1.5144, + "step": 751 + }, + { + "epoch": 0.9359054138145613, + "grad_norm": 2.810380697250366, + "learning_rate": 3.856088560885609e-06, + "loss": 1.4998, + "step": 752 + }, + { + "epoch": 0.9371499688861232, + "grad_norm": 2.6906750202178955, + "learning_rate": 3.853782287822879e-06, + "loss": 1.4748, + "step": 753 + }, + { + "epoch": 0.9383945239576851, + "grad_norm": 2.9030277729034424, + "learning_rate": 3.851476014760148e-06, + "loss": 1.615, + "step": 754 + }, + { + "epoch": 0.939639079029247, + "grad_norm": 2.8690176010131836, + "learning_rate": 3.849169741697417e-06, + "loss": 1.5083, + "step": 755 + }, + { + "epoch": 0.940883634100809, + "grad_norm": 2.669646978378296, + "learning_rate": 3.8468634686346865e-06, + "loss": 1.5572, + "step": 756 + }, + { + "epoch": 0.9421281891723708, + "grad_norm": 2.6517696380615234, + "learning_rate": 3.844557195571956e-06, + "loss": 1.5142, + "step": 757 + }, + { + "epoch": 0.9433727442439328, + "grad_norm": 2.5524444580078125, + "learning_rate": 3.842250922509225e-06, + "loss": 1.4924, + "step": 758 + }, + { + "epoch": 0.9446172993154948, + "grad_norm": 2.6532633304595947, + "learning_rate": 3.839944649446495e-06, + "loss": 1.484, + "step": 759 + }, + { + "epoch": 0.9458618543870566, + "grad_norm": 2.7779057025909424, + "learning_rate": 3.837638376383764e-06, + "loss": 1.5106, + "step": 760 + }, + { + "epoch": 0.9458618543870566, + "eval_loss": 1.6845752000808716, + "eval_runtime": 50.3415, + "eval_samples_per_second": 19.864, + "eval_steps_per_second": 0.834, + "step": 760 + }, + { + "epoch": 0.9471064094586186, + "grad_norm": 2.6192541122436523, + "learning_rate": 3.8353321033210336e-06, + "loss": 1.5454, + "step": 761 + }, + { + "epoch": 0.9483509645301804, + "grad_norm": 2.561861991882324, + "learning_rate": 3.833025830258303e-06, + "loss": 1.5141, + "step": 762 + }, + { + "epoch": 0.9495955196017424, + "grad_norm": 2.661829948425293, + "learning_rate": 3.830719557195572e-06, + "loss": 1.4997, + "step": 763 + }, + { + "epoch": 0.9508400746733043, + "grad_norm": 2.725275754928589, + "learning_rate": 3.828413284132842e-06, + "loss": 1.5224, + "step": 764 + }, + { + "epoch": 0.9520846297448662, + "grad_norm": 2.5664193630218506, + "learning_rate": 3.826107011070111e-06, + "loss": 1.5073, + "step": 765 + }, + { + "epoch": 0.9533291848164281, + "grad_norm": 2.595189332962036, + "learning_rate": 3.823800738007381e-06, + "loss": 1.5379, + "step": 766 + }, + { + "epoch": 0.95457373988799, + "grad_norm": 2.738060235977173, + "learning_rate": 3.821494464944649e-06, + "loss": 1.4814, + "step": 767 + }, + { + "epoch": 0.955818294959552, + "grad_norm": 2.601071357727051, + "learning_rate": 3.819188191881919e-06, + "loss": 1.4812, + "step": 768 + }, + { + "epoch": 0.9570628500311139, + "grad_norm": 2.657212257385254, + "learning_rate": 3.816881918819189e-06, + "loss": 1.4748, + "step": 769 + }, + { + "epoch": 0.9583074051026758, + "grad_norm": 2.7673351764678955, + "learning_rate": 3.814575645756458e-06, + "loss": 1.5535, + "step": 770 + }, + { + "epoch": 0.9583074051026758, + "eval_loss": 1.677243947982788, + "eval_runtime": 51.5537, + "eval_samples_per_second": 19.397, + "eval_steps_per_second": 0.815, + "step": 770 + }, + { + "epoch": 0.9595519601742377, + "grad_norm": 2.760890007019043, + "learning_rate": 3.812269372693727e-06, + "loss": 1.5654, + "step": 771 + }, + { + "epoch": 0.9607965152457997, + "grad_norm": 2.6934309005737305, + "learning_rate": 3.809963099630997e-06, + "loss": 1.5149, + "step": 772 + }, + { + "epoch": 0.9620410703173615, + "grad_norm": 2.729950428009033, + "learning_rate": 3.8076568265682662e-06, + "loss": 1.5579, + "step": 773 + }, + { + "epoch": 0.9632856253889235, + "grad_norm": 2.732926607131958, + "learning_rate": 3.8053505535055352e-06, + "loss": 1.5179, + "step": 774 + }, + { + "epoch": 0.9645301804604853, + "grad_norm": 2.745391368865967, + "learning_rate": 3.8030442804428046e-06, + "loss": 1.5124, + "step": 775 + }, + { + "epoch": 0.9657747355320473, + "grad_norm": 2.8050146102905273, + "learning_rate": 3.800738007380074e-06, + "loss": 1.519, + "step": 776 + }, + { + "epoch": 0.9670192906036092, + "grad_norm": 2.697171449661255, + "learning_rate": 3.798431734317343e-06, + "loss": 1.5219, + "step": 777 + }, + { + "epoch": 0.9682638456751711, + "grad_norm": 2.6375980377197266, + "learning_rate": 3.796125461254613e-06, + "loss": 1.5345, + "step": 778 + }, + { + "epoch": 0.9695084007467331, + "grad_norm": 2.586636781692505, + "learning_rate": 3.7938191881918823e-06, + "loss": 1.5182, + "step": 779 + }, + { + "epoch": 0.9707529558182949, + "grad_norm": 2.626453399658203, + "learning_rate": 3.7915129151291518e-06, + "loss": 1.4817, + "step": 780 + }, + { + "epoch": 0.9707529558182949, + "eval_loss": 1.6734713315963745, + "eval_runtime": 48.1436, + "eval_samples_per_second": 20.771, + "eval_steps_per_second": 0.872, + "step": 780 + }, + { + "epoch": 0.9719975108898569, + "grad_norm": 2.579970121383667, + "learning_rate": 3.7892066420664208e-06, + "loss": 1.4981, + "step": 781 + }, + { + "epoch": 0.9732420659614188, + "grad_norm": 2.6834843158721924, + "learning_rate": 3.7869003690036906e-06, + "loss": 1.5044, + "step": 782 + }, + { + "epoch": 0.9744866210329807, + "grad_norm": 2.586982488632202, + "learning_rate": 3.78459409594096e-06, + "loss": 1.5079, + "step": 783 + }, + { + "epoch": 0.9757311761045426, + "grad_norm": 2.910027503967285, + "learning_rate": 3.782287822878229e-06, + "loss": 1.5695, + "step": 784 + }, + { + "epoch": 0.9769757311761046, + "grad_norm": 2.7088494300842285, + "learning_rate": 3.7799815498154984e-06, + "loss": 1.4986, + "step": 785 + }, + { + "epoch": 0.9782202862476664, + "grad_norm": 2.581325054168701, + "learning_rate": 3.777675276752768e-06, + "loss": 1.5137, + "step": 786 + }, + { + "epoch": 0.9794648413192284, + "grad_norm": 2.700709819793701, + "learning_rate": 3.775369003690037e-06, + "loss": 1.4718, + "step": 787 + }, + { + "epoch": 0.9807093963907902, + "grad_norm": 2.6712708473205566, + "learning_rate": 3.7730627306273067e-06, + "loss": 1.5208, + "step": 788 + }, + { + "epoch": 0.9819539514623522, + "grad_norm": 2.6799817085266113, + "learning_rate": 3.770756457564576e-06, + "loss": 1.5374, + "step": 789 + }, + { + "epoch": 0.9831985065339142, + "grad_norm": 2.618988037109375, + "learning_rate": 3.768450184501845e-06, + "loss": 1.4997, + "step": 790 + }, + { + "epoch": 0.9831985065339142, + "eval_loss": 1.6860820055007935, + "eval_runtime": 51.7654, + "eval_samples_per_second": 19.318, + "eval_steps_per_second": 0.811, + "step": 790 + }, + { + "epoch": 0.984443061605476, + "grad_norm": 2.6899526119232178, + "learning_rate": 3.7661439114391146e-06, + "loss": 1.5216, + "step": 791 + }, + { + "epoch": 0.985687616677038, + "grad_norm": 2.8003487586975098, + "learning_rate": 3.763837638376384e-06, + "loss": 1.557, + "step": 792 + }, + { + "epoch": 0.9869321717485999, + "grad_norm": 2.744536876678467, + "learning_rate": 3.761531365313654e-06, + "loss": 1.5355, + "step": 793 + }, + { + "epoch": 0.9881767268201618, + "grad_norm": 2.587250232696533, + "learning_rate": 3.759225092250923e-06, + "loss": 1.5259, + "step": 794 + }, + { + "epoch": 0.9894212818917237, + "grad_norm": 2.616291046142578, + "learning_rate": 3.7569188191881922e-06, + "loss": 1.5176, + "step": 795 + }, + { + "epoch": 0.9906658369632856, + "grad_norm": 2.6410577297210693, + "learning_rate": 3.7546125461254617e-06, + "loss": 1.511, + "step": 796 + }, + { + "epoch": 0.9919103920348475, + "grad_norm": 2.577373504638672, + "learning_rate": 3.7523062730627307e-06, + "loss": 1.4715, + "step": 797 + }, + { + "epoch": 0.9931549471064095, + "grad_norm": 2.67305326461792, + "learning_rate": 3.7500000000000005e-06, + "loss": 1.4828, + "step": 798 + }, + { + "epoch": 0.9943995021779714, + "grad_norm": 3.1631500720977783, + "learning_rate": 3.74769372693727e-06, + "loss": 1.5602, + "step": 799 + }, + { + "epoch": 0.9956440572495333, + "grad_norm": 2.9222350120544434, + "learning_rate": 3.745387453874539e-06, + "loss": 1.4814, + "step": 800 + }, + { + "epoch": 0.9956440572495333, + "eval_loss": 1.6810544729232788, + "eval_runtime": 51.1262, + "eval_samples_per_second": 19.559, + "eval_steps_per_second": 0.821, + "step": 800 + }, + { + "epoch": 0.9968886123210952, + "grad_norm": 2.884312629699707, + "learning_rate": 3.7430811808118084e-06, + "loss": 1.4553, + "step": 801 + }, + { + "epoch": 0.9981331673926571, + "grad_norm": 2.712655782699585, + "learning_rate": 3.7407749077490778e-06, + "loss": 1.5275, + "step": 802 + }, + { + "epoch": 0.9993777224642191, + "grad_norm": 2.8700571060180664, + "learning_rate": 3.7384686346863468e-06, + "loss": 1.4923, + "step": 803 + }, + { + "epoch": 1.000622277535781, + "grad_norm": 2.7779245376586914, + "learning_rate": 3.7361623616236166e-06, + "loss": 1.5224, + "step": 804 + }, + { + "epoch": 1.0018668326073428, + "grad_norm": 2.7679810523986816, + "learning_rate": 3.733856088560886e-06, + "loss": 1.4977, + "step": 805 + }, + { + "epoch": 1.0031113876789048, + "grad_norm": 2.797023296356201, + "learning_rate": 3.7315498154981555e-06, + "loss": 1.4396, + "step": 806 + }, + { + "epoch": 1.0043559427504667, + "grad_norm": 3.007962465286255, + "learning_rate": 3.7292435424354245e-06, + "loss": 1.5015, + "step": 807 + }, + { + "epoch": 1.0056004978220285, + "grad_norm": 3.127639055252075, + "learning_rate": 3.726937269372694e-06, + "loss": 1.5176, + "step": 808 + }, + { + "epoch": 1.0068450528935906, + "grad_norm": 2.9001357555389404, + "learning_rate": 3.7246309963099637e-06, + "loss": 1.4735, + "step": 809 + }, + { + "epoch": 1.0080896079651525, + "grad_norm": 2.8245413303375244, + "learning_rate": 3.7223247232472327e-06, + "loss": 1.486, + "step": 810 + }, + { + "epoch": 1.0080896079651525, + "eval_loss": 1.6835130453109741, + "eval_runtime": 44.5682, + "eval_samples_per_second": 22.438, + "eval_steps_per_second": 0.942, + "step": 810 + }, + { + "epoch": 1.0093341630367143, + "grad_norm": 2.780172824859619, + "learning_rate": 3.720018450184502e-06, + "loss": 1.4437, + "step": 811 + }, + { + "epoch": 1.0105787181082764, + "grad_norm": 2.6013996601104736, + "learning_rate": 3.7177121771217716e-06, + "loss": 1.4529, + "step": 812 + }, + { + "epoch": 1.0118232731798382, + "grad_norm": 2.831015110015869, + "learning_rate": 3.7154059040590406e-06, + "loss": 1.4825, + "step": 813 + }, + { + "epoch": 1.0130678282514, + "grad_norm": 2.8341829776763916, + "learning_rate": 3.71309963099631e-06, + "loss": 1.4736, + "step": 814 + }, + { + "epoch": 1.0143123833229621, + "grad_norm": 2.769540309906006, + "learning_rate": 3.71079335793358e-06, + "loss": 1.4724, + "step": 815 + }, + { + "epoch": 1.015556938394524, + "grad_norm": 3.0083069801330566, + "learning_rate": 3.708487084870849e-06, + "loss": 1.4985, + "step": 816 + }, + { + "epoch": 1.0168014934660858, + "grad_norm": 3.052168607711792, + "learning_rate": 3.7061808118081183e-06, + "loss": 1.5193, + "step": 817 + }, + { + "epoch": 1.0180460485376477, + "grad_norm": 2.924877405166626, + "learning_rate": 3.7038745387453877e-06, + "loss": 1.5219, + "step": 818 + }, + { + "epoch": 1.0192906036092098, + "grad_norm": 3.150123357772827, + "learning_rate": 3.7015682656826576e-06, + "loss": 1.5026, + "step": 819 + }, + { + "epoch": 1.0205351586807716, + "grad_norm": 2.881655216217041, + "learning_rate": 3.6992619926199266e-06, + "loss": 1.5011, + "step": 820 + }, + { + "epoch": 1.0205351586807716, + "eval_loss": 1.6745303869247437, + "eval_runtime": 50.6791, + "eval_samples_per_second": 19.732, + "eval_steps_per_second": 0.829, + "step": 820 + }, + { + "epoch": 1.0217797137523335, + "grad_norm": 2.824249505996704, + "learning_rate": 3.696955719557196e-06, + "loss": 1.4642, + "step": 821 + }, + { + "epoch": 1.0230242688238955, + "grad_norm": 2.859144926071167, + "learning_rate": 3.6946494464944654e-06, + "loss": 1.4681, + "step": 822 + }, + { + "epoch": 1.0242688238954574, + "grad_norm": 2.90312123298645, + "learning_rate": 3.6923431734317344e-06, + "loss": 1.4926, + "step": 823 + }, + { + "epoch": 1.0255133789670192, + "grad_norm": 2.8340659141540527, + "learning_rate": 3.690036900369004e-06, + "loss": 1.4834, + "step": 824 + }, + { + "epoch": 1.0267579340385813, + "grad_norm": 2.9743151664733887, + "learning_rate": 3.6877306273062737e-06, + "loss": 1.4574, + "step": 825 + }, + { + "epoch": 1.0280024891101431, + "grad_norm": 3.032179594039917, + "learning_rate": 3.6854243542435427e-06, + "loss": 1.5571, + "step": 826 + }, + { + "epoch": 1.029247044181705, + "grad_norm": 2.816826343536377, + "learning_rate": 3.683118081180812e-06, + "loss": 1.455, + "step": 827 + }, + { + "epoch": 1.030491599253267, + "grad_norm": 3.046696186065674, + "learning_rate": 3.6808118081180815e-06, + "loss": 1.4871, + "step": 828 + }, + { + "epoch": 1.031736154324829, + "grad_norm": 2.8517391681671143, + "learning_rate": 3.678505535055351e-06, + "loss": 1.5292, + "step": 829 + }, + { + "epoch": 1.0329807093963908, + "grad_norm": 2.752811908721924, + "learning_rate": 3.67619926199262e-06, + "loss": 1.5091, + "step": 830 + }, + { + "epoch": 1.0329807093963908, + "eval_loss": 1.6692287921905518, + "eval_runtime": 50.8347, + "eval_samples_per_second": 19.672, + "eval_steps_per_second": 0.826, + "step": 830 + }, + { + "epoch": 1.0342252644679526, + "grad_norm": 2.9220356941223145, + "learning_rate": 3.6738929889298898e-06, + "loss": 1.4247, + "step": 831 + }, + { + "epoch": 1.0354698195395147, + "grad_norm": 2.8903002738952637, + "learning_rate": 3.671586715867159e-06, + "loss": 1.5595, + "step": 832 + }, + { + "epoch": 1.0367143746110765, + "grad_norm": 2.6495542526245117, + "learning_rate": 3.669280442804428e-06, + "loss": 1.4076, + "step": 833 + }, + { + "epoch": 1.0379589296826384, + "grad_norm": 2.880809783935547, + "learning_rate": 3.6669741697416976e-06, + "loss": 1.5285, + "step": 834 + }, + { + "epoch": 1.0392034847542004, + "grad_norm": 2.919142007827759, + "learning_rate": 3.6646678966789675e-06, + "loss": 1.5165, + "step": 835 + }, + { + "epoch": 1.0404480398257623, + "grad_norm": 3.1180522441864014, + "learning_rate": 3.6623616236162365e-06, + "loss": 1.5224, + "step": 836 + }, + { + "epoch": 1.0416925948973241, + "grad_norm": 2.767709255218506, + "learning_rate": 3.660055350553506e-06, + "loss": 1.4859, + "step": 837 + }, + { + "epoch": 1.0429371499688862, + "grad_norm": 2.708828926086426, + "learning_rate": 3.6577490774907753e-06, + "loss": 1.4803, + "step": 838 + }, + { + "epoch": 1.044181705040448, + "grad_norm": 2.688359260559082, + "learning_rate": 3.6554428044280443e-06, + "loss": 1.4571, + "step": 839 + }, + { + "epoch": 1.04542626011201, + "grad_norm": 2.691812753677368, + "learning_rate": 3.6531365313653137e-06, + "loss": 1.5043, + "step": 840 + }, + { + "epoch": 1.04542626011201, + "eval_loss": 1.6769559383392334, + "eval_runtime": 49.6233, + "eval_samples_per_second": 20.152, + "eval_steps_per_second": 0.846, + "step": 840 + }, + { + "epoch": 1.046670815183572, + "grad_norm": 2.686372995376587, + "learning_rate": 3.6508302583025836e-06, + "loss": 1.5221, + "step": 841 + }, + { + "epoch": 1.0479153702551338, + "grad_norm": 2.742859125137329, + "learning_rate": 3.648523985239853e-06, + "loss": 1.4685, + "step": 842 + }, + { + "epoch": 1.0491599253266957, + "grad_norm": 2.874018907546997, + "learning_rate": 3.646217712177122e-06, + "loss": 1.4813, + "step": 843 + }, + { + "epoch": 1.0504044803982575, + "grad_norm": 2.8159990310668945, + "learning_rate": 3.6439114391143914e-06, + "loss": 1.4421, + "step": 844 + }, + { + "epoch": 1.0516490354698196, + "grad_norm": 2.714787006378174, + "learning_rate": 3.641605166051661e-06, + "loss": 1.4879, + "step": 845 + }, + { + "epoch": 1.0528935905413814, + "grad_norm": 2.885148525238037, + "learning_rate": 3.63929889298893e-06, + "loss": 1.4725, + "step": 846 + }, + { + "epoch": 1.0541381456129433, + "grad_norm": 2.7272610664367676, + "learning_rate": 3.6369926199261997e-06, + "loss": 1.4276, + "step": 847 + }, + { + "epoch": 1.0553827006845053, + "grad_norm": 2.9114978313446045, + "learning_rate": 3.634686346863469e-06, + "loss": 1.4194, + "step": 848 + }, + { + "epoch": 1.0566272557560672, + "grad_norm": 3.019313335418701, + "learning_rate": 3.632380073800738e-06, + "loss": 1.5266, + "step": 849 + }, + { + "epoch": 1.057871810827629, + "grad_norm": 2.714751958847046, + "learning_rate": 3.6300738007380075e-06, + "loss": 1.4593, + "step": 850 + }, + { + "epoch": 1.057871810827629, + "eval_loss": 1.6760298013687134, + "eval_runtime": 52.1681, + "eval_samples_per_second": 19.169, + "eval_steps_per_second": 0.805, + "step": 850 + }, + { + "epoch": 1.0591163658991911, + "grad_norm": 3.0507936477661133, + "learning_rate": 3.627767527675277e-06, + "loss": 1.5064, + "step": 851 + }, + { + "epoch": 1.060360920970753, + "grad_norm": 2.8116295337677, + "learning_rate": 3.625461254612546e-06, + "loss": 1.5218, + "step": 852 + }, + { + "epoch": 1.0616054760423148, + "grad_norm": 2.8801465034484863, + "learning_rate": 3.623154981549816e-06, + "loss": 1.4641, + "step": 853 + }, + { + "epoch": 1.0628500311138769, + "grad_norm": 3.0581862926483154, + "learning_rate": 3.6208487084870852e-06, + "loss": 1.483, + "step": 854 + }, + { + "epoch": 1.0640945861854387, + "grad_norm": 2.898780584335327, + "learning_rate": 3.6185424354243547e-06, + "loss": 1.4623, + "step": 855 + }, + { + "epoch": 1.0653391412570006, + "grad_norm": 2.971820116043091, + "learning_rate": 3.6162361623616237e-06, + "loss": 1.4455, + "step": 856 + }, + { + "epoch": 1.0665836963285624, + "grad_norm": 3.0021913051605225, + "learning_rate": 3.6139298892988935e-06, + "loss": 1.4878, + "step": 857 + }, + { + "epoch": 1.0678282514001245, + "grad_norm": 3.0697457790374756, + "learning_rate": 3.611623616236163e-06, + "loss": 1.5008, + "step": 858 + }, + { + "epoch": 1.0690728064716863, + "grad_norm": 3.0787923336029053, + "learning_rate": 3.609317343173432e-06, + "loss": 1.4612, + "step": 859 + }, + { + "epoch": 1.0703173615432482, + "grad_norm": 2.789339780807495, + "learning_rate": 3.6070110701107014e-06, + "loss": 1.4287, + "step": 860 + }, + { + "epoch": 1.0703173615432482, + "eval_loss": 1.6747506856918335, + "eval_runtime": 51.4912, + "eval_samples_per_second": 19.421, + "eval_steps_per_second": 0.816, + "step": 860 + }, + { + "epoch": 1.0715619166148103, + "grad_norm": 2.9622788429260254, + "learning_rate": 3.6047047970479708e-06, + "loss": 1.4905, + "step": 861 + }, + { + "epoch": 1.072806471686372, + "grad_norm": 3.0675268173217773, + "learning_rate": 3.6023985239852398e-06, + "loss": 1.461, + "step": 862 + }, + { + "epoch": 1.074051026757934, + "grad_norm": 2.820996046066284, + "learning_rate": 3.6000922509225096e-06, + "loss": 1.4948, + "step": 863 + }, + { + "epoch": 1.075295581829496, + "grad_norm": 2.833761215209961, + "learning_rate": 3.597785977859779e-06, + "loss": 1.4683, + "step": 864 + }, + { + "epoch": 1.0765401369010579, + "grad_norm": 2.917313814163208, + "learning_rate": 3.595479704797048e-06, + "loss": 1.5191, + "step": 865 + }, + { + "epoch": 1.0777846919726197, + "grad_norm": 2.876166820526123, + "learning_rate": 3.5931734317343175e-06, + "loss": 1.4854, + "step": 866 + }, + { + "epoch": 1.0790292470441818, + "grad_norm": 3.139049768447876, + "learning_rate": 3.590867158671587e-06, + "loss": 1.5134, + "step": 867 + }, + { + "epoch": 1.0802738021157436, + "grad_norm": 2.9956753253936768, + "learning_rate": 3.5885608856088567e-06, + "loss": 1.5003, + "step": 868 + }, + { + "epoch": 1.0815183571873055, + "grad_norm": 2.957181930541992, + "learning_rate": 3.5862546125461257e-06, + "loss": 1.4588, + "step": 869 + }, + { + "epoch": 1.0827629122588673, + "grad_norm": 2.8576438426971436, + "learning_rate": 3.583948339483395e-06, + "loss": 1.48, + "step": 870 + }, + { + "epoch": 1.0827629122588673, + "eval_loss": 1.6724848747253418, + "eval_runtime": 50.9669, + "eval_samples_per_second": 19.621, + "eval_steps_per_second": 0.824, + "step": 870 + }, + { + "epoch": 1.0840074673304294, + "grad_norm": 2.8183560371398926, + "learning_rate": 3.5816420664206646e-06, + "loss": 1.4818, + "step": 871 + }, + { + "epoch": 1.0852520224019913, + "grad_norm": 2.7531797885894775, + "learning_rate": 3.5793357933579336e-06, + "loss": 1.4712, + "step": 872 + }, + { + "epoch": 1.086496577473553, + "grad_norm": 2.923962354660034, + "learning_rate": 3.5770295202952034e-06, + "loss": 1.4864, + "step": 873 + }, + { + "epoch": 1.0877411325451152, + "grad_norm": 2.851724147796631, + "learning_rate": 3.574723247232473e-06, + "loss": 1.4771, + "step": 874 + }, + { + "epoch": 1.088985687616677, + "grad_norm": 3.024634838104248, + "learning_rate": 3.572416974169742e-06, + "loss": 1.4643, + "step": 875 + }, + { + "epoch": 1.0902302426882389, + "grad_norm": 2.7100844383239746, + "learning_rate": 3.5701107011070113e-06, + "loss": 1.4773, + "step": 876 + }, + { + "epoch": 1.091474797759801, + "grad_norm": 2.730502128601074, + "learning_rate": 3.5678044280442807e-06, + "loss": 1.5051, + "step": 877 + }, + { + "epoch": 1.0927193528313628, + "grad_norm": 2.747082233428955, + "learning_rate": 3.5654981549815497e-06, + "loss": 1.5207, + "step": 878 + }, + { + "epoch": 1.0939639079029246, + "grad_norm": 2.718358039855957, + "learning_rate": 3.5631918819188195e-06, + "loss": 1.5067, + "step": 879 + }, + { + "epoch": 1.0952084629744867, + "grad_norm": 2.7794573307037354, + "learning_rate": 3.560885608856089e-06, + "loss": 1.4616, + "step": 880 + }, + { + "epoch": 1.0952084629744867, + "eval_loss": 1.6676132678985596, + "eval_runtime": 51.6157, + "eval_samples_per_second": 19.374, + "eval_steps_per_second": 0.814, + "step": 880 + }, + { + "epoch": 1.0964530180460486, + "grad_norm": 2.727416753768921, + "learning_rate": 3.5585793357933584e-06, + "loss": 1.4043, + "step": 881 + }, + { + "epoch": 1.0976975731176104, + "grad_norm": 2.856900930404663, + "learning_rate": 3.5562730627306274e-06, + "loss": 1.5198, + "step": 882 + }, + { + "epoch": 1.0989421281891725, + "grad_norm": 2.872823715209961, + "learning_rate": 3.553966789667897e-06, + "loss": 1.4981, + "step": 883 + }, + { + "epoch": 1.1001866832607343, + "grad_norm": 2.7724361419677734, + "learning_rate": 3.5516605166051667e-06, + "loss": 1.4819, + "step": 884 + }, + { + "epoch": 1.1014312383322962, + "grad_norm": 2.884434461593628, + "learning_rate": 3.5493542435424357e-06, + "loss": 1.433, + "step": 885 + }, + { + "epoch": 1.102675793403858, + "grad_norm": 2.887873411178589, + "learning_rate": 3.547047970479705e-06, + "loss": 1.4677, + "step": 886 + }, + { + "epoch": 1.10392034847542, + "grad_norm": 2.946845054626465, + "learning_rate": 3.5447416974169745e-06, + "loss": 1.4983, + "step": 887 + }, + { + "epoch": 1.105164903546982, + "grad_norm": 2.752552032470703, + "learning_rate": 3.5424354243542435e-06, + "loss": 1.4766, + "step": 888 + }, + { + "epoch": 1.1064094586185438, + "grad_norm": 2.788634777069092, + "learning_rate": 3.5401291512915133e-06, + "loss": 1.4747, + "step": 889 + }, + { + "epoch": 1.1076540136901059, + "grad_norm": 2.782792568206787, + "learning_rate": 3.5378228782287828e-06, + "loss": 1.5503, + "step": 890 + }, + { + "epoch": 1.1076540136901059, + "eval_loss": 1.6698333024978638, + "eval_runtime": 49.1373, + "eval_samples_per_second": 20.351, + "eval_steps_per_second": 0.855, + "step": 890 + }, + { + "epoch": 1.1088985687616677, + "grad_norm": 2.8351166248321533, + "learning_rate": 3.535516605166052e-06, + "loss": 1.4589, + "step": 891 + }, + { + "epoch": 1.1101431238332296, + "grad_norm": 2.798872709274292, + "learning_rate": 3.533210332103321e-06, + "loss": 1.4496, + "step": 892 + }, + { + "epoch": 1.1113876789047916, + "grad_norm": 3.0863239765167236, + "learning_rate": 3.5309040590405906e-06, + "loss": 1.4698, + "step": 893 + }, + { + "epoch": 1.1126322339763535, + "grad_norm": 2.853862762451172, + "learning_rate": 3.5285977859778605e-06, + "loss": 1.4429, + "step": 894 + }, + { + "epoch": 1.1138767890479153, + "grad_norm": 2.8946170806884766, + "learning_rate": 3.5262915129151295e-06, + "loss": 1.4697, + "step": 895 + }, + { + "epoch": 1.1151213441194774, + "grad_norm": 3.043823003768921, + "learning_rate": 3.523985239852399e-06, + "loss": 1.4666, + "step": 896 + }, + { + "epoch": 1.1163658991910392, + "grad_norm": 2.7822980880737305, + "learning_rate": 3.5216789667896683e-06, + "loss": 1.4523, + "step": 897 + }, + { + "epoch": 1.117610454262601, + "grad_norm": 2.9454171657562256, + "learning_rate": 3.5193726937269373e-06, + "loss": 1.5009, + "step": 898 + }, + { + "epoch": 1.1188550093341632, + "grad_norm": 2.8533740043640137, + "learning_rate": 3.5170664206642067e-06, + "loss": 1.4265, + "step": 899 + }, + { + "epoch": 1.120099564405725, + "grad_norm": 2.894937038421631, + "learning_rate": 3.5147601476014766e-06, + "loss": 1.4895, + "step": 900 + }, + { + "epoch": 1.120099564405725, + "eval_loss": 1.6701407432556152, + "eval_runtime": 45.477, + "eval_samples_per_second": 21.989, + "eval_steps_per_second": 0.924, + "step": 900 + }, + { + "epoch": 1.1213441194772868, + "grad_norm": 2.933821201324463, + "learning_rate": 3.5124538745387456e-06, + "loss": 1.468, + "step": 901 + }, + { + "epoch": 1.1225886745488487, + "grad_norm": 2.834265947341919, + "learning_rate": 3.510147601476015e-06, + "loss": 1.4621, + "step": 902 + }, + { + "epoch": 1.1238332296204108, + "grad_norm": 2.9064722061157227, + "learning_rate": 3.5078413284132844e-06, + "loss": 1.504, + "step": 903 + }, + { + "epoch": 1.1250777846919726, + "grad_norm": 2.8753886222839355, + "learning_rate": 3.505535055350554e-06, + "loss": 1.4643, + "step": 904 + }, + { + "epoch": 1.1263223397635345, + "grad_norm": 2.894659996032715, + "learning_rate": 3.503228782287823e-06, + "loss": 1.4909, + "step": 905 + }, + { + "epoch": 1.1275668948350965, + "grad_norm": 3.085977792739868, + "learning_rate": 3.5009225092250927e-06, + "loss": 1.459, + "step": 906 + }, + { + "epoch": 1.1288114499066584, + "grad_norm": 2.9958064556121826, + "learning_rate": 3.498616236162362e-06, + "loss": 1.4152, + "step": 907 + }, + { + "epoch": 1.1300560049782202, + "grad_norm": 2.9136762619018555, + "learning_rate": 3.496309963099631e-06, + "loss": 1.473, + "step": 908 + }, + { + "epoch": 1.1313005600497823, + "grad_norm": 2.956437826156616, + "learning_rate": 3.4940036900369005e-06, + "loss": 1.5167, + "step": 909 + }, + { + "epoch": 1.1325451151213441, + "grad_norm": 2.7354865074157715, + "learning_rate": 3.4916974169741704e-06, + "loss": 1.4413, + "step": 910 + }, + { + "epoch": 1.1325451151213441, + "eval_loss": 1.6629912853240967, + "eval_runtime": 41.5478, + "eval_samples_per_second": 24.069, + "eval_steps_per_second": 1.011, + "step": 910 + }, + { + "epoch": 1.133789670192906, + "grad_norm": 2.7443008422851562, + "learning_rate": 3.4893911439114394e-06, + "loss": 1.4475, + "step": 911 + }, + { + "epoch": 1.135034225264468, + "grad_norm": 2.7811238765716553, + "learning_rate": 3.487084870848709e-06, + "loss": 1.4257, + "step": 912 + }, + { + "epoch": 1.13627878033603, + "grad_norm": 2.7916479110717773, + "learning_rate": 3.4847785977859782e-06, + "loss": 1.4048, + "step": 913 + }, + { + "epoch": 1.1375233354075918, + "grad_norm": 2.7835633754730225, + "learning_rate": 3.4824723247232472e-06, + "loss": 1.3996, + "step": 914 + }, + { + "epoch": 1.1387678904791536, + "grad_norm": 2.8188705444335938, + "learning_rate": 3.4801660516605166e-06, + "loss": 1.4541, + "step": 915 + }, + { + "epoch": 1.1400124455507157, + "grad_norm": 3.0134100914001465, + "learning_rate": 3.4778597785977865e-06, + "loss": 1.4353, + "step": 916 + }, + { + "epoch": 1.1412570006222775, + "grad_norm": 2.6965606212615967, + "learning_rate": 3.475553505535056e-06, + "loss": 1.4308, + "step": 917 + }, + { + "epoch": 1.1425015556938394, + "grad_norm": 2.8179919719696045, + "learning_rate": 3.473247232472325e-06, + "loss": 1.5204, + "step": 918 + }, + { + "epoch": 1.1437461107654014, + "grad_norm": 2.7213902473449707, + "learning_rate": 3.4709409594095943e-06, + "loss": 1.4344, + "step": 919 + }, + { + "epoch": 1.1449906658369633, + "grad_norm": 2.850193738937378, + "learning_rate": 3.4686346863468638e-06, + "loss": 1.473, + "step": 920 + }, + { + "epoch": 1.1449906658369633, + "eval_loss": 1.663345217704773, + "eval_runtime": 43.8397, + "eval_samples_per_second": 22.81, + "eval_steps_per_second": 0.958, + "step": 920 + }, + { + "epoch": 1.1462352209085251, + "grad_norm": 2.780198574066162, + "learning_rate": 3.4663284132841328e-06, + "loss": 1.4472, + "step": 921 + }, + { + "epoch": 1.1474797759800872, + "grad_norm": 2.9373831748962402, + "learning_rate": 3.4640221402214026e-06, + "loss": 1.426, + "step": 922 + }, + { + "epoch": 1.148724331051649, + "grad_norm": 2.6986873149871826, + "learning_rate": 3.461715867158672e-06, + "loss": 1.3953, + "step": 923 + }, + { + "epoch": 1.149968886123211, + "grad_norm": 2.927441120147705, + "learning_rate": 3.459409594095941e-06, + "loss": 1.4578, + "step": 924 + }, + { + "epoch": 1.151213441194773, + "grad_norm": 2.8469648361206055, + "learning_rate": 3.4571033210332105e-06, + "loss": 1.4313, + "step": 925 + }, + { + "epoch": 1.1524579962663348, + "grad_norm": 2.8114013671875, + "learning_rate": 3.4547970479704803e-06, + "loss": 1.4267, + "step": 926 + }, + { + "epoch": 1.1537025513378967, + "grad_norm": 2.977605104446411, + "learning_rate": 3.4524907749077493e-06, + "loss": 1.4747, + "step": 927 + }, + { + "epoch": 1.1549471064094585, + "grad_norm": 2.9327852725982666, + "learning_rate": 3.4501845018450187e-06, + "loss": 1.4219, + "step": 928 + }, + { + "epoch": 1.1561916614810206, + "grad_norm": 2.844038963317871, + "learning_rate": 3.447878228782288e-06, + "loss": 1.5146, + "step": 929 + }, + { + "epoch": 1.1574362165525824, + "grad_norm": 2.9402201175689697, + "learning_rate": 3.4455719557195576e-06, + "loss": 1.45, + "step": 930 + }, + { + "epoch": 1.1574362165525824, + "eval_loss": 1.6620122194290161, + "eval_runtime": 46.1748, + "eval_samples_per_second": 21.657, + "eval_steps_per_second": 0.91, + "step": 930 + }, + { + "epoch": 1.1586807716241443, + "grad_norm": 2.842651128768921, + "learning_rate": 3.4432656826568266e-06, + "loss": 1.4415, + "step": 931 + }, + { + "epoch": 1.1599253266957064, + "grad_norm": 2.8909685611724854, + "learning_rate": 3.4409594095940964e-06, + "loss": 1.4529, + "step": 932 + }, + { + "epoch": 1.1611698817672682, + "grad_norm": 2.8025808334350586, + "learning_rate": 3.438653136531366e-06, + "loss": 1.4552, + "step": 933 + }, + { + "epoch": 1.16241443683883, + "grad_norm": 2.6995794773101807, + "learning_rate": 3.436346863468635e-06, + "loss": 1.4435, + "step": 934 + }, + { + "epoch": 1.1636589919103921, + "grad_norm": 2.7307121753692627, + "learning_rate": 3.4340405904059043e-06, + "loss": 1.4447, + "step": 935 + }, + { + "epoch": 1.164903546981954, + "grad_norm": 2.9532501697540283, + "learning_rate": 3.4317343173431737e-06, + "loss": 1.4747, + "step": 936 + }, + { + "epoch": 1.1661481020535158, + "grad_norm": 3.0135788917541504, + "learning_rate": 3.4294280442804427e-06, + "loss": 1.525, + "step": 937 + }, + { + "epoch": 1.167392657125078, + "grad_norm": 2.796355962753296, + "learning_rate": 3.4271217712177125e-06, + "loss": 1.4539, + "step": 938 + }, + { + "epoch": 1.1686372121966397, + "grad_norm": 2.9422309398651123, + "learning_rate": 3.424815498154982e-06, + "loss": 1.4664, + "step": 939 + }, + { + "epoch": 1.1698817672682016, + "grad_norm": 2.7680039405822754, + "learning_rate": 3.4225092250922514e-06, + "loss": 1.4837, + "step": 940 + }, + { + "epoch": 1.1698817672682016, + "eval_loss": 1.660908818244934, + "eval_runtime": 45.6998, + "eval_samples_per_second": 21.882, + "eval_steps_per_second": 0.919, + "step": 940 + }, + { + "epoch": 1.1711263223397634, + "grad_norm": 2.8888845443725586, + "learning_rate": 3.4202029520295204e-06, + "loss": 1.5315, + "step": 941 + }, + { + "epoch": 1.1723708774113255, + "grad_norm": 2.7124130725860596, + "learning_rate": 3.41789667896679e-06, + "loss": 1.4188, + "step": 942 + }, + { + "epoch": 1.1736154324828874, + "grad_norm": 2.75203013420105, + "learning_rate": 3.4155904059040596e-06, + "loss": 1.4533, + "step": 943 + }, + { + "epoch": 1.1748599875544492, + "grad_norm": 2.865480422973633, + "learning_rate": 3.4132841328413286e-06, + "loss": 1.4532, + "step": 944 + }, + { + "epoch": 1.1761045426260113, + "grad_norm": 2.8618321418762207, + "learning_rate": 3.410977859778598e-06, + "loss": 1.4386, + "step": 945 + }, + { + "epoch": 1.1773490976975731, + "grad_norm": 2.7656145095825195, + "learning_rate": 3.4086715867158675e-06, + "loss": 1.3996, + "step": 946 + }, + { + "epoch": 1.178593652769135, + "grad_norm": 2.76213002204895, + "learning_rate": 3.4063653136531365e-06, + "loss": 1.4979, + "step": 947 + }, + { + "epoch": 1.179838207840697, + "grad_norm": 2.8581957817077637, + "learning_rate": 3.4040590405904063e-06, + "loss": 1.4673, + "step": 948 + }, + { + "epoch": 1.1810827629122589, + "grad_norm": 2.7825586795806885, + "learning_rate": 3.4017527675276758e-06, + "loss": 1.4453, + "step": 949 + }, + { + "epoch": 1.1823273179838207, + "grad_norm": 2.9226131439208984, + "learning_rate": 3.3994464944649448e-06, + "loss": 1.4879, + "step": 950 + }, + { + "epoch": 1.1823273179838207, + "eval_loss": 1.6621263027191162, + "eval_runtime": 45.1222, + "eval_samples_per_second": 22.162, + "eval_steps_per_second": 0.931, + "step": 950 + }, + { + "epoch": 1.1835718730553828, + "grad_norm": 2.9997975826263428, + "learning_rate": 3.397140221402214e-06, + "loss": 1.505, + "step": 951 + }, + { + "epoch": 1.1848164281269447, + "grad_norm": 2.7225453853607178, + "learning_rate": 3.3948339483394836e-06, + "loss": 1.4695, + "step": 952 + }, + { + "epoch": 1.1860609831985065, + "grad_norm": 2.916473150253296, + "learning_rate": 3.3925276752767534e-06, + "loss": 1.4901, + "step": 953 + }, + { + "epoch": 1.1873055382700684, + "grad_norm": 2.8371644020080566, + "learning_rate": 3.3902214022140224e-06, + "loss": 1.4944, + "step": 954 + }, + { + "epoch": 1.1885500933416304, + "grad_norm": 2.813662528991699, + "learning_rate": 3.387915129151292e-06, + "loss": 1.4603, + "step": 955 + }, + { + "epoch": 1.1897946484131923, + "grad_norm": 2.991931676864624, + "learning_rate": 3.3856088560885613e-06, + "loss": 1.5496, + "step": 956 + }, + { + "epoch": 1.1910392034847541, + "grad_norm": 2.6252501010894775, + "learning_rate": 3.3833025830258303e-06, + "loss": 1.4405, + "step": 957 + }, + { + "epoch": 1.1922837585563162, + "grad_norm": 2.747725009918213, + "learning_rate": 3.3809963099630997e-06, + "loss": 1.4689, + "step": 958 + }, + { + "epoch": 1.193528313627878, + "grad_norm": 2.887763261795044, + "learning_rate": 3.3786900369003696e-06, + "loss": 1.4501, + "step": 959 + }, + { + "epoch": 1.1947728686994399, + "grad_norm": 2.9954354763031006, + "learning_rate": 3.3763837638376386e-06, + "loss": 1.4978, + "step": 960 + }, + { + "epoch": 1.1947728686994399, + "eval_loss": 1.664442539215088, + "eval_runtime": 49.1266, + "eval_samples_per_second": 20.356, + "eval_steps_per_second": 0.855, + "step": 960 + }, + { + "epoch": 1.196017423771002, + "grad_norm": 3.0174152851104736, + "learning_rate": 3.374077490774908e-06, + "loss": 1.4353, + "step": 961 + }, + { + "epoch": 1.1972619788425638, + "grad_norm": 2.9082882404327393, + "learning_rate": 3.3717712177121774e-06, + "loss": 1.4878, + "step": 962 + }, + { + "epoch": 1.1985065339141256, + "grad_norm": 2.897993326187134, + "learning_rate": 3.3694649446494464e-06, + "loss": 1.4975, + "step": 963 + }, + { + "epoch": 1.1997510889856877, + "grad_norm": 2.739470958709717, + "learning_rate": 3.3671586715867163e-06, + "loss": 1.4202, + "step": 964 + }, + { + "epoch": 1.2009956440572496, + "grad_norm": 2.817786693572998, + "learning_rate": 3.3648523985239857e-06, + "loss": 1.5195, + "step": 965 + }, + { + "epoch": 1.2022401991288114, + "grad_norm": 2.8476386070251465, + "learning_rate": 3.362546125461255e-06, + "loss": 1.444, + "step": 966 + }, + { + "epoch": 1.2034847542003733, + "grad_norm": 2.804047107696533, + "learning_rate": 3.360239852398524e-06, + "loss": 1.4442, + "step": 967 + }, + { + "epoch": 1.2047293092719353, + "grad_norm": 2.9285430908203125, + "learning_rate": 3.3579335793357935e-06, + "loss": 1.4877, + "step": 968 + }, + { + "epoch": 1.2059738643434972, + "grad_norm": 2.8814878463745117, + "learning_rate": 3.3556273062730634e-06, + "loss": 1.5078, + "step": 969 + }, + { + "epoch": 1.207218419415059, + "grad_norm": 2.970728635787964, + "learning_rate": 3.3533210332103324e-06, + "loss": 1.4166, + "step": 970 + }, + { + "epoch": 1.207218419415059, + "eval_loss": 1.6635315418243408, + "eval_runtime": 47.2321, + "eval_samples_per_second": 21.172, + "eval_steps_per_second": 0.889, + "step": 970 + }, + { + "epoch": 1.208462974486621, + "grad_norm": 2.9836292266845703, + "learning_rate": 3.3510147601476018e-06, + "loss": 1.4777, + "step": 971 + }, + { + "epoch": 1.209707529558183, + "grad_norm": 2.8495216369628906, + "learning_rate": 3.348708487084871e-06, + "loss": 1.4829, + "step": 972 + }, + { + "epoch": 1.2109520846297448, + "grad_norm": 2.7709312438964844, + "learning_rate": 3.34640221402214e-06, + "loss": 1.4242, + "step": 973 + }, + { + "epoch": 1.2121966397013069, + "grad_norm": 2.999462366104126, + "learning_rate": 3.3440959409594096e-06, + "loss": 1.4148, + "step": 974 + }, + { + "epoch": 1.2134411947728687, + "grad_norm": 2.8719074726104736, + "learning_rate": 3.3417896678966795e-06, + "loss": 1.4196, + "step": 975 + }, + { + "epoch": 1.2146857498444306, + "grad_norm": 2.8469338417053223, + "learning_rate": 3.3394833948339485e-06, + "loss": 1.4941, + "step": 976 + }, + { + "epoch": 1.2159303049159926, + "grad_norm": 2.9780938625335693, + "learning_rate": 3.337177121771218e-06, + "loss": 1.513, + "step": 977 + }, + { + "epoch": 1.2171748599875545, + "grad_norm": 2.814760684967041, + "learning_rate": 3.3348708487084873e-06, + "loss": 1.4204, + "step": 978 + }, + { + "epoch": 1.2184194150591163, + "grad_norm": 3.016261339187622, + "learning_rate": 3.332564575645757e-06, + "loss": 1.4389, + "step": 979 + }, + { + "epoch": 1.2196639701306782, + "grad_norm": 2.9385855197906494, + "learning_rate": 3.330258302583026e-06, + "loss": 1.4885, + "step": 980 + }, + { + "epoch": 1.2196639701306782, + "eval_loss": 1.6688873767852783, + "eval_runtime": 46.2942, + "eval_samples_per_second": 21.601, + "eval_steps_per_second": 0.907, + "step": 980 + }, + { + "epoch": 1.2209085252022402, + "grad_norm": 3.1434757709503174, + "learning_rate": 3.3279520295202956e-06, + "loss": 1.5157, + "step": 981 + }, + { + "epoch": 1.222153080273802, + "grad_norm": 3.0104827880859375, + "learning_rate": 3.325645756457565e-06, + "loss": 1.49, + "step": 982 + }, + { + "epoch": 1.223397635345364, + "grad_norm": 3.1913645267486572, + "learning_rate": 3.323339483394834e-06, + "loss": 1.4937, + "step": 983 + }, + { + "epoch": 1.224642190416926, + "grad_norm": 3.19496750831604, + "learning_rate": 3.3210332103321034e-06, + "loss": 1.5197, + "step": 984 + }, + { + "epoch": 1.2258867454884879, + "grad_norm": 3.046232223510742, + "learning_rate": 3.3187269372693733e-06, + "loss": 1.4497, + "step": 985 + }, + { + "epoch": 1.2271313005600497, + "grad_norm": 2.850675344467163, + "learning_rate": 3.3164206642066423e-06, + "loss": 1.4595, + "step": 986 + }, + { + "epoch": 1.2283758556316118, + "grad_norm": 2.7744576930999756, + "learning_rate": 3.3141143911439117e-06, + "loss": 1.4726, + "step": 987 + }, + { + "epoch": 1.2296204107031736, + "grad_norm": 3.017153024673462, + "learning_rate": 3.311808118081181e-06, + "loss": 1.493, + "step": 988 + }, + { + "epoch": 1.2308649657747355, + "grad_norm": 2.8821120262145996, + "learning_rate": 3.3095018450184506e-06, + "loss": 1.4734, + "step": 989 + }, + { + "epoch": 1.2321095208462975, + "grad_norm": 2.8805840015411377, + "learning_rate": 3.3071955719557196e-06, + "loss": 1.4562, + "step": 990 + }, + { + "epoch": 1.2321095208462975, + "eval_loss": 1.669345736503601, + "eval_runtime": 47.0893, + "eval_samples_per_second": 21.236, + "eval_steps_per_second": 0.892, + "step": 990 + }, + { + "epoch": 1.2333540759178594, + "grad_norm": 2.7599008083343506, + "learning_rate": 3.3048892988929894e-06, + "loss": 1.4218, + "step": 991 + }, + { + "epoch": 1.2345986309894212, + "grad_norm": 2.9835240840911865, + "learning_rate": 3.302583025830259e-06, + "loss": 1.4596, + "step": 992 + }, + { + "epoch": 1.235843186060983, + "grad_norm": 2.843358039855957, + "learning_rate": 3.300276752767528e-06, + "loss": 1.4561, + "step": 993 + }, + { + "epoch": 1.2370877411325452, + "grad_norm": 2.7386093139648438, + "learning_rate": 3.2979704797047972e-06, + "loss": 1.4475, + "step": 994 + }, + { + "epoch": 1.238332296204107, + "grad_norm": 2.668161392211914, + "learning_rate": 3.2956642066420667e-06, + "loss": 1.4661, + "step": 995 + }, + { + "epoch": 1.2395768512756689, + "grad_norm": 2.947235107421875, + "learning_rate": 3.2933579335793357e-06, + "loss": 1.4358, + "step": 996 + }, + { + "epoch": 1.240821406347231, + "grad_norm": 2.8159878253936768, + "learning_rate": 3.2910516605166055e-06, + "loss": 1.4432, + "step": 997 + }, + { + "epoch": 1.2420659614187928, + "grad_norm": 2.896597385406494, + "learning_rate": 3.288745387453875e-06, + "loss": 1.5585, + "step": 998 + }, + { + "epoch": 1.2433105164903546, + "grad_norm": 2.80241322517395, + "learning_rate": 3.286439114391144e-06, + "loss": 1.5076, + "step": 999 + }, + { + "epoch": 1.2445550715619167, + "grad_norm": 3.008939027786255, + "learning_rate": 3.2841328413284134e-06, + "loss": 1.4234, + "step": 1000 + }, + { + "epoch": 1.2445550715619167, + "eval_loss": 1.6621588468551636, + "eval_runtime": 50.4926, + "eval_samples_per_second": 19.805, + "eval_steps_per_second": 0.832, + "step": 1000 + }, + { + "epoch": 1.2457996266334785, + "grad_norm": 2.9872381687164307, + "learning_rate": 3.281826568265683e-06, + "loss": 1.453, + "step": 1001 + }, + { + "epoch": 1.2470441817050404, + "grad_norm": 2.704676628112793, + "learning_rate": 3.2795202952029526e-06, + "loss": 1.4711, + "step": 1002 + }, + { + "epoch": 1.2482887367766025, + "grad_norm": 2.8100757598876953, + "learning_rate": 3.2772140221402216e-06, + "loss": 1.3884, + "step": 1003 + }, + { + "epoch": 1.2495332918481643, + "grad_norm": 2.902953624725342, + "learning_rate": 3.274907749077491e-06, + "loss": 1.4986, + "step": 1004 + }, + { + "epoch": 1.2507778469197262, + "grad_norm": 2.980163097381592, + "learning_rate": 3.2726014760147605e-06, + "loss": 1.5067, + "step": 1005 + }, + { + "epoch": 1.252022401991288, + "grad_norm": 2.8820345401763916, + "learning_rate": 3.2702952029520295e-06, + "loss": 1.4995, + "step": 1006 + }, + { + "epoch": 1.25326695706285, + "grad_norm": 2.8219635486602783, + "learning_rate": 3.2679889298892993e-06, + "loss": 1.4881, + "step": 1007 + }, + { + "epoch": 1.254511512134412, + "grad_norm": 2.8593199253082275, + "learning_rate": 3.2656826568265687e-06, + "loss": 1.4469, + "step": 1008 + }, + { + "epoch": 1.255756067205974, + "grad_norm": 2.888357639312744, + "learning_rate": 3.2633763837638377e-06, + "loss": 1.4869, + "step": 1009 + }, + { + "epoch": 1.2570006222775358, + "grad_norm": 2.9985013008117676, + "learning_rate": 3.261070110701107e-06, + "loss": 1.4724, + "step": 1010 + }, + { + "epoch": 1.2570006222775358, + "eval_loss": 1.6492141485214233, + "eval_runtime": 45.2197, + "eval_samples_per_second": 22.114, + "eval_steps_per_second": 0.929, + "step": 1010 + }, + { + "epoch": 1.2582451773490977, + "grad_norm": 2.860780715942383, + "learning_rate": 3.2587638376383766e-06, + "loss": 1.4618, + "step": 1011 + }, + { + "epoch": 1.2594897324206595, + "grad_norm": 2.766479730606079, + "learning_rate": 3.2564575645756456e-06, + "loss": 1.4642, + "step": 1012 + }, + { + "epoch": 1.2607342874922216, + "grad_norm": 2.711759328842163, + "learning_rate": 3.2541512915129154e-06, + "loss": 1.4283, + "step": 1013 + }, + { + "epoch": 1.2619788425637835, + "grad_norm": 2.8365769386291504, + "learning_rate": 3.251845018450185e-06, + "loss": 1.4248, + "step": 1014 + }, + { + "epoch": 1.2632233976353453, + "grad_norm": 2.957620143890381, + "learning_rate": 3.2495387453874543e-06, + "loss": 1.4662, + "step": 1015 + }, + { + "epoch": 1.2644679527069074, + "grad_norm": 2.8066208362579346, + "learning_rate": 3.2472324723247233e-06, + "loss": 1.4447, + "step": 1016 + }, + { + "epoch": 1.2657125077784692, + "grad_norm": 2.902040958404541, + "learning_rate": 3.244926199261993e-06, + "loss": 1.4743, + "step": 1017 + }, + { + "epoch": 1.266957062850031, + "grad_norm": 2.7636687755584717, + "learning_rate": 3.2426199261992625e-06, + "loss": 1.4465, + "step": 1018 + }, + { + "epoch": 1.268201617921593, + "grad_norm": 2.95261549949646, + "learning_rate": 3.2403136531365315e-06, + "loss": 1.4769, + "step": 1019 + }, + { + "epoch": 1.269446172993155, + "grad_norm": 2.911128520965576, + "learning_rate": 3.238007380073801e-06, + "loss": 1.3843, + "step": 1020 + }, + { + "epoch": 1.269446172993155, + "eval_loss": 1.6526964902877808, + "eval_runtime": 48.2126, + "eval_samples_per_second": 20.741, + "eval_steps_per_second": 0.871, + "step": 1020 + }, + { + "epoch": 1.2706907280647168, + "grad_norm": 2.769951820373535, + "learning_rate": 3.2357011070110704e-06, + "loss": 1.4847, + "step": 1021 + }, + { + "epoch": 1.271935283136279, + "grad_norm": 3.0451674461364746, + "learning_rate": 3.2333948339483394e-06, + "loss": 1.4453, + "step": 1022 + }, + { + "epoch": 1.2731798382078408, + "grad_norm": 2.9543521404266357, + "learning_rate": 3.2310885608856092e-06, + "loss": 1.4789, + "step": 1023 + }, + { + "epoch": 1.2744243932794026, + "grad_norm": 2.9174394607543945, + "learning_rate": 3.2287822878228787e-06, + "loss": 1.4807, + "step": 1024 + }, + { + "epoch": 1.2756689483509644, + "grad_norm": 2.888258934020996, + "learning_rate": 3.2264760147601477e-06, + "loss": 1.4367, + "step": 1025 + }, + { + "epoch": 1.2769135034225265, + "grad_norm": 2.8346047401428223, + "learning_rate": 3.224169741697417e-06, + "loss": 1.4313, + "step": 1026 + }, + { + "epoch": 1.2781580584940884, + "grad_norm": 2.7531332969665527, + "learning_rate": 3.2218634686346865e-06, + "loss": 1.4018, + "step": 1027 + }, + { + "epoch": 1.2794026135656502, + "grad_norm": 3.136946439743042, + "learning_rate": 3.2195571955719564e-06, + "loss": 1.4417, + "step": 1028 + }, + { + "epoch": 1.2806471686372123, + "grad_norm": 2.9618165493011475, + "learning_rate": 3.2172509225092254e-06, + "loss": 1.5117, + "step": 1029 + }, + { + "epoch": 1.2818917237087741, + "grad_norm": 3.0740747451782227, + "learning_rate": 3.2149446494464948e-06, + "loss": 1.4641, + "step": 1030 + }, + { + "epoch": 1.2818917237087741, + "eval_loss": 1.6572761535644531, + "eval_runtime": 51.6696, + "eval_samples_per_second": 19.354, + "eval_steps_per_second": 0.813, + "step": 1030 + }, + { + "epoch": 1.283136278780336, + "grad_norm": 2.9225106239318848, + "learning_rate": 3.212638376383764e-06, + "loss": 1.4421, + "step": 1031 + }, + { + "epoch": 1.2843808338518978, + "grad_norm": 2.9530649185180664, + "learning_rate": 3.210332103321033e-06, + "loss": 1.5138, + "step": 1032 + }, + { + "epoch": 1.28562538892346, + "grad_norm": 2.8513238430023193, + "learning_rate": 3.208025830258303e-06, + "loss": 1.411, + "step": 1033 + }, + { + "epoch": 1.2868699439950217, + "grad_norm": 2.7440171241760254, + "learning_rate": 3.2057195571955725e-06, + "loss": 1.4182, + "step": 1034 + }, + { + "epoch": 1.2881144990665838, + "grad_norm": 2.8126630783081055, + "learning_rate": 3.2034132841328415e-06, + "loss": 1.4333, + "step": 1035 + }, + { + "epoch": 1.2893590541381457, + "grad_norm": 2.8598744869232178, + "learning_rate": 3.201107011070111e-06, + "loss": 1.439, + "step": 1036 + }, + { + "epoch": 1.2906036092097075, + "grad_norm": 2.901622772216797, + "learning_rate": 3.1988007380073803e-06, + "loss": 1.4461, + "step": 1037 + }, + { + "epoch": 1.2918481642812694, + "grad_norm": 2.7670180797576904, + "learning_rate": 3.1964944649446493e-06, + "loss": 1.4772, + "step": 1038 + }, + { + "epoch": 1.2930927193528314, + "grad_norm": 2.7860350608825684, + "learning_rate": 3.194188191881919e-06, + "loss": 1.4414, + "step": 1039 + }, + { + "epoch": 1.2943372744243933, + "grad_norm": 2.822052478790283, + "learning_rate": 3.1918819188191886e-06, + "loss": 1.4192, + "step": 1040 + }, + { + "epoch": 1.2943372744243933, + "eval_loss": 1.663284420967102, + "eval_runtime": 47.2822, + "eval_samples_per_second": 21.15, + "eval_steps_per_second": 0.888, + "step": 1040 + }, + { + "epoch": 1.2955818294959551, + "grad_norm": 2.909109115600586, + "learning_rate": 3.189575645756458e-06, + "loss": 1.4795, + "step": 1041 + }, + { + "epoch": 1.2968263845675172, + "grad_norm": 2.933673143386841, + "learning_rate": 3.187269372693727e-06, + "loss": 1.5038, + "step": 1042 + }, + { + "epoch": 1.298070939639079, + "grad_norm": 3.032518148422241, + "learning_rate": 3.1849630996309964e-06, + "loss": 1.5133, + "step": 1043 + }, + { + "epoch": 1.299315494710641, + "grad_norm": 2.826040506362915, + "learning_rate": 3.1826568265682663e-06, + "loss": 1.4395, + "step": 1044 + }, + { + "epoch": 1.3005600497822027, + "grad_norm": 2.8152658939361572, + "learning_rate": 3.1803505535055353e-06, + "loss": 1.4787, + "step": 1045 + }, + { + "epoch": 1.3018046048537648, + "grad_norm": 2.9410431385040283, + "learning_rate": 3.1780442804428047e-06, + "loss": 1.4375, + "step": 1046 + }, + { + "epoch": 1.3030491599253267, + "grad_norm": 2.948636531829834, + "learning_rate": 3.175738007380074e-06, + "loss": 1.4847, + "step": 1047 + }, + { + "epoch": 1.3042937149968887, + "grad_norm": 2.6287615299224854, + "learning_rate": 3.173431734317343e-06, + "loss": 1.4186, + "step": 1048 + }, + { + "epoch": 1.3055382700684506, + "grad_norm": 2.836029052734375, + "learning_rate": 3.1711254612546125e-06, + "loss": 1.4251, + "step": 1049 + }, + { + "epoch": 1.3067828251400124, + "grad_norm": 2.982081651687622, + "learning_rate": 3.1688191881918824e-06, + "loss": 1.4951, + "step": 1050 + }, + { + "epoch": 1.3067828251400124, + "eval_loss": 1.6578067541122437, + "eval_runtime": 47.5847, + "eval_samples_per_second": 21.015, + "eval_steps_per_second": 0.883, + "step": 1050 + }, + { + "epoch": 1.3080273802115743, + "grad_norm": 2.9297118186950684, + "learning_rate": 3.166512915129152e-06, + "loss": 1.4699, + "step": 1051 + }, + { + "epoch": 1.3092719352831363, + "grad_norm": 2.906435251235962, + "learning_rate": 3.164206642066421e-06, + "loss": 1.4562, + "step": 1052 + }, + { + "epoch": 1.3105164903546982, + "grad_norm": 2.8655896186828613, + "learning_rate": 3.1619003690036902e-06, + "loss": 1.4351, + "step": 1053 + }, + { + "epoch": 1.31176104542626, + "grad_norm": 2.779244899749756, + "learning_rate": 3.15959409594096e-06, + "loss": 1.4479, + "step": 1054 + }, + { + "epoch": 1.3130056004978221, + "grad_norm": 2.9863739013671875, + "learning_rate": 3.157287822878229e-06, + "loss": 1.4964, + "step": 1055 + }, + { + "epoch": 1.314250155569384, + "grad_norm": 2.821871042251587, + "learning_rate": 3.1549815498154985e-06, + "loss": 1.4173, + "step": 1056 + }, + { + "epoch": 1.3154947106409458, + "grad_norm": 2.8573882579803467, + "learning_rate": 3.152675276752768e-06, + "loss": 1.4877, + "step": 1057 + }, + { + "epoch": 1.3167392657125077, + "grad_norm": 2.9824137687683105, + "learning_rate": 3.150369003690037e-06, + "loss": 1.5124, + "step": 1058 + }, + { + "epoch": 1.3179838207840697, + "grad_norm": 2.881098747253418, + "learning_rate": 3.1480627306273063e-06, + "loss": 1.4346, + "step": 1059 + }, + { + "epoch": 1.3192283758556316, + "grad_norm": 2.918782949447632, + "learning_rate": 3.145756457564576e-06, + "loss": 1.4797, + "step": 1060 + }, + { + "epoch": 1.3192283758556316, + "eval_loss": 1.6510179042816162, + "eval_runtime": 53.6356, + "eval_samples_per_second": 18.644, + "eval_steps_per_second": 0.783, + "step": 1060 + }, + { + "epoch": 1.3204729309271936, + "grad_norm": 3.045776128768921, + "learning_rate": 3.143450184501845e-06, + "loss": 1.5039, + "step": 1061 + }, + { + "epoch": 1.3217174859987555, + "grad_norm": 3.041501760482788, + "learning_rate": 3.1411439114391146e-06, + "loss": 1.4878, + "step": 1062 + }, + { + "epoch": 1.3229620410703173, + "grad_norm": 2.917837142944336, + "learning_rate": 3.138837638376384e-06, + "loss": 1.4202, + "step": 1063 + }, + { + "epoch": 1.3242065961418792, + "grad_norm": 2.708070755004883, + "learning_rate": 3.1365313653136535e-06, + "loss": 1.4071, + "step": 1064 + }, + { + "epoch": 1.3254511512134413, + "grad_norm": 2.9146156311035156, + "learning_rate": 3.1342250922509225e-06, + "loss": 1.4428, + "step": 1065 + }, + { + "epoch": 1.326695706285003, + "grad_norm": 3.220919132232666, + "learning_rate": 3.1319188191881923e-06, + "loss": 1.4733, + "step": 1066 + }, + { + "epoch": 1.327940261356565, + "grad_norm": 2.9127554893493652, + "learning_rate": 3.1296125461254617e-06, + "loss": 1.4554, + "step": 1067 + }, + { + "epoch": 1.329184816428127, + "grad_norm": 3.053191661834717, + "learning_rate": 3.1273062730627307e-06, + "loss": 1.4044, + "step": 1068 + }, + { + "epoch": 1.3304293714996889, + "grad_norm": 3.2937021255493164, + "learning_rate": 3.125e-06, + "loss": 1.4949, + "step": 1069 + }, + { + "epoch": 1.3316739265712507, + "grad_norm": 2.976717948913574, + "learning_rate": 3.12269372693727e-06, + "loss": 1.446, + "step": 1070 + }, + { + "epoch": 1.3316739265712507, + "eval_loss": 1.6503105163574219, + "eval_runtime": 50.7579, + "eval_samples_per_second": 19.701, + "eval_steps_per_second": 0.827, + "step": 1070 + }, + { + "epoch": 1.3329184816428126, + "grad_norm": 3.0860936641693115, + "learning_rate": 3.120387453874539e-06, + "loss": 1.4055, + "step": 1071 + }, + { + "epoch": 1.3341630367143746, + "grad_norm": 3.0401217937469482, + "learning_rate": 3.1180811808118084e-06, + "loss": 1.4688, + "step": 1072 + }, + { + "epoch": 1.3354075917859365, + "grad_norm": 2.953253984451294, + "learning_rate": 3.115774907749078e-06, + "loss": 1.5001, + "step": 1073 + }, + { + "epoch": 1.3366521468574986, + "grad_norm": 3.0665059089660645, + "learning_rate": 3.113468634686347e-06, + "loss": 1.4217, + "step": 1074 + }, + { + "epoch": 1.3378967019290604, + "grad_norm": 3.121600389480591, + "learning_rate": 3.1111623616236163e-06, + "loss": 1.4508, + "step": 1075 + }, + { + "epoch": 1.3391412570006223, + "grad_norm": 2.8057339191436768, + "learning_rate": 3.108856088560886e-06, + "loss": 1.4241, + "step": 1076 + }, + { + "epoch": 1.340385812072184, + "grad_norm": 3.0147411823272705, + "learning_rate": 3.1065498154981555e-06, + "loss": 1.479, + "step": 1077 + }, + { + "epoch": 1.3416303671437462, + "grad_norm": 3.030010223388672, + "learning_rate": 3.1042435424354245e-06, + "loss": 1.4861, + "step": 1078 + }, + { + "epoch": 1.342874922215308, + "grad_norm": 3.027998208999634, + "learning_rate": 3.101937269372694e-06, + "loss": 1.4916, + "step": 1079 + }, + { + "epoch": 1.3441194772868699, + "grad_norm": 2.9252536296844482, + "learning_rate": 3.0996309963099634e-06, + "loss": 1.418, + "step": 1080 + }, + { + "epoch": 1.3441194772868699, + "eval_loss": 1.6490333080291748, + "eval_runtime": 47.4026, + "eval_samples_per_second": 21.096, + "eval_steps_per_second": 0.886, + "step": 1080 + }, + { + "epoch": 1.345364032358432, + "grad_norm": 3.110847234725952, + "learning_rate": 3.0973247232472324e-06, + "loss": 1.4127, + "step": 1081 + }, + { + "epoch": 1.3466085874299938, + "grad_norm": 2.846491575241089, + "learning_rate": 3.0950184501845022e-06, + "loss": 1.444, + "step": 1082 + }, + { + "epoch": 1.3478531425015556, + "grad_norm": 2.950533151626587, + "learning_rate": 3.0927121771217716e-06, + "loss": 1.4456, + "step": 1083 + }, + { + "epoch": 1.3490976975731175, + "grad_norm": 2.875964403152466, + "learning_rate": 3.0904059040590406e-06, + "loss": 1.4485, + "step": 1084 + }, + { + "epoch": 1.3503422526446796, + "grad_norm": 2.9612326622009277, + "learning_rate": 3.08809963099631e-06, + "loss": 1.4446, + "step": 1085 + }, + { + "epoch": 1.3515868077162414, + "grad_norm": 2.925297975540161, + "learning_rate": 3.0857933579335795e-06, + "loss": 1.4847, + "step": 1086 + }, + { + "epoch": 1.3528313627878035, + "grad_norm": 2.991616725921631, + "learning_rate": 3.0834870848708485e-06, + "loss": 1.5151, + "step": 1087 + }, + { + "epoch": 1.3540759178593653, + "grad_norm": 2.96329927444458, + "learning_rate": 3.0811808118081183e-06, + "loss": 1.4574, + "step": 1088 + }, + { + "epoch": 1.3553204729309272, + "grad_norm": 2.948702573776245, + "learning_rate": 3.0788745387453878e-06, + "loss": 1.4585, + "step": 1089 + }, + { + "epoch": 1.356565028002489, + "grad_norm": 3.218463659286499, + "learning_rate": 3.076568265682657e-06, + "loss": 1.4439, + "step": 1090 + }, + { + "epoch": 1.356565028002489, + "eval_loss": 1.645838975906372, + "eval_runtime": 47.048, + "eval_samples_per_second": 21.255, + "eval_steps_per_second": 0.893, + "step": 1090 + }, + { + "epoch": 1.357809583074051, + "grad_norm": 2.9799578189849854, + "learning_rate": 3.074261992619926e-06, + "loss": 1.4883, + "step": 1091 + }, + { + "epoch": 1.359054138145613, + "grad_norm": 2.7660505771636963, + "learning_rate": 3.071955719557196e-06, + "loss": 1.4349, + "step": 1092 + }, + { + "epoch": 1.3602986932171748, + "grad_norm": 3.0201916694641113, + "learning_rate": 3.0696494464944655e-06, + "loss": 1.418, + "step": 1093 + }, + { + "epoch": 1.3615432482887369, + "grad_norm": 2.844207763671875, + "learning_rate": 3.0673431734317345e-06, + "loss": 1.4769, + "step": 1094 + }, + { + "epoch": 1.3627878033602987, + "grad_norm": 2.8306753635406494, + "learning_rate": 3.065036900369004e-06, + "loss": 1.4653, + "step": 1095 + }, + { + "epoch": 1.3640323584318605, + "grad_norm": 2.805023431777954, + "learning_rate": 3.0627306273062733e-06, + "loss": 1.4771, + "step": 1096 + }, + { + "epoch": 1.3652769135034224, + "grad_norm": 3.020883083343506, + "learning_rate": 3.0604243542435423e-06, + "loss": 1.4237, + "step": 1097 + }, + { + "epoch": 1.3665214685749845, + "grad_norm": 2.8395731449127197, + "learning_rate": 3.058118081180812e-06, + "loss": 1.4647, + "step": 1098 + }, + { + "epoch": 1.3677660236465463, + "grad_norm": 2.871156692504883, + "learning_rate": 3.0558118081180816e-06, + "loss": 1.4196, + "step": 1099 + }, + { + "epoch": 1.3690105787181084, + "grad_norm": 2.974905014038086, + "learning_rate": 3.053505535055351e-06, + "loss": 1.4779, + "step": 1100 + }, + { + "epoch": 1.3690105787181084, + "eval_loss": 1.651503562927246, + "eval_runtime": 54.2713, + "eval_samples_per_second": 18.426, + "eval_steps_per_second": 0.774, + "step": 1100 + }, + { + "epoch": 1.3702551337896702, + "grad_norm": 2.7591328620910645, + "learning_rate": 3.05119926199262e-06, + "loss": 1.3899, + "step": 1101 + }, + { + "epoch": 1.371499688861232, + "grad_norm": 2.8575668334960938, + "learning_rate": 3.0488929889298894e-06, + "loss": 1.4482, + "step": 1102 + }, + { + "epoch": 1.372744243932794, + "grad_norm": 2.9049339294433594, + "learning_rate": 3.0465867158671593e-06, + "loss": 1.4785, + "step": 1103 + }, + { + "epoch": 1.373988799004356, + "grad_norm": 2.816453695297241, + "learning_rate": 3.0442804428044283e-06, + "loss": 1.4601, + "step": 1104 + }, + { + "epoch": 1.3752333540759178, + "grad_norm": 2.8844189643859863, + "learning_rate": 3.0419741697416977e-06, + "loss": 1.496, + "step": 1105 + }, + { + "epoch": 1.3764779091474797, + "grad_norm": 3.014744281768799, + "learning_rate": 3.039667896678967e-06, + "loss": 1.4098, + "step": 1106 + }, + { + "epoch": 1.3777224642190418, + "grad_norm": 2.8243560791015625, + "learning_rate": 3.037361623616236e-06, + "loss": 1.4086, + "step": 1107 + }, + { + "epoch": 1.3789670192906036, + "grad_norm": 2.8333253860473633, + "learning_rate": 3.035055350553506e-06, + "loss": 1.3516, + "step": 1108 + }, + { + "epoch": 1.3802115743621655, + "grad_norm": 2.926455020904541, + "learning_rate": 3.0327490774907754e-06, + "loss": 1.3976, + "step": 1109 + }, + { + "epoch": 1.3814561294337273, + "grad_norm": 2.900937080383301, + "learning_rate": 3.0304428044280444e-06, + "loss": 1.4336, + "step": 1110 + }, + { + "epoch": 1.3814561294337273, + "eval_loss": 1.647048830986023, + "eval_runtime": 42.3955, + "eval_samples_per_second": 23.587, + "eval_steps_per_second": 0.991, + "step": 1110 + }, + { + "epoch": 1.3827006845052894, + "grad_norm": 3.0756969451904297, + "learning_rate": 3.028136531365314e-06, + "loss": 1.4674, + "step": 1111 + }, + { + "epoch": 1.3839452395768512, + "grad_norm": 2.7754578590393066, + "learning_rate": 3.0258302583025832e-06, + "loss": 1.4435, + "step": 1112 + }, + { + "epoch": 1.3851897946484133, + "grad_norm": 2.8011393547058105, + "learning_rate": 3.023523985239853e-06, + "loss": 1.4212, + "step": 1113 + }, + { + "epoch": 1.3864343497199751, + "grad_norm": 2.7063710689544678, + "learning_rate": 3.021217712177122e-06, + "loss": 1.3769, + "step": 1114 + }, + { + "epoch": 1.387678904791537, + "grad_norm": 2.830411434173584, + "learning_rate": 3.0189114391143915e-06, + "loss": 1.4832, + "step": 1115 + }, + { + "epoch": 1.3889234598630988, + "grad_norm": 2.9075911045074463, + "learning_rate": 3.016605166051661e-06, + "loss": 1.486, + "step": 1116 + }, + { + "epoch": 1.390168014934661, + "grad_norm": 2.847822666168213, + "learning_rate": 3.01429889298893e-06, + "loss": 1.424, + "step": 1117 + }, + { + "epoch": 1.3914125700062228, + "grad_norm": 2.8822033405303955, + "learning_rate": 3.0119926199261993e-06, + "loss": 1.4218, + "step": 1118 + }, + { + "epoch": 1.3926571250777848, + "grad_norm": 2.8944740295410156, + "learning_rate": 3.009686346863469e-06, + "loss": 1.4283, + "step": 1119 + }, + { + "epoch": 1.3939016801493467, + "grad_norm": 2.765190839767456, + "learning_rate": 3.007380073800738e-06, + "loss": 1.4622, + "step": 1120 + }, + { + "epoch": 1.3939016801493467, + "eval_loss": 1.6451724767684937, + "eval_runtime": 43.7449, + "eval_samples_per_second": 22.86, + "eval_steps_per_second": 0.96, + "step": 1120 + }, + { + "epoch": 1.3951462352209085, + "grad_norm": 2.877002716064453, + "learning_rate": 3.0050738007380076e-06, + "loss": 1.4145, + "step": 1121 + }, + { + "epoch": 1.3963907902924704, + "grad_norm": 2.870389699935913, + "learning_rate": 3.002767527675277e-06, + "loss": 1.4401, + "step": 1122 + }, + { + "epoch": 1.3976353453640322, + "grad_norm": 2.7788920402526855, + "learning_rate": 3.000461254612546e-06, + "loss": 1.4514, + "step": 1123 + }, + { + "epoch": 1.3988799004355943, + "grad_norm": 2.9269332885742188, + "learning_rate": 2.998154981549816e-06, + "loss": 1.3961, + "step": 1124 + }, + { + "epoch": 1.4001244555071561, + "grad_norm": 3.013291835784912, + "learning_rate": 2.9958487084870853e-06, + "loss": 1.4597, + "step": 1125 + }, + { + "epoch": 1.4013690105787182, + "grad_norm": 2.9063880443573, + "learning_rate": 2.9935424354243547e-06, + "loss": 1.4668, + "step": 1126 + }, + { + "epoch": 1.40261356565028, + "grad_norm": 2.8926970958709717, + "learning_rate": 2.9912361623616237e-06, + "loss": 1.4332, + "step": 1127 + }, + { + "epoch": 1.403858120721842, + "grad_norm": 2.8563647270202637, + "learning_rate": 2.988929889298893e-06, + "loss": 1.407, + "step": 1128 + }, + { + "epoch": 1.4051026757934038, + "grad_norm": 2.8725359439849854, + "learning_rate": 2.986623616236163e-06, + "loss": 1.4434, + "step": 1129 + }, + { + "epoch": 1.4063472308649658, + "grad_norm": 2.906898260116577, + "learning_rate": 2.984317343173432e-06, + "loss": 1.4745, + "step": 1130 + }, + { + "epoch": 1.4063472308649658, + "eval_loss": 1.647884726524353, + "eval_runtime": 44.9221, + "eval_samples_per_second": 22.261, + "eval_steps_per_second": 0.935, + "step": 1130 + }, + { + "epoch": 1.4075917859365277, + "grad_norm": 2.8297719955444336, + "learning_rate": 2.9820110701107014e-06, + "loss": 1.4542, + "step": 1131 + }, + { + "epoch": 1.4088363410080897, + "grad_norm": 2.841031551361084, + "learning_rate": 2.979704797047971e-06, + "loss": 1.4105, + "step": 1132 + }, + { + "epoch": 1.4100808960796516, + "grad_norm": 2.867791175842285, + "learning_rate": 2.97739852398524e-06, + "loss": 1.4409, + "step": 1133 + }, + { + "epoch": 1.4113254511512134, + "grad_norm": 2.8156039714813232, + "learning_rate": 2.9750922509225093e-06, + "loss": 1.4284, + "step": 1134 + }, + { + "epoch": 1.4125700062227753, + "grad_norm": 2.825985908508301, + "learning_rate": 2.972785977859779e-06, + "loss": 1.4594, + "step": 1135 + }, + { + "epoch": 1.4138145612943374, + "grad_norm": 3.0134711265563965, + "learning_rate": 2.970479704797048e-06, + "loss": 1.4743, + "step": 1136 + }, + { + "epoch": 1.4150591163658992, + "grad_norm": 2.9829142093658447, + "learning_rate": 2.9681734317343175e-06, + "loss": 1.4935, + "step": 1137 + }, + { + "epoch": 1.416303671437461, + "grad_norm": 3.067620277404785, + "learning_rate": 2.965867158671587e-06, + "loss": 1.4266, + "step": 1138 + }, + { + "epoch": 1.4175482265090231, + "grad_norm": 2.8969919681549072, + "learning_rate": 2.9635608856088564e-06, + "loss": 1.451, + "step": 1139 + }, + { + "epoch": 1.418792781580585, + "grad_norm": 2.8706412315368652, + "learning_rate": 2.9612546125461254e-06, + "loss": 1.4834, + "step": 1140 + }, + { + "epoch": 1.418792781580585, + "eval_loss": 1.643480658531189, + "eval_runtime": 44.743, + "eval_samples_per_second": 22.35, + "eval_steps_per_second": 0.939, + "step": 1140 + }, + { + "epoch": 1.4200373366521468, + "grad_norm": 2.8801848888397217, + "learning_rate": 2.958948339483395e-06, + "loss": 1.4258, + "step": 1141 + }, + { + "epoch": 1.4212818917237087, + "grad_norm": 2.774635076522827, + "learning_rate": 2.9566420664206646e-06, + "loss": 1.416, + "step": 1142 + }, + { + "epoch": 1.4225264467952707, + "grad_norm": 2.8460140228271484, + "learning_rate": 2.9543357933579336e-06, + "loss": 1.4213, + "step": 1143 + }, + { + "epoch": 1.4237710018668326, + "grad_norm": 2.907888889312744, + "learning_rate": 2.952029520295203e-06, + "loss": 1.4507, + "step": 1144 + }, + { + "epoch": 1.4250155569383947, + "grad_norm": 2.789041757583618, + "learning_rate": 2.949723247232473e-06, + "loss": 1.4257, + "step": 1145 + }, + { + "epoch": 1.4262601120099565, + "grad_norm": 2.78859806060791, + "learning_rate": 2.947416974169742e-06, + "loss": 1.4377, + "step": 1146 + }, + { + "epoch": 1.4275046670815184, + "grad_norm": 2.840303421020508, + "learning_rate": 2.9451107011070113e-06, + "loss": 1.4684, + "step": 1147 + }, + { + "epoch": 1.4287492221530802, + "grad_norm": 2.8800530433654785, + "learning_rate": 2.9428044280442807e-06, + "loss": 1.4912, + "step": 1148 + }, + { + "epoch": 1.4299937772246423, + "grad_norm": 2.841092824935913, + "learning_rate": 2.9404981549815497e-06, + "loss": 1.4234, + "step": 1149 + }, + { + "epoch": 1.4312383322962041, + "grad_norm": 2.8226537704467773, + "learning_rate": 2.938191881918819e-06, + "loss": 1.4578, + "step": 1150 + }, + { + "epoch": 1.4312383322962041, + "eval_loss": 1.643249273300171, + "eval_runtime": 43.5586, + "eval_samples_per_second": 22.958, + "eval_steps_per_second": 0.964, + "step": 1150 + }, + { + "epoch": 1.432482887367766, + "grad_norm": 2.869736909866333, + "learning_rate": 2.935885608856089e-06, + "loss": 1.4701, + "step": 1151 + }, + { + "epoch": 1.433727442439328, + "grad_norm": 2.689211130142212, + "learning_rate": 2.9335793357933584e-06, + "loss": 1.4483, + "step": 1152 + }, + { + "epoch": 1.4349719975108899, + "grad_norm": 2.9267077445983887, + "learning_rate": 2.9312730627306274e-06, + "loss": 1.4446, + "step": 1153 + }, + { + "epoch": 1.4362165525824517, + "grad_norm": 2.906862497329712, + "learning_rate": 2.928966789667897e-06, + "loss": 1.4524, + "step": 1154 + }, + { + "epoch": 1.4374611076540136, + "grad_norm": 2.8249800205230713, + "learning_rate": 2.9266605166051663e-06, + "loss": 1.4208, + "step": 1155 + }, + { + "epoch": 1.4387056627255757, + "grad_norm": 2.9836032390594482, + "learning_rate": 2.9243542435424353e-06, + "loss": 1.4581, + "step": 1156 + }, + { + "epoch": 1.4399502177971375, + "grad_norm": 2.9595284461975098, + "learning_rate": 2.922047970479705e-06, + "loss": 1.4141, + "step": 1157 + }, + { + "epoch": 1.4411947728686996, + "grad_norm": 3.0363423824310303, + "learning_rate": 2.9197416974169746e-06, + "loss": 1.4859, + "step": 1158 + }, + { + "epoch": 1.4424393279402614, + "grad_norm": 2.911201000213623, + "learning_rate": 2.9174354243542436e-06, + "loss": 1.4346, + "step": 1159 + }, + { + "epoch": 1.4436838830118233, + "grad_norm": 2.8617193698883057, + "learning_rate": 2.915129151291513e-06, + "loss": 1.433, + "step": 1160 + }, + { + "epoch": 1.4436838830118233, + "eval_loss": 1.6437232494354248, + "eval_runtime": 43.961, + "eval_samples_per_second": 22.747, + "eval_steps_per_second": 0.955, + "step": 1160 + }, + { + "epoch": 1.4449284380833851, + "grad_norm": 2.9584295749664307, + "learning_rate": 2.912822878228783e-06, + "loss": 1.4847, + "step": 1161 + }, + { + "epoch": 1.4461729931549472, + "grad_norm": 2.8209400177001953, + "learning_rate": 2.9105166051660522e-06, + "loss": 1.421, + "step": 1162 + }, + { + "epoch": 1.447417548226509, + "grad_norm": 2.847637414932251, + "learning_rate": 2.9082103321033212e-06, + "loss": 1.4326, + "step": 1163 + }, + { + "epoch": 1.4486621032980709, + "grad_norm": 2.836228132247925, + "learning_rate": 2.9059040590405907e-06, + "loss": 1.4459, + "step": 1164 + }, + { + "epoch": 1.449906658369633, + "grad_norm": 2.8327207565307617, + "learning_rate": 2.90359778597786e-06, + "loss": 1.4189, + "step": 1165 + }, + { + "epoch": 1.4511512134411948, + "grad_norm": 2.880643606185913, + "learning_rate": 2.901291512915129e-06, + "loss": 1.4214, + "step": 1166 + }, + { + "epoch": 1.4523957685127566, + "grad_norm": 2.9637348651885986, + "learning_rate": 2.898985239852399e-06, + "loss": 1.4471, + "step": 1167 + }, + { + "epoch": 1.4536403235843185, + "grad_norm": 3.006145477294922, + "learning_rate": 2.8966789667896684e-06, + "loss": 1.4228, + "step": 1168 + }, + { + "epoch": 1.4548848786558806, + "grad_norm": 2.8547627925872803, + "learning_rate": 2.8943726937269374e-06, + "loss": 1.4707, + "step": 1169 + }, + { + "epoch": 1.4561294337274424, + "grad_norm": 2.973092794418335, + "learning_rate": 2.8920664206642068e-06, + "loss": 1.4804, + "step": 1170 + }, + { + "epoch": 1.4561294337274424, + "eval_loss": 1.6340677738189697, + "eval_runtime": 44.6094, + "eval_samples_per_second": 22.417, + "eval_steps_per_second": 0.942, + "step": 1170 + }, + { + "epoch": 1.4573739887990045, + "grad_norm": 2.9433603286743164, + "learning_rate": 2.889760147601476e-06, + "loss": 1.4354, + "step": 1171 + }, + { + "epoch": 1.4586185438705663, + "grad_norm": 2.872236728668213, + "learning_rate": 2.887453874538745e-06, + "loss": 1.418, + "step": 1172 + }, + { + "epoch": 1.4598630989421282, + "grad_norm": 2.847557783126831, + "learning_rate": 2.885147601476015e-06, + "loss": 1.4306, + "step": 1173 + }, + { + "epoch": 1.46110765401369, + "grad_norm": 2.7554969787597656, + "learning_rate": 2.8828413284132845e-06, + "loss": 1.4295, + "step": 1174 + }, + { + "epoch": 1.462352209085252, + "grad_norm": 2.8754842281341553, + "learning_rate": 2.880535055350554e-06, + "loss": 1.4482, + "step": 1175 + }, + { + "epoch": 1.463596764156814, + "grad_norm": 3.0115292072296143, + "learning_rate": 2.878228782287823e-06, + "loss": 1.4428, + "step": 1176 + }, + { + "epoch": 1.4648413192283758, + "grad_norm": 2.8976168632507324, + "learning_rate": 2.8759225092250923e-06, + "loss": 1.4707, + "step": 1177 + }, + { + "epoch": 1.4660858742999379, + "grad_norm": 2.7981812953948975, + "learning_rate": 2.873616236162362e-06, + "loss": 1.4693, + "step": 1178 + }, + { + "epoch": 1.4673304293714997, + "grad_norm": 2.8562371730804443, + "learning_rate": 2.871309963099631e-06, + "loss": 1.4242, + "step": 1179 + }, + { + "epoch": 1.4685749844430616, + "grad_norm": 2.8705570697784424, + "learning_rate": 2.8690036900369006e-06, + "loss": 1.4725, + "step": 1180 + }, + { + "epoch": 1.4685749844430616, + "eval_loss": 1.6338286399841309, + "eval_runtime": 44.7171, + "eval_samples_per_second": 22.363, + "eval_steps_per_second": 0.939, + "step": 1180 + }, + { + "epoch": 1.4698195395146234, + "grad_norm": 2.8025710582733154, + "learning_rate": 2.86669741697417e-06, + "loss": 1.427, + "step": 1181 + }, + { + "epoch": 1.4710640945861855, + "grad_norm": 2.845151901245117, + "learning_rate": 2.864391143911439e-06, + "loss": 1.4402, + "step": 1182 + }, + { + "epoch": 1.4723086496577473, + "grad_norm": 2.943373680114746, + "learning_rate": 2.862084870848709e-06, + "loss": 1.5194, + "step": 1183 + }, + { + "epoch": 1.4735532047293094, + "grad_norm": 2.896955966949463, + "learning_rate": 2.8597785977859783e-06, + "loss": 1.4242, + "step": 1184 + }, + { + "epoch": 1.4747977598008712, + "grad_norm": 3.131305456161499, + "learning_rate": 2.8574723247232473e-06, + "loss": 1.5008, + "step": 1185 + }, + { + "epoch": 1.476042314872433, + "grad_norm": 2.8605735301971436, + "learning_rate": 2.8551660516605167e-06, + "loss": 1.3533, + "step": 1186 + }, + { + "epoch": 1.477286869943995, + "grad_norm": 2.9158811569213867, + "learning_rate": 2.852859778597786e-06, + "loss": 1.4404, + "step": 1187 + }, + { + "epoch": 1.478531425015557, + "grad_norm": 2.7795321941375732, + "learning_rate": 2.850553505535056e-06, + "loss": 1.3977, + "step": 1188 + }, + { + "epoch": 1.4797759800871189, + "grad_norm": 2.8194212913513184, + "learning_rate": 2.848247232472325e-06, + "loss": 1.4111, + "step": 1189 + }, + { + "epoch": 1.4810205351586807, + "grad_norm": 2.983750104904175, + "learning_rate": 2.8459409594095944e-06, + "loss": 1.4061, + "step": 1190 + }, + { + "epoch": 1.4810205351586807, + "eval_loss": 1.6375409364700317, + "eval_runtime": 45.4026, + "eval_samples_per_second": 22.025, + "eval_steps_per_second": 0.925, + "step": 1190 + }, + { + "epoch": 1.4822650902302428, + "grad_norm": 3.0246665477752686, + "learning_rate": 2.843634686346864e-06, + "loss": 1.4532, + "step": 1191 + }, + { + "epoch": 1.4835096453018046, + "grad_norm": 2.8856449127197266, + "learning_rate": 2.841328413284133e-06, + "loss": 1.4528, + "step": 1192 + }, + { + "epoch": 1.4847542003733665, + "grad_norm": 2.848987102508545, + "learning_rate": 2.8390221402214022e-06, + "loss": 1.4369, + "step": 1193 + }, + { + "epoch": 1.4859987554449283, + "grad_norm": 2.970419406890869, + "learning_rate": 2.836715867158672e-06, + "loss": 1.4605, + "step": 1194 + }, + { + "epoch": 1.4872433105164904, + "grad_norm": 3.156613826751709, + "learning_rate": 2.834409594095941e-06, + "loss": 1.4489, + "step": 1195 + }, + { + "epoch": 1.4884878655880522, + "grad_norm": 3.095349073410034, + "learning_rate": 2.8321033210332105e-06, + "loss": 1.4604, + "step": 1196 + }, + { + "epoch": 1.4897324206596143, + "grad_norm": 2.870742082595825, + "learning_rate": 2.82979704797048e-06, + "loss": 1.4284, + "step": 1197 + }, + { + "epoch": 1.4909769757311762, + "grad_norm": 2.9415056705474854, + "learning_rate": 2.827490774907749e-06, + "loss": 1.4715, + "step": 1198 + }, + { + "epoch": 1.492221530802738, + "grad_norm": 2.8297767639160156, + "learning_rate": 2.8251845018450188e-06, + "loss": 1.4508, + "step": 1199 + }, + { + "epoch": 1.4934660858742999, + "grad_norm": 2.872086524963379, + "learning_rate": 2.822878228782288e-06, + "loss": 1.5098, + "step": 1200 + }, + { + "epoch": 1.4934660858742999, + "eval_loss": 1.6357526779174805, + "eval_runtime": 51.422, + "eval_samples_per_second": 19.447, + "eval_steps_per_second": 0.817, + "step": 1200 + } + ], + "logging_steps": 1, + "max_steps": 2409, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 100, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3.740771291234304e+17, + "train_batch_size": 3, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/training_args.bin b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..6cdf93817e8f6d23b60d79888b226cc66ed2e88b --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a50e4918f3c49370326e14a7ed2f73b48b05e521f4b3399b53438fac6abee926 +size 6011 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/zero_to_fp32.py b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..24cc342e78d1a006c782b3a4cd68d9ce786d8fd8 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1200/zero_to_fp32.py @@ -0,0 +1,604 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: python zero_to_fp32.py . pytorch_model.bin + +import argparse +import torch +import glob +import math +import os +import re +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + + total_files = len(files) + state_dicts = [] + for f in files: + state_dict = torch.load(f, map_location=device) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + if zero_stage <= 2: + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + elif zero_stage == 3: + # if there is more than one param group, there will be multiple flattened tensors - one + # flattened tensor per group - for simplicity merge them into a single tensor + # + # XXX: could make the script more memory efficient for when there are multiple groups - it + # will require matching the sub-lists of param_shapes for each param group flattened tensor + + fp32_flat_groups = [ + torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts)) + ] + + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = fp32_flat_groups[0].numel() * world_size + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + for name, shape in param_shapes.items(): + + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # XXX: memory usage doubles here + state_dict[name] = torch.cat( + tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)), + 0).narrow(0, 0, unpartitioned_numel).view(shape) + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + + Returns: + - pytorch ``state_dict`` + + Note: this approach may not work if your application doesn't have sufficient free CPU memory and + you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters) + print(f"Saving fp32 state dict to {output_file}") + torch.save(state_dict, output_file) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info(f"Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info(f"Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument( + "output_file", + type=str, + help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_file, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fb0f9a53a507ffcdb60410deb71d60ef801bf350 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/config.json @@ -0,0 +1,36 @@ +{ + "_name_or_path": "meta-llama/Llama-3.2-3B", + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "eos_token_id": 128001, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 3072, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 24, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "torch_dtype": "float16", + "transformers_version": "4.45.1", + "use_cache": true, + "vocab_size": 128256 +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/generation_config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9c389b841a9a29ddd904e02d1eb0e08dcce82ad9 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": 128001, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.45.1" +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/latest b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/latest new file mode 100644 index 0000000000000000000000000000000000000000..d71c4c79bfb8e74842ee005fbfc27bedb903c39b --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/latest @@ -0,0 +1 @@ +global_step1300 \ No newline at end of file diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/model.safetensors.index.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..ed64de846d720b9a7859dc20575fea8e8ca51940 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/model.safetensors.index.json @@ -0,0 +1,262 @@ +{ + "metadata": { + "total_size": 7213504512 + }, + "weight_map": { + "lm_head.weight": "model-00002-of-00002.safetensors", + "model.embed_tokens.weight": "model-00001-of-00002.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.norm.weight": "model-00002-of-00002.safetensors" + } +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/rng_state_0.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..3429b5bbf5e65631c22532fed743338bede54d6b --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7020c2ba12cf7959bf7033d7ad83dcf637e59ddbd3ff5b17fc4b9a1ebed3202 +size 16567 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/rng_state_1.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..cbba424cb2109da26bffbb01960eb52b2dc47b0d --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f29ab0036e394c39267ad86bf247bf1558e2213add95383fce34a980f109b42e +size 16567 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/rng_state_2.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..b47aeb757c57446497f8f5d1181a50c3faf2dd79 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14413139ea5b777791c35297ed8a0a717bb3d6a3553268cc92b15fa61be1ae39 +size 16567 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/scheduler.pt b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..7a30969f5d2b2999f9a40b5f8aa0f3c8ac393445 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed4432bd76a70895d1eed9ffb9bcf3d55a555c7ee8371297518a9bf289187660 +size 627 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/special_tokens_map.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..6949c5975ee0e961ef61cf31010dce04df0a03f8 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "[PAD]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/tokenizer.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f28ecaeab53ae07feed29ccf8624d2b0a8344df9 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19fb2e1e3cdd6f7433d89fd6d62c82042599dd4984f342efe7fec6e159e6a8f6 +size 17210734 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/tokenizer_config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8991b49e9c2a43fc527dab9e09ad8171f0cc5943 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/tokenizer_config.json @@ -0,0 +1,2086 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "[PAD]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128257": { + "content": "🅂", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "128258": { + "content": "🄿", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "[PAD]", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/trainer_state.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..b6203b875fe82a91ab33c148fa47c147a48ff59e --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/trainer_state.json @@ -0,0 +1,10173 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.6179215930304915, + "eval_steps": 10, + "global_step": 1300, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0012445550715619166, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8368, + "step": 1 + }, + { + "epoch": 0.002489110143123833, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.86, + "step": 2 + }, + { + "epoch": 0.00373366521468575, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8265, + "step": 3 + }, + { + "epoch": 0.004978220286247666, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8271, + "step": 4 + }, + { + "epoch": 0.006222775357809583, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8768, + "step": 5 + }, + { + "epoch": 0.0074673304293715, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8402, + "step": 6 + }, + { + "epoch": 0.008711885500933417, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.785, + "step": 7 + }, + { + "epoch": 0.009956440572495333, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8466, + "step": 8 + }, + { + "epoch": 0.01120099564405725, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.7673, + "step": 9 + }, + { + "epoch": 0.012445550715619166, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8074, + "step": 10 + }, + { + "epoch": 0.012445550715619166, + "eval_loss": 2.8535053730010986, + "eval_runtime": 42.446, + "eval_samples_per_second": 23.559, + "eval_steps_per_second": 0.989, + "step": 10 + }, + { + "epoch": 0.013690105787181083, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8592, + "step": 11 + }, + { + "epoch": 0.014934660858743, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.804, + "step": 12 + }, + { + "epoch": 0.016179215930304917, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8076, + "step": 13 + }, + { + "epoch": 0.017423771001866834, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8219, + "step": 14 + }, + { + "epoch": 0.018668326073428748, + "grad_norm": 5.105710506439209, + "learning_rate": 2.0746887966804982e-08, + "loss": 2.7995, + "step": 15 + }, + { + "epoch": 0.019912881144990666, + "grad_norm": 5.156588077545166, + "learning_rate": 4.1493775933609963e-08, + "loss": 2.8577, + "step": 16 + }, + { + "epoch": 0.021157436216552583, + "grad_norm": 5.709373950958252, + "learning_rate": 6.224066390041494e-08, + "loss": 2.8433, + "step": 17 + }, + { + "epoch": 0.0224019912881145, + "grad_norm": 5.64658784866333, + "learning_rate": 8.298755186721993e-08, + "loss": 2.8428, + "step": 18 + }, + { + "epoch": 0.023646546359676415, + "grad_norm": 5.541937351226807, + "learning_rate": 1.037344398340249e-07, + "loss": 2.7785, + "step": 19 + }, + { + "epoch": 0.024891101431238332, + "grad_norm": 5.233374118804932, + "learning_rate": 1.2448132780082988e-07, + "loss": 2.8599, + "step": 20 + }, + { + "epoch": 0.024891101431238332, + "eval_loss": 2.8521182537078857, + "eval_runtime": 43.5664, + "eval_samples_per_second": 22.953, + "eval_steps_per_second": 0.964, + "step": 20 + }, + { + "epoch": 0.02613565650280025, + "grad_norm": 5.407598972320557, + "learning_rate": 1.4522821576763488e-07, + "loss": 2.8081, + "step": 21 + }, + { + "epoch": 0.027380211574362167, + "grad_norm": 5.0483317375183105, + "learning_rate": 1.6597510373443985e-07, + "loss": 2.8489, + "step": 22 + }, + { + "epoch": 0.02862476664592408, + "grad_norm": 4.985588550567627, + "learning_rate": 1.8672199170124483e-07, + "loss": 2.8375, + "step": 23 + }, + { + "epoch": 0.029869321717486, + "grad_norm": 5.201700687408447, + "learning_rate": 2.074688796680498e-07, + "loss": 2.8355, + "step": 24 + }, + { + "epoch": 0.031113876789047916, + "grad_norm": 5.009235382080078, + "learning_rate": 2.2821576763485478e-07, + "loss": 2.7774, + "step": 25 + }, + { + "epoch": 0.03235843186060983, + "grad_norm": 4.509210109710693, + "learning_rate": 2.4896265560165975e-07, + "loss": 2.7992, + "step": 26 + }, + { + "epoch": 0.03360298693217175, + "grad_norm": 4.529262065887451, + "learning_rate": 2.6970954356846476e-07, + "loss": 2.836, + "step": 27 + }, + { + "epoch": 0.03484754200373367, + "grad_norm": 4.526787281036377, + "learning_rate": 2.9045643153526976e-07, + "loss": 2.8128, + "step": 28 + }, + { + "epoch": 0.03609209707529558, + "grad_norm": 4.016609191894531, + "learning_rate": 3.112033195020747e-07, + "loss": 2.7947, + "step": 29 + }, + { + "epoch": 0.037336652146857496, + "grad_norm": 3.4784562587738037, + "learning_rate": 3.319502074688797e-07, + "loss": 2.7854, + "step": 30 + }, + { + "epoch": 0.037336652146857496, + "eval_loss": 2.8020100593566895, + "eval_runtime": 44.8464, + "eval_samples_per_second": 22.298, + "eval_steps_per_second": 0.937, + "step": 30 + }, + { + "epoch": 0.038581207218419414, + "grad_norm": 3.4887945652008057, + "learning_rate": 3.5269709543568466e-07, + "loss": 2.7908, + "step": 31 + }, + { + "epoch": 0.03982576228998133, + "grad_norm": 3.4533374309539795, + "learning_rate": 3.7344398340248966e-07, + "loss": 2.7358, + "step": 32 + }, + { + "epoch": 0.04107031736154325, + "grad_norm": 3.270954132080078, + "learning_rate": 3.941908713692946e-07, + "loss": 2.8013, + "step": 33 + }, + { + "epoch": 0.042314872433105166, + "grad_norm": 3.61322283744812, + "learning_rate": 4.149377593360996e-07, + "loss": 2.7521, + "step": 34 + }, + { + "epoch": 0.043559427504667084, + "grad_norm": 3.158719062805176, + "learning_rate": 4.3568464730290456e-07, + "loss": 2.7573, + "step": 35 + }, + { + "epoch": 0.044803982576229, + "grad_norm": 2.888746500015259, + "learning_rate": 4.5643153526970956e-07, + "loss": 2.7795, + "step": 36 + }, + { + "epoch": 0.04604853764779091, + "grad_norm": 3.022629499435425, + "learning_rate": 4.771784232365145e-07, + "loss": 2.7891, + "step": 37 + }, + { + "epoch": 0.04729309271935283, + "grad_norm": 3.038097381591797, + "learning_rate": 4.979253112033195e-07, + "loss": 2.7376, + "step": 38 + }, + { + "epoch": 0.04853764779091475, + "grad_norm": 2.9992239475250244, + "learning_rate": 5.186721991701245e-07, + "loss": 2.7279, + "step": 39 + }, + { + "epoch": 0.049782202862476664, + "grad_norm": 2.908842086791992, + "learning_rate": 5.394190871369295e-07, + "loss": 2.6582, + "step": 40 + }, + { + "epoch": 0.049782202862476664, + "eval_loss": 2.725968837738037, + "eval_runtime": 44.9498, + "eval_samples_per_second": 22.247, + "eval_steps_per_second": 0.934, + "step": 40 + }, + { + "epoch": 0.05102675793403858, + "grad_norm": 2.8025174140930176, + "learning_rate": 5.601659751037345e-07, + "loss": 2.6809, + "step": 41 + }, + { + "epoch": 0.0522713130056005, + "grad_norm": 2.7555699348449707, + "learning_rate": 5.809128630705395e-07, + "loss": 2.6954, + "step": 42 + }, + { + "epoch": 0.053515868077162417, + "grad_norm": 2.4732933044433594, + "learning_rate": 6.016597510373444e-07, + "loss": 2.7307, + "step": 43 + }, + { + "epoch": 0.054760423148724334, + "grad_norm": 2.451366424560547, + "learning_rate": 6.224066390041494e-07, + "loss": 2.7064, + "step": 44 + }, + { + "epoch": 0.056004978220286245, + "grad_norm": 2.5768678188323975, + "learning_rate": 6.431535269709543e-07, + "loss": 2.6466, + "step": 45 + }, + { + "epoch": 0.05724953329184816, + "grad_norm": 2.579332113265991, + "learning_rate": 6.639004149377594e-07, + "loss": 2.6127, + "step": 46 + }, + { + "epoch": 0.05849408836341008, + "grad_norm": 2.231207847595215, + "learning_rate": 6.846473029045644e-07, + "loss": 2.6429, + "step": 47 + }, + { + "epoch": 0.059738643434972, + "grad_norm": 2.134375810623169, + "learning_rate": 7.053941908713693e-07, + "loss": 2.6423, + "step": 48 + }, + { + "epoch": 0.060983198506533914, + "grad_norm": 2.0485494136810303, + "learning_rate": 7.261410788381744e-07, + "loss": 2.581, + "step": 49 + }, + { + "epoch": 0.06222775357809583, + "grad_norm": 2.02909517288208, + "learning_rate": 7.468879668049793e-07, + "loss": 2.6357, + "step": 50 + }, + { + "epoch": 0.06222775357809583, + "eval_loss": 2.632822036743164, + "eval_runtime": 47.0104, + "eval_samples_per_second": 21.272, + "eval_steps_per_second": 0.893, + "step": 50 + }, + { + "epoch": 0.06347230864965775, + "grad_norm": 1.892500877380371, + "learning_rate": 7.676348547717843e-07, + "loss": 2.5945, + "step": 51 + }, + { + "epoch": 0.06471686372121967, + "grad_norm": 2.107921600341797, + "learning_rate": 7.883817427385892e-07, + "loss": 2.5867, + "step": 52 + }, + { + "epoch": 0.06596141879278158, + "grad_norm": 1.8287527561187744, + "learning_rate": 8.091286307053943e-07, + "loss": 2.5862, + "step": 53 + }, + { + "epoch": 0.0672059738643435, + "grad_norm": 1.7342065572738647, + "learning_rate": 8.298755186721992e-07, + "loss": 2.5735, + "step": 54 + }, + { + "epoch": 0.06845052893590542, + "grad_norm": 1.7905038595199585, + "learning_rate": 8.506224066390042e-07, + "loss": 2.572, + "step": 55 + }, + { + "epoch": 0.06969508400746734, + "grad_norm": 1.9715144634246826, + "learning_rate": 8.713692946058091e-07, + "loss": 2.5797, + "step": 56 + }, + { + "epoch": 0.07093963907902924, + "grad_norm": 1.788966417312622, + "learning_rate": 8.921161825726142e-07, + "loss": 2.5567, + "step": 57 + }, + { + "epoch": 0.07218419415059116, + "grad_norm": 1.8978915214538574, + "learning_rate": 9.128630705394191e-07, + "loss": 2.5776, + "step": 58 + }, + { + "epoch": 0.07342874922215308, + "grad_norm": 1.6401689052581787, + "learning_rate": 9.336099585062241e-07, + "loss": 2.5662, + "step": 59 + }, + { + "epoch": 0.07467330429371499, + "grad_norm": 1.8200798034667969, + "learning_rate": 9.54356846473029e-07, + "loss": 2.5001, + "step": 60 + }, + { + "epoch": 0.07467330429371499, + "eval_loss": 2.569110870361328, + "eval_runtime": 47.3292, + "eval_samples_per_second": 21.129, + "eval_steps_per_second": 0.887, + "step": 60 + }, + { + "epoch": 0.07591785936527691, + "grad_norm": 1.5468674898147583, + "learning_rate": 9.751037344398341e-07, + "loss": 2.5308, + "step": 61 + }, + { + "epoch": 0.07716241443683883, + "grad_norm": 1.723684310913086, + "learning_rate": 9.95850622406639e-07, + "loss": 2.4907, + "step": 62 + }, + { + "epoch": 0.07840696950840075, + "grad_norm": 1.6346293687820435, + "learning_rate": 1.0165975103734441e-06, + "loss": 2.514, + "step": 63 + }, + { + "epoch": 0.07965152457996266, + "grad_norm": 1.8902088403701782, + "learning_rate": 1.037344398340249e-06, + "loss": 2.4859, + "step": 64 + }, + { + "epoch": 0.08089607965152458, + "grad_norm": 2.092611312866211, + "learning_rate": 1.058091286307054e-06, + "loss": 2.4888, + "step": 65 + }, + { + "epoch": 0.0821406347230865, + "grad_norm": 1.758482575416565, + "learning_rate": 1.078838174273859e-06, + "loss": 2.4537, + "step": 66 + }, + { + "epoch": 0.08338518979464841, + "grad_norm": 1.8941714763641357, + "learning_rate": 1.099585062240664e-06, + "loss": 2.5416, + "step": 67 + }, + { + "epoch": 0.08462974486621033, + "grad_norm": 1.7030234336853027, + "learning_rate": 1.120331950207469e-06, + "loss": 2.4791, + "step": 68 + }, + { + "epoch": 0.08587429993777225, + "grad_norm": 1.7602269649505615, + "learning_rate": 1.141078838174274e-06, + "loss": 2.4316, + "step": 69 + }, + { + "epoch": 0.08711885500933417, + "grad_norm": 1.7635431289672852, + "learning_rate": 1.161825726141079e-06, + "loss": 2.4674, + "step": 70 + }, + { + "epoch": 0.08711885500933417, + "eval_loss": 2.4922080039978027, + "eval_runtime": 45.3028, + "eval_samples_per_second": 22.074, + "eval_steps_per_second": 0.927, + "step": 70 + }, + { + "epoch": 0.08836341008089608, + "grad_norm": 2.0913803577423096, + "learning_rate": 1.182572614107884e-06, + "loss": 2.4341, + "step": 71 + }, + { + "epoch": 0.089607965152458, + "grad_norm": 1.8292183876037598, + "learning_rate": 1.2033195020746888e-06, + "loss": 2.4291, + "step": 72 + }, + { + "epoch": 0.09085252022401992, + "grad_norm": 2.206770181655884, + "learning_rate": 1.224066390041494e-06, + "loss": 2.3764, + "step": 73 + }, + { + "epoch": 0.09209707529558182, + "grad_norm": 1.8263559341430664, + "learning_rate": 1.2448132780082988e-06, + "loss": 2.4082, + "step": 74 + }, + { + "epoch": 0.09334163036714374, + "grad_norm": 1.9405455589294434, + "learning_rate": 1.2655601659751037e-06, + "loss": 2.4673, + "step": 75 + }, + { + "epoch": 0.09458618543870566, + "grad_norm": 2.138108015060425, + "learning_rate": 1.2863070539419086e-06, + "loss": 2.3879, + "step": 76 + }, + { + "epoch": 0.09583074051026758, + "grad_norm": 1.9024745225906372, + "learning_rate": 1.307053941908714e-06, + "loss": 2.4344, + "step": 77 + }, + { + "epoch": 0.0970752955818295, + "grad_norm": 1.9136689901351929, + "learning_rate": 1.3278008298755188e-06, + "loss": 2.474, + "step": 78 + }, + { + "epoch": 0.09831985065339141, + "grad_norm": 2.4752795696258545, + "learning_rate": 1.3485477178423237e-06, + "loss": 2.3412, + "step": 79 + }, + { + "epoch": 0.09956440572495333, + "grad_norm": 1.8720006942749023, + "learning_rate": 1.3692946058091288e-06, + "loss": 2.3438, + "step": 80 + }, + { + "epoch": 0.09956440572495333, + "eval_loss": 2.3980512619018555, + "eval_runtime": 48.2788, + "eval_samples_per_second": 20.713, + "eval_steps_per_second": 0.87, + "step": 80 + }, + { + "epoch": 0.10080896079651525, + "grad_norm": 2.671691656112671, + "learning_rate": 1.3900414937759337e-06, + "loss": 2.3336, + "step": 81 + }, + { + "epoch": 0.10205351586807716, + "grad_norm": 2.2953391075134277, + "learning_rate": 1.4107883817427386e-06, + "loss": 2.377, + "step": 82 + }, + { + "epoch": 0.10329807093963908, + "grad_norm": 3.009018898010254, + "learning_rate": 1.4315352697095435e-06, + "loss": 2.2977, + "step": 83 + }, + { + "epoch": 0.104542626011201, + "grad_norm": 2.664454936981201, + "learning_rate": 1.4522821576763488e-06, + "loss": 2.3271, + "step": 84 + }, + { + "epoch": 0.10578718108276292, + "grad_norm": 3.017303705215454, + "learning_rate": 1.4730290456431537e-06, + "loss": 2.3251, + "step": 85 + }, + { + "epoch": 0.10703173615432483, + "grad_norm": 2.634716510772705, + "learning_rate": 1.4937759336099586e-06, + "loss": 2.332, + "step": 86 + }, + { + "epoch": 0.10827629122588675, + "grad_norm": 3.059644937515259, + "learning_rate": 1.5145228215767635e-06, + "loss": 2.3478, + "step": 87 + }, + { + "epoch": 0.10952084629744867, + "grad_norm": 2.6962637901306152, + "learning_rate": 1.5352697095435686e-06, + "loss": 2.2792, + "step": 88 + }, + { + "epoch": 0.11076540136901059, + "grad_norm": 3.419729709625244, + "learning_rate": 1.5560165975103735e-06, + "loss": 2.2571, + "step": 89 + }, + { + "epoch": 0.11200995644057249, + "grad_norm": 2.740781545639038, + "learning_rate": 1.5767634854771784e-06, + "loss": 2.2875, + "step": 90 + }, + { + "epoch": 0.11200995644057249, + "eval_loss": 2.30843186378479, + "eval_runtime": 49.9264, + "eval_samples_per_second": 20.029, + "eval_steps_per_second": 0.841, + "step": 90 + }, + { + "epoch": 0.1132545115121344, + "grad_norm": 2.5608789920806885, + "learning_rate": 1.5975103734439833e-06, + "loss": 2.2592, + "step": 91 + }, + { + "epoch": 0.11449906658369632, + "grad_norm": 2.687999963760376, + "learning_rate": 1.6182572614107886e-06, + "loss": 2.2546, + "step": 92 + }, + { + "epoch": 0.11574362165525824, + "grad_norm": 2.695909023284912, + "learning_rate": 1.6390041493775935e-06, + "loss": 2.2525, + "step": 93 + }, + { + "epoch": 0.11698817672682016, + "grad_norm": 2.818357467651367, + "learning_rate": 1.6597510373443984e-06, + "loss": 2.216, + "step": 94 + }, + { + "epoch": 0.11823273179838208, + "grad_norm": 2.884119987487793, + "learning_rate": 1.6804979253112035e-06, + "loss": 2.2321, + "step": 95 + }, + { + "epoch": 0.119477286869944, + "grad_norm": 2.52104115486145, + "learning_rate": 1.7012448132780084e-06, + "loss": 2.199, + "step": 96 + }, + { + "epoch": 0.12072184194150591, + "grad_norm": 2.420313596725464, + "learning_rate": 1.7219917012448133e-06, + "loss": 2.1862, + "step": 97 + }, + { + "epoch": 0.12196639701306783, + "grad_norm": 2.8047542572021484, + "learning_rate": 1.7427385892116182e-06, + "loss": 2.1793, + "step": 98 + }, + { + "epoch": 0.12321095208462975, + "grad_norm": 2.836482286453247, + "learning_rate": 1.7634854771784235e-06, + "loss": 2.2271, + "step": 99 + }, + { + "epoch": 0.12445550715619166, + "grad_norm": 2.5282301902770996, + "learning_rate": 1.7842323651452284e-06, + "loss": 2.1768, + "step": 100 + }, + { + "epoch": 0.12445550715619166, + "eval_loss": 2.2312686443328857, + "eval_runtime": 49.2382, + "eval_samples_per_second": 20.309, + "eval_steps_per_second": 0.853, + "step": 100 + }, + { + "epoch": 0.12570006222775357, + "grad_norm": 3.3407280445098877, + "learning_rate": 1.8049792531120333e-06, + "loss": 2.1666, + "step": 101 + }, + { + "epoch": 0.1269446172993155, + "grad_norm": 2.4754133224487305, + "learning_rate": 1.8257261410788382e-06, + "loss": 2.1768, + "step": 102 + }, + { + "epoch": 0.1281891723708774, + "grad_norm": 3.430889129638672, + "learning_rate": 1.8464730290456433e-06, + "loss": 2.1953, + "step": 103 + }, + { + "epoch": 0.12943372744243933, + "grad_norm": 2.835294246673584, + "learning_rate": 1.8672199170124482e-06, + "loss": 2.146, + "step": 104 + }, + { + "epoch": 0.13067828251400124, + "grad_norm": 3.1532323360443115, + "learning_rate": 1.8879668049792531e-06, + "loss": 2.1729, + "step": 105 + }, + { + "epoch": 0.13192283758556317, + "grad_norm": 3.2278342247009277, + "learning_rate": 1.908713692946058e-06, + "loss": 2.1336, + "step": 106 + }, + { + "epoch": 0.13316739265712507, + "grad_norm": 2.7892515659332275, + "learning_rate": 1.929460580912863e-06, + "loss": 2.141, + "step": 107 + }, + { + "epoch": 0.134411947728687, + "grad_norm": 3.0179977416992188, + "learning_rate": 1.9502074688796682e-06, + "loss": 2.1255, + "step": 108 + }, + { + "epoch": 0.1356565028002489, + "grad_norm": 2.977935552597046, + "learning_rate": 1.970954356846473e-06, + "loss": 2.0986, + "step": 109 + }, + { + "epoch": 0.13690105787181084, + "grad_norm": 3.3042492866516113, + "learning_rate": 1.991701244813278e-06, + "loss": 2.0992, + "step": 110 + }, + { + "epoch": 0.13690105787181084, + "eval_loss": 2.171299695968628, + "eval_runtime": 42.9511, + "eval_samples_per_second": 23.282, + "eval_steps_per_second": 0.978, + "step": 110 + }, + { + "epoch": 0.13814561294337274, + "grad_norm": 2.9750890731811523, + "learning_rate": 2.012448132780083e-06, + "loss": 2.1116, + "step": 111 + }, + { + "epoch": 0.13939016801493467, + "grad_norm": 2.8199286460876465, + "learning_rate": 2.0331950207468883e-06, + "loss": 2.0822, + "step": 112 + }, + { + "epoch": 0.14063472308649658, + "grad_norm": 2.930532217025757, + "learning_rate": 2.053941908713693e-06, + "loss": 2.161, + "step": 113 + }, + { + "epoch": 0.14187927815805848, + "grad_norm": 3.3400321006774902, + "learning_rate": 2.074688796680498e-06, + "loss": 2.0953, + "step": 114 + }, + { + "epoch": 0.1431238332296204, + "grad_norm": 2.716564655303955, + "learning_rate": 2.095435684647303e-06, + "loss": 2.0757, + "step": 115 + }, + { + "epoch": 0.14436838830118232, + "grad_norm": 2.7586236000061035, + "learning_rate": 2.116182572614108e-06, + "loss": 2.0979, + "step": 116 + }, + { + "epoch": 0.14561294337274425, + "grad_norm": 2.622126340866089, + "learning_rate": 2.136929460580913e-06, + "loss": 2.0748, + "step": 117 + }, + { + "epoch": 0.14685749844430615, + "grad_norm": 2.646477460861206, + "learning_rate": 2.157676348547718e-06, + "loss": 2.0605, + "step": 118 + }, + { + "epoch": 0.14810205351586808, + "grad_norm": 2.8882410526275635, + "learning_rate": 2.178423236514523e-06, + "loss": 2.0501, + "step": 119 + }, + { + "epoch": 0.14934660858742999, + "grad_norm": 2.8302247524261475, + "learning_rate": 2.199170124481328e-06, + "loss": 2.0592, + "step": 120 + }, + { + "epoch": 0.14934660858742999, + "eval_loss": 2.1291966438293457, + "eval_runtime": 47.5399, + "eval_samples_per_second": 21.035, + "eval_steps_per_second": 0.883, + "step": 120 + }, + { + "epoch": 0.15059116365899192, + "grad_norm": 3.26979660987854, + "learning_rate": 2.219917012448133e-06, + "loss": 2.0422, + "step": 121 + }, + { + "epoch": 0.15183571873055382, + "grad_norm": 2.7483913898468018, + "learning_rate": 2.240663900414938e-06, + "loss": 2.0872, + "step": 122 + }, + { + "epoch": 0.15308027380211575, + "grad_norm": 3.1252024173736572, + "learning_rate": 2.2614107883817427e-06, + "loss": 2.0684, + "step": 123 + }, + { + "epoch": 0.15432482887367766, + "grad_norm": 2.9175283908843994, + "learning_rate": 2.282157676348548e-06, + "loss": 2.0522, + "step": 124 + }, + { + "epoch": 0.1555693839452396, + "grad_norm": 2.7950754165649414, + "learning_rate": 2.302904564315353e-06, + "loss": 2.065, + "step": 125 + }, + { + "epoch": 0.1568139390168015, + "grad_norm": 2.84049654006958, + "learning_rate": 2.323651452282158e-06, + "loss": 2.0781, + "step": 126 + }, + { + "epoch": 0.15805849408836342, + "grad_norm": 2.7396647930145264, + "learning_rate": 2.3443983402489627e-06, + "loss": 2.1758, + "step": 127 + }, + { + "epoch": 0.15930304915992533, + "grad_norm": 3.0765926837921143, + "learning_rate": 2.365145228215768e-06, + "loss": 2.0172, + "step": 128 + }, + { + "epoch": 0.16054760423148726, + "grad_norm": 3.217189073562622, + "learning_rate": 2.385892116182573e-06, + "loss": 2.0286, + "step": 129 + }, + { + "epoch": 0.16179215930304916, + "grad_norm": 3.141545295715332, + "learning_rate": 2.4066390041493776e-06, + "loss": 2.0583, + "step": 130 + }, + { + "epoch": 0.16179215930304916, + "eval_loss": 2.0946149826049805, + "eval_runtime": 54.92, + "eval_samples_per_second": 18.208, + "eval_steps_per_second": 0.765, + "step": 130 + }, + { + "epoch": 0.16303671437461106, + "grad_norm": 2.850052833557129, + "learning_rate": 2.4273858921161828e-06, + "loss": 2.0746, + "step": 131 + }, + { + "epoch": 0.164281269446173, + "grad_norm": 3.28913950920105, + "learning_rate": 2.448132780082988e-06, + "loss": 2.0539, + "step": 132 + }, + { + "epoch": 0.1655258245177349, + "grad_norm": 2.8819124698638916, + "learning_rate": 2.468879668049793e-06, + "loss": 1.9843, + "step": 133 + }, + { + "epoch": 0.16677037958929683, + "grad_norm": 3.6254632472991943, + "learning_rate": 2.4896265560165977e-06, + "loss": 2.0233, + "step": 134 + }, + { + "epoch": 0.16801493466085873, + "grad_norm": 2.7385146617889404, + "learning_rate": 2.5103734439834028e-06, + "loss": 1.9667, + "step": 135 + }, + { + "epoch": 0.16925948973242066, + "grad_norm": 2.9722647666931152, + "learning_rate": 2.5311203319502074e-06, + "loss": 1.9527, + "step": 136 + }, + { + "epoch": 0.17050404480398257, + "grad_norm": 3.139526605606079, + "learning_rate": 2.5518672199170125e-06, + "loss": 1.9967, + "step": 137 + }, + { + "epoch": 0.1717485998755445, + "grad_norm": 3.399920701980591, + "learning_rate": 2.5726141078838172e-06, + "loss": 1.9798, + "step": 138 + }, + { + "epoch": 0.1729931549471064, + "grad_norm": 2.8017327785491943, + "learning_rate": 2.5933609958506228e-06, + "loss": 1.983, + "step": 139 + }, + { + "epoch": 0.17423771001866833, + "grad_norm": 3.9967198371887207, + "learning_rate": 2.614107883817428e-06, + "loss": 1.977, + "step": 140 + }, + { + "epoch": 0.17423771001866833, + "eval_loss": 2.059037685394287, + "eval_runtime": 52.6232, + "eval_samples_per_second": 19.003, + "eval_steps_per_second": 0.798, + "step": 140 + }, + { + "epoch": 0.17548226509023024, + "grad_norm": 2.893092393875122, + "learning_rate": 2.6348547717842326e-06, + "loss": 1.9888, + "step": 141 + }, + { + "epoch": 0.17672682016179217, + "grad_norm": 3.634352207183838, + "learning_rate": 2.6556016597510377e-06, + "loss": 2.0726, + "step": 142 + }, + { + "epoch": 0.17797137523335407, + "grad_norm": 3.3651444911956787, + "learning_rate": 2.6763485477178423e-06, + "loss": 1.973, + "step": 143 + }, + { + "epoch": 0.179215930304916, + "grad_norm": 3.968986988067627, + "learning_rate": 2.6970954356846475e-06, + "loss": 1.9948, + "step": 144 + }, + { + "epoch": 0.1804604853764779, + "grad_norm": 3.20105242729187, + "learning_rate": 2.717842323651452e-06, + "loss": 1.9238, + "step": 145 + }, + { + "epoch": 0.18170504044803984, + "grad_norm": 3.648339033126831, + "learning_rate": 2.7385892116182577e-06, + "loss": 1.9923, + "step": 146 + }, + { + "epoch": 0.18294959551960174, + "grad_norm": 3.0856316089630127, + "learning_rate": 2.7593360995850628e-06, + "loss": 1.9749, + "step": 147 + }, + { + "epoch": 0.18419415059116365, + "grad_norm": 3.2681071758270264, + "learning_rate": 2.7800829875518675e-06, + "loss": 1.9304, + "step": 148 + }, + { + "epoch": 0.18543870566272558, + "grad_norm": 2.634958267211914, + "learning_rate": 2.8008298755186726e-06, + "loss": 1.9237, + "step": 149 + }, + { + "epoch": 0.18668326073428748, + "grad_norm": 2.769491672515869, + "learning_rate": 2.8215767634854773e-06, + "loss": 1.8963, + "step": 150 + }, + { + "epoch": 0.18668326073428748, + "eval_loss": 2.0394654273986816, + "eval_runtime": 55.2136, + "eval_samples_per_second": 18.111, + "eval_steps_per_second": 0.761, + "step": 150 + }, + { + "epoch": 0.1879278158058494, + "grad_norm": 3.4345781803131104, + "learning_rate": 2.8423236514522824e-06, + "loss": 1.9836, + "step": 151 + }, + { + "epoch": 0.18917237087741132, + "grad_norm": 3.115727424621582, + "learning_rate": 2.863070539419087e-06, + "loss": 1.9352, + "step": 152 + }, + { + "epoch": 0.19041692594897325, + "grad_norm": 3.050652503967285, + "learning_rate": 2.883817427385892e-06, + "loss": 1.9031, + "step": 153 + }, + { + "epoch": 0.19166148102053515, + "grad_norm": 2.9404428005218506, + "learning_rate": 2.9045643153526977e-06, + "loss": 1.9196, + "step": 154 + }, + { + "epoch": 0.19290603609209708, + "grad_norm": 3.080810785293579, + "learning_rate": 2.9253112033195024e-06, + "loss": 1.9405, + "step": 155 + }, + { + "epoch": 0.194150591163659, + "grad_norm": 3.058558702468872, + "learning_rate": 2.9460580912863075e-06, + "loss": 1.9052, + "step": 156 + }, + { + "epoch": 0.19539514623522092, + "grad_norm": 3.307955026626587, + "learning_rate": 2.966804979253112e-06, + "loss": 1.9245, + "step": 157 + }, + { + "epoch": 0.19663970130678282, + "grad_norm": 2.845506191253662, + "learning_rate": 2.9875518672199173e-06, + "loss": 1.9461, + "step": 158 + }, + { + "epoch": 0.19788425637834475, + "grad_norm": 2.758654832839966, + "learning_rate": 3.008298755186722e-06, + "loss": 1.8885, + "step": 159 + }, + { + "epoch": 0.19912881144990666, + "grad_norm": 3.161252737045288, + "learning_rate": 3.029045643153527e-06, + "loss": 1.8535, + "step": 160 + }, + { + "epoch": 0.19912881144990666, + "eval_loss": 2.018101453781128, + "eval_runtime": 51.0744, + "eval_samples_per_second": 19.579, + "eval_steps_per_second": 0.822, + "step": 160 + }, + { + "epoch": 0.2003733665214686, + "grad_norm": 3.196847677230835, + "learning_rate": 3.0497925311203326e-06, + "loss": 1.8853, + "step": 161 + }, + { + "epoch": 0.2016179215930305, + "grad_norm": 2.8364663124084473, + "learning_rate": 3.0705394190871373e-06, + "loss": 1.9463, + "step": 162 + }, + { + "epoch": 0.2028624766645924, + "grad_norm": 3.0874054431915283, + "learning_rate": 3.0912863070539424e-06, + "loss": 1.9254, + "step": 163 + }, + { + "epoch": 0.20410703173615433, + "grad_norm": 2.7914493083953857, + "learning_rate": 3.112033195020747e-06, + "loss": 1.9213, + "step": 164 + }, + { + "epoch": 0.20535158680771623, + "grad_norm": 3.3871428966522217, + "learning_rate": 3.132780082987552e-06, + "loss": 1.8993, + "step": 165 + }, + { + "epoch": 0.20659614187927816, + "grad_norm": 3.096653461456299, + "learning_rate": 3.153526970954357e-06, + "loss": 1.8827, + "step": 166 + }, + { + "epoch": 0.20784069695084006, + "grad_norm": 2.8276076316833496, + "learning_rate": 3.174273858921162e-06, + "loss": 1.912, + "step": 167 + }, + { + "epoch": 0.209085252022402, + "grad_norm": 3.3058435916900635, + "learning_rate": 3.1950207468879666e-06, + "loss": 1.8381, + "step": 168 + }, + { + "epoch": 0.2103298070939639, + "grad_norm": 3.5017333030700684, + "learning_rate": 3.215767634854772e-06, + "loss": 1.9544, + "step": 169 + }, + { + "epoch": 0.21157436216552583, + "grad_norm": 3.457296133041382, + "learning_rate": 3.2365145228215773e-06, + "loss": 1.922, + "step": 170 + }, + { + "epoch": 0.21157436216552583, + "eval_loss": 1.988455891609192, + "eval_runtime": 56.2099, + "eval_samples_per_second": 17.79, + "eval_steps_per_second": 0.747, + "step": 170 + }, + { + "epoch": 0.21281891723708773, + "grad_norm": 3.1964566707611084, + "learning_rate": 3.257261410788382e-06, + "loss": 1.8925, + "step": 171 + }, + { + "epoch": 0.21406347230864967, + "grad_norm": 3.234652042388916, + "learning_rate": 3.278008298755187e-06, + "loss": 1.884, + "step": 172 + }, + { + "epoch": 0.21530802738021157, + "grad_norm": 3.14414119720459, + "learning_rate": 3.2987551867219918e-06, + "loss": 1.8905, + "step": 173 + }, + { + "epoch": 0.2165525824517735, + "grad_norm": 3.606379508972168, + "learning_rate": 3.319502074688797e-06, + "loss": 1.946, + "step": 174 + }, + { + "epoch": 0.2177971375233354, + "grad_norm": 2.986646890640259, + "learning_rate": 3.3402489626556016e-06, + "loss": 1.8691, + "step": 175 + }, + { + "epoch": 0.21904169259489734, + "grad_norm": 3.7298269271850586, + "learning_rate": 3.360995850622407e-06, + "loss": 1.8717, + "step": 176 + }, + { + "epoch": 0.22028624766645924, + "grad_norm": 3.422295093536377, + "learning_rate": 3.381742738589212e-06, + "loss": 1.8568, + "step": 177 + }, + { + "epoch": 0.22153080273802117, + "grad_norm": 3.2293782234191895, + "learning_rate": 3.402489626556017e-06, + "loss": 1.8471, + "step": 178 + }, + { + "epoch": 0.22277535780958307, + "grad_norm": 3.2293782234191895, + "learning_rate": 3.402489626556017e-06, + "loss": 1.9303, + "step": 179 + }, + { + "epoch": 0.22401991288114498, + "grad_norm": 3.506223440170288, + "learning_rate": 3.423236514522822e-06, + "loss": 1.8237, + "step": 180 + }, + { + "epoch": 0.22401991288114498, + "eval_loss": 1.9734643697738647, + "eval_runtime": 52.2815, + "eval_samples_per_second": 19.127, + "eval_steps_per_second": 0.803, + "step": 180 + }, + { + "epoch": 0.2252644679527069, + "grad_norm": 3.0156619548797607, + "learning_rate": 3.4439834024896267e-06, + "loss": 1.908, + "step": 181 + }, + { + "epoch": 0.2265090230242688, + "grad_norm": 3.55517315864563, + "learning_rate": 3.4647302904564318e-06, + "loss": 1.9104, + "step": 182 + }, + { + "epoch": 0.22775357809583074, + "grad_norm": 3.144984006881714, + "learning_rate": 3.4854771784232365e-06, + "loss": 1.8198, + "step": 183 + }, + { + "epoch": 0.22899813316739265, + "grad_norm": 4.302074432373047, + "learning_rate": 3.5062240663900416e-06, + "loss": 1.8237, + "step": 184 + }, + { + "epoch": 0.23024268823895458, + "grad_norm": 3.0522446632385254, + "learning_rate": 3.526970954356847e-06, + "loss": 1.8409, + "step": 185 + }, + { + "epoch": 0.23148724331051648, + "grad_norm": 3.8607394695281982, + "learning_rate": 3.5477178423236518e-06, + "loss": 1.8807, + "step": 186 + }, + { + "epoch": 0.23273179838207841, + "grad_norm": 2.9236302375793457, + "learning_rate": 3.568464730290457e-06, + "loss": 1.7873, + "step": 187 + }, + { + "epoch": 0.23397635345364032, + "grad_norm": 4.013780117034912, + "learning_rate": 3.5892116182572616e-06, + "loss": 1.7909, + "step": 188 + }, + { + "epoch": 0.23522090852520225, + "grad_norm": 3.0933122634887695, + "learning_rate": 3.6099585062240667e-06, + "loss": 1.8469, + "step": 189 + }, + { + "epoch": 0.23646546359676415, + "grad_norm": 3.487816095352173, + "learning_rate": 3.6307053941908714e-06, + "loss": 1.8469, + "step": 190 + }, + { + "epoch": 0.23646546359676415, + "eval_loss": 1.9604240655899048, + "eval_runtime": 53.4604, + "eval_samples_per_second": 18.705, + "eval_steps_per_second": 0.786, + "step": 190 + }, + { + "epoch": 0.23771001866832608, + "grad_norm": 2.829219341278076, + "learning_rate": 3.6514522821576765e-06, + "loss": 1.8639, + "step": 191 + }, + { + "epoch": 0.238954573739888, + "grad_norm": 3.595534324645996, + "learning_rate": 3.672199170124482e-06, + "loss": 1.8988, + "step": 192 + }, + { + "epoch": 0.24019912881144992, + "grad_norm": 2.9022483825683594, + "learning_rate": 3.6929460580912867e-06, + "loss": 1.8972, + "step": 193 + }, + { + "epoch": 0.24144368388301182, + "grad_norm": 3.0667994022369385, + "learning_rate": 3.713692946058092e-06, + "loss": 1.8499, + "step": 194 + }, + { + "epoch": 0.24268823895457373, + "grad_norm": 3.0498485565185547, + "learning_rate": 3.7344398340248965e-06, + "loss": 1.8341, + "step": 195 + }, + { + "epoch": 0.24393279402613566, + "grad_norm": 4.058084964752197, + "learning_rate": 3.7551867219917016e-06, + "loss": 1.8641, + "step": 196 + }, + { + "epoch": 0.24517734909769756, + "grad_norm": 2.9995028972625732, + "learning_rate": 3.7759336099585063e-06, + "loss": 1.7885, + "step": 197 + }, + { + "epoch": 0.2464219041692595, + "grad_norm": 3.254704475402832, + "learning_rate": 3.7966804979253114e-06, + "loss": 1.7544, + "step": 198 + }, + { + "epoch": 0.2476664592408214, + "grad_norm": 2.9126744270324707, + "learning_rate": 3.817427385892116e-06, + "loss": 1.8461, + "step": 199 + }, + { + "epoch": 0.24891101431238333, + "grad_norm": 3.475247621536255, + "learning_rate": 3.838174273858922e-06, + "loss": 1.8095, + "step": 200 + }, + { + "epoch": 0.24891101431238333, + "eval_loss": 1.9373760223388672, + "eval_runtime": 51.8733, + "eval_samples_per_second": 19.278, + "eval_steps_per_second": 0.81, + "step": 200 + }, + { + "epoch": 0.25015556938394523, + "grad_norm": 3.2663731575012207, + "learning_rate": 3.858921161825726e-06, + "loss": 1.8076, + "step": 201 + }, + { + "epoch": 0.25140012445550713, + "grad_norm": 3.1151726245880127, + "learning_rate": 3.879668049792531e-06, + "loss": 1.8143, + "step": 202 + }, + { + "epoch": 0.2526446795270691, + "grad_norm": 3.102038860321045, + "learning_rate": 3.9004149377593365e-06, + "loss": 1.8279, + "step": 203 + }, + { + "epoch": 0.253889234598631, + "grad_norm": 3.370642900466919, + "learning_rate": 3.921161825726142e-06, + "loss": 1.8522, + "step": 204 + }, + { + "epoch": 0.2551337896701929, + "grad_norm": 3.116128921508789, + "learning_rate": 3.941908713692946e-06, + "loss": 1.8104, + "step": 205 + }, + { + "epoch": 0.2563783447417548, + "grad_norm": 3.12028431892395, + "learning_rate": 3.962655601659751e-06, + "loss": 1.7723, + "step": 206 + }, + { + "epoch": 0.25762289981331676, + "grad_norm": 3.1251418590545654, + "learning_rate": 3.983402489626556e-06, + "loss": 1.863, + "step": 207 + }, + { + "epoch": 0.25886745488487867, + "grad_norm": 3.073702812194824, + "learning_rate": 4.004149377593361e-06, + "loss": 1.805, + "step": 208 + }, + { + "epoch": 0.26011200995644057, + "grad_norm": 2.9772469997406006, + "learning_rate": 4.024896265560166e-06, + "loss": 1.8251, + "step": 209 + }, + { + "epoch": 0.2613565650280025, + "grad_norm": 2.9970037937164307, + "learning_rate": 4.045643153526971e-06, + "loss": 1.8386, + "step": 210 + }, + { + "epoch": 0.2613565650280025, + "eval_loss": 1.9167065620422363, + "eval_runtime": 42.3328, + "eval_samples_per_second": 23.622, + "eval_steps_per_second": 0.992, + "step": 210 + }, + { + "epoch": 0.26260112009956443, + "grad_norm": 3.1391561031341553, + "learning_rate": 4.0663900414937765e-06, + "loss": 1.8109, + "step": 211 + }, + { + "epoch": 0.26384567517112634, + "grad_norm": 3.161538600921631, + "learning_rate": 4.087136929460581e-06, + "loss": 1.8441, + "step": 212 + }, + { + "epoch": 0.26509023024268824, + "grad_norm": 3.12412166595459, + "learning_rate": 4.107883817427386e-06, + "loss": 1.7736, + "step": 213 + }, + { + "epoch": 0.26633478531425014, + "grad_norm": 3.3241145610809326, + "learning_rate": 4.128630705394191e-06, + "loss": 1.8067, + "step": 214 + }, + { + "epoch": 0.26757934038581205, + "grad_norm": 3.503307819366455, + "learning_rate": 4.149377593360996e-06, + "loss": 1.7858, + "step": 215 + }, + { + "epoch": 0.268823895457374, + "grad_norm": 3.1009578704833984, + "learning_rate": 4.170124481327801e-06, + "loss": 1.7542, + "step": 216 + }, + { + "epoch": 0.2700684505289359, + "grad_norm": 3.2236011028289795, + "learning_rate": 4.190871369294606e-06, + "loss": 1.8006, + "step": 217 + }, + { + "epoch": 0.2713130056004978, + "grad_norm": 3.048935651779175, + "learning_rate": 4.211618257261411e-06, + "loss": 1.7706, + "step": 218 + }, + { + "epoch": 0.2725575606720597, + "grad_norm": 3.3416147232055664, + "learning_rate": 4.232365145228216e-06, + "loss": 1.8108, + "step": 219 + }, + { + "epoch": 0.2738021157436217, + "grad_norm": 3.2010738849639893, + "learning_rate": 4.253112033195021e-06, + "loss": 1.8561, + "step": 220 + }, + { + "epoch": 0.2738021157436217, + "eval_loss": 1.9111930131912231, + "eval_runtime": 50.4077, + "eval_samples_per_second": 19.838, + "eval_steps_per_second": 0.833, + "step": 220 + }, + { + "epoch": 0.2750466708151836, + "grad_norm": 3.4033374786376953, + "learning_rate": 4.273858921161826e-06, + "loss": 1.8218, + "step": 221 + }, + { + "epoch": 0.2762912258867455, + "grad_norm": 3.2086002826690674, + "learning_rate": 4.294605809128631e-06, + "loss": 1.8213, + "step": 222 + }, + { + "epoch": 0.2775357809583074, + "grad_norm": 4.090956211090088, + "learning_rate": 4.315352697095436e-06, + "loss": 1.8455, + "step": 223 + }, + { + "epoch": 0.27878033602986935, + "grad_norm": 3.1142985820770264, + "learning_rate": 4.336099585062241e-06, + "loss": 1.7737, + "step": 224 + }, + { + "epoch": 0.28002489110143125, + "grad_norm": 3.369669198989868, + "learning_rate": 4.356846473029046e-06, + "loss": 1.824, + "step": 225 + }, + { + "epoch": 0.28126944617299315, + "grad_norm": 3.165672779083252, + "learning_rate": 4.3775933609958506e-06, + "loss": 1.7768, + "step": 226 + }, + { + "epoch": 0.28251400124455506, + "grad_norm": 3.3343470096588135, + "learning_rate": 4.398340248962656e-06, + "loss": 1.7278, + "step": 227 + }, + { + "epoch": 0.28375855631611696, + "grad_norm": 3.2574994564056396, + "learning_rate": 4.419087136929461e-06, + "loss": 1.8439, + "step": 228 + }, + { + "epoch": 0.2850031113876789, + "grad_norm": 3.043928623199463, + "learning_rate": 4.439834024896266e-06, + "loss": 1.818, + "step": 229 + }, + { + "epoch": 0.2862476664592408, + "grad_norm": 2.9703125953674316, + "learning_rate": 4.460580912863071e-06, + "loss": 1.773, + "step": 230 + }, + { + "epoch": 0.2862476664592408, + "eval_loss": 1.898109793663025, + "eval_runtime": 45.6023, + "eval_samples_per_second": 21.929, + "eval_steps_per_second": 0.921, + "step": 230 + }, + { + "epoch": 0.2874922215308027, + "grad_norm": 2.9997220039367676, + "learning_rate": 4.481327800829876e-06, + "loss": 1.7727, + "step": 231 + }, + { + "epoch": 0.28873677660236463, + "grad_norm": 3.1358482837677, + "learning_rate": 4.502074688796681e-06, + "loss": 1.7764, + "step": 232 + }, + { + "epoch": 0.2899813316739266, + "grad_norm": 3.395747661590576, + "learning_rate": 4.5228215767634855e-06, + "loss": 1.7826, + "step": 233 + }, + { + "epoch": 0.2912258867454885, + "grad_norm": 3.7494754791259766, + "learning_rate": 4.543568464730291e-06, + "loss": 1.7181, + "step": 234 + }, + { + "epoch": 0.2924704418170504, + "grad_norm": 3.42293381690979, + "learning_rate": 4.564315352697096e-06, + "loss": 1.7773, + "step": 235 + }, + { + "epoch": 0.2937149968886123, + "grad_norm": 3.2524514198303223, + "learning_rate": 4.585062240663901e-06, + "loss": 1.7294, + "step": 236 + }, + { + "epoch": 0.29495955196017426, + "grad_norm": 3.657869577407837, + "learning_rate": 4.605809128630706e-06, + "loss": 1.762, + "step": 237 + }, + { + "epoch": 0.29620410703173616, + "grad_norm": 3.127372980117798, + "learning_rate": 4.626556016597511e-06, + "loss": 1.7623, + "step": 238 + }, + { + "epoch": 0.29744866210329807, + "grad_norm": 4.166962146759033, + "learning_rate": 4.647302904564316e-06, + "loss": 1.6995, + "step": 239 + }, + { + "epoch": 0.29869321717485997, + "grad_norm": 3.094264030456543, + "learning_rate": 4.66804979253112e-06, + "loss": 1.7469, + "step": 240 + }, + { + "epoch": 0.29869321717485997, + "eval_loss": 1.8908178806304932, + "eval_runtime": 44.6332, + "eval_samples_per_second": 22.405, + "eval_steps_per_second": 0.941, + "step": 240 + }, + { + "epoch": 0.29993777224642193, + "grad_norm": 3.8644745349884033, + "learning_rate": 4.6887966804979255e-06, + "loss": 1.7644, + "step": 241 + }, + { + "epoch": 0.30118232731798383, + "grad_norm": 3.1488852500915527, + "learning_rate": 4.709543568464731e-06, + "loss": 1.772, + "step": 242 + }, + { + "epoch": 0.30242688238954574, + "grad_norm": 3.3179638385772705, + "learning_rate": 4.730290456431536e-06, + "loss": 1.7588, + "step": 243 + }, + { + "epoch": 0.30367143746110764, + "grad_norm": 3.328355550765991, + "learning_rate": 4.751037344398341e-06, + "loss": 1.7417, + "step": 244 + }, + { + "epoch": 0.30491599253266954, + "grad_norm": 3.2690482139587402, + "learning_rate": 4.771784232365146e-06, + "loss": 1.7562, + "step": 245 + }, + { + "epoch": 0.3061605476042315, + "grad_norm": 3.2759454250335693, + "learning_rate": 4.792531120331951e-06, + "loss": 1.7536, + "step": 246 + }, + { + "epoch": 0.3074051026757934, + "grad_norm": 2.937964916229248, + "learning_rate": 4.813278008298755e-06, + "loss": 1.7518, + "step": 247 + }, + { + "epoch": 0.3086496577473553, + "grad_norm": 3.0995302200317383, + "learning_rate": 4.83402489626556e-06, + "loss": 1.7593, + "step": 248 + }, + { + "epoch": 0.3098942128189172, + "grad_norm": 3.230459213256836, + "learning_rate": 4.8547717842323655e-06, + "loss": 1.6962, + "step": 249 + }, + { + "epoch": 0.3111387678904792, + "grad_norm": 3.242576837539673, + "learning_rate": 4.875518672199171e-06, + "loss": 1.7307, + "step": 250 + }, + { + "epoch": 0.3111387678904792, + "eval_loss": 1.8805371522903442, + "eval_runtime": 45.9886, + "eval_samples_per_second": 21.745, + "eval_steps_per_second": 0.913, + "step": 250 + }, + { + "epoch": 0.3123833229620411, + "grad_norm": 3.097045421600342, + "learning_rate": 4.896265560165976e-06, + "loss": 1.7438, + "step": 251 + }, + { + "epoch": 0.313627878033603, + "grad_norm": 3.2428948879241943, + "learning_rate": 4.91701244813278e-06, + "loss": 1.7179, + "step": 252 + }, + { + "epoch": 0.3148724331051649, + "grad_norm": 3.196274518966675, + "learning_rate": 4.937759336099586e-06, + "loss": 1.7855, + "step": 253 + }, + { + "epoch": 0.31611698817672684, + "grad_norm": 2.978203535079956, + "learning_rate": 4.95850622406639e-06, + "loss": 1.7144, + "step": 254 + }, + { + "epoch": 0.31736154324828875, + "grad_norm": 3.2641701698303223, + "learning_rate": 4.979253112033195e-06, + "loss": 1.7427, + "step": 255 + }, + { + "epoch": 0.31860609831985065, + "grad_norm": 2.7441232204437256, + "learning_rate": 5e-06, + "loss": 1.7619, + "step": 256 + }, + { + "epoch": 0.31985065339141255, + "grad_norm": 3.0723495483398438, + "learning_rate": 4.99769372693727e-06, + "loss": 1.8017, + "step": 257 + }, + { + "epoch": 0.3210952084629745, + "grad_norm": 3.104752540588379, + "learning_rate": 4.995387453874539e-06, + "loss": 1.7761, + "step": 258 + }, + { + "epoch": 0.3223397635345364, + "grad_norm": 3.138627052307129, + "learning_rate": 4.993081180811809e-06, + "loss": 1.7879, + "step": 259 + }, + { + "epoch": 0.3235843186060983, + "grad_norm": 2.8567333221435547, + "learning_rate": 4.990774907749078e-06, + "loss": 1.7561, + "step": 260 + }, + { + "epoch": 0.3235843186060983, + "eval_loss": 1.8606494665145874, + "eval_runtime": 46.453, + "eval_samples_per_second": 21.527, + "eval_steps_per_second": 0.904, + "step": 260 + }, + { + "epoch": 0.3248288736776602, + "grad_norm": 3.2696540355682373, + "learning_rate": 4.988468634686347e-06, + "loss": 1.7201, + "step": 261 + }, + { + "epoch": 0.3260734287492221, + "grad_norm": 2.6692731380462646, + "learning_rate": 4.986162361623617e-06, + "loss": 1.7264, + "step": 262 + }, + { + "epoch": 0.3273179838207841, + "grad_norm": 3.047549247741699, + "learning_rate": 4.983856088560886e-06, + "loss": 1.7362, + "step": 263 + }, + { + "epoch": 0.328562538892346, + "grad_norm": 3.0667457580566406, + "learning_rate": 4.981549815498156e-06, + "loss": 1.7321, + "step": 264 + }, + { + "epoch": 0.3298070939639079, + "grad_norm": 2.560047149658203, + "learning_rate": 4.979243542435424e-06, + "loss": 1.7508, + "step": 265 + }, + { + "epoch": 0.3310516490354698, + "grad_norm": 3.26595401763916, + "learning_rate": 4.976937269372694e-06, + "loss": 1.7248, + "step": 266 + }, + { + "epoch": 0.33229620410703176, + "grad_norm": 2.929210662841797, + "learning_rate": 4.974630996309964e-06, + "loss": 1.697, + "step": 267 + }, + { + "epoch": 0.33354075917859366, + "grad_norm": 3.1526286602020264, + "learning_rate": 4.972324723247233e-06, + "loss": 1.7385, + "step": 268 + }, + { + "epoch": 0.33478531425015556, + "grad_norm": 3.001619577407837, + "learning_rate": 4.970018450184502e-06, + "loss": 1.7072, + "step": 269 + }, + { + "epoch": 0.33602986932171747, + "grad_norm": 3.2464189529418945, + "learning_rate": 4.9677121771217715e-06, + "loss": 1.7361, + "step": 270 + }, + { + "epoch": 0.33602986932171747, + "eval_loss": 1.860226035118103, + "eval_runtime": 46.6881, + "eval_samples_per_second": 21.419, + "eval_steps_per_second": 0.9, + "step": 270 + }, + { + "epoch": 0.3372744243932794, + "grad_norm": 3.2104530334472656, + "learning_rate": 4.965405904059041e-06, + "loss": 1.7171, + "step": 271 + }, + { + "epoch": 0.33851897946484133, + "grad_norm": 3.146847724914551, + "learning_rate": 4.96309963099631e-06, + "loss": 1.7311, + "step": 272 + }, + { + "epoch": 0.33976353453640323, + "grad_norm": 3.1431286334991455, + "learning_rate": 4.96079335793358e-06, + "loss": 1.7345, + "step": 273 + }, + { + "epoch": 0.34100808960796514, + "grad_norm": 2.847163677215576, + "learning_rate": 4.958487084870849e-06, + "loss": 1.7262, + "step": 274 + }, + { + "epoch": 0.3422526446795271, + "grad_norm": 3.1987810134887695, + "learning_rate": 4.956180811808119e-06, + "loss": 1.7945, + "step": 275 + }, + { + "epoch": 0.343497199751089, + "grad_norm": 2.7475385665893555, + "learning_rate": 4.953874538745388e-06, + "loss": 1.76, + "step": 276 + }, + { + "epoch": 0.3447417548226509, + "grad_norm": 3.0427663326263428, + "learning_rate": 4.9515682656826574e-06, + "loss": 1.7324, + "step": 277 + }, + { + "epoch": 0.3459863098942128, + "grad_norm": 3.605212688446045, + "learning_rate": 4.949261992619927e-06, + "loss": 1.7712, + "step": 278 + }, + { + "epoch": 0.3472308649657747, + "grad_norm": 3.0564935207366943, + "learning_rate": 4.946955719557196e-06, + "loss": 1.6792, + "step": 279 + }, + { + "epoch": 0.34847542003733667, + "grad_norm": 3.1069741249084473, + "learning_rate": 4.944649446494466e-06, + "loss": 1.7145, + "step": 280 + }, + { + "epoch": 0.34847542003733667, + "eval_loss": 1.8537051677703857, + "eval_runtime": 47.4813, + "eval_samples_per_second": 21.061, + "eval_steps_per_second": 0.885, + "step": 280 + }, + { + "epoch": 0.3497199751088986, + "grad_norm": 2.8021512031555176, + "learning_rate": 4.942343173431734e-06, + "loss": 1.7145, + "step": 281 + }, + { + "epoch": 0.3509645301804605, + "grad_norm": 3.5549023151397705, + "learning_rate": 4.940036900369004e-06, + "loss": 1.7642, + "step": 282 + }, + { + "epoch": 0.3522090852520224, + "grad_norm": 2.7648985385894775, + "learning_rate": 4.937730627306274e-06, + "loss": 1.7255, + "step": 283 + }, + { + "epoch": 0.35345364032358434, + "grad_norm": 3.0815863609313965, + "learning_rate": 4.9354243542435426e-06, + "loss": 1.7055, + "step": 284 + }, + { + "epoch": 0.35469819539514624, + "grad_norm": 2.9009227752685547, + "learning_rate": 4.933118081180812e-06, + "loss": 1.7019, + "step": 285 + }, + { + "epoch": 0.35594275046670815, + "grad_norm": 2.618429660797119, + "learning_rate": 4.930811808118081e-06, + "loss": 1.6318, + "step": 286 + }, + { + "epoch": 0.35718730553827005, + "grad_norm": 3.368230104446411, + "learning_rate": 4.928505535055351e-06, + "loss": 1.7304, + "step": 287 + }, + { + "epoch": 0.358431860609832, + "grad_norm": 3.256889820098877, + "learning_rate": 4.92619926199262e-06, + "loss": 1.7019, + "step": 288 + }, + { + "epoch": 0.3596764156813939, + "grad_norm": 2.9366304874420166, + "learning_rate": 4.92389298892989e-06, + "loss": 1.734, + "step": 289 + }, + { + "epoch": 0.3609209707529558, + "grad_norm": 3.0393142700195312, + "learning_rate": 4.921586715867159e-06, + "loss": 1.7504, + "step": 290 + }, + { + "epoch": 0.3609209707529558, + "eval_loss": 1.8359886407852173, + "eval_runtime": 49.7622, + "eval_samples_per_second": 20.096, + "eval_steps_per_second": 0.844, + "step": 290 + }, + { + "epoch": 0.3621655258245177, + "grad_norm": 2.7288589477539062, + "learning_rate": 4.9192804428044285e-06, + "loss": 1.7004, + "step": 291 + }, + { + "epoch": 0.3634100808960797, + "grad_norm": 3.2793378829956055, + "learning_rate": 4.916974169741698e-06, + "loss": 1.7028, + "step": 292 + }, + { + "epoch": 0.3646546359676416, + "grad_norm": 2.960880756378174, + "learning_rate": 4.914667896678967e-06, + "loss": 1.6759, + "step": 293 + }, + { + "epoch": 0.3658991910392035, + "grad_norm": 2.836421012878418, + "learning_rate": 4.912361623616237e-06, + "loss": 1.6475, + "step": 294 + }, + { + "epoch": 0.3671437461107654, + "grad_norm": 2.9495465755462646, + "learning_rate": 4.910055350553506e-06, + "loss": 1.7076, + "step": 295 + }, + { + "epoch": 0.3683883011823273, + "grad_norm": 2.954730272293091, + "learning_rate": 4.907749077490776e-06, + "loss": 1.7261, + "step": 296 + }, + { + "epoch": 0.36963285625388925, + "grad_norm": 2.8167543411254883, + "learning_rate": 4.905442804428044e-06, + "loss": 1.6939, + "step": 297 + }, + { + "epoch": 0.37087741132545116, + "grad_norm": 2.857316017150879, + "learning_rate": 4.903136531365314e-06, + "loss": 1.7362, + "step": 298 + }, + { + "epoch": 0.37212196639701306, + "grad_norm": 2.8612918853759766, + "learning_rate": 4.900830258302584e-06, + "loss": 1.6489, + "step": 299 + }, + { + "epoch": 0.37336652146857496, + "grad_norm": 2.9618914127349854, + "learning_rate": 4.898523985239853e-06, + "loss": 1.718, + "step": 300 + }, + { + "epoch": 0.37336652146857496, + "eval_loss": 1.827571988105774, + "eval_runtime": 46.4552, + "eval_samples_per_second": 21.526, + "eval_steps_per_second": 0.904, + "step": 300 + }, + { + "epoch": 0.3746110765401369, + "grad_norm": 2.8706953525543213, + "learning_rate": 4.896217712177122e-06, + "loss": 1.6925, + "step": 301 + }, + { + "epoch": 0.3758556316116988, + "grad_norm": 3.2565090656280518, + "learning_rate": 4.893911439114391e-06, + "loss": 1.7369, + "step": 302 + }, + { + "epoch": 0.37710018668326073, + "grad_norm": 2.928858995437622, + "learning_rate": 4.891605166051661e-06, + "loss": 1.6882, + "step": 303 + }, + { + "epoch": 0.37834474175482263, + "grad_norm": 3.042314052581787, + "learning_rate": 4.88929889298893e-06, + "loss": 1.6466, + "step": 304 + }, + { + "epoch": 0.3795892968263846, + "grad_norm": 2.7509143352508545, + "learning_rate": 4.8869926199262e-06, + "loss": 1.685, + "step": 305 + }, + { + "epoch": 0.3808338518979465, + "grad_norm": 2.9601588249206543, + "learning_rate": 4.884686346863469e-06, + "loss": 1.6772, + "step": 306 + }, + { + "epoch": 0.3820784069695084, + "grad_norm": 2.618608236312866, + "learning_rate": 4.8823800738007384e-06, + "loss": 1.7127, + "step": 307 + }, + { + "epoch": 0.3833229620410703, + "grad_norm": 3.047581195831299, + "learning_rate": 4.880073800738008e-06, + "loss": 1.7174, + "step": 308 + }, + { + "epoch": 0.3845675171126322, + "grad_norm": 2.87735652923584, + "learning_rate": 4.877767527675277e-06, + "loss": 1.6677, + "step": 309 + }, + { + "epoch": 0.38581207218419417, + "grad_norm": 2.7169721126556396, + "learning_rate": 4.875461254612546e-06, + "loss": 1.7486, + "step": 310 + }, + { + "epoch": 0.38581207218419417, + "eval_loss": 1.8221794366836548, + "eval_runtime": 43.1681, + "eval_samples_per_second": 23.165, + "eval_steps_per_second": 0.973, + "step": 310 + }, + { + "epoch": 0.38705662725575607, + "grad_norm": 2.7094991207122803, + "learning_rate": 4.873154981549816e-06, + "loss": 1.7348, + "step": 311 + }, + { + "epoch": 0.388301182327318, + "grad_norm": 2.8989078998565674, + "learning_rate": 4.8708487084870856e-06, + "loss": 1.7352, + "step": 312 + }, + { + "epoch": 0.3895457373988799, + "grad_norm": 2.9565601348876953, + "learning_rate": 4.868542435424355e-06, + "loss": 1.7063, + "step": 313 + }, + { + "epoch": 0.39079029247044184, + "grad_norm": 2.8354082107543945, + "learning_rate": 4.8662361623616235e-06, + "loss": 1.6808, + "step": 314 + }, + { + "epoch": 0.39203484754200374, + "grad_norm": 2.8287479877471924, + "learning_rate": 4.863929889298894e-06, + "loss": 1.6731, + "step": 315 + }, + { + "epoch": 0.39327940261356564, + "grad_norm": 2.6537821292877197, + "learning_rate": 4.861623616236163e-06, + "loss": 1.695, + "step": 316 + }, + { + "epoch": 0.39452395768512755, + "grad_norm": 2.7135190963745117, + "learning_rate": 4.859317343173432e-06, + "loss": 1.6897, + "step": 317 + }, + { + "epoch": 0.3957685127566895, + "grad_norm": 3.1019129753112793, + "learning_rate": 4.857011070110701e-06, + "loss": 1.7219, + "step": 318 + }, + { + "epoch": 0.3970130678282514, + "grad_norm": 2.945737838745117, + "learning_rate": 4.854704797047971e-06, + "loss": 1.7316, + "step": 319 + }, + { + "epoch": 0.3982576228998133, + "grad_norm": 2.7668251991271973, + "learning_rate": 4.85239852398524e-06, + "loss": 1.6461, + "step": 320 + }, + { + "epoch": 0.3982576228998133, + "eval_loss": 1.8210570812225342, + "eval_runtime": 46.2552, + "eval_samples_per_second": 21.619, + "eval_steps_per_second": 0.908, + "step": 320 + }, + { + "epoch": 0.3995021779713752, + "grad_norm": 2.738109588623047, + "learning_rate": 4.8500922509225095e-06, + "loss": 1.6907, + "step": 321 + }, + { + "epoch": 0.4007467330429372, + "grad_norm": 3.0195062160491943, + "learning_rate": 4.847785977859779e-06, + "loss": 1.7099, + "step": 322 + }, + { + "epoch": 0.4019912881144991, + "grad_norm": 2.8672404289245605, + "learning_rate": 4.845479704797048e-06, + "loss": 1.6509, + "step": 323 + }, + { + "epoch": 0.403235843186061, + "grad_norm": 2.9041008949279785, + "learning_rate": 4.843173431734318e-06, + "loss": 1.5818, + "step": 324 + }, + { + "epoch": 0.4044803982576229, + "grad_norm": 2.9105985164642334, + "learning_rate": 4.840867158671587e-06, + "loss": 1.7002, + "step": 325 + }, + { + "epoch": 0.4057249533291848, + "grad_norm": 2.749009847640991, + "learning_rate": 4.838560885608857e-06, + "loss": 1.7021, + "step": 326 + }, + { + "epoch": 0.40696950840074675, + "grad_norm": 2.6998822689056396, + "learning_rate": 4.836254612546126e-06, + "loss": 1.6853, + "step": 327 + }, + { + "epoch": 0.40821406347230865, + "grad_norm": 2.678201675415039, + "learning_rate": 4.8339483394833955e-06, + "loss": 1.6934, + "step": 328 + }, + { + "epoch": 0.40945861854387056, + "grad_norm": 2.774845600128174, + "learning_rate": 4.831642066420665e-06, + "loss": 1.657, + "step": 329 + }, + { + "epoch": 0.41070317361543246, + "grad_norm": 2.6363000869750977, + "learning_rate": 4.8293357933579335e-06, + "loss": 1.6555, + "step": 330 + }, + { + "epoch": 0.41070317361543246, + "eval_loss": 1.8126581907272339, + "eval_runtime": 52.7717, + "eval_samples_per_second": 18.95, + "eval_steps_per_second": 0.796, + "step": 330 + }, + { + "epoch": 0.4119477286869944, + "grad_norm": 3.011659622192383, + "learning_rate": 4.827029520295204e-06, + "loss": 1.7202, + "step": 331 + }, + { + "epoch": 0.4131922837585563, + "grad_norm": 2.8322594165802, + "learning_rate": 4.824723247232473e-06, + "loss": 1.6736, + "step": 332 + }, + { + "epoch": 0.4144368388301182, + "grad_norm": 2.9360146522521973, + "learning_rate": 4.822416974169742e-06, + "loss": 1.7213, + "step": 333 + }, + { + "epoch": 0.41568139390168013, + "grad_norm": 2.810206413269043, + "learning_rate": 4.820110701107011e-06, + "loss": 1.6867, + "step": 334 + }, + { + "epoch": 0.4169259489732421, + "grad_norm": 2.6864097118377686, + "learning_rate": 4.817804428044281e-06, + "loss": 1.6854, + "step": 335 + }, + { + "epoch": 0.418170504044804, + "grad_norm": 2.7396061420440674, + "learning_rate": 4.81549815498155e-06, + "loss": 1.68, + "step": 336 + }, + { + "epoch": 0.4194150591163659, + "grad_norm": 2.602771520614624, + "learning_rate": 4.8131918819188194e-06, + "loss": 1.6269, + "step": 337 + }, + { + "epoch": 0.4206596141879278, + "grad_norm": 2.8174757957458496, + "learning_rate": 4.810885608856089e-06, + "loss": 1.6981, + "step": 338 + }, + { + "epoch": 0.42190416925948976, + "grad_norm": 3.2398130893707275, + "learning_rate": 4.808579335793358e-06, + "loss": 1.7061, + "step": 339 + }, + { + "epoch": 0.42314872433105166, + "grad_norm": 2.7119736671447754, + "learning_rate": 4.806273062730628e-06, + "loss": 1.6875, + "step": 340 + }, + { + "epoch": 0.42314872433105166, + "eval_loss": 1.7973068952560425, + "eval_runtime": 53.5791, + "eval_samples_per_second": 18.664, + "eval_steps_per_second": 0.784, + "step": 340 + }, + { + "epoch": 0.42439327940261357, + "grad_norm": 2.868533134460449, + "learning_rate": 4.803966789667897e-06, + "loss": 1.6967, + "step": 341 + }, + { + "epoch": 0.42563783447417547, + "grad_norm": 2.528083562850952, + "learning_rate": 4.8016605166051665e-06, + "loss": 1.6195, + "step": 342 + }, + { + "epoch": 0.4268823895457374, + "grad_norm": 2.7885115146636963, + "learning_rate": 4.799354243542436e-06, + "loss": 1.6197, + "step": 343 + }, + { + "epoch": 0.42812694461729933, + "grad_norm": 2.768247604370117, + "learning_rate": 4.797047970479705e-06, + "loss": 1.6271, + "step": 344 + }, + { + "epoch": 0.42937149968886124, + "grad_norm": 2.7710275650024414, + "learning_rate": 4.794741697416975e-06, + "loss": 1.6768, + "step": 345 + }, + { + "epoch": 0.43061605476042314, + "grad_norm": 2.968337297439575, + "learning_rate": 4.792435424354243e-06, + "loss": 1.6433, + "step": 346 + }, + { + "epoch": 0.43186060983198504, + "grad_norm": 2.5739808082580566, + "learning_rate": 4.790129151291514e-06, + "loss": 1.6327, + "step": 347 + }, + { + "epoch": 0.433105164903547, + "grad_norm": 2.7676408290863037, + "learning_rate": 4.787822878228783e-06, + "loss": 1.6178, + "step": 348 + }, + { + "epoch": 0.4343497199751089, + "grad_norm": 2.959059476852417, + "learning_rate": 4.7855166051660525e-06, + "loss": 1.6809, + "step": 349 + }, + { + "epoch": 0.4355942750466708, + "grad_norm": 2.559967279434204, + "learning_rate": 4.783210332103321e-06, + "loss": 1.6704, + "step": 350 + }, + { + "epoch": 0.4355942750466708, + "eval_loss": 1.8013949394226074, + "eval_runtime": 50.4649, + "eval_samples_per_second": 19.816, + "eval_steps_per_second": 0.832, + "step": 350 + }, + { + "epoch": 0.4368388301182327, + "grad_norm": 2.817901611328125, + "learning_rate": 4.7809040590405905e-06, + "loss": 1.6296, + "step": 351 + }, + { + "epoch": 0.43808338518979467, + "grad_norm": 2.7116055488586426, + "learning_rate": 4.778597785977861e-06, + "loss": 1.6652, + "step": 352 + }, + { + "epoch": 0.4393279402613566, + "grad_norm": 2.544750452041626, + "learning_rate": 4.776291512915129e-06, + "loss": 1.5668, + "step": 353 + }, + { + "epoch": 0.4405724953329185, + "grad_norm": 2.720534086227417, + "learning_rate": 4.773985239852399e-06, + "loss": 1.6152, + "step": 354 + }, + { + "epoch": 0.4418170504044804, + "grad_norm": 2.7270126342773438, + "learning_rate": 4.771678966789668e-06, + "loss": 1.5624, + "step": 355 + }, + { + "epoch": 0.44306160547604234, + "grad_norm": 2.7986624240875244, + "learning_rate": 4.769372693726938e-06, + "loss": 1.6621, + "step": 356 + }, + { + "epoch": 0.44430616054760425, + "grad_norm": 2.943107843399048, + "learning_rate": 4.767066420664207e-06, + "loss": 1.6713, + "step": 357 + }, + { + "epoch": 0.44555071561916615, + "grad_norm": 2.739898681640625, + "learning_rate": 4.7647601476014765e-06, + "loss": 1.6363, + "step": 358 + }, + { + "epoch": 0.44679527069072805, + "grad_norm": 2.8729329109191895, + "learning_rate": 4.762453874538746e-06, + "loss": 1.6605, + "step": 359 + }, + { + "epoch": 0.44803982576228996, + "grad_norm": 2.909029483795166, + "learning_rate": 4.760147601476015e-06, + "loss": 1.6277, + "step": 360 + }, + { + "epoch": 0.44803982576228996, + "eval_loss": 1.8110179901123047, + "eval_runtime": 47.924, + "eval_samples_per_second": 20.866, + "eval_steps_per_second": 0.876, + "step": 360 + }, + { + "epoch": 0.4492843808338519, + "grad_norm": 3.3826682567596436, + "learning_rate": 4.757841328413285e-06, + "loss": 1.7155, + "step": 361 + }, + { + "epoch": 0.4505289359054138, + "grad_norm": 3.0586729049682617, + "learning_rate": 4.755535055350554e-06, + "loss": 1.5991, + "step": 362 + }, + { + "epoch": 0.4517734909769757, + "grad_norm": 2.645576238632202, + "learning_rate": 4.753228782287823e-06, + "loss": 1.6504, + "step": 363 + }, + { + "epoch": 0.4530180460485376, + "grad_norm": 2.705967664718628, + "learning_rate": 4.750922509225093e-06, + "loss": 1.6761, + "step": 364 + }, + { + "epoch": 0.4542626011200996, + "grad_norm": 2.7651922702789307, + "learning_rate": 4.748616236162362e-06, + "loss": 1.61, + "step": 365 + }, + { + "epoch": 0.4555071561916615, + "grad_norm": 2.5654966831207275, + "learning_rate": 4.746309963099631e-06, + "loss": 1.6335, + "step": 366 + }, + { + "epoch": 0.4567517112632234, + "grad_norm": 2.6606173515319824, + "learning_rate": 4.7440036900369e-06, + "loss": 1.6207, + "step": 367 + }, + { + "epoch": 0.4579962663347853, + "grad_norm": 2.724755048751831, + "learning_rate": 4.741697416974171e-06, + "loss": 1.6329, + "step": 368 + }, + { + "epoch": 0.45924082140634725, + "grad_norm": 2.7019615173339844, + "learning_rate": 4.739391143911439e-06, + "loss": 1.6763, + "step": 369 + }, + { + "epoch": 0.46048537647790916, + "grad_norm": 2.6594254970550537, + "learning_rate": 4.737084870848709e-06, + "loss": 1.6174, + "step": 370 + }, + { + "epoch": 0.46048537647790916, + "eval_loss": 1.7924141883850098, + "eval_runtime": 53.0349, + "eval_samples_per_second": 18.855, + "eval_steps_per_second": 0.792, + "step": 370 + }, + { + "epoch": 0.46172993154947106, + "grad_norm": 2.668421745300293, + "learning_rate": 4.734778597785978e-06, + "loss": 1.6421, + "step": 371 + }, + { + "epoch": 0.46297448662103297, + "grad_norm": 2.7148995399475098, + "learning_rate": 4.7324723247232475e-06, + "loss": 1.6635, + "step": 372 + }, + { + "epoch": 0.4642190416925949, + "grad_norm": 3.156646490097046, + "learning_rate": 4.730166051660517e-06, + "loss": 1.6679, + "step": 373 + }, + { + "epoch": 0.46546359676415683, + "grad_norm": 2.8170645236968994, + "learning_rate": 4.727859778597786e-06, + "loss": 1.6492, + "step": 374 + }, + { + "epoch": 0.46670815183571873, + "grad_norm": 2.9796457290649414, + "learning_rate": 4.725553505535056e-06, + "loss": 1.6025, + "step": 375 + }, + { + "epoch": 0.46795270690728064, + "grad_norm": 2.6493406295776367, + "learning_rate": 4.723247232472325e-06, + "loss": 1.6398, + "step": 376 + }, + { + "epoch": 0.46919726197884254, + "grad_norm": 3.2643542289733887, + "learning_rate": 4.720940959409595e-06, + "loss": 1.6413, + "step": 377 + }, + { + "epoch": 0.4704418170504045, + "grad_norm": 2.7613179683685303, + "learning_rate": 4.718634686346864e-06, + "loss": 1.6251, + "step": 378 + }, + { + "epoch": 0.4716863721219664, + "grad_norm": 3.1216518878936768, + "learning_rate": 4.716328413284133e-06, + "loss": 1.6268, + "step": 379 + }, + { + "epoch": 0.4729309271935283, + "grad_norm": 2.571647882461548, + "learning_rate": 4.714022140221403e-06, + "loss": 1.6245, + "step": 380 + }, + { + "epoch": 0.4729309271935283, + "eval_loss": 1.7875893115997314, + "eval_runtime": 49.3058, + "eval_samples_per_second": 20.282, + "eval_steps_per_second": 0.852, + "step": 380 + }, + { + "epoch": 0.4741754822650902, + "grad_norm": 3.443906545639038, + "learning_rate": 4.711715867158672e-06, + "loss": 1.6799, + "step": 381 + }, + { + "epoch": 0.47542003733665217, + "grad_norm": 2.7887957096099854, + "learning_rate": 4.709409594095941e-06, + "loss": 1.6589, + "step": 382 + }, + { + "epoch": 0.47666459240821407, + "grad_norm": 3.2833151817321777, + "learning_rate": 4.70710332103321e-06, + "loss": 1.6064, + "step": 383 + }, + { + "epoch": 0.477909147479776, + "grad_norm": 2.6975057125091553, + "learning_rate": 4.704797047970481e-06, + "loss": 1.6029, + "step": 384 + }, + { + "epoch": 0.4791537025513379, + "grad_norm": 2.745246171951294, + "learning_rate": 4.702490774907749e-06, + "loss": 1.7127, + "step": 385 + }, + { + "epoch": 0.48039825762289984, + "grad_norm": 3.0522775650024414, + "learning_rate": 4.700184501845019e-06, + "loss": 1.6508, + "step": 386 + }, + { + "epoch": 0.48164281269446174, + "grad_norm": 2.689218044281006, + "learning_rate": 4.697878228782288e-06, + "loss": 1.6971, + "step": 387 + }, + { + "epoch": 0.48288736776602365, + "grad_norm": 3.060361862182617, + "learning_rate": 4.6955719557195575e-06, + "loss": 1.6464, + "step": 388 + }, + { + "epoch": 0.48413192283758555, + "grad_norm": 2.872262477874756, + "learning_rate": 4.693265682656827e-06, + "loss": 1.6293, + "step": 389 + }, + { + "epoch": 0.48537647790914745, + "grad_norm": 2.980050563812256, + "learning_rate": 4.690959409594096e-06, + "loss": 1.6435, + "step": 390 + }, + { + "epoch": 0.48537647790914745, + "eval_loss": 1.7791496515274048, + "eval_runtime": 50.0299, + "eval_samples_per_second": 19.988, + "eval_steps_per_second": 0.839, + "step": 390 + }, + { + "epoch": 0.4866210329807094, + "grad_norm": 2.764827251434326, + "learning_rate": 4.688653136531366e-06, + "loss": 1.6279, + "step": 391 + }, + { + "epoch": 0.4878655880522713, + "grad_norm": 2.6912333965301514, + "learning_rate": 4.686346863468635e-06, + "loss": 1.7276, + "step": 392 + }, + { + "epoch": 0.4891101431238332, + "grad_norm": 2.8358330726623535, + "learning_rate": 4.6840405904059046e-06, + "loss": 1.6482, + "step": 393 + }, + { + "epoch": 0.4903546981953951, + "grad_norm": 2.6581575870513916, + "learning_rate": 4.681734317343174e-06, + "loss": 1.6853, + "step": 394 + }, + { + "epoch": 0.4915992532669571, + "grad_norm": 2.941425323486328, + "learning_rate": 4.6794280442804426e-06, + "loss": 1.6075, + "step": 395 + }, + { + "epoch": 0.492843808338519, + "grad_norm": 3.2334814071655273, + "learning_rate": 4.677121771217713e-06, + "loss": 1.6157, + "step": 396 + }, + { + "epoch": 0.4940883634100809, + "grad_norm": 2.984410524368286, + "learning_rate": 4.674815498154982e-06, + "loss": 1.6257, + "step": 397 + }, + { + "epoch": 0.4953329184816428, + "grad_norm": 3.0650789737701416, + "learning_rate": 4.672509225092252e-06, + "loss": 1.6573, + "step": 398 + }, + { + "epoch": 0.49657747355320475, + "grad_norm": 2.798664093017578, + "learning_rate": 4.67020295202952e-06, + "loss": 1.638, + "step": 399 + }, + { + "epoch": 0.49782202862476665, + "grad_norm": 2.8266706466674805, + "learning_rate": 4.66789667896679e-06, + "loss": 1.6484, + "step": 400 + }, + { + "epoch": 0.49782202862476665, + "eval_loss": 1.7714862823486328, + "eval_runtime": 48.0657, + "eval_samples_per_second": 20.805, + "eval_steps_per_second": 0.874, + "step": 400 + }, + { + "epoch": 0.49906658369632856, + "grad_norm": 2.9928388595581055, + "learning_rate": 4.66559040590406e-06, + "loss": 1.6721, + "step": 401 + }, + { + "epoch": 0.5003111387678905, + "grad_norm": 2.573514461517334, + "learning_rate": 4.6632841328413285e-06, + "loss": 1.679, + "step": 402 + }, + { + "epoch": 0.5015556938394524, + "grad_norm": 3.1736996173858643, + "learning_rate": 4.660977859778598e-06, + "loss": 1.7124, + "step": 403 + }, + { + "epoch": 0.5028002489110143, + "grad_norm": 2.858849287033081, + "learning_rate": 4.658671586715867e-06, + "loss": 1.6622, + "step": 404 + }, + { + "epoch": 0.5040448039825762, + "grad_norm": 2.825698137283325, + "learning_rate": 4.656365313653137e-06, + "loss": 1.6464, + "step": 405 + }, + { + "epoch": 0.5052893590541382, + "grad_norm": 3.783891439437866, + "learning_rate": 4.654059040590406e-06, + "loss": 1.6665, + "step": 406 + }, + { + "epoch": 0.5065339141257, + "grad_norm": 2.789813756942749, + "learning_rate": 4.651752767527676e-06, + "loss": 1.6064, + "step": 407 + }, + { + "epoch": 0.507778469197262, + "grad_norm": 3.3106348514556885, + "learning_rate": 4.649446494464945e-06, + "loss": 1.6542, + "step": 408 + }, + { + "epoch": 0.5090230242688238, + "grad_norm": 2.8805112838745117, + "learning_rate": 4.6471402214022145e-06, + "loss": 1.6058, + "step": 409 + }, + { + "epoch": 0.5102675793403858, + "grad_norm": 3.30898118019104, + "learning_rate": 4.644833948339484e-06, + "loss": 1.5933, + "step": 410 + }, + { + "epoch": 0.5102675793403858, + "eval_loss": 1.7787818908691406, + "eval_runtime": 44.0876, + "eval_samples_per_second": 22.682, + "eval_steps_per_second": 0.953, + "step": 410 + }, + { + "epoch": 0.5115121344119478, + "grad_norm": 3.3563625812530518, + "learning_rate": 4.642527675276753e-06, + "loss": 1.6751, + "step": 411 + }, + { + "epoch": 0.5127566894835096, + "grad_norm": 2.968120574951172, + "learning_rate": 4.640221402214023e-06, + "loss": 1.663, + "step": 412 + }, + { + "epoch": 0.5140012445550716, + "grad_norm": 3.310248374938965, + "learning_rate": 4.637915129151292e-06, + "loss": 1.658, + "step": 413 + }, + { + "epoch": 0.5152457996266335, + "grad_norm": 2.7011170387268066, + "learning_rate": 4.635608856088562e-06, + "loss": 1.6894, + "step": 414 + }, + { + "epoch": 0.5164903546981954, + "grad_norm": 2.804901361465454, + "learning_rate": 4.63330258302583e-06, + "loss": 1.602, + "step": 415 + }, + { + "epoch": 0.5177349097697573, + "grad_norm": 3.2412750720977783, + "learning_rate": 4.6309963099631e-06, + "loss": 1.6158, + "step": 416 + }, + { + "epoch": 0.5189794648413192, + "grad_norm": 2.699909210205078, + "learning_rate": 4.62869003690037e-06, + "loss": 1.5842, + "step": 417 + }, + { + "epoch": 0.5202240199128811, + "grad_norm": 3.065929412841797, + "learning_rate": 4.6263837638376384e-06, + "loss": 1.6434, + "step": 418 + }, + { + "epoch": 0.5214685749844431, + "grad_norm": 2.895305633544922, + "learning_rate": 4.624077490774908e-06, + "loss": 1.5969, + "step": 419 + }, + { + "epoch": 0.522713130056005, + "grad_norm": 3.1590664386749268, + "learning_rate": 4.621771217712177e-06, + "loss": 1.6211, + "step": 420 + }, + { + "epoch": 0.522713130056005, + "eval_loss": 1.7674189805984497, + "eval_runtime": 53.328, + "eval_samples_per_second": 18.752, + "eval_steps_per_second": 0.788, + "step": 420 + }, + { + "epoch": 0.5239576851275669, + "grad_norm": 2.7126312255859375, + "learning_rate": 4.619464944649447e-06, + "loss": 1.6246, + "step": 421 + }, + { + "epoch": 0.5252022401991289, + "grad_norm": 2.918508768081665, + "learning_rate": 4.617158671586716e-06, + "loss": 1.6937, + "step": 422 + }, + { + "epoch": 0.5264467952706907, + "grad_norm": 3.008610725402832, + "learning_rate": 4.6148523985239856e-06, + "loss": 1.6864, + "step": 423 + }, + { + "epoch": 0.5276913503422527, + "grad_norm": 2.6612043380737305, + "learning_rate": 4.612546125461255e-06, + "loss": 1.5785, + "step": 424 + }, + { + "epoch": 0.5289359054138145, + "grad_norm": 2.850679874420166, + "learning_rate": 4.610239852398524e-06, + "loss": 1.5899, + "step": 425 + }, + { + "epoch": 0.5301804604853765, + "grad_norm": 2.9205031394958496, + "learning_rate": 4.607933579335794e-06, + "loss": 1.5867, + "step": 426 + }, + { + "epoch": 0.5314250155569384, + "grad_norm": 2.929532527923584, + "learning_rate": 4.605627306273063e-06, + "loss": 1.6182, + "step": 427 + }, + { + "epoch": 0.5326695706285003, + "grad_norm": 2.874936103820801, + "learning_rate": 4.603321033210333e-06, + "loss": 1.5789, + "step": 428 + }, + { + "epoch": 0.5339141257000622, + "grad_norm": 2.8703525066375732, + "learning_rate": 4.601014760147602e-06, + "loss": 1.5859, + "step": 429 + }, + { + "epoch": 0.5351586807716241, + "grad_norm": 2.7076902389526367, + "learning_rate": 4.5987084870848715e-06, + "loss": 1.644, + "step": 430 + }, + { + "epoch": 0.5351586807716241, + "eval_loss": 1.7592095136642456, + "eval_runtime": 47.9438, + "eval_samples_per_second": 20.858, + "eval_steps_per_second": 0.876, + "step": 430 + }, + { + "epoch": 0.536403235843186, + "grad_norm": 2.839160680770874, + "learning_rate": 4.59640221402214e-06, + "loss": 1.6357, + "step": 431 + }, + { + "epoch": 0.537647790914748, + "grad_norm": 2.674025535583496, + "learning_rate": 4.5940959409594095e-06, + "loss": 1.676, + "step": 432 + }, + { + "epoch": 0.5388923459863099, + "grad_norm": 3.0619537830352783, + "learning_rate": 4.59178966789668e-06, + "loss": 1.5617, + "step": 433 + }, + { + "epoch": 0.5401369010578718, + "grad_norm": 3.058418035507202, + "learning_rate": 4.589483394833948e-06, + "loss": 1.5951, + "step": 434 + }, + { + "epoch": 0.5413814561294338, + "grad_norm": 3.0046396255493164, + "learning_rate": 4.587177121771218e-06, + "loss": 1.6473, + "step": 435 + }, + { + "epoch": 0.5426260112009956, + "grad_norm": 2.7736752033233643, + "learning_rate": 4.584870848708487e-06, + "loss": 1.6262, + "step": 436 + }, + { + "epoch": 0.5438705662725576, + "grad_norm": 2.7804994583129883, + "learning_rate": 4.5825645756457575e-06, + "loss": 1.6886, + "step": 437 + }, + { + "epoch": 0.5451151213441194, + "grad_norm": 3.0717954635620117, + "learning_rate": 4.580258302583026e-06, + "loss": 1.6287, + "step": 438 + }, + { + "epoch": 0.5463596764156814, + "grad_norm": 2.800243854522705, + "learning_rate": 4.5779520295202955e-06, + "loss": 1.6169, + "step": 439 + }, + { + "epoch": 0.5476042314872434, + "grad_norm": 2.9581644535064697, + "learning_rate": 4.575645756457565e-06, + "loss": 1.6617, + "step": 440 + }, + { + "epoch": 0.5476042314872434, + "eval_loss": 1.7642868757247925, + "eval_runtime": 51.2646, + "eval_samples_per_second": 19.507, + "eval_steps_per_second": 0.819, + "step": 440 + }, + { + "epoch": 0.5488487865588052, + "grad_norm": 3.0067460536956787, + "learning_rate": 4.573339483394834e-06, + "loss": 1.6482, + "step": 441 + }, + { + "epoch": 0.5500933416303672, + "grad_norm": 2.9075019359588623, + "learning_rate": 4.571033210332104e-06, + "loss": 1.6595, + "step": 442 + }, + { + "epoch": 0.551337896701929, + "grad_norm": 2.7788755893707275, + "learning_rate": 4.568726937269373e-06, + "loss": 1.5733, + "step": 443 + }, + { + "epoch": 0.552582451773491, + "grad_norm": 2.7775425910949707, + "learning_rate": 4.566420664206643e-06, + "loss": 1.6925, + "step": 444 + }, + { + "epoch": 0.5538270068450529, + "grad_norm": 2.6949567794799805, + "learning_rate": 4.564114391143912e-06, + "loss": 1.5738, + "step": 445 + }, + { + "epoch": 0.5550715619166148, + "grad_norm": 2.7093007564544678, + "learning_rate": 4.5618081180811814e-06, + "loss": 1.5848, + "step": 446 + }, + { + "epoch": 0.5563161169881767, + "grad_norm": 2.6920173168182373, + "learning_rate": 4.559501845018451e-06, + "loss": 1.6092, + "step": 447 + }, + { + "epoch": 0.5575606720597387, + "grad_norm": 2.733731985092163, + "learning_rate": 4.5571955719557194e-06, + "loss": 1.6056, + "step": 448 + }, + { + "epoch": 0.5588052271313005, + "grad_norm": 2.9156274795532227, + "learning_rate": 4.55488929889299e-06, + "loss": 1.6436, + "step": 449 + }, + { + "epoch": 0.5600497822028625, + "grad_norm": 2.8802616596221924, + "learning_rate": 4.552583025830259e-06, + "loss": 1.6309, + "step": 450 + }, + { + "epoch": 0.5600497822028625, + "eval_loss": 1.7486340999603271, + "eval_runtime": 47.2219, + "eval_samples_per_second": 21.177, + "eval_steps_per_second": 0.889, + "step": 450 + }, + { + "epoch": 0.5612943372744243, + "grad_norm": 2.729743242263794, + "learning_rate": 4.550276752767528e-06, + "loss": 1.7171, + "step": 451 + }, + { + "epoch": 0.5625388923459863, + "grad_norm": 2.800049066543579, + "learning_rate": 4.547970479704797e-06, + "loss": 1.6098, + "step": 452 + }, + { + "epoch": 0.5637834474175483, + "grad_norm": 2.9659311771392822, + "learning_rate": 4.5456642066420666e-06, + "loss": 1.6646, + "step": 453 + }, + { + "epoch": 0.5650280024891101, + "grad_norm": 2.5417754650115967, + "learning_rate": 4.543357933579336e-06, + "loss": 1.5847, + "step": 454 + }, + { + "epoch": 0.5662725575606721, + "grad_norm": 2.8471338748931885, + "learning_rate": 4.541051660516605e-06, + "loss": 1.6235, + "step": 455 + }, + { + "epoch": 0.5675171126322339, + "grad_norm": 2.7039637565612793, + "learning_rate": 4.538745387453875e-06, + "loss": 1.5613, + "step": 456 + }, + { + "epoch": 0.5687616677037959, + "grad_norm": 3.0383341312408447, + "learning_rate": 4.536439114391144e-06, + "loss": 1.6485, + "step": 457 + }, + { + "epoch": 0.5700062227753578, + "grad_norm": 2.528388261795044, + "learning_rate": 4.534132841328414e-06, + "loss": 1.6117, + "step": 458 + }, + { + "epoch": 0.5712507778469197, + "grad_norm": 2.7623119354248047, + "learning_rate": 4.531826568265683e-06, + "loss": 1.6017, + "step": 459 + }, + { + "epoch": 0.5724953329184816, + "grad_norm": 2.9213945865631104, + "learning_rate": 4.5295202952029525e-06, + "loss": 1.643, + "step": 460 + }, + { + "epoch": 0.5724953329184816, + "eval_loss": 1.747575044631958, + "eval_runtime": 45.2844, + "eval_samples_per_second": 22.083, + "eval_steps_per_second": 0.927, + "step": 460 + }, + { + "epoch": 0.5737398879900436, + "grad_norm": 2.664275884628296, + "learning_rate": 4.527214022140222e-06, + "loss": 1.6154, + "step": 461 + }, + { + "epoch": 0.5749844430616055, + "grad_norm": 2.5891788005828857, + "learning_rate": 4.524907749077491e-06, + "loss": 1.5975, + "step": 462 + }, + { + "epoch": 0.5762289981331674, + "grad_norm": 2.8126487731933594, + "learning_rate": 4.522601476014761e-06, + "loss": 1.6306, + "step": 463 + }, + { + "epoch": 0.5774735532047293, + "grad_norm": 2.799391031265259, + "learning_rate": 4.520295202952029e-06, + "loss": 1.6139, + "step": 464 + }, + { + "epoch": 0.5787181082762912, + "grad_norm": 2.7323718070983887, + "learning_rate": 4.5179889298893e-06, + "loss": 1.632, + "step": 465 + }, + { + "epoch": 0.5799626633478532, + "grad_norm": 2.601386785507202, + "learning_rate": 4.515682656826569e-06, + "loss": 1.6139, + "step": 466 + }, + { + "epoch": 0.581207218419415, + "grad_norm": 2.562162399291992, + "learning_rate": 4.513376383763838e-06, + "loss": 1.5168, + "step": 467 + }, + { + "epoch": 0.582451773490977, + "grad_norm": 2.6912002563476562, + "learning_rate": 4.511070110701107e-06, + "loss": 1.6249, + "step": 468 + }, + { + "epoch": 0.583696328562539, + "grad_norm": 2.6971495151519775, + "learning_rate": 4.5087638376383765e-06, + "loss": 1.616, + "step": 469 + }, + { + "epoch": 0.5849408836341008, + "grad_norm": 2.6898910999298096, + "learning_rate": 4.506457564575646e-06, + "loss": 1.614, + "step": 470 + }, + { + "epoch": 0.5849408836341008, + "eval_loss": 1.7460769414901733, + "eval_runtime": 45.3055, + "eval_samples_per_second": 22.072, + "eval_steps_per_second": 0.927, + "step": 470 + }, + { + "epoch": 0.5861854387056628, + "grad_norm": 2.698180675506592, + "learning_rate": 4.504151291512915e-06, + "loss": 1.5993, + "step": 471 + }, + { + "epoch": 0.5874299937772246, + "grad_norm": 2.7125210762023926, + "learning_rate": 4.501845018450185e-06, + "loss": 1.6113, + "step": 472 + }, + { + "epoch": 0.5886745488487866, + "grad_norm": 2.754445791244507, + "learning_rate": 4.499538745387454e-06, + "loss": 1.6719, + "step": 473 + }, + { + "epoch": 0.5899191039203485, + "grad_norm": 2.6979362964630127, + "learning_rate": 4.497232472324724e-06, + "loss": 1.608, + "step": 474 + }, + { + "epoch": 0.5911636589919104, + "grad_norm": 2.7694528102874756, + "learning_rate": 4.494926199261993e-06, + "loss": 1.5695, + "step": 475 + }, + { + "epoch": 0.5924082140634723, + "grad_norm": 2.653353214263916, + "learning_rate": 4.4926199261992624e-06, + "loss": 1.5523, + "step": 476 + }, + { + "epoch": 0.5936527691350342, + "grad_norm": 2.6632070541381836, + "learning_rate": 4.490313653136532e-06, + "loss": 1.5675, + "step": 477 + }, + { + "epoch": 0.5948973242065961, + "grad_norm": 3.038543939590454, + "learning_rate": 4.488007380073801e-06, + "loss": 1.5692, + "step": 478 + }, + { + "epoch": 0.5961418792781581, + "grad_norm": 2.8123953342437744, + "learning_rate": 4.485701107011071e-06, + "loss": 1.6223, + "step": 479 + }, + { + "epoch": 0.5973864343497199, + "grad_norm": 2.5845773220062256, + "learning_rate": 4.483394833948339e-06, + "loss": 1.5773, + "step": 480 + }, + { + "epoch": 0.5973864343497199, + "eval_loss": 1.7419319152832031, + "eval_runtime": 44.6842, + "eval_samples_per_second": 22.379, + "eval_steps_per_second": 0.94, + "step": 480 + }, + { + "epoch": 0.5986309894212819, + "grad_norm": 3.069936752319336, + "learning_rate": 4.4810885608856096e-06, + "loss": 1.6323, + "step": 481 + }, + { + "epoch": 0.5998755444928439, + "grad_norm": 2.9085781574249268, + "learning_rate": 4.478782287822879e-06, + "loss": 1.5845, + "step": 482 + }, + { + "epoch": 0.6011200995644057, + "grad_norm": 2.8568010330200195, + "learning_rate": 4.4764760147601476e-06, + "loss": 1.5898, + "step": 483 + }, + { + "epoch": 0.6023646546359677, + "grad_norm": 3.089081048965454, + "learning_rate": 4.474169741697417e-06, + "loss": 1.6622, + "step": 484 + }, + { + "epoch": 0.6036092097075295, + "grad_norm": 2.8515470027923584, + "learning_rate": 4.471863468634686e-06, + "loss": 1.5859, + "step": 485 + }, + { + "epoch": 0.6048537647790915, + "grad_norm": 2.647765874862671, + "learning_rate": 4.469557195571957e-06, + "loss": 1.5926, + "step": 486 + }, + { + "epoch": 0.6060983198506534, + "grad_norm": 2.61676025390625, + "learning_rate": 4.467250922509225e-06, + "loss": 1.5684, + "step": 487 + }, + { + "epoch": 0.6073428749222153, + "grad_norm": 2.7127039432525635, + "learning_rate": 4.464944649446495e-06, + "loss": 1.5702, + "step": 488 + }, + { + "epoch": 0.6085874299937772, + "grad_norm": 2.9932055473327637, + "learning_rate": 4.462638376383764e-06, + "loss": 1.5976, + "step": 489 + }, + { + "epoch": 0.6098319850653391, + "grad_norm": 2.6508774757385254, + "learning_rate": 4.4603321033210335e-06, + "loss": 1.586, + "step": 490 + }, + { + "epoch": 0.6098319850653391, + "eval_loss": 1.7357326745986938, + "eval_runtime": 45.393, + "eval_samples_per_second": 22.03, + "eval_steps_per_second": 0.925, + "step": 490 + }, + { + "epoch": 0.611076540136901, + "grad_norm": 2.7626044750213623, + "learning_rate": 4.458025830258303e-06, + "loss": 1.5773, + "step": 491 + }, + { + "epoch": 0.612321095208463, + "grad_norm": 2.595003604888916, + "learning_rate": 4.455719557195572e-06, + "loss": 1.601, + "step": 492 + }, + { + "epoch": 0.6135656502800249, + "grad_norm": 2.595767021179199, + "learning_rate": 4.453413284132842e-06, + "loss": 1.6287, + "step": 493 + }, + { + "epoch": 0.6148102053515868, + "grad_norm": 2.755845308303833, + "learning_rate": 4.451107011070111e-06, + "loss": 1.593, + "step": 494 + }, + { + "epoch": 0.6160547604231488, + "grad_norm": 2.7302653789520264, + "learning_rate": 4.448800738007381e-06, + "loss": 1.5768, + "step": 495 + }, + { + "epoch": 0.6172993154947106, + "grad_norm": 2.6493024826049805, + "learning_rate": 4.446494464944649e-06, + "loss": 1.6059, + "step": 496 + }, + { + "epoch": 0.6185438705662726, + "grad_norm": 2.8633735179901123, + "learning_rate": 4.4441881918819195e-06, + "loss": 1.598, + "step": 497 + }, + { + "epoch": 0.6197884256378344, + "grad_norm": 2.65639066696167, + "learning_rate": 4.441881918819189e-06, + "loss": 1.6036, + "step": 498 + }, + { + "epoch": 0.6210329807093964, + "grad_norm": 2.6867759227752686, + "learning_rate": 4.439575645756458e-06, + "loss": 1.5676, + "step": 499 + }, + { + "epoch": 0.6222775357809583, + "grad_norm": 2.7986443042755127, + "learning_rate": 4.437269372693727e-06, + "loss": 1.524, + "step": 500 + }, + { + "epoch": 0.6222775357809583, + "eval_loss": 1.7449125051498413, + "eval_runtime": 43.9329, + "eval_samples_per_second": 22.762, + "eval_steps_per_second": 0.956, + "step": 500 + }, + { + "epoch": 0.6235220908525202, + "grad_norm": 2.665905714035034, + "learning_rate": 4.434963099630996e-06, + "loss": 1.5635, + "step": 501 + }, + { + "epoch": 0.6247666459240822, + "grad_norm": 2.902435541152954, + "learning_rate": 4.432656826568267e-06, + "loss": 1.6377, + "step": 502 + }, + { + "epoch": 0.6260112009956441, + "grad_norm": 2.6872262954711914, + "learning_rate": 4.430350553505535e-06, + "loss": 1.5553, + "step": 503 + }, + { + "epoch": 0.627255756067206, + "grad_norm": 2.676621913909912, + "learning_rate": 4.428044280442805e-06, + "loss": 1.6101, + "step": 504 + }, + { + "epoch": 0.6285003111387679, + "grad_norm": 2.756347179412842, + "learning_rate": 4.425738007380074e-06, + "loss": 1.5884, + "step": 505 + }, + { + "epoch": 0.6297448662103298, + "grad_norm": 2.856882333755493, + "learning_rate": 4.4234317343173434e-06, + "loss": 1.5635, + "step": 506 + }, + { + "epoch": 0.6309894212818917, + "grad_norm": 2.7708330154418945, + "learning_rate": 4.421125461254613e-06, + "loss": 1.6028, + "step": 507 + }, + { + "epoch": 0.6322339763534537, + "grad_norm": 2.8167600631713867, + "learning_rate": 4.418819188191882e-06, + "loss": 1.6026, + "step": 508 + }, + { + "epoch": 0.6334785314250155, + "grad_norm": 2.54194974899292, + "learning_rate": 4.416512915129152e-06, + "loss": 1.6001, + "step": 509 + }, + { + "epoch": 0.6347230864965775, + "grad_norm": 2.683037519454956, + "learning_rate": 4.414206642066421e-06, + "loss": 1.6084, + "step": 510 + }, + { + "epoch": 0.6347230864965775, + "eval_loss": 1.7376186847686768, + "eval_runtime": 42.3026, + "eval_samples_per_second": 23.639, + "eval_steps_per_second": 0.993, + "step": 510 + }, + { + "epoch": 0.6359676415681393, + "grad_norm": 2.82627010345459, + "learning_rate": 4.4119003690036905e-06, + "loss": 1.6258, + "step": 511 + }, + { + "epoch": 0.6372121966397013, + "grad_norm": 2.6139848232269287, + "learning_rate": 4.40959409594096e-06, + "loss": 1.6029, + "step": 512 + }, + { + "epoch": 0.6384567517112633, + "grad_norm": 2.646712303161621, + "learning_rate": 4.407287822878229e-06, + "loss": 1.5602, + "step": 513 + }, + { + "epoch": 0.6397013067828251, + "grad_norm": 2.910935163497925, + "learning_rate": 4.404981549815499e-06, + "loss": 1.5757, + "step": 514 + }, + { + "epoch": 0.6409458618543871, + "grad_norm": 2.6001148223876953, + "learning_rate": 4.402675276752768e-06, + "loss": 1.5857, + "step": 515 + }, + { + "epoch": 0.642190416925949, + "grad_norm": 2.8525235652923584, + "learning_rate": 4.400369003690037e-06, + "loss": 1.6092, + "step": 516 + }, + { + "epoch": 0.6434349719975109, + "grad_norm": 2.5335488319396973, + "learning_rate": 4.398062730627306e-06, + "loss": 1.5849, + "step": 517 + }, + { + "epoch": 0.6446795270690728, + "grad_norm": 2.5787103176116943, + "learning_rate": 4.3957564575645765e-06, + "loss": 1.634, + "step": 518 + }, + { + "epoch": 0.6459240821406347, + "grad_norm": 2.6188197135925293, + "learning_rate": 4.393450184501845e-06, + "loss": 1.5477, + "step": 519 + }, + { + "epoch": 0.6471686372121966, + "grad_norm": 2.6548666954040527, + "learning_rate": 4.3911439114391145e-06, + "loss": 1.6133, + "step": 520 + }, + { + "epoch": 0.6471686372121966, + "eval_loss": 1.7393039464950562, + "eval_runtime": 45.9234, + "eval_samples_per_second": 21.775, + "eval_steps_per_second": 0.915, + "step": 520 + }, + { + "epoch": 0.6484131922837586, + "grad_norm": 2.907928228378296, + "learning_rate": 4.388837638376384e-06, + "loss": 1.587, + "step": 521 + }, + { + "epoch": 0.6496577473553204, + "grad_norm": 2.7899692058563232, + "learning_rate": 4.386531365313653e-06, + "loss": 1.6126, + "step": 522 + }, + { + "epoch": 0.6509023024268824, + "grad_norm": 2.680147886276245, + "learning_rate": 4.384225092250923e-06, + "loss": 1.565, + "step": 523 + }, + { + "epoch": 0.6521468574984443, + "grad_norm": 2.6590754985809326, + "learning_rate": 4.381918819188192e-06, + "loss": 1.6127, + "step": 524 + }, + { + "epoch": 0.6533914125700062, + "grad_norm": 2.6638906002044678, + "learning_rate": 4.379612546125462e-06, + "loss": 1.5615, + "step": 525 + }, + { + "epoch": 0.6546359676415682, + "grad_norm": 2.6518194675445557, + "learning_rate": 4.377306273062731e-06, + "loss": 1.6202, + "step": 526 + }, + { + "epoch": 0.65588052271313, + "grad_norm": 2.5323619842529297, + "learning_rate": 4.3750000000000005e-06, + "loss": 1.5305, + "step": 527 + }, + { + "epoch": 0.657125077784692, + "grad_norm": 2.6871442794799805, + "learning_rate": 4.37269372693727e-06, + "loss": 1.6043, + "step": 528 + }, + { + "epoch": 0.6583696328562539, + "grad_norm": 2.760746717453003, + "learning_rate": 4.370387453874539e-06, + "loss": 1.6059, + "step": 529 + }, + { + "epoch": 0.6596141879278158, + "grad_norm": 2.780482769012451, + "learning_rate": 4.368081180811809e-06, + "loss": 1.6082, + "step": 530 + }, + { + "epoch": 0.6596141879278158, + "eval_loss": 1.7219713926315308, + "eval_runtime": 51.8478, + "eval_samples_per_second": 19.287, + "eval_steps_per_second": 0.81, + "step": 530 + }, + { + "epoch": 0.6608587429993777, + "grad_norm": 2.605890989303589, + "learning_rate": 4.365774907749078e-06, + "loss": 1.5554, + "step": 531 + }, + { + "epoch": 0.6621032980709396, + "grad_norm": 2.731555461883545, + "learning_rate": 4.363468634686347e-06, + "loss": 1.606, + "step": 532 + }, + { + "epoch": 0.6633478531425016, + "grad_norm": 2.8356943130493164, + "learning_rate": 4.361162361623616e-06, + "loss": 1.5601, + "step": 533 + }, + { + "epoch": 0.6645924082140635, + "grad_norm": 2.7196593284606934, + "learning_rate": 4.3588560885608864e-06, + "loss": 1.5722, + "step": 534 + }, + { + "epoch": 0.6658369632856254, + "grad_norm": 2.621371269226074, + "learning_rate": 4.356549815498156e-06, + "loss": 1.6194, + "step": 535 + }, + { + "epoch": 0.6670815183571873, + "grad_norm": 2.653916120529175, + "learning_rate": 4.354243542435424e-06, + "loss": 1.5592, + "step": 536 + }, + { + "epoch": 0.6683260734287492, + "grad_norm": 2.88431453704834, + "learning_rate": 4.351937269372694e-06, + "loss": 1.6296, + "step": 537 + }, + { + "epoch": 0.6695706285003111, + "grad_norm": 2.667130470275879, + "learning_rate": 4.349630996309963e-06, + "loss": 1.5624, + "step": 538 + }, + { + "epoch": 0.6708151835718731, + "grad_norm": 2.6453566551208496, + "learning_rate": 4.347324723247233e-06, + "loss": 1.6025, + "step": 539 + }, + { + "epoch": 0.6720597386434349, + "grad_norm": 3.032271146774292, + "learning_rate": 4.345018450184502e-06, + "loss": 1.6771, + "step": 540 + }, + { + "epoch": 0.6720597386434349, + "eval_loss": 1.721895456314087, + "eval_runtime": 50.62, + "eval_samples_per_second": 19.755, + "eval_steps_per_second": 0.83, + "step": 540 + }, + { + "epoch": 0.6733042937149969, + "grad_norm": 2.8476362228393555, + "learning_rate": 4.3427121771217715e-06, + "loss": 1.5663, + "step": 541 + }, + { + "epoch": 0.6745488487865589, + "grad_norm": 2.615602970123291, + "learning_rate": 4.340405904059041e-06, + "loss": 1.5815, + "step": 542 + }, + { + "epoch": 0.6757934038581207, + "grad_norm": 3.221571683883667, + "learning_rate": 4.33809963099631e-06, + "loss": 1.5955, + "step": 543 + }, + { + "epoch": 0.6770379589296827, + "grad_norm": 2.602994203567505, + "learning_rate": 4.33579335793358e-06, + "loss": 1.5667, + "step": 544 + }, + { + "epoch": 0.6782825140012445, + "grad_norm": 2.9380433559417725, + "learning_rate": 4.333487084870848e-06, + "loss": 1.5651, + "step": 545 + }, + { + "epoch": 0.6795270690728065, + "grad_norm": 3.0689425468444824, + "learning_rate": 4.331180811808119e-06, + "loss": 1.6003, + "step": 546 + }, + { + "epoch": 0.6807716241443684, + "grad_norm": 2.770317316055298, + "learning_rate": 4.328874538745388e-06, + "loss": 1.5576, + "step": 547 + }, + { + "epoch": 0.6820161792159303, + "grad_norm": 2.8631815910339355, + "learning_rate": 4.3265682656826575e-06, + "loss": 1.5308, + "step": 548 + }, + { + "epoch": 0.6832607342874922, + "grad_norm": 2.785576820373535, + "learning_rate": 4.324261992619926e-06, + "loss": 1.6113, + "step": 549 + }, + { + "epoch": 0.6845052893590542, + "grad_norm": 2.723919630050659, + "learning_rate": 4.321955719557196e-06, + "loss": 1.5606, + "step": 550 + }, + { + "epoch": 0.6845052893590542, + "eval_loss": 1.721803069114685, + "eval_runtime": 51.6036, + "eval_samples_per_second": 19.378, + "eval_steps_per_second": 0.814, + "step": 550 + }, + { + "epoch": 0.685749844430616, + "grad_norm": 3.081198215484619, + "learning_rate": 4.319649446494466e-06, + "loss": 1.5643, + "step": 551 + }, + { + "epoch": 0.686994399502178, + "grad_norm": 2.757392168045044, + "learning_rate": 4.317343173431734e-06, + "loss": 1.5909, + "step": 552 + }, + { + "epoch": 0.6882389545737398, + "grad_norm": 3.038753032684326, + "learning_rate": 4.315036900369004e-06, + "loss": 1.6023, + "step": 553 + }, + { + "epoch": 0.6894835096453018, + "grad_norm": 2.8246452808380127, + "learning_rate": 4.312730627306273e-06, + "loss": 1.5548, + "step": 554 + }, + { + "epoch": 0.6907280647168638, + "grad_norm": 2.589320182800293, + "learning_rate": 4.310424354243543e-06, + "loss": 1.5424, + "step": 555 + }, + { + "epoch": 0.6919726197884256, + "grad_norm": 2.6413373947143555, + "learning_rate": 4.308118081180812e-06, + "loss": 1.6243, + "step": 556 + }, + { + "epoch": 0.6932171748599876, + "grad_norm": 2.764784336090088, + "learning_rate": 4.3058118081180815e-06, + "loss": 1.589, + "step": 557 + }, + { + "epoch": 0.6944617299315494, + "grad_norm": 2.806821823120117, + "learning_rate": 4.303505535055351e-06, + "loss": 1.621, + "step": 558 + }, + { + "epoch": 0.6957062850031114, + "grad_norm": 2.718017578125, + "learning_rate": 4.30119926199262e-06, + "loss": 1.5773, + "step": 559 + }, + { + "epoch": 0.6969508400746733, + "grad_norm": 2.7620160579681396, + "learning_rate": 4.29889298892989e-06, + "loss": 1.5868, + "step": 560 + }, + { + "epoch": 0.6969508400746733, + "eval_loss": 1.7152249813079834, + "eval_runtime": 52.0463, + "eval_samples_per_second": 19.214, + "eval_steps_per_second": 0.807, + "step": 560 + }, + { + "epoch": 0.6981953951462352, + "grad_norm": 2.66686749458313, + "learning_rate": 4.296586715867159e-06, + "loss": 1.534, + "step": 561 + }, + { + "epoch": 0.6994399502177971, + "grad_norm": 2.812577247619629, + "learning_rate": 4.2942804428044286e-06, + "loss": 1.5501, + "step": 562 + }, + { + "epoch": 0.7006845052893591, + "grad_norm": 2.578508138656616, + "learning_rate": 4.291974169741698e-06, + "loss": 1.6286, + "step": 563 + }, + { + "epoch": 0.701929060360921, + "grad_norm": 2.524249315261841, + "learning_rate": 4.289667896678967e-06, + "loss": 1.5719, + "step": 564 + }, + { + "epoch": 0.7031736154324829, + "grad_norm": 2.827235460281372, + "learning_rate": 4.287361623616236e-06, + "loss": 1.5333, + "step": 565 + }, + { + "epoch": 0.7044181705040448, + "grad_norm": 2.6359963417053223, + "learning_rate": 4.285055350553506e-06, + "loss": 1.5849, + "step": 566 + }, + { + "epoch": 0.7056627255756067, + "grad_norm": 2.930530071258545, + "learning_rate": 4.282749077490776e-06, + "loss": 1.5672, + "step": 567 + }, + { + "epoch": 0.7069072806471687, + "grad_norm": 2.750102996826172, + "learning_rate": 4.280442804428044e-06, + "loss": 1.5696, + "step": 568 + }, + { + "epoch": 0.7081518357187305, + "grad_norm": 2.869690418243408, + "learning_rate": 4.278136531365314e-06, + "loss": 1.5689, + "step": 569 + }, + { + "epoch": 0.7093963907902925, + "grad_norm": 2.954852819442749, + "learning_rate": 4.275830258302583e-06, + "loss": 1.5931, + "step": 570 + }, + { + "epoch": 0.7093963907902925, + "eval_loss": 1.7143300771713257, + "eval_runtime": 52.7293, + "eval_samples_per_second": 18.965, + "eval_steps_per_second": 0.797, + "step": 570 + }, + { + "epoch": 0.7106409458618543, + "grad_norm": 2.702223539352417, + "learning_rate": 4.273523985239853e-06, + "loss": 1.5635, + "step": 571 + }, + { + "epoch": 0.7118855009334163, + "grad_norm": 2.689995050430298, + "learning_rate": 4.271217712177122e-06, + "loss": 1.5545, + "step": 572 + }, + { + "epoch": 0.7131300560049783, + "grad_norm": 2.68979811668396, + "learning_rate": 4.268911439114391e-06, + "loss": 1.5404, + "step": 573 + }, + { + "epoch": 0.7143746110765401, + "grad_norm": 2.7477986812591553, + "learning_rate": 4.266605166051661e-06, + "loss": 1.5719, + "step": 574 + }, + { + "epoch": 0.7156191661481021, + "grad_norm": 2.975778341293335, + "learning_rate": 4.26429889298893e-06, + "loss": 1.5587, + "step": 575 + }, + { + "epoch": 0.716863721219664, + "grad_norm": 2.658170700073242, + "learning_rate": 4.2619926199262e-06, + "loss": 1.5746, + "step": 576 + }, + { + "epoch": 0.7181082762912259, + "grad_norm": 2.8825011253356934, + "learning_rate": 4.259686346863469e-06, + "loss": 1.5246, + "step": 577 + }, + { + "epoch": 0.7193528313627878, + "grad_norm": 2.845280170440674, + "learning_rate": 4.2573800738007385e-06, + "loss": 1.5703, + "step": 578 + }, + { + "epoch": 0.7205973864343497, + "grad_norm": 2.660616636276245, + "learning_rate": 4.255073800738008e-06, + "loss": 1.5373, + "step": 579 + }, + { + "epoch": 0.7218419415059116, + "grad_norm": 2.749447822570801, + "learning_rate": 4.252767527675277e-06, + "loss": 1.59, + "step": 580 + }, + { + "epoch": 0.7218419415059116, + "eval_loss": 1.7149444818496704, + "eval_runtime": 52.2099, + "eval_samples_per_second": 19.153, + "eval_steps_per_second": 0.804, + "step": 580 + }, + { + "epoch": 0.7230864965774736, + "grad_norm": 2.813328742980957, + "learning_rate": 4.250461254612546e-06, + "loss": 1.5694, + "step": 581 + }, + { + "epoch": 0.7243310516490354, + "grad_norm": 2.823866844177246, + "learning_rate": 4.248154981549816e-06, + "loss": 1.5143, + "step": 582 + }, + { + "epoch": 0.7255756067205974, + "grad_norm": 2.71337890625, + "learning_rate": 4.245848708487086e-06, + "loss": 1.5915, + "step": 583 + }, + { + "epoch": 0.7268201617921594, + "grad_norm": 2.718085765838623, + "learning_rate": 4.243542435424355e-06, + "loss": 1.604, + "step": 584 + }, + { + "epoch": 0.7280647168637212, + "grad_norm": 2.5741796493530273, + "learning_rate": 4.241236162361624e-06, + "loss": 1.5398, + "step": 585 + }, + { + "epoch": 0.7293092719352832, + "grad_norm": 2.727114200592041, + "learning_rate": 4.238929889298893e-06, + "loss": 1.5886, + "step": 586 + }, + { + "epoch": 0.730553827006845, + "grad_norm": 2.6703338623046875, + "learning_rate": 4.236623616236163e-06, + "loss": 1.5959, + "step": 587 + }, + { + "epoch": 0.731798382078407, + "grad_norm": 2.750326633453369, + "learning_rate": 4.234317343173432e-06, + "loss": 1.5916, + "step": 588 + }, + { + "epoch": 0.7330429371499689, + "grad_norm": 2.6680715084075928, + "learning_rate": 4.232011070110701e-06, + "loss": 1.562, + "step": 589 + }, + { + "epoch": 0.7342874922215308, + "grad_norm": 2.6603832244873047, + "learning_rate": 4.229704797047971e-06, + "loss": 1.6121, + "step": 590 + }, + { + "epoch": 0.7342874922215308, + "eval_loss": 1.7144734859466553, + "eval_runtime": 50.5317, + "eval_samples_per_second": 19.79, + "eval_steps_per_second": 0.831, + "step": 590 + }, + { + "epoch": 0.7355320472930927, + "grad_norm": 2.7868523597717285, + "learning_rate": 4.22739852398524e-06, + "loss": 1.5488, + "step": 591 + }, + { + "epoch": 0.7367766023646546, + "grad_norm": 2.755270004272461, + "learning_rate": 4.2250922509225096e-06, + "loss": 1.5801, + "step": 592 + }, + { + "epoch": 0.7380211574362165, + "grad_norm": 2.7614126205444336, + "learning_rate": 4.222785977859779e-06, + "loss": 1.5507, + "step": 593 + }, + { + "epoch": 0.7392657125077785, + "grad_norm": 2.8674862384796143, + "learning_rate": 4.220479704797048e-06, + "loss": 1.5807, + "step": 594 + }, + { + "epoch": 0.7405102675793404, + "grad_norm": 2.7366859912872314, + "learning_rate": 4.218173431734318e-06, + "loss": 1.6594, + "step": 595 + }, + { + "epoch": 0.7417548226509023, + "grad_norm": 2.6231045722961426, + "learning_rate": 4.215867158671587e-06, + "loss": 1.5609, + "step": 596 + }, + { + "epoch": 0.7429993777224643, + "grad_norm": 2.7694146633148193, + "learning_rate": 4.213560885608857e-06, + "loss": 1.5889, + "step": 597 + }, + { + "epoch": 0.7442439327940261, + "grad_norm": 2.6275386810302734, + "learning_rate": 4.211254612546125e-06, + "loss": 1.575, + "step": 598 + }, + { + "epoch": 0.7454884878655881, + "grad_norm": 2.9413866996765137, + "learning_rate": 4.2089483394833955e-06, + "loss": 1.589, + "step": 599 + }, + { + "epoch": 0.7467330429371499, + "grad_norm": 2.768606185913086, + "learning_rate": 4.206642066420665e-06, + "loss": 1.5738, + "step": 600 + }, + { + "epoch": 0.7467330429371499, + "eval_loss": 1.7148027420043945, + "eval_runtime": 54.7031, + "eval_samples_per_second": 18.28, + "eval_steps_per_second": 0.768, + "step": 600 + }, + { + "epoch": 0.7479775980087119, + "grad_norm": 2.7917706966400146, + "learning_rate": 4.2043357933579335e-06, + "loss": 1.499, + "step": 601 + }, + { + "epoch": 0.7492221530802738, + "grad_norm": 2.5982208251953125, + "learning_rate": 4.202029520295203e-06, + "loss": 1.5113, + "step": 602 + }, + { + "epoch": 0.7504667081518357, + "grad_norm": 2.692032814025879, + "learning_rate": 4.199723247232473e-06, + "loss": 1.4721, + "step": 603 + }, + { + "epoch": 0.7517112632233977, + "grad_norm": 2.975860834121704, + "learning_rate": 4.197416974169742e-06, + "loss": 1.5147, + "step": 604 + }, + { + "epoch": 0.7529558182949595, + "grad_norm": 2.758610248565674, + "learning_rate": 4.195110701107011e-06, + "loss": 1.5674, + "step": 605 + }, + { + "epoch": 0.7542003733665215, + "grad_norm": 2.7620184421539307, + "learning_rate": 4.192804428044281e-06, + "loss": 1.5669, + "step": 606 + }, + { + "epoch": 0.7554449284380834, + "grad_norm": 3.043940305709839, + "learning_rate": 4.19049815498155e-06, + "loss": 1.587, + "step": 607 + }, + { + "epoch": 0.7566894835096453, + "grad_norm": 2.7482151985168457, + "learning_rate": 4.1881918819188195e-06, + "loss": 1.5451, + "step": 608 + }, + { + "epoch": 0.7579340385812072, + "grad_norm": 2.7259294986724854, + "learning_rate": 4.185885608856089e-06, + "loss": 1.5295, + "step": 609 + }, + { + "epoch": 0.7591785936527692, + "grad_norm": 2.6340832710266113, + "learning_rate": 4.183579335793358e-06, + "loss": 1.5335, + "step": 610 + }, + { + "epoch": 0.7591785936527692, + "eval_loss": 1.7038393020629883, + "eval_runtime": 44.0792, + "eval_samples_per_second": 22.686, + "eval_steps_per_second": 0.953, + "step": 610 + }, + { + "epoch": 0.760423148724331, + "grad_norm": 2.6144909858703613, + "learning_rate": 4.181273062730628e-06, + "loss": 1.5907, + "step": 611 + }, + { + "epoch": 0.761667703795893, + "grad_norm": 2.7754175662994385, + "learning_rate": 4.178966789667897e-06, + "loss": 1.5773, + "step": 612 + }, + { + "epoch": 0.7629122588674548, + "grad_norm": 2.6313252449035645, + "learning_rate": 4.176660516605167e-06, + "loss": 1.5675, + "step": 613 + }, + { + "epoch": 0.7641568139390168, + "grad_norm": 2.549074649810791, + "learning_rate": 4.174354243542435e-06, + "loss": 1.595, + "step": 614 + }, + { + "epoch": 0.7654013690105788, + "grad_norm": 2.611804246902466, + "learning_rate": 4.1720479704797054e-06, + "loss": 1.5416, + "step": 615 + }, + { + "epoch": 0.7666459240821406, + "grad_norm": 2.6322927474975586, + "learning_rate": 4.169741697416975e-06, + "loss": 1.604, + "step": 616 + }, + { + "epoch": 0.7678904791537026, + "grad_norm": 2.5792219638824463, + "learning_rate": 4.1674354243542434e-06, + "loss": 1.5976, + "step": 617 + }, + { + "epoch": 0.7691350342252644, + "grad_norm": 2.845416307449341, + "learning_rate": 4.165129151291513e-06, + "loss": 1.5945, + "step": 618 + }, + { + "epoch": 0.7703795892968264, + "grad_norm": 2.8647871017456055, + "learning_rate": 4.162822878228783e-06, + "loss": 1.5637, + "step": 619 + }, + { + "epoch": 0.7716241443683883, + "grad_norm": 2.590719699859619, + "learning_rate": 4.1605166051660526e-06, + "loss": 1.556, + "step": 620 + }, + { + "epoch": 0.7716241443683883, + "eval_loss": 1.7115222215652466, + "eval_runtime": 50.3905, + "eval_samples_per_second": 19.845, + "eval_steps_per_second": 0.833, + "step": 620 + }, + { + "epoch": 0.7728686994399502, + "grad_norm": 2.626723527908325, + "learning_rate": 4.158210332103321e-06, + "loss": 1.5486, + "step": 621 + }, + { + "epoch": 0.7741132545115121, + "grad_norm": 2.713069438934326, + "learning_rate": 4.1559040590405906e-06, + "loss": 1.5474, + "step": 622 + }, + { + "epoch": 0.7753578095830741, + "grad_norm": 2.685563087463379, + "learning_rate": 4.15359778597786e-06, + "loss": 1.5529, + "step": 623 + }, + { + "epoch": 0.776602364654636, + "grad_norm": 2.690586805343628, + "learning_rate": 4.151291512915129e-06, + "loss": 1.5293, + "step": 624 + }, + { + "epoch": 0.7778469197261979, + "grad_norm": 2.865345001220703, + "learning_rate": 4.148985239852399e-06, + "loss": 1.5095, + "step": 625 + }, + { + "epoch": 0.7790914747977598, + "grad_norm": 2.92393159866333, + "learning_rate": 4.146678966789668e-06, + "loss": 1.5561, + "step": 626 + }, + { + "epoch": 0.7803360298693217, + "grad_norm": 2.554480791091919, + "learning_rate": 4.144372693726938e-06, + "loss": 1.4876, + "step": 627 + }, + { + "epoch": 0.7815805849408837, + "grad_norm": 2.6128833293914795, + "learning_rate": 4.142066420664207e-06, + "loss": 1.5378, + "step": 628 + }, + { + "epoch": 0.7828251400124455, + "grad_norm": 3.0110888481140137, + "learning_rate": 4.1397601476014765e-06, + "loss": 1.5859, + "step": 629 + }, + { + "epoch": 0.7840696950840075, + "grad_norm": 2.8152706623077393, + "learning_rate": 4.137453874538745e-06, + "loss": 1.5841, + "step": 630 + }, + { + "epoch": 0.7840696950840075, + "eval_loss": 1.702181339263916, + "eval_runtime": 53.6926, + "eval_samples_per_second": 18.625, + "eval_steps_per_second": 0.782, + "step": 630 + }, + { + "epoch": 0.7853142501555694, + "grad_norm": 2.8634233474731445, + "learning_rate": 4.135147601476015e-06, + "loss": 1.564, + "step": 631 + }, + { + "epoch": 0.7865588052271313, + "grad_norm": 2.8939666748046875, + "learning_rate": 4.132841328413285e-06, + "loss": 1.5859, + "step": 632 + }, + { + "epoch": 0.7878033602986932, + "grad_norm": 2.6560444831848145, + "learning_rate": 4.130535055350554e-06, + "loss": 1.5199, + "step": 633 + }, + { + "epoch": 0.7890479153702551, + "grad_norm": 2.964721202850342, + "learning_rate": 4.128228782287823e-06, + "loss": 1.5888, + "step": 634 + }, + { + "epoch": 0.790292470441817, + "grad_norm": 2.74668288230896, + "learning_rate": 4.125922509225092e-06, + "loss": 1.5847, + "step": 635 + }, + { + "epoch": 0.791537025513379, + "grad_norm": 2.723123550415039, + "learning_rate": 4.1236162361623625e-06, + "loss": 1.5166, + "step": 636 + }, + { + "epoch": 0.7927815805849409, + "grad_norm": 2.7265713214874268, + "learning_rate": 4.121309963099631e-06, + "loss": 1.5954, + "step": 637 + }, + { + "epoch": 0.7940261356565028, + "grad_norm": 2.979126214981079, + "learning_rate": 4.1190036900369005e-06, + "loss": 1.5583, + "step": 638 + }, + { + "epoch": 0.7952706907280647, + "grad_norm": 2.844376802444458, + "learning_rate": 4.11669741697417e-06, + "loss": 1.5564, + "step": 639 + }, + { + "epoch": 0.7965152457996266, + "grad_norm": 2.7583703994750977, + "learning_rate": 4.114391143911439e-06, + "loss": 1.5094, + "step": 640 + }, + { + "epoch": 0.7965152457996266, + "eval_loss": 1.6986565589904785, + "eval_runtime": 47.3549, + "eval_samples_per_second": 21.117, + "eval_steps_per_second": 0.887, + "step": 640 + }, + { + "epoch": 0.7977598008711886, + "grad_norm": 2.8942224979400635, + "learning_rate": 4.112084870848709e-06, + "loss": 1.5504, + "step": 641 + }, + { + "epoch": 0.7990043559427504, + "grad_norm": 2.6590495109558105, + "learning_rate": 4.109778597785978e-06, + "loss": 1.5227, + "step": 642 + }, + { + "epoch": 0.8002489110143124, + "grad_norm": 2.5988378524780273, + "learning_rate": 4.107472324723248e-06, + "loss": 1.5054, + "step": 643 + }, + { + "epoch": 0.8014934660858744, + "grad_norm": 2.787335157394409, + "learning_rate": 4.105166051660517e-06, + "loss": 1.5844, + "step": 644 + }, + { + "epoch": 0.8027380211574362, + "grad_norm": 2.8202896118164062, + "learning_rate": 4.1028597785977864e-06, + "loss": 1.5596, + "step": 645 + }, + { + "epoch": 0.8039825762289982, + "grad_norm": 2.65376615524292, + "learning_rate": 4.100553505535056e-06, + "loss": 1.5893, + "step": 646 + }, + { + "epoch": 0.80522713130056, + "grad_norm": 2.5933308601379395, + "learning_rate": 4.098247232472325e-06, + "loss": 1.5403, + "step": 647 + }, + { + "epoch": 0.806471686372122, + "grad_norm": 2.777070999145508, + "learning_rate": 4.095940959409595e-06, + "loss": 1.5231, + "step": 648 + }, + { + "epoch": 0.8077162414436839, + "grad_norm": 2.6427664756774902, + "learning_rate": 4.093634686346864e-06, + "loss": 1.5937, + "step": 649 + }, + { + "epoch": 0.8089607965152458, + "grad_norm": 2.651561737060547, + "learning_rate": 4.091328413284133e-06, + "loss": 1.4847, + "step": 650 + }, + { + "epoch": 0.8089607965152458, + "eval_loss": 1.7089996337890625, + "eval_runtime": 47.4346, + "eval_samples_per_second": 21.082, + "eval_steps_per_second": 0.885, + "step": 650 + }, + { + "epoch": 0.8102053515868077, + "grad_norm": 2.844067335128784, + "learning_rate": 4.089022140221402e-06, + "loss": 1.488, + "step": 651 + }, + { + "epoch": 0.8114499066583696, + "grad_norm": 2.603480577468872, + "learning_rate": 4.086715867158672e-06, + "loss": 1.5248, + "step": 652 + }, + { + "epoch": 0.8126944617299315, + "grad_norm": 2.789322853088379, + "learning_rate": 4.084409594095941e-06, + "loss": 1.5119, + "step": 653 + }, + { + "epoch": 0.8139390168014935, + "grad_norm": 2.7183773517608643, + "learning_rate": 4.08210332103321e-06, + "loss": 1.5123, + "step": 654 + }, + { + "epoch": 0.8151835718730553, + "grad_norm": 2.80613374710083, + "learning_rate": 4.07979704797048e-06, + "loss": 1.5526, + "step": 655 + }, + { + "epoch": 0.8164281269446173, + "grad_norm": 2.641035795211792, + "learning_rate": 4.077490774907749e-06, + "loss": 1.6127, + "step": 656 + }, + { + "epoch": 0.8176726820161793, + "grad_norm": 2.773684024810791, + "learning_rate": 4.075184501845019e-06, + "loss": 1.5404, + "step": 657 + }, + { + "epoch": 0.8189172370877411, + "grad_norm": 2.6951382160186768, + "learning_rate": 4.072878228782288e-06, + "loss": 1.4928, + "step": 658 + }, + { + "epoch": 0.8201617921593031, + "grad_norm": 2.931771755218506, + "learning_rate": 4.0705719557195575e-06, + "loss": 1.5949, + "step": 659 + }, + { + "epoch": 0.8214063472308649, + "grad_norm": 2.8786075115203857, + "learning_rate": 4.068265682656827e-06, + "loss": 1.5582, + "step": 660 + }, + { + "epoch": 0.8214063472308649, + "eval_loss": 1.698564052581787, + "eval_runtime": 46.6299, + "eval_samples_per_second": 21.445, + "eval_steps_per_second": 0.901, + "step": 660 + }, + { + "epoch": 0.8226509023024269, + "grad_norm": 2.7402451038360596, + "learning_rate": 4.065959409594096e-06, + "loss": 1.5562, + "step": 661 + }, + { + "epoch": 0.8238954573739888, + "grad_norm": 2.7968692779541016, + "learning_rate": 4.063653136531366e-06, + "loss": 1.556, + "step": 662 + }, + { + "epoch": 0.8251400124455507, + "grad_norm": 2.8233132362365723, + "learning_rate": 4.061346863468635e-06, + "loss": 1.5527, + "step": 663 + }, + { + "epoch": 0.8263845675171126, + "grad_norm": 2.713390827178955, + "learning_rate": 4.059040590405905e-06, + "loss": 1.6012, + "step": 664 + }, + { + "epoch": 0.8276291225886746, + "grad_norm": 2.7783989906311035, + "learning_rate": 4.056734317343174e-06, + "loss": 1.5247, + "step": 665 + }, + { + "epoch": 0.8288736776602365, + "grad_norm": 2.808554172515869, + "learning_rate": 4.054428044280443e-06, + "loss": 1.5412, + "step": 666 + }, + { + "epoch": 0.8301182327317984, + "grad_norm": 2.8476831912994385, + "learning_rate": 4.052121771217712e-06, + "loss": 1.5307, + "step": 667 + }, + { + "epoch": 0.8313627878033603, + "grad_norm": 2.8287601470947266, + "learning_rate": 4.049815498154982e-06, + "loss": 1.6021, + "step": 668 + }, + { + "epoch": 0.8326073428749222, + "grad_norm": 2.975510835647583, + "learning_rate": 4.047509225092252e-06, + "loss": 1.5474, + "step": 669 + }, + { + "epoch": 0.8338518979464842, + "grad_norm": 2.723236560821533, + "learning_rate": 4.04520295202952e-06, + "loss": 1.5242, + "step": 670 + }, + { + "epoch": 0.8338518979464842, + "eval_loss": 1.7005239725112915, + "eval_runtime": 49.4504, + "eval_samples_per_second": 20.222, + "eval_steps_per_second": 0.849, + "step": 670 + }, + { + "epoch": 0.835096453018046, + "grad_norm": 2.7521748542785645, + "learning_rate": 4.04289667896679e-06, + "loss": 1.5467, + "step": 671 + }, + { + "epoch": 0.836341008089608, + "grad_norm": 2.7770044803619385, + "learning_rate": 4.04059040590406e-06, + "loss": 1.5407, + "step": 672 + }, + { + "epoch": 0.8375855631611698, + "grad_norm": 2.744323492050171, + "learning_rate": 4.038284132841329e-06, + "loss": 1.5422, + "step": 673 + }, + { + "epoch": 0.8388301182327318, + "grad_norm": 2.6699817180633545, + "learning_rate": 4.035977859778598e-06, + "loss": 1.5508, + "step": 674 + }, + { + "epoch": 0.8400746733042938, + "grad_norm": 2.705273151397705, + "learning_rate": 4.0336715867158674e-06, + "loss": 1.5099, + "step": 675 + }, + { + "epoch": 0.8413192283758556, + "grad_norm": 2.9485747814178467, + "learning_rate": 4.031365313653137e-06, + "loss": 1.5529, + "step": 676 + }, + { + "epoch": 0.8425637834474176, + "grad_norm": 3.1750423908233643, + "learning_rate": 4.029059040590406e-06, + "loss": 1.5894, + "step": 677 + }, + { + "epoch": 0.8438083385189795, + "grad_norm": 2.76448655128479, + "learning_rate": 4.026752767527676e-06, + "loss": 1.5752, + "step": 678 + }, + { + "epoch": 0.8450528935905414, + "grad_norm": 2.676708698272705, + "learning_rate": 4.024446494464945e-06, + "loss": 1.5487, + "step": 679 + }, + { + "epoch": 0.8462974486621033, + "grad_norm": 2.6832051277160645, + "learning_rate": 4.0221402214022145e-06, + "loss": 1.5055, + "step": 680 + }, + { + "epoch": 0.8462974486621033, + "eval_loss": 1.6945017576217651, + "eval_runtime": 43.9369, + "eval_samples_per_second": 22.76, + "eval_steps_per_second": 0.956, + "step": 680 + }, + { + "epoch": 0.8475420037336652, + "grad_norm": 2.6812336444854736, + "learning_rate": 4.019833948339484e-06, + "loss": 1.5315, + "step": 681 + }, + { + "epoch": 0.8487865588052271, + "grad_norm": 2.9394326210021973, + "learning_rate": 4.017527675276753e-06, + "loss": 1.5636, + "step": 682 + }, + { + "epoch": 0.8500311138767891, + "grad_norm": 2.944952964782715, + "learning_rate": 4.015221402214022e-06, + "loss": 1.56, + "step": 683 + }, + { + "epoch": 0.8512756689483509, + "grad_norm": 2.7475314140319824, + "learning_rate": 4.012915129151292e-06, + "loss": 1.5442, + "step": 684 + }, + { + "epoch": 0.8525202240199129, + "grad_norm": 3.0572879314422607, + "learning_rate": 4.010608856088562e-06, + "loss": 1.5023, + "step": 685 + }, + { + "epoch": 0.8537647790914747, + "grad_norm": 2.75365948677063, + "learning_rate": 4.00830258302583e-06, + "loss": 1.5414, + "step": 686 + }, + { + "epoch": 0.8550093341630367, + "grad_norm": 2.5714449882507324, + "learning_rate": 4.0059963099631e-06, + "loss": 1.5027, + "step": 687 + }, + { + "epoch": 0.8562538892345987, + "grad_norm": 2.755167007446289, + "learning_rate": 4.003690036900369e-06, + "loss": 1.5788, + "step": 688 + }, + { + "epoch": 0.8574984443061605, + "grad_norm": 2.798967123031616, + "learning_rate": 4.0013837638376385e-06, + "loss": 1.5162, + "step": 689 + }, + { + "epoch": 0.8587429993777225, + "grad_norm": 2.803614854812622, + "learning_rate": 3.999077490774908e-06, + "loss": 1.5079, + "step": 690 + }, + { + "epoch": 0.8587429993777225, + "eval_loss": 1.6924962997436523, + "eval_runtime": 44.47, + "eval_samples_per_second": 22.487, + "eval_steps_per_second": 0.944, + "step": 690 + }, + { + "epoch": 0.8599875544492844, + "grad_norm": 2.6898066997528076, + "learning_rate": 3.996771217712177e-06, + "loss": 1.5478, + "step": 691 + }, + { + "epoch": 0.8612321095208463, + "grad_norm": 2.6376144886016846, + "learning_rate": 3.994464944649447e-06, + "loss": 1.5362, + "step": 692 + }, + { + "epoch": 0.8624766645924082, + "grad_norm": 2.7240405082702637, + "learning_rate": 3.992158671586716e-06, + "loss": 1.5502, + "step": 693 + }, + { + "epoch": 0.8637212196639701, + "grad_norm": 2.929445266723633, + "learning_rate": 3.989852398523986e-06, + "loss": 1.5799, + "step": 694 + }, + { + "epoch": 0.864965774735532, + "grad_norm": 2.593223810195923, + "learning_rate": 3.987546125461255e-06, + "loss": 1.5352, + "step": 695 + }, + { + "epoch": 0.866210329807094, + "grad_norm": 2.7710154056549072, + "learning_rate": 3.9852398523985245e-06, + "loss": 1.5048, + "step": 696 + }, + { + "epoch": 0.8674548848786559, + "grad_norm": 2.7370848655700684, + "learning_rate": 3.982933579335794e-06, + "loss": 1.5233, + "step": 697 + }, + { + "epoch": 0.8686994399502178, + "grad_norm": 2.8632168769836426, + "learning_rate": 3.980627306273063e-06, + "loss": 1.5435, + "step": 698 + }, + { + "epoch": 0.8699439950217797, + "grad_norm": 2.752298593521118, + "learning_rate": 3.978321033210332e-06, + "loss": 1.5728, + "step": 699 + }, + { + "epoch": 0.8711885500933416, + "grad_norm": 2.8345625400543213, + "learning_rate": 3.976014760147602e-06, + "loss": 1.5727, + "step": 700 + }, + { + "epoch": 0.8711885500933416, + "eval_loss": 1.6855305433273315, + "eval_runtime": 43.3683, + "eval_samples_per_second": 23.058, + "eval_steps_per_second": 0.968, + "step": 700 + }, + { + "epoch": 0.8724331051649036, + "grad_norm": 2.8135335445404053, + "learning_rate": 3.973708487084872e-06, + "loss": 1.5086, + "step": 701 + }, + { + "epoch": 0.8736776602364654, + "grad_norm": 2.7416603565216064, + "learning_rate": 3.97140221402214e-06, + "loss": 1.545, + "step": 702 + }, + { + "epoch": 0.8749222153080274, + "grad_norm": 2.6902289390563965, + "learning_rate": 3.96909594095941e-06, + "loss": 1.5509, + "step": 703 + }, + { + "epoch": 0.8761667703795893, + "grad_norm": 2.6537084579467773, + "learning_rate": 3.966789667896679e-06, + "loss": 1.562, + "step": 704 + }, + { + "epoch": 0.8774113254511512, + "grad_norm": 2.6979284286499023, + "learning_rate": 3.964483394833948e-06, + "loss": 1.4829, + "step": 705 + }, + { + "epoch": 0.8786558805227132, + "grad_norm": 2.5795764923095703, + "learning_rate": 3.962177121771218e-06, + "loss": 1.4677, + "step": 706 + }, + { + "epoch": 0.879900435594275, + "grad_norm": 3.0185086727142334, + "learning_rate": 3.959870848708487e-06, + "loss": 1.5608, + "step": 707 + }, + { + "epoch": 0.881144990665837, + "grad_norm": 2.6458113193511963, + "learning_rate": 3.957564575645757e-06, + "loss": 1.5342, + "step": 708 + }, + { + "epoch": 0.8823895457373989, + "grad_norm": 2.7985665798187256, + "learning_rate": 3.955258302583026e-06, + "loss": 1.527, + "step": 709 + }, + { + "epoch": 0.8836341008089608, + "grad_norm": 2.9069082736968994, + "learning_rate": 3.9529520295202955e-06, + "loss": 1.5156, + "step": 710 + }, + { + "epoch": 0.8836341008089608, + "eval_loss": 1.6898822784423828, + "eval_runtime": 42.8749, + "eval_samples_per_second": 23.324, + "eval_steps_per_second": 0.98, + "step": 710 + }, + { + "epoch": 0.8848786558805227, + "grad_norm": 2.760143995285034, + "learning_rate": 3.950645756457565e-06, + "loss": 1.5124, + "step": 711 + }, + { + "epoch": 0.8861232109520847, + "grad_norm": 2.8085529804229736, + "learning_rate": 3.948339483394834e-06, + "loss": 1.5862, + "step": 712 + }, + { + "epoch": 0.8873677660236465, + "grad_norm": 2.909905195236206, + "learning_rate": 3.946033210332104e-06, + "loss": 1.5557, + "step": 713 + }, + { + "epoch": 0.8886123210952085, + "grad_norm": 2.826899290084839, + "learning_rate": 3.943726937269373e-06, + "loss": 1.5277, + "step": 714 + }, + { + "epoch": 0.8898568761667703, + "grad_norm": 2.6269052028656006, + "learning_rate": 3.941420664206642e-06, + "loss": 1.4819, + "step": 715 + }, + { + "epoch": 0.8911014312383323, + "grad_norm": 2.6687698364257812, + "learning_rate": 3.939114391143912e-06, + "loss": 1.5156, + "step": 716 + }, + { + "epoch": 0.8923459863098943, + "grad_norm": 2.9605629444122314, + "learning_rate": 3.9368081180811815e-06, + "loss": 1.5358, + "step": 717 + }, + { + "epoch": 0.8935905413814561, + "grad_norm": 3.0367166996002197, + "learning_rate": 3.934501845018451e-06, + "loss": 1.5847, + "step": 718 + }, + { + "epoch": 0.8948350964530181, + "grad_norm": 2.528796434402466, + "learning_rate": 3.9321955719557195e-06, + "loss": 1.4851, + "step": 719 + }, + { + "epoch": 0.8960796515245799, + "grad_norm": 2.7116613388061523, + "learning_rate": 3.929889298892989e-06, + "loss": 1.4706, + "step": 720 + }, + { + "epoch": 0.8960796515245799, + "eval_loss": 1.6845883131027222, + "eval_runtime": 44.4734, + "eval_samples_per_second": 22.485, + "eval_steps_per_second": 0.944, + "step": 720 + }, + { + "epoch": 0.8973242065961419, + "grad_norm": 2.771864414215088, + "learning_rate": 3.927583025830259e-06, + "loss": 1.5444, + "step": 721 + }, + { + "epoch": 0.8985687616677038, + "grad_norm": 2.773746967315674, + "learning_rate": 3.925276752767528e-06, + "loss": 1.564, + "step": 722 + }, + { + "epoch": 0.8998133167392657, + "grad_norm": 2.7069854736328125, + "learning_rate": 3.922970479704797e-06, + "loss": 1.5601, + "step": 723 + }, + { + "epoch": 0.9010578718108276, + "grad_norm": 2.840261697769165, + "learning_rate": 3.920664206642067e-06, + "loss": 1.5733, + "step": 724 + }, + { + "epoch": 0.9023024268823896, + "grad_norm": 2.709897041320801, + "learning_rate": 3.918357933579336e-06, + "loss": 1.528, + "step": 725 + }, + { + "epoch": 0.9035469819539514, + "grad_norm": 2.662367343902588, + "learning_rate": 3.9160516605166055e-06, + "loss": 1.5532, + "step": 726 + }, + { + "epoch": 0.9047915370255134, + "grad_norm": 2.6775717735290527, + "learning_rate": 3.913745387453875e-06, + "loss": 1.5464, + "step": 727 + }, + { + "epoch": 0.9060360920970753, + "grad_norm": 2.617842674255371, + "learning_rate": 3.911439114391144e-06, + "loss": 1.4539, + "step": 728 + }, + { + "epoch": 0.9072806471686372, + "grad_norm": 2.668649435043335, + "learning_rate": 3.909132841328414e-06, + "loss": 1.5627, + "step": 729 + }, + { + "epoch": 0.9085252022401992, + "grad_norm": 2.5331950187683105, + "learning_rate": 3.906826568265683e-06, + "loss": 1.5165, + "step": 730 + }, + { + "epoch": 0.9085252022401992, + "eval_loss": 1.6860331296920776, + "eval_runtime": 46.094, + "eval_samples_per_second": 21.695, + "eval_steps_per_second": 0.911, + "step": 730 + }, + { + "epoch": 0.909769757311761, + "grad_norm": 2.602492332458496, + "learning_rate": 3.9045202952029526e-06, + "loss": 1.5145, + "step": 731 + }, + { + "epoch": 0.911014312383323, + "grad_norm": 2.632782459259033, + "learning_rate": 3.902214022140222e-06, + "loss": 1.4413, + "step": 732 + }, + { + "epoch": 0.9122588674548848, + "grad_norm": 2.6025912761688232, + "learning_rate": 3.899907749077491e-06, + "loss": 1.5263, + "step": 733 + }, + { + "epoch": 0.9135034225264468, + "grad_norm": 2.770116090774536, + "learning_rate": 3.897601476014761e-06, + "loss": 1.5514, + "step": 734 + }, + { + "epoch": 0.9147479775980087, + "grad_norm": 2.7822961807250977, + "learning_rate": 3.895295202952029e-06, + "loss": 1.5296, + "step": 735 + }, + { + "epoch": 0.9159925326695706, + "grad_norm": 2.6689720153808594, + "learning_rate": 3.892988929889299e-06, + "loss": 1.4552, + "step": 736 + }, + { + "epoch": 0.9172370877411326, + "grad_norm": 2.8021798133850098, + "learning_rate": 3.890682656826569e-06, + "loss": 1.5308, + "step": 737 + }, + { + "epoch": 0.9184816428126945, + "grad_norm": 2.575313091278076, + "learning_rate": 3.888376383763838e-06, + "loss": 1.4807, + "step": 738 + }, + { + "epoch": 0.9197261978842564, + "grad_norm": 2.5955779552459717, + "learning_rate": 3.886070110701107e-06, + "loss": 1.4222, + "step": 739 + }, + { + "epoch": 0.9209707529558183, + "grad_norm": 2.647939920425415, + "learning_rate": 3.8837638376383765e-06, + "loss": 1.5285, + "step": 740 + }, + { + "epoch": 0.9209707529558183, + "eval_loss": 1.6907480955123901, + "eval_runtime": 45.7678, + "eval_samples_per_second": 21.849, + "eval_steps_per_second": 0.918, + "step": 740 + }, + { + "epoch": 0.9222153080273802, + "grad_norm": 2.7664523124694824, + "learning_rate": 3.881457564575646e-06, + "loss": 1.5349, + "step": 741 + }, + { + "epoch": 0.9234598630989421, + "grad_norm": 2.5770998001098633, + "learning_rate": 3.879151291512915e-06, + "loss": 1.5071, + "step": 742 + }, + { + "epoch": 0.9247044181705041, + "grad_norm": 2.502567768096924, + "learning_rate": 3.876845018450185e-06, + "loss": 1.492, + "step": 743 + }, + { + "epoch": 0.9259489732420659, + "grad_norm": 2.6531119346618652, + "learning_rate": 3.874538745387454e-06, + "loss": 1.5045, + "step": 744 + }, + { + "epoch": 0.9271935283136279, + "grad_norm": 2.7343876361846924, + "learning_rate": 3.872232472324724e-06, + "loss": 1.5194, + "step": 745 + }, + { + "epoch": 0.9284380833851898, + "grad_norm": 2.842313528060913, + "learning_rate": 3.869926199261993e-06, + "loss": 1.5588, + "step": 746 + }, + { + "epoch": 0.9296826384567517, + "grad_norm": 2.718245267868042, + "learning_rate": 3.8676199261992625e-06, + "loss": 1.5252, + "step": 747 + }, + { + "epoch": 0.9309271935283137, + "grad_norm": 2.695392370223999, + "learning_rate": 3.865313653136532e-06, + "loss": 1.5596, + "step": 748 + }, + { + "epoch": 0.9321717485998755, + "grad_norm": 2.7859959602355957, + "learning_rate": 3.863007380073801e-06, + "loss": 1.5151, + "step": 749 + }, + { + "epoch": 0.9334163036714375, + "grad_norm": 2.5199174880981445, + "learning_rate": 3.860701107011071e-06, + "loss": 1.4948, + "step": 750 + }, + { + "epoch": 0.9334163036714375, + "eval_loss": 1.6868674755096436, + "eval_runtime": 46.4115, + "eval_samples_per_second": 21.546, + "eval_steps_per_second": 0.905, + "step": 750 + }, + { + "epoch": 0.9346608587429994, + "grad_norm": 2.8194046020507812, + "learning_rate": 3.858394833948339e-06, + "loss": 1.5144, + "step": 751 + }, + { + "epoch": 0.9359054138145613, + "grad_norm": 2.810380697250366, + "learning_rate": 3.856088560885609e-06, + "loss": 1.4998, + "step": 752 + }, + { + "epoch": 0.9371499688861232, + "grad_norm": 2.6906750202178955, + "learning_rate": 3.853782287822879e-06, + "loss": 1.4748, + "step": 753 + }, + { + "epoch": 0.9383945239576851, + "grad_norm": 2.9030277729034424, + "learning_rate": 3.851476014760148e-06, + "loss": 1.615, + "step": 754 + }, + { + "epoch": 0.939639079029247, + "grad_norm": 2.8690176010131836, + "learning_rate": 3.849169741697417e-06, + "loss": 1.5083, + "step": 755 + }, + { + "epoch": 0.940883634100809, + "grad_norm": 2.669646978378296, + "learning_rate": 3.8468634686346865e-06, + "loss": 1.5572, + "step": 756 + }, + { + "epoch": 0.9421281891723708, + "grad_norm": 2.6517696380615234, + "learning_rate": 3.844557195571956e-06, + "loss": 1.5142, + "step": 757 + }, + { + "epoch": 0.9433727442439328, + "grad_norm": 2.5524444580078125, + "learning_rate": 3.842250922509225e-06, + "loss": 1.4924, + "step": 758 + }, + { + "epoch": 0.9446172993154948, + "grad_norm": 2.6532633304595947, + "learning_rate": 3.839944649446495e-06, + "loss": 1.484, + "step": 759 + }, + { + "epoch": 0.9458618543870566, + "grad_norm": 2.7779057025909424, + "learning_rate": 3.837638376383764e-06, + "loss": 1.5106, + "step": 760 + }, + { + "epoch": 0.9458618543870566, + "eval_loss": 1.6845752000808716, + "eval_runtime": 50.3415, + "eval_samples_per_second": 19.864, + "eval_steps_per_second": 0.834, + "step": 760 + }, + { + "epoch": 0.9471064094586186, + "grad_norm": 2.6192541122436523, + "learning_rate": 3.8353321033210336e-06, + "loss": 1.5454, + "step": 761 + }, + { + "epoch": 0.9483509645301804, + "grad_norm": 2.561861991882324, + "learning_rate": 3.833025830258303e-06, + "loss": 1.5141, + "step": 762 + }, + { + "epoch": 0.9495955196017424, + "grad_norm": 2.661829948425293, + "learning_rate": 3.830719557195572e-06, + "loss": 1.4997, + "step": 763 + }, + { + "epoch": 0.9508400746733043, + "grad_norm": 2.725275754928589, + "learning_rate": 3.828413284132842e-06, + "loss": 1.5224, + "step": 764 + }, + { + "epoch": 0.9520846297448662, + "grad_norm": 2.5664193630218506, + "learning_rate": 3.826107011070111e-06, + "loss": 1.5073, + "step": 765 + }, + { + "epoch": 0.9533291848164281, + "grad_norm": 2.595189332962036, + "learning_rate": 3.823800738007381e-06, + "loss": 1.5379, + "step": 766 + }, + { + "epoch": 0.95457373988799, + "grad_norm": 2.738060235977173, + "learning_rate": 3.821494464944649e-06, + "loss": 1.4814, + "step": 767 + }, + { + "epoch": 0.955818294959552, + "grad_norm": 2.601071357727051, + "learning_rate": 3.819188191881919e-06, + "loss": 1.4812, + "step": 768 + }, + { + "epoch": 0.9570628500311139, + "grad_norm": 2.657212257385254, + "learning_rate": 3.816881918819189e-06, + "loss": 1.4748, + "step": 769 + }, + { + "epoch": 0.9583074051026758, + "grad_norm": 2.7673351764678955, + "learning_rate": 3.814575645756458e-06, + "loss": 1.5535, + "step": 770 + }, + { + "epoch": 0.9583074051026758, + "eval_loss": 1.677243947982788, + "eval_runtime": 51.5537, + "eval_samples_per_second": 19.397, + "eval_steps_per_second": 0.815, + "step": 770 + }, + { + "epoch": 0.9595519601742377, + "grad_norm": 2.760890007019043, + "learning_rate": 3.812269372693727e-06, + "loss": 1.5654, + "step": 771 + }, + { + "epoch": 0.9607965152457997, + "grad_norm": 2.6934309005737305, + "learning_rate": 3.809963099630997e-06, + "loss": 1.5149, + "step": 772 + }, + { + "epoch": 0.9620410703173615, + "grad_norm": 2.729950428009033, + "learning_rate": 3.8076568265682662e-06, + "loss": 1.5579, + "step": 773 + }, + { + "epoch": 0.9632856253889235, + "grad_norm": 2.732926607131958, + "learning_rate": 3.8053505535055352e-06, + "loss": 1.5179, + "step": 774 + }, + { + "epoch": 0.9645301804604853, + "grad_norm": 2.745391368865967, + "learning_rate": 3.8030442804428046e-06, + "loss": 1.5124, + "step": 775 + }, + { + "epoch": 0.9657747355320473, + "grad_norm": 2.8050146102905273, + "learning_rate": 3.800738007380074e-06, + "loss": 1.519, + "step": 776 + }, + { + "epoch": 0.9670192906036092, + "grad_norm": 2.697171449661255, + "learning_rate": 3.798431734317343e-06, + "loss": 1.5219, + "step": 777 + }, + { + "epoch": 0.9682638456751711, + "grad_norm": 2.6375980377197266, + "learning_rate": 3.796125461254613e-06, + "loss": 1.5345, + "step": 778 + }, + { + "epoch": 0.9695084007467331, + "grad_norm": 2.586636781692505, + "learning_rate": 3.7938191881918823e-06, + "loss": 1.5182, + "step": 779 + }, + { + "epoch": 0.9707529558182949, + "grad_norm": 2.626453399658203, + "learning_rate": 3.7915129151291518e-06, + "loss": 1.4817, + "step": 780 + }, + { + "epoch": 0.9707529558182949, + "eval_loss": 1.6734713315963745, + "eval_runtime": 48.1436, + "eval_samples_per_second": 20.771, + "eval_steps_per_second": 0.872, + "step": 780 + }, + { + "epoch": 0.9719975108898569, + "grad_norm": 2.579970121383667, + "learning_rate": 3.7892066420664208e-06, + "loss": 1.4981, + "step": 781 + }, + { + "epoch": 0.9732420659614188, + "grad_norm": 2.6834843158721924, + "learning_rate": 3.7869003690036906e-06, + "loss": 1.5044, + "step": 782 + }, + { + "epoch": 0.9744866210329807, + "grad_norm": 2.586982488632202, + "learning_rate": 3.78459409594096e-06, + "loss": 1.5079, + "step": 783 + }, + { + "epoch": 0.9757311761045426, + "grad_norm": 2.910027503967285, + "learning_rate": 3.782287822878229e-06, + "loss": 1.5695, + "step": 784 + }, + { + "epoch": 0.9769757311761046, + "grad_norm": 2.7088494300842285, + "learning_rate": 3.7799815498154984e-06, + "loss": 1.4986, + "step": 785 + }, + { + "epoch": 0.9782202862476664, + "grad_norm": 2.581325054168701, + "learning_rate": 3.777675276752768e-06, + "loss": 1.5137, + "step": 786 + }, + { + "epoch": 0.9794648413192284, + "grad_norm": 2.700709819793701, + "learning_rate": 3.775369003690037e-06, + "loss": 1.4718, + "step": 787 + }, + { + "epoch": 0.9807093963907902, + "grad_norm": 2.6712708473205566, + "learning_rate": 3.7730627306273067e-06, + "loss": 1.5208, + "step": 788 + }, + { + "epoch": 0.9819539514623522, + "grad_norm": 2.6799817085266113, + "learning_rate": 3.770756457564576e-06, + "loss": 1.5374, + "step": 789 + }, + { + "epoch": 0.9831985065339142, + "grad_norm": 2.618988037109375, + "learning_rate": 3.768450184501845e-06, + "loss": 1.4997, + "step": 790 + }, + { + "epoch": 0.9831985065339142, + "eval_loss": 1.6860820055007935, + "eval_runtime": 51.7654, + "eval_samples_per_second": 19.318, + "eval_steps_per_second": 0.811, + "step": 790 + }, + { + "epoch": 0.984443061605476, + "grad_norm": 2.6899526119232178, + "learning_rate": 3.7661439114391146e-06, + "loss": 1.5216, + "step": 791 + }, + { + "epoch": 0.985687616677038, + "grad_norm": 2.8003487586975098, + "learning_rate": 3.763837638376384e-06, + "loss": 1.557, + "step": 792 + }, + { + "epoch": 0.9869321717485999, + "grad_norm": 2.744536876678467, + "learning_rate": 3.761531365313654e-06, + "loss": 1.5355, + "step": 793 + }, + { + "epoch": 0.9881767268201618, + "grad_norm": 2.587250232696533, + "learning_rate": 3.759225092250923e-06, + "loss": 1.5259, + "step": 794 + }, + { + "epoch": 0.9894212818917237, + "grad_norm": 2.616291046142578, + "learning_rate": 3.7569188191881922e-06, + "loss": 1.5176, + "step": 795 + }, + { + "epoch": 0.9906658369632856, + "grad_norm": 2.6410577297210693, + "learning_rate": 3.7546125461254617e-06, + "loss": 1.511, + "step": 796 + }, + { + "epoch": 0.9919103920348475, + "grad_norm": 2.577373504638672, + "learning_rate": 3.7523062730627307e-06, + "loss": 1.4715, + "step": 797 + }, + { + "epoch": 0.9931549471064095, + "grad_norm": 2.67305326461792, + "learning_rate": 3.7500000000000005e-06, + "loss": 1.4828, + "step": 798 + }, + { + "epoch": 0.9943995021779714, + "grad_norm": 3.1631500720977783, + "learning_rate": 3.74769372693727e-06, + "loss": 1.5602, + "step": 799 + }, + { + "epoch": 0.9956440572495333, + "grad_norm": 2.9222350120544434, + "learning_rate": 3.745387453874539e-06, + "loss": 1.4814, + "step": 800 + }, + { + "epoch": 0.9956440572495333, + "eval_loss": 1.6810544729232788, + "eval_runtime": 51.1262, + "eval_samples_per_second": 19.559, + "eval_steps_per_second": 0.821, + "step": 800 + }, + { + "epoch": 0.9968886123210952, + "grad_norm": 2.884312629699707, + "learning_rate": 3.7430811808118084e-06, + "loss": 1.4553, + "step": 801 + }, + { + "epoch": 0.9981331673926571, + "grad_norm": 2.712655782699585, + "learning_rate": 3.7407749077490778e-06, + "loss": 1.5275, + "step": 802 + }, + { + "epoch": 0.9993777224642191, + "grad_norm": 2.8700571060180664, + "learning_rate": 3.7384686346863468e-06, + "loss": 1.4923, + "step": 803 + }, + { + "epoch": 1.000622277535781, + "grad_norm": 2.7779245376586914, + "learning_rate": 3.7361623616236166e-06, + "loss": 1.5224, + "step": 804 + }, + { + "epoch": 1.0018668326073428, + "grad_norm": 2.7679810523986816, + "learning_rate": 3.733856088560886e-06, + "loss": 1.4977, + "step": 805 + }, + { + "epoch": 1.0031113876789048, + "grad_norm": 2.797023296356201, + "learning_rate": 3.7315498154981555e-06, + "loss": 1.4396, + "step": 806 + }, + { + "epoch": 1.0043559427504667, + "grad_norm": 3.007962465286255, + "learning_rate": 3.7292435424354245e-06, + "loss": 1.5015, + "step": 807 + }, + { + "epoch": 1.0056004978220285, + "grad_norm": 3.127639055252075, + "learning_rate": 3.726937269372694e-06, + "loss": 1.5176, + "step": 808 + }, + { + "epoch": 1.0068450528935906, + "grad_norm": 2.9001357555389404, + "learning_rate": 3.7246309963099637e-06, + "loss": 1.4735, + "step": 809 + }, + { + "epoch": 1.0080896079651525, + "grad_norm": 2.8245413303375244, + "learning_rate": 3.7223247232472327e-06, + "loss": 1.486, + "step": 810 + }, + { + "epoch": 1.0080896079651525, + "eval_loss": 1.6835130453109741, + "eval_runtime": 44.5682, + "eval_samples_per_second": 22.438, + "eval_steps_per_second": 0.942, + "step": 810 + }, + { + "epoch": 1.0093341630367143, + "grad_norm": 2.780172824859619, + "learning_rate": 3.720018450184502e-06, + "loss": 1.4437, + "step": 811 + }, + { + "epoch": 1.0105787181082764, + "grad_norm": 2.6013996601104736, + "learning_rate": 3.7177121771217716e-06, + "loss": 1.4529, + "step": 812 + }, + { + "epoch": 1.0118232731798382, + "grad_norm": 2.831015110015869, + "learning_rate": 3.7154059040590406e-06, + "loss": 1.4825, + "step": 813 + }, + { + "epoch": 1.0130678282514, + "grad_norm": 2.8341829776763916, + "learning_rate": 3.71309963099631e-06, + "loss": 1.4736, + "step": 814 + }, + { + "epoch": 1.0143123833229621, + "grad_norm": 2.769540309906006, + "learning_rate": 3.71079335793358e-06, + "loss": 1.4724, + "step": 815 + }, + { + "epoch": 1.015556938394524, + "grad_norm": 3.0083069801330566, + "learning_rate": 3.708487084870849e-06, + "loss": 1.4985, + "step": 816 + }, + { + "epoch": 1.0168014934660858, + "grad_norm": 3.052168607711792, + "learning_rate": 3.7061808118081183e-06, + "loss": 1.5193, + "step": 817 + }, + { + "epoch": 1.0180460485376477, + "grad_norm": 2.924877405166626, + "learning_rate": 3.7038745387453877e-06, + "loss": 1.5219, + "step": 818 + }, + { + "epoch": 1.0192906036092098, + "grad_norm": 3.150123357772827, + "learning_rate": 3.7015682656826576e-06, + "loss": 1.5026, + "step": 819 + }, + { + "epoch": 1.0205351586807716, + "grad_norm": 2.881655216217041, + "learning_rate": 3.6992619926199266e-06, + "loss": 1.5011, + "step": 820 + }, + { + "epoch": 1.0205351586807716, + "eval_loss": 1.6745303869247437, + "eval_runtime": 50.6791, + "eval_samples_per_second": 19.732, + "eval_steps_per_second": 0.829, + "step": 820 + }, + { + "epoch": 1.0217797137523335, + "grad_norm": 2.824249505996704, + "learning_rate": 3.696955719557196e-06, + "loss": 1.4642, + "step": 821 + }, + { + "epoch": 1.0230242688238955, + "grad_norm": 2.859144926071167, + "learning_rate": 3.6946494464944654e-06, + "loss": 1.4681, + "step": 822 + }, + { + "epoch": 1.0242688238954574, + "grad_norm": 2.90312123298645, + "learning_rate": 3.6923431734317344e-06, + "loss": 1.4926, + "step": 823 + }, + { + "epoch": 1.0255133789670192, + "grad_norm": 2.8340659141540527, + "learning_rate": 3.690036900369004e-06, + "loss": 1.4834, + "step": 824 + }, + { + "epoch": 1.0267579340385813, + "grad_norm": 2.9743151664733887, + "learning_rate": 3.6877306273062737e-06, + "loss": 1.4574, + "step": 825 + }, + { + "epoch": 1.0280024891101431, + "grad_norm": 3.032179594039917, + "learning_rate": 3.6854243542435427e-06, + "loss": 1.5571, + "step": 826 + }, + { + "epoch": 1.029247044181705, + "grad_norm": 2.816826343536377, + "learning_rate": 3.683118081180812e-06, + "loss": 1.455, + "step": 827 + }, + { + "epoch": 1.030491599253267, + "grad_norm": 3.046696186065674, + "learning_rate": 3.6808118081180815e-06, + "loss": 1.4871, + "step": 828 + }, + { + "epoch": 1.031736154324829, + "grad_norm": 2.8517391681671143, + "learning_rate": 3.678505535055351e-06, + "loss": 1.5292, + "step": 829 + }, + { + "epoch": 1.0329807093963908, + "grad_norm": 2.752811908721924, + "learning_rate": 3.67619926199262e-06, + "loss": 1.5091, + "step": 830 + }, + { + "epoch": 1.0329807093963908, + "eval_loss": 1.6692287921905518, + "eval_runtime": 50.8347, + "eval_samples_per_second": 19.672, + "eval_steps_per_second": 0.826, + "step": 830 + }, + { + "epoch": 1.0342252644679526, + "grad_norm": 2.9220356941223145, + "learning_rate": 3.6738929889298898e-06, + "loss": 1.4247, + "step": 831 + }, + { + "epoch": 1.0354698195395147, + "grad_norm": 2.8903002738952637, + "learning_rate": 3.671586715867159e-06, + "loss": 1.5595, + "step": 832 + }, + { + "epoch": 1.0367143746110765, + "grad_norm": 2.6495542526245117, + "learning_rate": 3.669280442804428e-06, + "loss": 1.4076, + "step": 833 + }, + { + "epoch": 1.0379589296826384, + "grad_norm": 2.880809783935547, + "learning_rate": 3.6669741697416976e-06, + "loss": 1.5285, + "step": 834 + }, + { + "epoch": 1.0392034847542004, + "grad_norm": 2.919142007827759, + "learning_rate": 3.6646678966789675e-06, + "loss": 1.5165, + "step": 835 + }, + { + "epoch": 1.0404480398257623, + "grad_norm": 3.1180522441864014, + "learning_rate": 3.6623616236162365e-06, + "loss": 1.5224, + "step": 836 + }, + { + "epoch": 1.0416925948973241, + "grad_norm": 2.767709255218506, + "learning_rate": 3.660055350553506e-06, + "loss": 1.4859, + "step": 837 + }, + { + "epoch": 1.0429371499688862, + "grad_norm": 2.708828926086426, + "learning_rate": 3.6577490774907753e-06, + "loss": 1.4803, + "step": 838 + }, + { + "epoch": 1.044181705040448, + "grad_norm": 2.688359260559082, + "learning_rate": 3.6554428044280443e-06, + "loss": 1.4571, + "step": 839 + }, + { + "epoch": 1.04542626011201, + "grad_norm": 2.691812753677368, + "learning_rate": 3.6531365313653137e-06, + "loss": 1.5043, + "step": 840 + }, + { + "epoch": 1.04542626011201, + "eval_loss": 1.6769559383392334, + "eval_runtime": 49.6233, + "eval_samples_per_second": 20.152, + "eval_steps_per_second": 0.846, + "step": 840 + }, + { + "epoch": 1.046670815183572, + "grad_norm": 2.686372995376587, + "learning_rate": 3.6508302583025836e-06, + "loss": 1.5221, + "step": 841 + }, + { + "epoch": 1.0479153702551338, + "grad_norm": 2.742859125137329, + "learning_rate": 3.648523985239853e-06, + "loss": 1.4685, + "step": 842 + }, + { + "epoch": 1.0491599253266957, + "grad_norm": 2.874018907546997, + "learning_rate": 3.646217712177122e-06, + "loss": 1.4813, + "step": 843 + }, + { + "epoch": 1.0504044803982575, + "grad_norm": 2.8159990310668945, + "learning_rate": 3.6439114391143914e-06, + "loss": 1.4421, + "step": 844 + }, + { + "epoch": 1.0516490354698196, + "grad_norm": 2.714787006378174, + "learning_rate": 3.641605166051661e-06, + "loss": 1.4879, + "step": 845 + }, + { + "epoch": 1.0528935905413814, + "grad_norm": 2.885148525238037, + "learning_rate": 3.63929889298893e-06, + "loss": 1.4725, + "step": 846 + }, + { + "epoch": 1.0541381456129433, + "grad_norm": 2.7272610664367676, + "learning_rate": 3.6369926199261997e-06, + "loss": 1.4276, + "step": 847 + }, + { + "epoch": 1.0553827006845053, + "grad_norm": 2.9114978313446045, + "learning_rate": 3.634686346863469e-06, + "loss": 1.4194, + "step": 848 + }, + { + "epoch": 1.0566272557560672, + "grad_norm": 3.019313335418701, + "learning_rate": 3.632380073800738e-06, + "loss": 1.5266, + "step": 849 + }, + { + "epoch": 1.057871810827629, + "grad_norm": 2.714751958847046, + "learning_rate": 3.6300738007380075e-06, + "loss": 1.4593, + "step": 850 + }, + { + "epoch": 1.057871810827629, + "eval_loss": 1.6760298013687134, + "eval_runtime": 52.1681, + "eval_samples_per_second": 19.169, + "eval_steps_per_second": 0.805, + "step": 850 + }, + { + "epoch": 1.0591163658991911, + "grad_norm": 3.0507936477661133, + "learning_rate": 3.627767527675277e-06, + "loss": 1.5064, + "step": 851 + }, + { + "epoch": 1.060360920970753, + "grad_norm": 2.8116295337677, + "learning_rate": 3.625461254612546e-06, + "loss": 1.5218, + "step": 852 + }, + { + "epoch": 1.0616054760423148, + "grad_norm": 2.8801465034484863, + "learning_rate": 3.623154981549816e-06, + "loss": 1.4641, + "step": 853 + }, + { + "epoch": 1.0628500311138769, + "grad_norm": 3.0581862926483154, + "learning_rate": 3.6208487084870852e-06, + "loss": 1.483, + "step": 854 + }, + { + "epoch": 1.0640945861854387, + "grad_norm": 2.898780584335327, + "learning_rate": 3.6185424354243547e-06, + "loss": 1.4623, + "step": 855 + }, + { + "epoch": 1.0653391412570006, + "grad_norm": 2.971820116043091, + "learning_rate": 3.6162361623616237e-06, + "loss": 1.4455, + "step": 856 + }, + { + "epoch": 1.0665836963285624, + "grad_norm": 3.0021913051605225, + "learning_rate": 3.6139298892988935e-06, + "loss": 1.4878, + "step": 857 + }, + { + "epoch": 1.0678282514001245, + "grad_norm": 3.0697457790374756, + "learning_rate": 3.611623616236163e-06, + "loss": 1.5008, + "step": 858 + }, + { + "epoch": 1.0690728064716863, + "grad_norm": 3.0787923336029053, + "learning_rate": 3.609317343173432e-06, + "loss": 1.4612, + "step": 859 + }, + { + "epoch": 1.0703173615432482, + "grad_norm": 2.789339780807495, + "learning_rate": 3.6070110701107014e-06, + "loss": 1.4287, + "step": 860 + }, + { + "epoch": 1.0703173615432482, + "eval_loss": 1.6747506856918335, + "eval_runtime": 51.4912, + "eval_samples_per_second": 19.421, + "eval_steps_per_second": 0.816, + "step": 860 + }, + { + "epoch": 1.0715619166148103, + "grad_norm": 2.9622788429260254, + "learning_rate": 3.6047047970479708e-06, + "loss": 1.4905, + "step": 861 + }, + { + "epoch": 1.072806471686372, + "grad_norm": 3.0675268173217773, + "learning_rate": 3.6023985239852398e-06, + "loss": 1.461, + "step": 862 + }, + { + "epoch": 1.074051026757934, + "grad_norm": 2.820996046066284, + "learning_rate": 3.6000922509225096e-06, + "loss": 1.4948, + "step": 863 + }, + { + "epoch": 1.075295581829496, + "grad_norm": 2.833761215209961, + "learning_rate": 3.597785977859779e-06, + "loss": 1.4683, + "step": 864 + }, + { + "epoch": 1.0765401369010579, + "grad_norm": 2.917313814163208, + "learning_rate": 3.595479704797048e-06, + "loss": 1.5191, + "step": 865 + }, + { + "epoch": 1.0777846919726197, + "grad_norm": 2.876166820526123, + "learning_rate": 3.5931734317343175e-06, + "loss": 1.4854, + "step": 866 + }, + { + "epoch": 1.0790292470441818, + "grad_norm": 3.139049768447876, + "learning_rate": 3.590867158671587e-06, + "loss": 1.5134, + "step": 867 + }, + { + "epoch": 1.0802738021157436, + "grad_norm": 2.9956753253936768, + "learning_rate": 3.5885608856088567e-06, + "loss": 1.5003, + "step": 868 + }, + { + "epoch": 1.0815183571873055, + "grad_norm": 2.957181930541992, + "learning_rate": 3.5862546125461257e-06, + "loss": 1.4588, + "step": 869 + }, + { + "epoch": 1.0827629122588673, + "grad_norm": 2.8576438426971436, + "learning_rate": 3.583948339483395e-06, + "loss": 1.48, + "step": 870 + }, + { + "epoch": 1.0827629122588673, + "eval_loss": 1.6724848747253418, + "eval_runtime": 50.9669, + "eval_samples_per_second": 19.621, + "eval_steps_per_second": 0.824, + "step": 870 + }, + { + "epoch": 1.0840074673304294, + "grad_norm": 2.8183560371398926, + "learning_rate": 3.5816420664206646e-06, + "loss": 1.4818, + "step": 871 + }, + { + "epoch": 1.0852520224019913, + "grad_norm": 2.7531797885894775, + "learning_rate": 3.5793357933579336e-06, + "loss": 1.4712, + "step": 872 + }, + { + "epoch": 1.086496577473553, + "grad_norm": 2.923962354660034, + "learning_rate": 3.5770295202952034e-06, + "loss": 1.4864, + "step": 873 + }, + { + "epoch": 1.0877411325451152, + "grad_norm": 2.851724147796631, + "learning_rate": 3.574723247232473e-06, + "loss": 1.4771, + "step": 874 + }, + { + "epoch": 1.088985687616677, + "grad_norm": 3.024634838104248, + "learning_rate": 3.572416974169742e-06, + "loss": 1.4643, + "step": 875 + }, + { + "epoch": 1.0902302426882389, + "grad_norm": 2.7100844383239746, + "learning_rate": 3.5701107011070113e-06, + "loss": 1.4773, + "step": 876 + }, + { + "epoch": 1.091474797759801, + "grad_norm": 2.730502128601074, + "learning_rate": 3.5678044280442807e-06, + "loss": 1.5051, + "step": 877 + }, + { + "epoch": 1.0927193528313628, + "grad_norm": 2.747082233428955, + "learning_rate": 3.5654981549815497e-06, + "loss": 1.5207, + "step": 878 + }, + { + "epoch": 1.0939639079029246, + "grad_norm": 2.718358039855957, + "learning_rate": 3.5631918819188195e-06, + "loss": 1.5067, + "step": 879 + }, + { + "epoch": 1.0952084629744867, + "grad_norm": 2.7794573307037354, + "learning_rate": 3.560885608856089e-06, + "loss": 1.4616, + "step": 880 + }, + { + "epoch": 1.0952084629744867, + "eval_loss": 1.6676132678985596, + "eval_runtime": 51.6157, + "eval_samples_per_second": 19.374, + "eval_steps_per_second": 0.814, + "step": 880 + }, + { + "epoch": 1.0964530180460486, + "grad_norm": 2.727416753768921, + "learning_rate": 3.5585793357933584e-06, + "loss": 1.4043, + "step": 881 + }, + { + "epoch": 1.0976975731176104, + "grad_norm": 2.856900930404663, + "learning_rate": 3.5562730627306274e-06, + "loss": 1.5198, + "step": 882 + }, + { + "epoch": 1.0989421281891725, + "grad_norm": 2.872823715209961, + "learning_rate": 3.553966789667897e-06, + "loss": 1.4981, + "step": 883 + }, + { + "epoch": 1.1001866832607343, + "grad_norm": 2.7724361419677734, + "learning_rate": 3.5516605166051667e-06, + "loss": 1.4819, + "step": 884 + }, + { + "epoch": 1.1014312383322962, + "grad_norm": 2.884434461593628, + "learning_rate": 3.5493542435424357e-06, + "loss": 1.433, + "step": 885 + }, + { + "epoch": 1.102675793403858, + "grad_norm": 2.887873411178589, + "learning_rate": 3.547047970479705e-06, + "loss": 1.4677, + "step": 886 + }, + { + "epoch": 1.10392034847542, + "grad_norm": 2.946845054626465, + "learning_rate": 3.5447416974169745e-06, + "loss": 1.4983, + "step": 887 + }, + { + "epoch": 1.105164903546982, + "grad_norm": 2.752552032470703, + "learning_rate": 3.5424354243542435e-06, + "loss": 1.4766, + "step": 888 + }, + { + "epoch": 1.1064094586185438, + "grad_norm": 2.788634777069092, + "learning_rate": 3.5401291512915133e-06, + "loss": 1.4747, + "step": 889 + }, + { + "epoch": 1.1076540136901059, + "grad_norm": 2.782792568206787, + "learning_rate": 3.5378228782287828e-06, + "loss": 1.5503, + "step": 890 + }, + { + "epoch": 1.1076540136901059, + "eval_loss": 1.6698333024978638, + "eval_runtime": 49.1373, + "eval_samples_per_second": 20.351, + "eval_steps_per_second": 0.855, + "step": 890 + }, + { + "epoch": 1.1088985687616677, + "grad_norm": 2.8351166248321533, + "learning_rate": 3.535516605166052e-06, + "loss": 1.4589, + "step": 891 + }, + { + "epoch": 1.1101431238332296, + "grad_norm": 2.798872709274292, + "learning_rate": 3.533210332103321e-06, + "loss": 1.4496, + "step": 892 + }, + { + "epoch": 1.1113876789047916, + "grad_norm": 3.0863239765167236, + "learning_rate": 3.5309040590405906e-06, + "loss": 1.4698, + "step": 893 + }, + { + "epoch": 1.1126322339763535, + "grad_norm": 2.853862762451172, + "learning_rate": 3.5285977859778605e-06, + "loss": 1.4429, + "step": 894 + }, + { + "epoch": 1.1138767890479153, + "grad_norm": 2.8946170806884766, + "learning_rate": 3.5262915129151295e-06, + "loss": 1.4697, + "step": 895 + }, + { + "epoch": 1.1151213441194774, + "grad_norm": 3.043823003768921, + "learning_rate": 3.523985239852399e-06, + "loss": 1.4666, + "step": 896 + }, + { + "epoch": 1.1163658991910392, + "grad_norm": 2.7822980880737305, + "learning_rate": 3.5216789667896683e-06, + "loss": 1.4523, + "step": 897 + }, + { + "epoch": 1.117610454262601, + "grad_norm": 2.9454171657562256, + "learning_rate": 3.5193726937269373e-06, + "loss": 1.5009, + "step": 898 + }, + { + "epoch": 1.1188550093341632, + "grad_norm": 2.8533740043640137, + "learning_rate": 3.5170664206642067e-06, + "loss": 1.4265, + "step": 899 + }, + { + "epoch": 1.120099564405725, + "grad_norm": 2.894937038421631, + "learning_rate": 3.5147601476014766e-06, + "loss": 1.4895, + "step": 900 + }, + { + "epoch": 1.120099564405725, + "eval_loss": 1.6701407432556152, + "eval_runtime": 45.477, + "eval_samples_per_second": 21.989, + "eval_steps_per_second": 0.924, + "step": 900 + }, + { + "epoch": 1.1213441194772868, + "grad_norm": 2.933821201324463, + "learning_rate": 3.5124538745387456e-06, + "loss": 1.468, + "step": 901 + }, + { + "epoch": 1.1225886745488487, + "grad_norm": 2.834265947341919, + "learning_rate": 3.510147601476015e-06, + "loss": 1.4621, + "step": 902 + }, + { + "epoch": 1.1238332296204108, + "grad_norm": 2.9064722061157227, + "learning_rate": 3.5078413284132844e-06, + "loss": 1.504, + "step": 903 + }, + { + "epoch": 1.1250777846919726, + "grad_norm": 2.8753886222839355, + "learning_rate": 3.505535055350554e-06, + "loss": 1.4643, + "step": 904 + }, + { + "epoch": 1.1263223397635345, + "grad_norm": 2.894659996032715, + "learning_rate": 3.503228782287823e-06, + "loss": 1.4909, + "step": 905 + }, + { + "epoch": 1.1275668948350965, + "grad_norm": 3.085977792739868, + "learning_rate": 3.5009225092250927e-06, + "loss": 1.459, + "step": 906 + }, + { + "epoch": 1.1288114499066584, + "grad_norm": 2.9958064556121826, + "learning_rate": 3.498616236162362e-06, + "loss": 1.4152, + "step": 907 + }, + { + "epoch": 1.1300560049782202, + "grad_norm": 2.9136762619018555, + "learning_rate": 3.496309963099631e-06, + "loss": 1.473, + "step": 908 + }, + { + "epoch": 1.1313005600497823, + "grad_norm": 2.956437826156616, + "learning_rate": 3.4940036900369005e-06, + "loss": 1.5167, + "step": 909 + }, + { + "epoch": 1.1325451151213441, + "grad_norm": 2.7354865074157715, + "learning_rate": 3.4916974169741704e-06, + "loss": 1.4413, + "step": 910 + }, + { + "epoch": 1.1325451151213441, + "eval_loss": 1.6629912853240967, + "eval_runtime": 41.5478, + "eval_samples_per_second": 24.069, + "eval_steps_per_second": 1.011, + "step": 910 + }, + { + "epoch": 1.133789670192906, + "grad_norm": 2.7443008422851562, + "learning_rate": 3.4893911439114394e-06, + "loss": 1.4475, + "step": 911 + }, + { + "epoch": 1.135034225264468, + "grad_norm": 2.7811238765716553, + "learning_rate": 3.487084870848709e-06, + "loss": 1.4257, + "step": 912 + }, + { + "epoch": 1.13627878033603, + "grad_norm": 2.7916479110717773, + "learning_rate": 3.4847785977859782e-06, + "loss": 1.4048, + "step": 913 + }, + { + "epoch": 1.1375233354075918, + "grad_norm": 2.7835633754730225, + "learning_rate": 3.4824723247232472e-06, + "loss": 1.3996, + "step": 914 + }, + { + "epoch": 1.1387678904791536, + "grad_norm": 2.8188705444335938, + "learning_rate": 3.4801660516605166e-06, + "loss": 1.4541, + "step": 915 + }, + { + "epoch": 1.1400124455507157, + "grad_norm": 3.0134100914001465, + "learning_rate": 3.4778597785977865e-06, + "loss": 1.4353, + "step": 916 + }, + { + "epoch": 1.1412570006222775, + "grad_norm": 2.6965606212615967, + "learning_rate": 3.475553505535056e-06, + "loss": 1.4308, + "step": 917 + }, + { + "epoch": 1.1425015556938394, + "grad_norm": 2.8179919719696045, + "learning_rate": 3.473247232472325e-06, + "loss": 1.5204, + "step": 918 + }, + { + "epoch": 1.1437461107654014, + "grad_norm": 2.7213902473449707, + "learning_rate": 3.4709409594095943e-06, + "loss": 1.4344, + "step": 919 + }, + { + "epoch": 1.1449906658369633, + "grad_norm": 2.850193738937378, + "learning_rate": 3.4686346863468638e-06, + "loss": 1.473, + "step": 920 + }, + { + "epoch": 1.1449906658369633, + "eval_loss": 1.663345217704773, + "eval_runtime": 43.8397, + "eval_samples_per_second": 22.81, + "eval_steps_per_second": 0.958, + "step": 920 + }, + { + "epoch": 1.1462352209085251, + "grad_norm": 2.780198574066162, + "learning_rate": 3.4663284132841328e-06, + "loss": 1.4472, + "step": 921 + }, + { + "epoch": 1.1474797759800872, + "grad_norm": 2.9373831748962402, + "learning_rate": 3.4640221402214026e-06, + "loss": 1.426, + "step": 922 + }, + { + "epoch": 1.148724331051649, + "grad_norm": 2.6986873149871826, + "learning_rate": 3.461715867158672e-06, + "loss": 1.3953, + "step": 923 + }, + { + "epoch": 1.149968886123211, + "grad_norm": 2.927441120147705, + "learning_rate": 3.459409594095941e-06, + "loss": 1.4578, + "step": 924 + }, + { + "epoch": 1.151213441194773, + "grad_norm": 2.8469648361206055, + "learning_rate": 3.4571033210332105e-06, + "loss": 1.4313, + "step": 925 + }, + { + "epoch": 1.1524579962663348, + "grad_norm": 2.8114013671875, + "learning_rate": 3.4547970479704803e-06, + "loss": 1.4267, + "step": 926 + }, + { + "epoch": 1.1537025513378967, + "grad_norm": 2.977605104446411, + "learning_rate": 3.4524907749077493e-06, + "loss": 1.4747, + "step": 927 + }, + { + "epoch": 1.1549471064094585, + "grad_norm": 2.9327852725982666, + "learning_rate": 3.4501845018450187e-06, + "loss": 1.4219, + "step": 928 + }, + { + "epoch": 1.1561916614810206, + "grad_norm": 2.844038963317871, + "learning_rate": 3.447878228782288e-06, + "loss": 1.5146, + "step": 929 + }, + { + "epoch": 1.1574362165525824, + "grad_norm": 2.9402201175689697, + "learning_rate": 3.4455719557195576e-06, + "loss": 1.45, + "step": 930 + }, + { + "epoch": 1.1574362165525824, + "eval_loss": 1.6620122194290161, + "eval_runtime": 46.1748, + "eval_samples_per_second": 21.657, + "eval_steps_per_second": 0.91, + "step": 930 + }, + { + "epoch": 1.1586807716241443, + "grad_norm": 2.842651128768921, + "learning_rate": 3.4432656826568266e-06, + "loss": 1.4415, + "step": 931 + }, + { + "epoch": 1.1599253266957064, + "grad_norm": 2.8909685611724854, + "learning_rate": 3.4409594095940964e-06, + "loss": 1.4529, + "step": 932 + }, + { + "epoch": 1.1611698817672682, + "grad_norm": 2.8025808334350586, + "learning_rate": 3.438653136531366e-06, + "loss": 1.4552, + "step": 933 + }, + { + "epoch": 1.16241443683883, + "grad_norm": 2.6995794773101807, + "learning_rate": 3.436346863468635e-06, + "loss": 1.4435, + "step": 934 + }, + { + "epoch": 1.1636589919103921, + "grad_norm": 2.7307121753692627, + "learning_rate": 3.4340405904059043e-06, + "loss": 1.4447, + "step": 935 + }, + { + "epoch": 1.164903546981954, + "grad_norm": 2.9532501697540283, + "learning_rate": 3.4317343173431737e-06, + "loss": 1.4747, + "step": 936 + }, + { + "epoch": 1.1661481020535158, + "grad_norm": 3.0135788917541504, + "learning_rate": 3.4294280442804427e-06, + "loss": 1.525, + "step": 937 + }, + { + "epoch": 1.167392657125078, + "grad_norm": 2.796355962753296, + "learning_rate": 3.4271217712177125e-06, + "loss": 1.4539, + "step": 938 + }, + { + "epoch": 1.1686372121966397, + "grad_norm": 2.9422309398651123, + "learning_rate": 3.424815498154982e-06, + "loss": 1.4664, + "step": 939 + }, + { + "epoch": 1.1698817672682016, + "grad_norm": 2.7680039405822754, + "learning_rate": 3.4225092250922514e-06, + "loss": 1.4837, + "step": 940 + }, + { + "epoch": 1.1698817672682016, + "eval_loss": 1.660908818244934, + "eval_runtime": 45.6998, + "eval_samples_per_second": 21.882, + "eval_steps_per_second": 0.919, + "step": 940 + }, + { + "epoch": 1.1711263223397634, + "grad_norm": 2.8888845443725586, + "learning_rate": 3.4202029520295204e-06, + "loss": 1.5315, + "step": 941 + }, + { + "epoch": 1.1723708774113255, + "grad_norm": 2.7124130725860596, + "learning_rate": 3.41789667896679e-06, + "loss": 1.4188, + "step": 942 + }, + { + "epoch": 1.1736154324828874, + "grad_norm": 2.75203013420105, + "learning_rate": 3.4155904059040596e-06, + "loss": 1.4533, + "step": 943 + }, + { + "epoch": 1.1748599875544492, + "grad_norm": 2.865480422973633, + "learning_rate": 3.4132841328413286e-06, + "loss": 1.4532, + "step": 944 + }, + { + "epoch": 1.1761045426260113, + "grad_norm": 2.8618321418762207, + "learning_rate": 3.410977859778598e-06, + "loss": 1.4386, + "step": 945 + }, + { + "epoch": 1.1773490976975731, + "grad_norm": 2.7656145095825195, + "learning_rate": 3.4086715867158675e-06, + "loss": 1.3996, + "step": 946 + }, + { + "epoch": 1.178593652769135, + "grad_norm": 2.76213002204895, + "learning_rate": 3.4063653136531365e-06, + "loss": 1.4979, + "step": 947 + }, + { + "epoch": 1.179838207840697, + "grad_norm": 2.8581957817077637, + "learning_rate": 3.4040590405904063e-06, + "loss": 1.4673, + "step": 948 + }, + { + "epoch": 1.1810827629122589, + "grad_norm": 2.7825586795806885, + "learning_rate": 3.4017527675276758e-06, + "loss": 1.4453, + "step": 949 + }, + { + "epoch": 1.1823273179838207, + "grad_norm": 2.9226131439208984, + "learning_rate": 3.3994464944649448e-06, + "loss": 1.4879, + "step": 950 + }, + { + "epoch": 1.1823273179838207, + "eval_loss": 1.6621263027191162, + "eval_runtime": 45.1222, + "eval_samples_per_second": 22.162, + "eval_steps_per_second": 0.931, + "step": 950 + }, + { + "epoch": 1.1835718730553828, + "grad_norm": 2.9997975826263428, + "learning_rate": 3.397140221402214e-06, + "loss": 1.505, + "step": 951 + }, + { + "epoch": 1.1848164281269447, + "grad_norm": 2.7225453853607178, + "learning_rate": 3.3948339483394836e-06, + "loss": 1.4695, + "step": 952 + }, + { + "epoch": 1.1860609831985065, + "grad_norm": 2.916473150253296, + "learning_rate": 3.3925276752767534e-06, + "loss": 1.4901, + "step": 953 + }, + { + "epoch": 1.1873055382700684, + "grad_norm": 2.8371644020080566, + "learning_rate": 3.3902214022140224e-06, + "loss": 1.4944, + "step": 954 + }, + { + "epoch": 1.1885500933416304, + "grad_norm": 2.813662528991699, + "learning_rate": 3.387915129151292e-06, + "loss": 1.4603, + "step": 955 + }, + { + "epoch": 1.1897946484131923, + "grad_norm": 2.991931676864624, + "learning_rate": 3.3856088560885613e-06, + "loss": 1.5496, + "step": 956 + }, + { + "epoch": 1.1910392034847541, + "grad_norm": 2.6252501010894775, + "learning_rate": 3.3833025830258303e-06, + "loss": 1.4405, + "step": 957 + }, + { + "epoch": 1.1922837585563162, + "grad_norm": 2.747725009918213, + "learning_rate": 3.3809963099630997e-06, + "loss": 1.4689, + "step": 958 + }, + { + "epoch": 1.193528313627878, + "grad_norm": 2.887763261795044, + "learning_rate": 3.3786900369003696e-06, + "loss": 1.4501, + "step": 959 + }, + { + "epoch": 1.1947728686994399, + "grad_norm": 2.9954354763031006, + "learning_rate": 3.3763837638376386e-06, + "loss": 1.4978, + "step": 960 + }, + { + "epoch": 1.1947728686994399, + "eval_loss": 1.664442539215088, + "eval_runtime": 49.1266, + "eval_samples_per_second": 20.356, + "eval_steps_per_second": 0.855, + "step": 960 + }, + { + "epoch": 1.196017423771002, + "grad_norm": 3.0174152851104736, + "learning_rate": 3.374077490774908e-06, + "loss": 1.4353, + "step": 961 + }, + { + "epoch": 1.1972619788425638, + "grad_norm": 2.9082882404327393, + "learning_rate": 3.3717712177121774e-06, + "loss": 1.4878, + "step": 962 + }, + { + "epoch": 1.1985065339141256, + "grad_norm": 2.897993326187134, + "learning_rate": 3.3694649446494464e-06, + "loss": 1.4975, + "step": 963 + }, + { + "epoch": 1.1997510889856877, + "grad_norm": 2.739470958709717, + "learning_rate": 3.3671586715867163e-06, + "loss": 1.4202, + "step": 964 + }, + { + "epoch": 1.2009956440572496, + "grad_norm": 2.817786693572998, + "learning_rate": 3.3648523985239857e-06, + "loss": 1.5195, + "step": 965 + }, + { + "epoch": 1.2022401991288114, + "grad_norm": 2.8476386070251465, + "learning_rate": 3.362546125461255e-06, + "loss": 1.444, + "step": 966 + }, + { + "epoch": 1.2034847542003733, + "grad_norm": 2.804047107696533, + "learning_rate": 3.360239852398524e-06, + "loss": 1.4442, + "step": 967 + }, + { + "epoch": 1.2047293092719353, + "grad_norm": 2.9285430908203125, + "learning_rate": 3.3579335793357935e-06, + "loss": 1.4877, + "step": 968 + }, + { + "epoch": 1.2059738643434972, + "grad_norm": 2.8814878463745117, + "learning_rate": 3.3556273062730634e-06, + "loss": 1.5078, + "step": 969 + }, + { + "epoch": 1.207218419415059, + "grad_norm": 2.970728635787964, + "learning_rate": 3.3533210332103324e-06, + "loss": 1.4166, + "step": 970 + }, + { + "epoch": 1.207218419415059, + "eval_loss": 1.6635315418243408, + "eval_runtime": 47.2321, + "eval_samples_per_second": 21.172, + "eval_steps_per_second": 0.889, + "step": 970 + }, + { + "epoch": 1.208462974486621, + "grad_norm": 2.9836292266845703, + "learning_rate": 3.3510147601476018e-06, + "loss": 1.4777, + "step": 971 + }, + { + "epoch": 1.209707529558183, + "grad_norm": 2.8495216369628906, + "learning_rate": 3.348708487084871e-06, + "loss": 1.4829, + "step": 972 + }, + { + "epoch": 1.2109520846297448, + "grad_norm": 2.7709312438964844, + "learning_rate": 3.34640221402214e-06, + "loss": 1.4242, + "step": 973 + }, + { + "epoch": 1.2121966397013069, + "grad_norm": 2.999462366104126, + "learning_rate": 3.3440959409594096e-06, + "loss": 1.4148, + "step": 974 + }, + { + "epoch": 1.2134411947728687, + "grad_norm": 2.8719074726104736, + "learning_rate": 3.3417896678966795e-06, + "loss": 1.4196, + "step": 975 + }, + { + "epoch": 1.2146857498444306, + "grad_norm": 2.8469338417053223, + "learning_rate": 3.3394833948339485e-06, + "loss": 1.4941, + "step": 976 + }, + { + "epoch": 1.2159303049159926, + "grad_norm": 2.9780938625335693, + "learning_rate": 3.337177121771218e-06, + "loss": 1.513, + "step": 977 + }, + { + "epoch": 1.2171748599875545, + "grad_norm": 2.814760684967041, + "learning_rate": 3.3348708487084873e-06, + "loss": 1.4204, + "step": 978 + }, + { + "epoch": 1.2184194150591163, + "grad_norm": 3.016261339187622, + "learning_rate": 3.332564575645757e-06, + "loss": 1.4389, + "step": 979 + }, + { + "epoch": 1.2196639701306782, + "grad_norm": 2.9385855197906494, + "learning_rate": 3.330258302583026e-06, + "loss": 1.4885, + "step": 980 + }, + { + "epoch": 1.2196639701306782, + "eval_loss": 1.6688873767852783, + "eval_runtime": 46.2942, + "eval_samples_per_second": 21.601, + "eval_steps_per_second": 0.907, + "step": 980 + }, + { + "epoch": 1.2209085252022402, + "grad_norm": 3.1434757709503174, + "learning_rate": 3.3279520295202956e-06, + "loss": 1.5157, + "step": 981 + }, + { + "epoch": 1.222153080273802, + "grad_norm": 3.0104827880859375, + "learning_rate": 3.325645756457565e-06, + "loss": 1.49, + "step": 982 + }, + { + "epoch": 1.223397635345364, + "grad_norm": 3.1913645267486572, + "learning_rate": 3.323339483394834e-06, + "loss": 1.4937, + "step": 983 + }, + { + "epoch": 1.224642190416926, + "grad_norm": 3.19496750831604, + "learning_rate": 3.3210332103321034e-06, + "loss": 1.5197, + "step": 984 + }, + { + "epoch": 1.2258867454884879, + "grad_norm": 3.046232223510742, + "learning_rate": 3.3187269372693733e-06, + "loss": 1.4497, + "step": 985 + }, + { + "epoch": 1.2271313005600497, + "grad_norm": 2.850675344467163, + "learning_rate": 3.3164206642066423e-06, + "loss": 1.4595, + "step": 986 + }, + { + "epoch": 1.2283758556316118, + "grad_norm": 2.7744576930999756, + "learning_rate": 3.3141143911439117e-06, + "loss": 1.4726, + "step": 987 + }, + { + "epoch": 1.2296204107031736, + "grad_norm": 3.017153024673462, + "learning_rate": 3.311808118081181e-06, + "loss": 1.493, + "step": 988 + }, + { + "epoch": 1.2308649657747355, + "grad_norm": 2.8821120262145996, + "learning_rate": 3.3095018450184506e-06, + "loss": 1.4734, + "step": 989 + }, + { + "epoch": 1.2321095208462975, + "grad_norm": 2.8805840015411377, + "learning_rate": 3.3071955719557196e-06, + "loss": 1.4562, + "step": 990 + }, + { + "epoch": 1.2321095208462975, + "eval_loss": 1.669345736503601, + "eval_runtime": 47.0893, + "eval_samples_per_second": 21.236, + "eval_steps_per_second": 0.892, + "step": 990 + }, + { + "epoch": 1.2333540759178594, + "grad_norm": 2.7599008083343506, + "learning_rate": 3.3048892988929894e-06, + "loss": 1.4218, + "step": 991 + }, + { + "epoch": 1.2345986309894212, + "grad_norm": 2.9835240840911865, + "learning_rate": 3.302583025830259e-06, + "loss": 1.4596, + "step": 992 + }, + { + "epoch": 1.235843186060983, + "grad_norm": 2.843358039855957, + "learning_rate": 3.300276752767528e-06, + "loss": 1.4561, + "step": 993 + }, + { + "epoch": 1.2370877411325452, + "grad_norm": 2.7386093139648438, + "learning_rate": 3.2979704797047972e-06, + "loss": 1.4475, + "step": 994 + }, + { + "epoch": 1.238332296204107, + "grad_norm": 2.668161392211914, + "learning_rate": 3.2956642066420667e-06, + "loss": 1.4661, + "step": 995 + }, + { + "epoch": 1.2395768512756689, + "grad_norm": 2.947235107421875, + "learning_rate": 3.2933579335793357e-06, + "loss": 1.4358, + "step": 996 + }, + { + "epoch": 1.240821406347231, + "grad_norm": 2.8159878253936768, + "learning_rate": 3.2910516605166055e-06, + "loss": 1.4432, + "step": 997 + }, + { + "epoch": 1.2420659614187928, + "grad_norm": 2.896597385406494, + "learning_rate": 3.288745387453875e-06, + "loss": 1.5585, + "step": 998 + }, + { + "epoch": 1.2433105164903546, + "grad_norm": 2.80241322517395, + "learning_rate": 3.286439114391144e-06, + "loss": 1.5076, + "step": 999 + }, + { + "epoch": 1.2445550715619167, + "grad_norm": 3.008939027786255, + "learning_rate": 3.2841328413284134e-06, + "loss": 1.4234, + "step": 1000 + }, + { + "epoch": 1.2445550715619167, + "eval_loss": 1.6621588468551636, + "eval_runtime": 50.4926, + "eval_samples_per_second": 19.805, + "eval_steps_per_second": 0.832, + "step": 1000 + }, + { + "epoch": 1.2457996266334785, + "grad_norm": 2.9872381687164307, + "learning_rate": 3.281826568265683e-06, + "loss": 1.453, + "step": 1001 + }, + { + "epoch": 1.2470441817050404, + "grad_norm": 2.704676628112793, + "learning_rate": 3.2795202952029526e-06, + "loss": 1.4711, + "step": 1002 + }, + { + "epoch": 1.2482887367766025, + "grad_norm": 2.8100757598876953, + "learning_rate": 3.2772140221402216e-06, + "loss": 1.3884, + "step": 1003 + }, + { + "epoch": 1.2495332918481643, + "grad_norm": 2.902953624725342, + "learning_rate": 3.274907749077491e-06, + "loss": 1.4986, + "step": 1004 + }, + { + "epoch": 1.2507778469197262, + "grad_norm": 2.980163097381592, + "learning_rate": 3.2726014760147605e-06, + "loss": 1.5067, + "step": 1005 + }, + { + "epoch": 1.252022401991288, + "grad_norm": 2.8820345401763916, + "learning_rate": 3.2702952029520295e-06, + "loss": 1.4995, + "step": 1006 + }, + { + "epoch": 1.25326695706285, + "grad_norm": 2.8219635486602783, + "learning_rate": 3.2679889298892993e-06, + "loss": 1.4881, + "step": 1007 + }, + { + "epoch": 1.254511512134412, + "grad_norm": 2.8593199253082275, + "learning_rate": 3.2656826568265687e-06, + "loss": 1.4469, + "step": 1008 + }, + { + "epoch": 1.255756067205974, + "grad_norm": 2.888357639312744, + "learning_rate": 3.2633763837638377e-06, + "loss": 1.4869, + "step": 1009 + }, + { + "epoch": 1.2570006222775358, + "grad_norm": 2.9985013008117676, + "learning_rate": 3.261070110701107e-06, + "loss": 1.4724, + "step": 1010 + }, + { + "epoch": 1.2570006222775358, + "eval_loss": 1.6492141485214233, + "eval_runtime": 45.2197, + "eval_samples_per_second": 22.114, + "eval_steps_per_second": 0.929, + "step": 1010 + }, + { + "epoch": 1.2582451773490977, + "grad_norm": 2.860780715942383, + "learning_rate": 3.2587638376383766e-06, + "loss": 1.4618, + "step": 1011 + }, + { + "epoch": 1.2594897324206595, + "grad_norm": 2.766479730606079, + "learning_rate": 3.2564575645756456e-06, + "loss": 1.4642, + "step": 1012 + }, + { + "epoch": 1.2607342874922216, + "grad_norm": 2.711759328842163, + "learning_rate": 3.2541512915129154e-06, + "loss": 1.4283, + "step": 1013 + }, + { + "epoch": 1.2619788425637835, + "grad_norm": 2.8365769386291504, + "learning_rate": 3.251845018450185e-06, + "loss": 1.4248, + "step": 1014 + }, + { + "epoch": 1.2632233976353453, + "grad_norm": 2.957620143890381, + "learning_rate": 3.2495387453874543e-06, + "loss": 1.4662, + "step": 1015 + }, + { + "epoch": 1.2644679527069074, + "grad_norm": 2.8066208362579346, + "learning_rate": 3.2472324723247233e-06, + "loss": 1.4447, + "step": 1016 + }, + { + "epoch": 1.2657125077784692, + "grad_norm": 2.902040958404541, + "learning_rate": 3.244926199261993e-06, + "loss": 1.4743, + "step": 1017 + }, + { + "epoch": 1.266957062850031, + "grad_norm": 2.7636687755584717, + "learning_rate": 3.2426199261992625e-06, + "loss": 1.4465, + "step": 1018 + }, + { + "epoch": 1.268201617921593, + "grad_norm": 2.95261549949646, + "learning_rate": 3.2403136531365315e-06, + "loss": 1.4769, + "step": 1019 + }, + { + "epoch": 1.269446172993155, + "grad_norm": 2.911128520965576, + "learning_rate": 3.238007380073801e-06, + "loss": 1.3843, + "step": 1020 + }, + { + "epoch": 1.269446172993155, + "eval_loss": 1.6526964902877808, + "eval_runtime": 48.2126, + "eval_samples_per_second": 20.741, + "eval_steps_per_second": 0.871, + "step": 1020 + }, + { + "epoch": 1.2706907280647168, + "grad_norm": 2.769951820373535, + "learning_rate": 3.2357011070110704e-06, + "loss": 1.4847, + "step": 1021 + }, + { + "epoch": 1.271935283136279, + "grad_norm": 3.0451674461364746, + "learning_rate": 3.2333948339483394e-06, + "loss": 1.4453, + "step": 1022 + }, + { + "epoch": 1.2731798382078408, + "grad_norm": 2.9543521404266357, + "learning_rate": 3.2310885608856092e-06, + "loss": 1.4789, + "step": 1023 + }, + { + "epoch": 1.2744243932794026, + "grad_norm": 2.9174394607543945, + "learning_rate": 3.2287822878228787e-06, + "loss": 1.4807, + "step": 1024 + }, + { + "epoch": 1.2756689483509644, + "grad_norm": 2.888258934020996, + "learning_rate": 3.2264760147601477e-06, + "loss": 1.4367, + "step": 1025 + }, + { + "epoch": 1.2769135034225265, + "grad_norm": 2.8346047401428223, + "learning_rate": 3.224169741697417e-06, + "loss": 1.4313, + "step": 1026 + }, + { + "epoch": 1.2781580584940884, + "grad_norm": 2.7531332969665527, + "learning_rate": 3.2218634686346865e-06, + "loss": 1.4018, + "step": 1027 + }, + { + "epoch": 1.2794026135656502, + "grad_norm": 3.136946439743042, + "learning_rate": 3.2195571955719564e-06, + "loss": 1.4417, + "step": 1028 + }, + { + "epoch": 1.2806471686372123, + "grad_norm": 2.9618165493011475, + "learning_rate": 3.2172509225092254e-06, + "loss": 1.5117, + "step": 1029 + }, + { + "epoch": 1.2818917237087741, + "grad_norm": 3.0740747451782227, + "learning_rate": 3.2149446494464948e-06, + "loss": 1.4641, + "step": 1030 + }, + { + "epoch": 1.2818917237087741, + "eval_loss": 1.6572761535644531, + "eval_runtime": 51.6696, + "eval_samples_per_second": 19.354, + "eval_steps_per_second": 0.813, + "step": 1030 + }, + { + "epoch": 1.283136278780336, + "grad_norm": 2.9225106239318848, + "learning_rate": 3.212638376383764e-06, + "loss": 1.4421, + "step": 1031 + }, + { + "epoch": 1.2843808338518978, + "grad_norm": 2.9530649185180664, + "learning_rate": 3.210332103321033e-06, + "loss": 1.5138, + "step": 1032 + }, + { + "epoch": 1.28562538892346, + "grad_norm": 2.8513238430023193, + "learning_rate": 3.208025830258303e-06, + "loss": 1.411, + "step": 1033 + }, + { + "epoch": 1.2868699439950217, + "grad_norm": 2.7440171241760254, + "learning_rate": 3.2057195571955725e-06, + "loss": 1.4182, + "step": 1034 + }, + { + "epoch": 1.2881144990665838, + "grad_norm": 2.8126630783081055, + "learning_rate": 3.2034132841328415e-06, + "loss": 1.4333, + "step": 1035 + }, + { + "epoch": 1.2893590541381457, + "grad_norm": 2.8598744869232178, + "learning_rate": 3.201107011070111e-06, + "loss": 1.439, + "step": 1036 + }, + { + "epoch": 1.2906036092097075, + "grad_norm": 2.901622772216797, + "learning_rate": 3.1988007380073803e-06, + "loss": 1.4461, + "step": 1037 + }, + { + "epoch": 1.2918481642812694, + "grad_norm": 2.7670180797576904, + "learning_rate": 3.1964944649446493e-06, + "loss": 1.4772, + "step": 1038 + }, + { + "epoch": 1.2930927193528314, + "grad_norm": 2.7860350608825684, + "learning_rate": 3.194188191881919e-06, + "loss": 1.4414, + "step": 1039 + }, + { + "epoch": 1.2943372744243933, + "grad_norm": 2.822052478790283, + "learning_rate": 3.1918819188191886e-06, + "loss": 1.4192, + "step": 1040 + }, + { + "epoch": 1.2943372744243933, + "eval_loss": 1.663284420967102, + "eval_runtime": 47.2822, + "eval_samples_per_second": 21.15, + "eval_steps_per_second": 0.888, + "step": 1040 + }, + { + "epoch": 1.2955818294959551, + "grad_norm": 2.909109115600586, + "learning_rate": 3.189575645756458e-06, + "loss": 1.4795, + "step": 1041 + }, + { + "epoch": 1.2968263845675172, + "grad_norm": 2.933673143386841, + "learning_rate": 3.187269372693727e-06, + "loss": 1.5038, + "step": 1042 + }, + { + "epoch": 1.298070939639079, + "grad_norm": 3.032518148422241, + "learning_rate": 3.1849630996309964e-06, + "loss": 1.5133, + "step": 1043 + }, + { + "epoch": 1.299315494710641, + "grad_norm": 2.826040506362915, + "learning_rate": 3.1826568265682663e-06, + "loss": 1.4395, + "step": 1044 + }, + { + "epoch": 1.3005600497822027, + "grad_norm": 2.8152658939361572, + "learning_rate": 3.1803505535055353e-06, + "loss": 1.4787, + "step": 1045 + }, + { + "epoch": 1.3018046048537648, + "grad_norm": 2.9410431385040283, + "learning_rate": 3.1780442804428047e-06, + "loss": 1.4375, + "step": 1046 + }, + { + "epoch": 1.3030491599253267, + "grad_norm": 2.948636531829834, + "learning_rate": 3.175738007380074e-06, + "loss": 1.4847, + "step": 1047 + }, + { + "epoch": 1.3042937149968887, + "grad_norm": 2.6287615299224854, + "learning_rate": 3.173431734317343e-06, + "loss": 1.4186, + "step": 1048 + }, + { + "epoch": 1.3055382700684506, + "grad_norm": 2.836029052734375, + "learning_rate": 3.1711254612546125e-06, + "loss": 1.4251, + "step": 1049 + }, + { + "epoch": 1.3067828251400124, + "grad_norm": 2.982081651687622, + "learning_rate": 3.1688191881918824e-06, + "loss": 1.4951, + "step": 1050 + }, + { + "epoch": 1.3067828251400124, + "eval_loss": 1.6578067541122437, + "eval_runtime": 47.5847, + "eval_samples_per_second": 21.015, + "eval_steps_per_second": 0.883, + "step": 1050 + }, + { + "epoch": 1.3080273802115743, + "grad_norm": 2.9297118186950684, + "learning_rate": 3.166512915129152e-06, + "loss": 1.4699, + "step": 1051 + }, + { + "epoch": 1.3092719352831363, + "grad_norm": 2.906435251235962, + "learning_rate": 3.164206642066421e-06, + "loss": 1.4562, + "step": 1052 + }, + { + "epoch": 1.3105164903546982, + "grad_norm": 2.8655896186828613, + "learning_rate": 3.1619003690036902e-06, + "loss": 1.4351, + "step": 1053 + }, + { + "epoch": 1.31176104542626, + "grad_norm": 2.779244899749756, + "learning_rate": 3.15959409594096e-06, + "loss": 1.4479, + "step": 1054 + }, + { + "epoch": 1.3130056004978221, + "grad_norm": 2.9863739013671875, + "learning_rate": 3.157287822878229e-06, + "loss": 1.4964, + "step": 1055 + }, + { + "epoch": 1.314250155569384, + "grad_norm": 2.821871042251587, + "learning_rate": 3.1549815498154985e-06, + "loss": 1.4173, + "step": 1056 + }, + { + "epoch": 1.3154947106409458, + "grad_norm": 2.8573882579803467, + "learning_rate": 3.152675276752768e-06, + "loss": 1.4877, + "step": 1057 + }, + { + "epoch": 1.3167392657125077, + "grad_norm": 2.9824137687683105, + "learning_rate": 3.150369003690037e-06, + "loss": 1.5124, + "step": 1058 + }, + { + "epoch": 1.3179838207840697, + "grad_norm": 2.881098747253418, + "learning_rate": 3.1480627306273063e-06, + "loss": 1.4346, + "step": 1059 + }, + { + "epoch": 1.3192283758556316, + "grad_norm": 2.918782949447632, + "learning_rate": 3.145756457564576e-06, + "loss": 1.4797, + "step": 1060 + }, + { + "epoch": 1.3192283758556316, + "eval_loss": 1.6510179042816162, + "eval_runtime": 53.6356, + "eval_samples_per_second": 18.644, + "eval_steps_per_second": 0.783, + "step": 1060 + }, + { + "epoch": 1.3204729309271936, + "grad_norm": 3.045776128768921, + "learning_rate": 3.143450184501845e-06, + "loss": 1.5039, + "step": 1061 + }, + { + "epoch": 1.3217174859987555, + "grad_norm": 3.041501760482788, + "learning_rate": 3.1411439114391146e-06, + "loss": 1.4878, + "step": 1062 + }, + { + "epoch": 1.3229620410703173, + "grad_norm": 2.917837142944336, + "learning_rate": 3.138837638376384e-06, + "loss": 1.4202, + "step": 1063 + }, + { + "epoch": 1.3242065961418792, + "grad_norm": 2.708070755004883, + "learning_rate": 3.1365313653136535e-06, + "loss": 1.4071, + "step": 1064 + }, + { + "epoch": 1.3254511512134413, + "grad_norm": 2.9146156311035156, + "learning_rate": 3.1342250922509225e-06, + "loss": 1.4428, + "step": 1065 + }, + { + "epoch": 1.326695706285003, + "grad_norm": 3.220919132232666, + "learning_rate": 3.1319188191881923e-06, + "loss": 1.4733, + "step": 1066 + }, + { + "epoch": 1.327940261356565, + "grad_norm": 2.9127554893493652, + "learning_rate": 3.1296125461254617e-06, + "loss": 1.4554, + "step": 1067 + }, + { + "epoch": 1.329184816428127, + "grad_norm": 3.053191661834717, + "learning_rate": 3.1273062730627307e-06, + "loss": 1.4044, + "step": 1068 + }, + { + "epoch": 1.3304293714996889, + "grad_norm": 3.2937021255493164, + "learning_rate": 3.125e-06, + "loss": 1.4949, + "step": 1069 + }, + { + "epoch": 1.3316739265712507, + "grad_norm": 2.976717948913574, + "learning_rate": 3.12269372693727e-06, + "loss": 1.446, + "step": 1070 + }, + { + "epoch": 1.3316739265712507, + "eval_loss": 1.6503105163574219, + "eval_runtime": 50.7579, + "eval_samples_per_second": 19.701, + "eval_steps_per_second": 0.827, + "step": 1070 + }, + { + "epoch": 1.3329184816428126, + "grad_norm": 3.0860936641693115, + "learning_rate": 3.120387453874539e-06, + "loss": 1.4055, + "step": 1071 + }, + { + "epoch": 1.3341630367143746, + "grad_norm": 3.0401217937469482, + "learning_rate": 3.1180811808118084e-06, + "loss": 1.4688, + "step": 1072 + }, + { + "epoch": 1.3354075917859365, + "grad_norm": 2.953253984451294, + "learning_rate": 3.115774907749078e-06, + "loss": 1.5001, + "step": 1073 + }, + { + "epoch": 1.3366521468574986, + "grad_norm": 3.0665059089660645, + "learning_rate": 3.113468634686347e-06, + "loss": 1.4217, + "step": 1074 + }, + { + "epoch": 1.3378967019290604, + "grad_norm": 3.121600389480591, + "learning_rate": 3.1111623616236163e-06, + "loss": 1.4508, + "step": 1075 + }, + { + "epoch": 1.3391412570006223, + "grad_norm": 2.8057339191436768, + "learning_rate": 3.108856088560886e-06, + "loss": 1.4241, + "step": 1076 + }, + { + "epoch": 1.340385812072184, + "grad_norm": 3.0147411823272705, + "learning_rate": 3.1065498154981555e-06, + "loss": 1.479, + "step": 1077 + }, + { + "epoch": 1.3416303671437462, + "grad_norm": 3.030010223388672, + "learning_rate": 3.1042435424354245e-06, + "loss": 1.4861, + "step": 1078 + }, + { + "epoch": 1.342874922215308, + "grad_norm": 3.027998208999634, + "learning_rate": 3.101937269372694e-06, + "loss": 1.4916, + "step": 1079 + }, + { + "epoch": 1.3441194772868699, + "grad_norm": 2.9252536296844482, + "learning_rate": 3.0996309963099634e-06, + "loss": 1.418, + "step": 1080 + }, + { + "epoch": 1.3441194772868699, + "eval_loss": 1.6490333080291748, + "eval_runtime": 47.4026, + "eval_samples_per_second": 21.096, + "eval_steps_per_second": 0.886, + "step": 1080 + }, + { + "epoch": 1.345364032358432, + "grad_norm": 3.110847234725952, + "learning_rate": 3.0973247232472324e-06, + "loss": 1.4127, + "step": 1081 + }, + { + "epoch": 1.3466085874299938, + "grad_norm": 2.846491575241089, + "learning_rate": 3.0950184501845022e-06, + "loss": 1.444, + "step": 1082 + }, + { + "epoch": 1.3478531425015556, + "grad_norm": 2.950533151626587, + "learning_rate": 3.0927121771217716e-06, + "loss": 1.4456, + "step": 1083 + }, + { + "epoch": 1.3490976975731175, + "grad_norm": 2.875964403152466, + "learning_rate": 3.0904059040590406e-06, + "loss": 1.4485, + "step": 1084 + }, + { + "epoch": 1.3503422526446796, + "grad_norm": 2.9612326622009277, + "learning_rate": 3.08809963099631e-06, + "loss": 1.4446, + "step": 1085 + }, + { + "epoch": 1.3515868077162414, + "grad_norm": 2.925297975540161, + "learning_rate": 3.0857933579335795e-06, + "loss": 1.4847, + "step": 1086 + }, + { + "epoch": 1.3528313627878035, + "grad_norm": 2.991616725921631, + "learning_rate": 3.0834870848708485e-06, + "loss": 1.5151, + "step": 1087 + }, + { + "epoch": 1.3540759178593653, + "grad_norm": 2.96329927444458, + "learning_rate": 3.0811808118081183e-06, + "loss": 1.4574, + "step": 1088 + }, + { + "epoch": 1.3553204729309272, + "grad_norm": 2.948702573776245, + "learning_rate": 3.0788745387453878e-06, + "loss": 1.4585, + "step": 1089 + }, + { + "epoch": 1.356565028002489, + "grad_norm": 3.218463659286499, + "learning_rate": 3.076568265682657e-06, + "loss": 1.4439, + "step": 1090 + }, + { + "epoch": 1.356565028002489, + "eval_loss": 1.645838975906372, + "eval_runtime": 47.048, + "eval_samples_per_second": 21.255, + "eval_steps_per_second": 0.893, + "step": 1090 + }, + { + "epoch": 1.357809583074051, + "grad_norm": 2.9799578189849854, + "learning_rate": 3.074261992619926e-06, + "loss": 1.4883, + "step": 1091 + }, + { + "epoch": 1.359054138145613, + "grad_norm": 2.7660505771636963, + "learning_rate": 3.071955719557196e-06, + "loss": 1.4349, + "step": 1092 + }, + { + "epoch": 1.3602986932171748, + "grad_norm": 3.0201916694641113, + "learning_rate": 3.0696494464944655e-06, + "loss": 1.418, + "step": 1093 + }, + { + "epoch": 1.3615432482887369, + "grad_norm": 2.844207763671875, + "learning_rate": 3.0673431734317345e-06, + "loss": 1.4769, + "step": 1094 + }, + { + "epoch": 1.3627878033602987, + "grad_norm": 2.8306753635406494, + "learning_rate": 3.065036900369004e-06, + "loss": 1.4653, + "step": 1095 + }, + { + "epoch": 1.3640323584318605, + "grad_norm": 2.805023431777954, + "learning_rate": 3.0627306273062733e-06, + "loss": 1.4771, + "step": 1096 + }, + { + "epoch": 1.3652769135034224, + "grad_norm": 3.020883083343506, + "learning_rate": 3.0604243542435423e-06, + "loss": 1.4237, + "step": 1097 + }, + { + "epoch": 1.3665214685749845, + "grad_norm": 2.8395731449127197, + "learning_rate": 3.058118081180812e-06, + "loss": 1.4647, + "step": 1098 + }, + { + "epoch": 1.3677660236465463, + "grad_norm": 2.871156692504883, + "learning_rate": 3.0558118081180816e-06, + "loss": 1.4196, + "step": 1099 + }, + { + "epoch": 1.3690105787181084, + "grad_norm": 2.974905014038086, + "learning_rate": 3.053505535055351e-06, + "loss": 1.4779, + "step": 1100 + }, + { + "epoch": 1.3690105787181084, + "eval_loss": 1.651503562927246, + "eval_runtime": 54.2713, + "eval_samples_per_second": 18.426, + "eval_steps_per_second": 0.774, + "step": 1100 + }, + { + "epoch": 1.3702551337896702, + "grad_norm": 2.7591328620910645, + "learning_rate": 3.05119926199262e-06, + "loss": 1.3899, + "step": 1101 + }, + { + "epoch": 1.371499688861232, + "grad_norm": 2.8575668334960938, + "learning_rate": 3.0488929889298894e-06, + "loss": 1.4482, + "step": 1102 + }, + { + "epoch": 1.372744243932794, + "grad_norm": 2.9049339294433594, + "learning_rate": 3.0465867158671593e-06, + "loss": 1.4785, + "step": 1103 + }, + { + "epoch": 1.373988799004356, + "grad_norm": 2.816453695297241, + "learning_rate": 3.0442804428044283e-06, + "loss": 1.4601, + "step": 1104 + }, + { + "epoch": 1.3752333540759178, + "grad_norm": 2.8844189643859863, + "learning_rate": 3.0419741697416977e-06, + "loss": 1.496, + "step": 1105 + }, + { + "epoch": 1.3764779091474797, + "grad_norm": 3.014744281768799, + "learning_rate": 3.039667896678967e-06, + "loss": 1.4098, + "step": 1106 + }, + { + "epoch": 1.3777224642190418, + "grad_norm": 2.8243560791015625, + "learning_rate": 3.037361623616236e-06, + "loss": 1.4086, + "step": 1107 + }, + { + "epoch": 1.3789670192906036, + "grad_norm": 2.8333253860473633, + "learning_rate": 3.035055350553506e-06, + "loss": 1.3516, + "step": 1108 + }, + { + "epoch": 1.3802115743621655, + "grad_norm": 2.926455020904541, + "learning_rate": 3.0327490774907754e-06, + "loss": 1.3976, + "step": 1109 + }, + { + "epoch": 1.3814561294337273, + "grad_norm": 2.900937080383301, + "learning_rate": 3.0304428044280444e-06, + "loss": 1.4336, + "step": 1110 + }, + { + "epoch": 1.3814561294337273, + "eval_loss": 1.647048830986023, + "eval_runtime": 42.3955, + "eval_samples_per_second": 23.587, + "eval_steps_per_second": 0.991, + "step": 1110 + }, + { + "epoch": 1.3827006845052894, + "grad_norm": 3.0756969451904297, + "learning_rate": 3.028136531365314e-06, + "loss": 1.4674, + "step": 1111 + }, + { + "epoch": 1.3839452395768512, + "grad_norm": 2.7754578590393066, + "learning_rate": 3.0258302583025832e-06, + "loss": 1.4435, + "step": 1112 + }, + { + "epoch": 1.3851897946484133, + "grad_norm": 2.8011393547058105, + "learning_rate": 3.023523985239853e-06, + "loss": 1.4212, + "step": 1113 + }, + { + "epoch": 1.3864343497199751, + "grad_norm": 2.7063710689544678, + "learning_rate": 3.021217712177122e-06, + "loss": 1.3769, + "step": 1114 + }, + { + "epoch": 1.387678904791537, + "grad_norm": 2.830411434173584, + "learning_rate": 3.0189114391143915e-06, + "loss": 1.4832, + "step": 1115 + }, + { + "epoch": 1.3889234598630988, + "grad_norm": 2.9075911045074463, + "learning_rate": 3.016605166051661e-06, + "loss": 1.486, + "step": 1116 + }, + { + "epoch": 1.390168014934661, + "grad_norm": 2.847822666168213, + "learning_rate": 3.01429889298893e-06, + "loss": 1.424, + "step": 1117 + }, + { + "epoch": 1.3914125700062228, + "grad_norm": 2.8822033405303955, + "learning_rate": 3.0119926199261993e-06, + "loss": 1.4218, + "step": 1118 + }, + { + "epoch": 1.3926571250777848, + "grad_norm": 2.8944740295410156, + "learning_rate": 3.009686346863469e-06, + "loss": 1.4283, + "step": 1119 + }, + { + "epoch": 1.3939016801493467, + "grad_norm": 2.765190839767456, + "learning_rate": 3.007380073800738e-06, + "loss": 1.4622, + "step": 1120 + }, + { + "epoch": 1.3939016801493467, + "eval_loss": 1.6451724767684937, + "eval_runtime": 43.7449, + "eval_samples_per_second": 22.86, + "eval_steps_per_second": 0.96, + "step": 1120 + }, + { + "epoch": 1.3951462352209085, + "grad_norm": 2.877002716064453, + "learning_rate": 3.0050738007380076e-06, + "loss": 1.4145, + "step": 1121 + }, + { + "epoch": 1.3963907902924704, + "grad_norm": 2.870389699935913, + "learning_rate": 3.002767527675277e-06, + "loss": 1.4401, + "step": 1122 + }, + { + "epoch": 1.3976353453640322, + "grad_norm": 2.7788920402526855, + "learning_rate": 3.000461254612546e-06, + "loss": 1.4514, + "step": 1123 + }, + { + "epoch": 1.3988799004355943, + "grad_norm": 2.9269332885742188, + "learning_rate": 2.998154981549816e-06, + "loss": 1.3961, + "step": 1124 + }, + { + "epoch": 1.4001244555071561, + "grad_norm": 3.013291835784912, + "learning_rate": 2.9958487084870853e-06, + "loss": 1.4597, + "step": 1125 + }, + { + "epoch": 1.4013690105787182, + "grad_norm": 2.9063880443573, + "learning_rate": 2.9935424354243547e-06, + "loss": 1.4668, + "step": 1126 + }, + { + "epoch": 1.40261356565028, + "grad_norm": 2.8926970958709717, + "learning_rate": 2.9912361623616237e-06, + "loss": 1.4332, + "step": 1127 + }, + { + "epoch": 1.403858120721842, + "grad_norm": 2.8563647270202637, + "learning_rate": 2.988929889298893e-06, + "loss": 1.407, + "step": 1128 + }, + { + "epoch": 1.4051026757934038, + "grad_norm": 2.8725359439849854, + "learning_rate": 2.986623616236163e-06, + "loss": 1.4434, + "step": 1129 + }, + { + "epoch": 1.4063472308649658, + "grad_norm": 2.906898260116577, + "learning_rate": 2.984317343173432e-06, + "loss": 1.4745, + "step": 1130 + }, + { + "epoch": 1.4063472308649658, + "eval_loss": 1.647884726524353, + "eval_runtime": 44.9221, + "eval_samples_per_second": 22.261, + "eval_steps_per_second": 0.935, + "step": 1130 + }, + { + "epoch": 1.4075917859365277, + "grad_norm": 2.8297719955444336, + "learning_rate": 2.9820110701107014e-06, + "loss": 1.4542, + "step": 1131 + }, + { + "epoch": 1.4088363410080897, + "grad_norm": 2.841031551361084, + "learning_rate": 2.979704797047971e-06, + "loss": 1.4105, + "step": 1132 + }, + { + "epoch": 1.4100808960796516, + "grad_norm": 2.867791175842285, + "learning_rate": 2.97739852398524e-06, + "loss": 1.4409, + "step": 1133 + }, + { + "epoch": 1.4113254511512134, + "grad_norm": 2.8156039714813232, + "learning_rate": 2.9750922509225093e-06, + "loss": 1.4284, + "step": 1134 + }, + { + "epoch": 1.4125700062227753, + "grad_norm": 2.825985908508301, + "learning_rate": 2.972785977859779e-06, + "loss": 1.4594, + "step": 1135 + }, + { + "epoch": 1.4138145612943374, + "grad_norm": 3.0134711265563965, + "learning_rate": 2.970479704797048e-06, + "loss": 1.4743, + "step": 1136 + }, + { + "epoch": 1.4150591163658992, + "grad_norm": 2.9829142093658447, + "learning_rate": 2.9681734317343175e-06, + "loss": 1.4935, + "step": 1137 + }, + { + "epoch": 1.416303671437461, + "grad_norm": 3.067620277404785, + "learning_rate": 2.965867158671587e-06, + "loss": 1.4266, + "step": 1138 + }, + { + "epoch": 1.4175482265090231, + "grad_norm": 2.8969919681549072, + "learning_rate": 2.9635608856088564e-06, + "loss": 1.451, + "step": 1139 + }, + { + "epoch": 1.418792781580585, + "grad_norm": 2.8706412315368652, + "learning_rate": 2.9612546125461254e-06, + "loss": 1.4834, + "step": 1140 + }, + { + "epoch": 1.418792781580585, + "eval_loss": 1.643480658531189, + "eval_runtime": 44.743, + "eval_samples_per_second": 22.35, + "eval_steps_per_second": 0.939, + "step": 1140 + }, + { + "epoch": 1.4200373366521468, + "grad_norm": 2.8801848888397217, + "learning_rate": 2.958948339483395e-06, + "loss": 1.4258, + "step": 1141 + }, + { + "epoch": 1.4212818917237087, + "grad_norm": 2.774635076522827, + "learning_rate": 2.9566420664206646e-06, + "loss": 1.416, + "step": 1142 + }, + { + "epoch": 1.4225264467952707, + "grad_norm": 2.8460140228271484, + "learning_rate": 2.9543357933579336e-06, + "loss": 1.4213, + "step": 1143 + }, + { + "epoch": 1.4237710018668326, + "grad_norm": 2.907888889312744, + "learning_rate": 2.952029520295203e-06, + "loss": 1.4507, + "step": 1144 + }, + { + "epoch": 1.4250155569383947, + "grad_norm": 2.789041757583618, + "learning_rate": 2.949723247232473e-06, + "loss": 1.4257, + "step": 1145 + }, + { + "epoch": 1.4262601120099565, + "grad_norm": 2.78859806060791, + "learning_rate": 2.947416974169742e-06, + "loss": 1.4377, + "step": 1146 + }, + { + "epoch": 1.4275046670815184, + "grad_norm": 2.840303421020508, + "learning_rate": 2.9451107011070113e-06, + "loss": 1.4684, + "step": 1147 + }, + { + "epoch": 1.4287492221530802, + "grad_norm": 2.8800530433654785, + "learning_rate": 2.9428044280442807e-06, + "loss": 1.4912, + "step": 1148 + }, + { + "epoch": 1.4299937772246423, + "grad_norm": 2.841092824935913, + "learning_rate": 2.9404981549815497e-06, + "loss": 1.4234, + "step": 1149 + }, + { + "epoch": 1.4312383322962041, + "grad_norm": 2.8226537704467773, + "learning_rate": 2.938191881918819e-06, + "loss": 1.4578, + "step": 1150 + }, + { + "epoch": 1.4312383322962041, + "eval_loss": 1.643249273300171, + "eval_runtime": 43.5586, + "eval_samples_per_second": 22.958, + "eval_steps_per_second": 0.964, + "step": 1150 + }, + { + "epoch": 1.432482887367766, + "grad_norm": 2.869736909866333, + "learning_rate": 2.935885608856089e-06, + "loss": 1.4701, + "step": 1151 + }, + { + "epoch": 1.433727442439328, + "grad_norm": 2.689211130142212, + "learning_rate": 2.9335793357933584e-06, + "loss": 1.4483, + "step": 1152 + }, + { + "epoch": 1.4349719975108899, + "grad_norm": 2.9267077445983887, + "learning_rate": 2.9312730627306274e-06, + "loss": 1.4446, + "step": 1153 + }, + { + "epoch": 1.4362165525824517, + "grad_norm": 2.906862497329712, + "learning_rate": 2.928966789667897e-06, + "loss": 1.4524, + "step": 1154 + }, + { + "epoch": 1.4374611076540136, + "grad_norm": 2.8249800205230713, + "learning_rate": 2.9266605166051663e-06, + "loss": 1.4208, + "step": 1155 + }, + { + "epoch": 1.4387056627255757, + "grad_norm": 2.9836032390594482, + "learning_rate": 2.9243542435424353e-06, + "loss": 1.4581, + "step": 1156 + }, + { + "epoch": 1.4399502177971375, + "grad_norm": 2.9595284461975098, + "learning_rate": 2.922047970479705e-06, + "loss": 1.4141, + "step": 1157 + }, + { + "epoch": 1.4411947728686996, + "grad_norm": 3.0363423824310303, + "learning_rate": 2.9197416974169746e-06, + "loss": 1.4859, + "step": 1158 + }, + { + "epoch": 1.4424393279402614, + "grad_norm": 2.911201000213623, + "learning_rate": 2.9174354243542436e-06, + "loss": 1.4346, + "step": 1159 + }, + { + "epoch": 1.4436838830118233, + "grad_norm": 2.8617193698883057, + "learning_rate": 2.915129151291513e-06, + "loss": 1.433, + "step": 1160 + }, + { + "epoch": 1.4436838830118233, + "eval_loss": 1.6437232494354248, + "eval_runtime": 43.961, + "eval_samples_per_second": 22.747, + "eval_steps_per_second": 0.955, + "step": 1160 + }, + { + "epoch": 1.4449284380833851, + "grad_norm": 2.9584295749664307, + "learning_rate": 2.912822878228783e-06, + "loss": 1.4847, + "step": 1161 + }, + { + "epoch": 1.4461729931549472, + "grad_norm": 2.8209400177001953, + "learning_rate": 2.9105166051660522e-06, + "loss": 1.421, + "step": 1162 + }, + { + "epoch": 1.447417548226509, + "grad_norm": 2.847637414932251, + "learning_rate": 2.9082103321033212e-06, + "loss": 1.4326, + "step": 1163 + }, + { + "epoch": 1.4486621032980709, + "grad_norm": 2.836228132247925, + "learning_rate": 2.9059040590405907e-06, + "loss": 1.4459, + "step": 1164 + }, + { + "epoch": 1.449906658369633, + "grad_norm": 2.8327207565307617, + "learning_rate": 2.90359778597786e-06, + "loss": 1.4189, + "step": 1165 + }, + { + "epoch": 1.4511512134411948, + "grad_norm": 2.880643606185913, + "learning_rate": 2.901291512915129e-06, + "loss": 1.4214, + "step": 1166 + }, + { + "epoch": 1.4523957685127566, + "grad_norm": 2.9637348651885986, + "learning_rate": 2.898985239852399e-06, + "loss": 1.4471, + "step": 1167 + }, + { + "epoch": 1.4536403235843185, + "grad_norm": 3.006145477294922, + "learning_rate": 2.8966789667896684e-06, + "loss": 1.4228, + "step": 1168 + }, + { + "epoch": 1.4548848786558806, + "grad_norm": 2.8547627925872803, + "learning_rate": 2.8943726937269374e-06, + "loss": 1.4707, + "step": 1169 + }, + { + "epoch": 1.4561294337274424, + "grad_norm": 2.973092794418335, + "learning_rate": 2.8920664206642068e-06, + "loss": 1.4804, + "step": 1170 + }, + { + "epoch": 1.4561294337274424, + "eval_loss": 1.6340677738189697, + "eval_runtime": 44.6094, + "eval_samples_per_second": 22.417, + "eval_steps_per_second": 0.942, + "step": 1170 + }, + { + "epoch": 1.4573739887990045, + "grad_norm": 2.9433603286743164, + "learning_rate": 2.889760147601476e-06, + "loss": 1.4354, + "step": 1171 + }, + { + "epoch": 1.4586185438705663, + "grad_norm": 2.872236728668213, + "learning_rate": 2.887453874538745e-06, + "loss": 1.418, + "step": 1172 + }, + { + "epoch": 1.4598630989421282, + "grad_norm": 2.847557783126831, + "learning_rate": 2.885147601476015e-06, + "loss": 1.4306, + "step": 1173 + }, + { + "epoch": 1.46110765401369, + "grad_norm": 2.7554969787597656, + "learning_rate": 2.8828413284132845e-06, + "loss": 1.4295, + "step": 1174 + }, + { + "epoch": 1.462352209085252, + "grad_norm": 2.8754842281341553, + "learning_rate": 2.880535055350554e-06, + "loss": 1.4482, + "step": 1175 + }, + { + "epoch": 1.463596764156814, + "grad_norm": 3.0115292072296143, + "learning_rate": 2.878228782287823e-06, + "loss": 1.4428, + "step": 1176 + }, + { + "epoch": 1.4648413192283758, + "grad_norm": 2.8976168632507324, + "learning_rate": 2.8759225092250923e-06, + "loss": 1.4707, + "step": 1177 + }, + { + "epoch": 1.4660858742999379, + "grad_norm": 2.7981812953948975, + "learning_rate": 2.873616236162362e-06, + "loss": 1.4693, + "step": 1178 + }, + { + "epoch": 1.4673304293714997, + "grad_norm": 2.8562371730804443, + "learning_rate": 2.871309963099631e-06, + "loss": 1.4242, + "step": 1179 + }, + { + "epoch": 1.4685749844430616, + "grad_norm": 2.8705570697784424, + "learning_rate": 2.8690036900369006e-06, + "loss": 1.4725, + "step": 1180 + }, + { + "epoch": 1.4685749844430616, + "eval_loss": 1.6338286399841309, + "eval_runtime": 44.7171, + "eval_samples_per_second": 22.363, + "eval_steps_per_second": 0.939, + "step": 1180 + }, + { + "epoch": 1.4698195395146234, + "grad_norm": 2.8025710582733154, + "learning_rate": 2.86669741697417e-06, + "loss": 1.427, + "step": 1181 + }, + { + "epoch": 1.4710640945861855, + "grad_norm": 2.845151901245117, + "learning_rate": 2.864391143911439e-06, + "loss": 1.4402, + "step": 1182 + }, + { + "epoch": 1.4723086496577473, + "grad_norm": 2.943373680114746, + "learning_rate": 2.862084870848709e-06, + "loss": 1.5194, + "step": 1183 + }, + { + "epoch": 1.4735532047293094, + "grad_norm": 2.896955966949463, + "learning_rate": 2.8597785977859783e-06, + "loss": 1.4242, + "step": 1184 + }, + { + "epoch": 1.4747977598008712, + "grad_norm": 3.131305456161499, + "learning_rate": 2.8574723247232473e-06, + "loss": 1.5008, + "step": 1185 + }, + { + "epoch": 1.476042314872433, + "grad_norm": 2.8605735301971436, + "learning_rate": 2.8551660516605167e-06, + "loss": 1.3533, + "step": 1186 + }, + { + "epoch": 1.477286869943995, + "grad_norm": 2.9158811569213867, + "learning_rate": 2.852859778597786e-06, + "loss": 1.4404, + "step": 1187 + }, + { + "epoch": 1.478531425015557, + "grad_norm": 2.7795321941375732, + "learning_rate": 2.850553505535056e-06, + "loss": 1.3977, + "step": 1188 + }, + { + "epoch": 1.4797759800871189, + "grad_norm": 2.8194212913513184, + "learning_rate": 2.848247232472325e-06, + "loss": 1.4111, + "step": 1189 + }, + { + "epoch": 1.4810205351586807, + "grad_norm": 2.983750104904175, + "learning_rate": 2.8459409594095944e-06, + "loss": 1.4061, + "step": 1190 + }, + { + "epoch": 1.4810205351586807, + "eval_loss": 1.6375409364700317, + "eval_runtime": 45.4026, + "eval_samples_per_second": 22.025, + "eval_steps_per_second": 0.925, + "step": 1190 + }, + { + "epoch": 1.4822650902302428, + "grad_norm": 3.0246665477752686, + "learning_rate": 2.843634686346864e-06, + "loss": 1.4532, + "step": 1191 + }, + { + "epoch": 1.4835096453018046, + "grad_norm": 2.8856449127197266, + "learning_rate": 2.841328413284133e-06, + "loss": 1.4528, + "step": 1192 + }, + { + "epoch": 1.4847542003733665, + "grad_norm": 2.848987102508545, + "learning_rate": 2.8390221402214022e-06, + "loss": 1.4369, + "step": 1193 + }, + { + "epoch": 1.4859987554449283, + "grad_norm": 2.970419406890869, + "learning_rate": 2.836715867158672e-06, + "loss": 1.4605, + "step": 1194 + }, + { + "epoch": 1.4872433105164904, + "grad_norm": 3.156613826751709, + "learning_rate": 2.834409594095941e-06, + "loss": 1.4489, + "step": 1195 + }, + { + "epoch": 1.4884878655880522, + "grad_norm": 3.095349073410034, + "learning_rate": 2.8321033210332105e-06, + "loss": 1.4604, + "step": 1196 + }, + { + "epoch": 1.4897324206596143, + "grad_norm": 2.870742082595825, + "learning_rate": 2.82979704797048e-06, + "loss": 1.4284, + "step": 1197 + }, + { + "epoch": 1.4909769757311762, + "grad_norm": 2.9415056705474854, + "learning_rate": 2.827490774907749e-06, + "loss": 1.4715, + "step": 1198 + }, + { + "epoch": 1.492221530802738, + "grad_norm": 2.8297767639160156, + "learning_rate": 2.8251845018450188e-06, + "loss": 1.4508, + "step": 1199 + }, + { + "epoch": 1.4934660858742999, + "grad_norm": 2.872086524963379, + "learning_rate": 2.822878228782288e-06, + "loss": 1.5098, + "step": 1200 + }, + { + "epoch": 1.4934660858742999, + "eval_loss": 1.6357526779174805, + "eval_runtime": 51.422, + "eval_samples_per_second": 19.447, + "eval_steps_per_second": 0.817, + "step": 1200 + }, + { + "epoch": 1.494710640945862, + "grad_norm": 2.7152082920074463, + "learning_rate": 2.8205719557195576e-06, + "loss": 1.4329, + "step": 1201 + }, + { + "epoch": 1.4959551960174238, + "grad_norm": 2.866170883178711, + "learning_rate": 2.8182656826568266e-06, + "loss": 1.4455, + "step": 1202 + }, + { + "epoch": 1.4971997510889856, + "grad_norm": 2.9010684490203857, + "learning_rate": 2.815959409594096e-06, + "loss": 1.423, + "step": 1203 + }, + { + "epoch": 1.4984443061605477, + "grad_norm": 2.8588502407073975, + "learning_rate": 2.813653136531366e-06, + "loss": 1.4041, + "step": 1204 + }, + { + "epoch": 1.4996888612321095, + "grad_norm": 2.932544231414795, + "learning_rate": 2.811346863468635e-06, + "loss": 1.4559, + "step": 1205 + }, + { + "epoch": 1.5009334163036714, + "grad_norm": 2.7486014366149902, + "learning_rate": 2.8090405904059043e-06, + "loss": 1.4337, + "step": 1206 + }, + { + "epoch": 1.5021779713752332, + "grad_norm": 3.0866589546203613, + "learning_rate": 2.8067343173431737e-06, + "loss": 1.4186, + "step": 1207 + }, + { + "epoch": 1.5034225264467953, + "grad_norm": 2.9710276126861572, + "learning_rate": 2.8044280442804427e-06, + "loss": 1.419, + "step": 1208 + }, + { + "epoch": 1.5046670815183572, + "grad_norm": 2.8379950523376465, + "learning_rate": 2.802121771217712e-06, + "loss": 1.4922, + "step": 1209 + }, + { + "epoch": 1.5059116365899192, + "grad_norm": 2.939629554748535, + "learning_rate": 2.799815498154982e-06, + "loss": 1.4402, + "step": 1210 + }, + { + "epoch": 1.5059116365899192, + "eval_loss": 1.6419442892074585, + "eval_runtime": 43.1202, + "eval_samples_per_second": 23.191, + "eval_steps_per_second": 0.974, + "step": 1210 + }, + { + "epoch": 1.507156191661481, + "grad_norm": 3.041389226913452, + "learning_rate": 2.7975092250922514e-06, + "loss": 1.4873, + "step": 1211 + }, + { + "epoch": 1.508400746733043, + "grad_norm": 2.9776995182037354, + "learning_rate": 2.7952029520295204e-06, + "loss": 1.4422, + "step": 1212 + }, + { + "epoch": 1.5096453018046048, + "grad_norm": 2.8798792362213135, + "learning_rate": 2.79289667896679e-06, + "loss": 1.4535, + "step": 1213 + }, + { + "epoch": 1.5108898568761666, + "grad_norm": 2.9016385078430176, + "learning_rate": 2.7905904059040597e-06, + "loss": 1.4462, + "step": 1214 + }, + { + "epoch": 1.5121344119477287, + "grad_norm": 2.8987581729888916, + "learning_rate": 2.7882841328413287e-06, + "loss": 1.4477, + "step": 1215 + }, + { + "epoch": 1.5133789670192908, + "grad_norm": 2.960266351699829, + "learning_rate": 2.785977859778598e-06, + "loss": 1.4811, + "step": 1216 + }, + { + "epoch": 1.5146235220908526, + "grad_norm": 2.8965611457824707, + "learning_rate": 2.7836715867158675e-06, + "loss": 1.4366, + "step": 1217 + }, + { + "epoch": 1.5158680771624145, + "grad_norm": 3.0667364597320557, + "learning_rate": 2.7813653136531365e-06, + "loss": 1.3828, + "step": 1218 + }, + { + "epoch": 1.5171126322339763, + "grad_norm": 2.952362537384033, + "learning_rate": 2.779059040590406e-06, + "loss": 1.3903, + "step": 1219 + }, + { + "epoch": 1.5183571873055381, + "grad_norm": 2.9899795055389404, + "learning_rate": 2.776752767527676e-06, + "loss": 1.4312, + "step": 1220 + }, + { + "epoch": 1.5183571873055381, + "eval_loss": 1.640383005142212, + "eval_runtime": 45.4128, + "eval_samples_per_second": 22.02, + "eval_steps_per_second": 0.925, + "step": 1220 + }, + { + "epoch": 1.5196017423771002, + "grad_norm": 3.2202725410461426, + "learning_rate": 2.774446494464945e-06, + "loss": 1.4344, + "step": 1221 + }, + { + "epoch": 1.520846297448662, + "grad_norm": 3.2202725410461426, + "learning_rate": 2.774446494464945e-06, + "loss": 1.4386, + "step": 1222 + }, + { + "epoch": 1.5220908525202241, + "grad_norm": 2.883223295211792, + "learning_rate": 2.7721402214022142e-06, + "loss": 1.4401, + "step": 1223 + }, + { + "epoch": 1.523335407591786, + "grad_norm": 2.8346424102783203, + "learning_rate": 2.7698339483394837e-06, + "loss": 1.3926, + "step": 1224 + }, + { + "epoch": 1.5245799626633478, + "grad_norm": 2.8898870944976807, + "learning_rate": 2.767527675276753e-06, + "loss": 1.4128, + "step": 1225 + }, + { + "epoch": 1.5258245177349097, + "grad_norm": 2.828594923019409, + "learning_rate": 2.765221402214022e-06, + "loss": 1.4103, + "step": 1226 + }, + { + "epoch": 1.5270690728064715, + "grad_norm": 2.882558822631836, + "learning_rate": 2.762915129151292e-06, + "loss": 1.4186, + "step": 1227 + }, + { + "epoch": 1.5283136278780336, + "grad_norm": 2.955760955810547, + "learning_rate": 2.7606088560885613e-06, + "loss": 1.3705, + "step": 1228 + }, + { + "epoch": 1.5295581829495957, + "grad_norm": 2.862934112548828, + "learning_rate": 2.7583025830258303e-06, + "loss": 1.5012, + "step": 1229 + }, + { + "epoch": 1.5308027380211575, + "grad_norm": 2.926501989364624, + "learning_rate": 2.7559963099630998e-06, + "loss": 1.3875, + "step": 1230 + }, + { + "epoch": 1.5308027380211575, + "eval_loss": 1.6376254558563232, + "eval_runtime": 47.9535, + "eval_samples_per_second": 20.854, + "eval_steps_per_second": 0.876, + "step": 1230 + }, + { + "epoch": 1.5320472930927194, + "grad_norm": 2.7648823261260986, + "learning_rate": 2.753690036900369e-06, + "loss": 1.3781, + "step": 1231 + }, + { + "epoch": 1.5332918481642812, + "grad_norm": 2.927103281021118, + "learning_rate": 2.751383763837638e-06, + "loss": 1.4091, + "step": 1232 + }, + { + "epoch": 1.534536403235843, + "grad_norm": 2.964823007583618, + "learning_rate": 2.749077490774908e-06, + "loss": 1.5075, + "step": 1233 + }, + { + "epoch": 1.5357809583074051, + "grad_norm": 2.7944016456604004, + "learning_rate": 2.7467712177121775e-06, + "loss": 1.3888, + "step": 1234 + }, + { + "epoch": 1.537025513378967, + "grad_norm": 2.891592025756836, + "learning_rate": 2.7444649446494465e-06, + "loss": 1.4825, + "step": 1235 + }, + { + "epoch": 1.538270068450529, + "grad_norm": 2.8120031356811523, + "learning_rate": 2.742158671586716e-06, + "loss": 1.4246, + "step": 1236 + }, + { + "epoch": 1.539514623522091, + "grad_norm": 2.9384255409240723, + "learning_rate": 2.7398523985239857e-06, + "loss": 1.4657, + "step": 1237 + }, + { + "epoch": 1.5407591785936527, + "grad_norm": 2.73111891746521, + "learning_rate": 2.737546125461255e-06, + "loss": 1.4689, + "step": 1238 + }, + { + "epoch": 1.5420037336652146, + "grad_norm": 2.858963966369629, + "learning_rate": 2.735239852398524e-06, + "loss": 1.4626, + "step": 1239 + }, + { + "epoch": 1.5432482887367764, + "grad_norm": 2.9148507118225098, + "learning_rate": 2.7329335793357936e-06, + "loss": 1.4787, + "step": 1240 + }, + { + "epoch": 1.5432482887367764, + "eval_loss": 1.6436412334442139, + "eval_runtime": 47.3139, + "eval_samples_per_second": 21.135, + "eval_steps_per_second": 0.888, + "step": 1240 + }, + { + "epoch": 1.5444928438083385, + "grad_norm": 2.8731327056884766, + "learning_rate": 2.730627306273063e-06, + "loss": 1.4748, + "step": 1241 + }, + { + "epoch": 1.5457373988799006, + "grad_norm": 2.990265369415283, + "learning_rate": 2.728321033210332e-06, + "loss": 1.4375, + "step": 1242 + }, + { + "epoch": 1.5469819539514624, + "grad_norm": 2.874321460723877, + "learning_rate": 2.726014760147602e-06, + "loss": 1.4154, + "step": 1243 + }, + { + "epoch": 1.5482265090230243, + "grad_norm": 2.9316225051879883, + "learning_rate": 2.7237084870848713e-06, + "loss": 1.4285, + "step": 1244 + }, + { + "epoch": 1.5494710640945861, + "grad_norm": 2.783966302871704, + "learning_rate": 2.7214022140221403e-06, + "loss": 1.409, + "step": 1245 + }, + { + "epoch": 1.550715619166148, + "grad_norm": 2.9578707218170166, + "learning_rate": 2.7190959409594097e-06, + "loss": 1.4325, + "step": 1246 + }, + { + "epoch": 1.55196017423771, + "grad_norm": 2.869893789291382, + "learning_rate": 2.716789667896679e-06, + "loss": 1.3989, + "step": 1247 + }, + { + "epoch": 1.553204729309272, + "grad_norm": 2.962944269180298, + "learning_rate": 2.714483394833948e-06, + "loss": 1.4302, + "step": 1248 + }, + { + "epoch": 1.554449284380834, + "grad_norm": 2.974494695663452, + "learning_rate": 2.712177121771218e-06, + "loss": 1.46, + "step": 1249 + }, + { + "epoch": 1.5556938394523958, + "grad_norm": 2.9154603481292725, + "learning_rate": 2.7098708487084874e-06, + "loss": 1.4388, + "step": 1250 + }, + { + "epoch": 1.5556938394523958, + "eval_loss": 1.6357773542404175, + "eval_runtime": 47.6085, + "eval_samples_per_second": 21.005, + "eval_steps_per_second": 0.882, + "step": 1250 + }, + { + "epoch": 1.5569383945239577, + "grad_norm": 3.0333547592163086, + "learning_rate": 2.707564575645757e-06, + "loss": 1.4846, + "step": 1251 + }, + { + "epoch": 1.5581829495955195, + "grad_norm": 3.06097674369812, + "learning_rate": 2.705258302583026e-06, + "loss": 1.4355, + "step": 1252 + }, + { + "epoch": 1.5594275046670814, + "grad_norm": 2.872680425643921, + "learning_rate": 2.7029520295202956e-06, + "loss": 1.4219, + "step": 1253 + }, + { + "epoch": 1.5606720597386434, + "grad_norm": 2.9066731929779053, + "learning_rate": 2.700645756457565e-06, + "loss": 1.384, + "step": 1254 + }, + { + "epoch": 1.5619166148102055, + "grad_norm": 3.0570194721221924, + "learning_rate": 2.698339483394834e-06, + "loss": 1.4687, + "step": 1255 + }, + { + "epoch": 1.5631611698817673, + "grad_norm": 3.1955857276916504, + "learning_rate": 2.6960332103321035e-06, + "loss": 1.4736, + "step": 1256 + }, + { + "epoch": 1.5644057249533292, + "grad_norm": 2.9686198234558105, + "learning_rate": 2.693726937269373e-06, + "loss": 1.4176, + "step": 1257 + }, + { + "epoch": 1.565650280024891, + "grad_norm": 3.1161272525787354, + "learning_rate": 2.691420664206642e-06, + "loss": 1.443, + "step": 1258 + }, + { + "epoch": 1.5668948350964529, + "grad_norm": 2.9031193256378174, + "learning_rate": 2.6891143911439118e-06, + "loss": 1.3871, + "step": 1259 + }, + { + "epoch": 1.568139390168015, + "grad_norm": 2.9120476245880127, + "learning_rate": 2.686808118081181e-06, + "loss": 1.4406, + "step": 1260 + }, + { + "epoch": 1.568139390168015, + "eval_loss": 1.6291446685791016, + "eval_runtime": 49.7006, + "eval_samples_per_second": 20.12, + "eval_steps_per_second": 0.845, + "step": 1260 + }, + { + "epoch": 1.5693839452395768, + "grad_norm": 3.3181824684143066, + "learning_rate": 2.6845018450184506e-06, + "loss": 1.5278, + "step": 1261 + }, + { + "epoch": 1.5706285003111389, + "grad_norm": 2.9628148078918457, + "learning_rate": 2.6821955719557196e-06, + "loss": 1.4206, + "step": 1262 + }, + { + "epoch": 1.5718730553827007, + "grad_norm": 2.9828474521636963, + "learning_rate": 2.679889298892989e-06, + "loss": 1.5478, + "step": 1263 + }, + { + "epoch": 1.5731176104542626, + "grad_norm": 3.018711805343628, + "learning_rate": 2.677583025830259e-06, + "loss": 1.4442, + "step": 1264 + }, + { + "epoch": 1.5743621655258244, + "grad_norm": 2.8596792221069336, + "learning_rate": 2.675276752767528e-06, + "loss": 1.4325, + "step": 1265 + }, + { + "epoch": 1.5756067205973865, + "grad_norm": 2.9592769145965576, + "learning_rate": 2.6729704797047973e-06, + "loss": 1.4059, + "step": 1266 + }, + { + "epoch": 1.5768512756689483, + "grad_norm": 2.773589611053467, + "learning_rate": 2.6706642066420667e-06, + "loss": 1.3608, + "step": 1267 + }, + { + "epoch": 1.5780958307405104, + "grad_norm": 3.313255548477173, + "learning_rate": 2.6683579335793357e-06, + "loss": 1.4552, + "step": 1268 + }, + { + "epoch": 1.5793403858120723, + "grad_norm": 2.977649211883545, + "learning_rate": 2.6660516605166056e-06, + "loss": 1.4221, + "step": 1269 + }, + { + "epoch": 1.580584940883634, + "grad_norm": 3.0672051906585693, + "learning_rate": 2.663745387453875e-06, + "loss": 1.4568, + "step": 1270 + }, + { + "epoch": 1.580584940883634, + "eval_loss": 1.6317907571792603, + "eval_runtime": 47.2354, + "eval_samples_per_second": 21.171, + "eval_steps_per_second": 0.889, + "step": 1270 + }, + { + "epoch": 1.581829495955196, + "grad_norm": 3.029320240020752, + "learning_rate": 2.661439114391144e-06, + "loss": 1.4432, + "step": 1271 + }, + { + "epoch": 1.5830740510267578, + "grad_norm": 3.0183846950531006, + "learning_rate": 2.6591328413284134e-06, + "loss": 1.4332, + "step": 1272 + }, + { + "epoch": 1.5843186060983199, + "grad_norm": 2.856551170349121, + "learning_rate": 2.656826568265683e-06, + "loss": 1.4291, + "step": 1273 + }, + { + "epoch": 1.5855631611698817, + "grad_norm": 3.124971389770508, + "learning_rate": 2.6545202952029527e-06, + "loss": 1.4363, + "step": 1274 + }, + { + "epoch": 1.5868077162414438, + "grad_norm": 2.9470791816711426, + "learning_rate": 2.6522140221402217e-06, + "loss": 1.4502, + "step": 1275 + }, + { + "epoch": 1.5880522713130056, + "grad_norm": 2.9116358757019043, + "learning_rate": 2.649907749077491e-06, + "loss": 1.4314, + "step": 1276 + }, + { + "epoch": 1.5892968263845675, + "grad_norm": 3.2575323581695557, + "learning_rate": 2.6476014760147605e-06, + "loss": 1.4749, + "step": 1277 + }, + { + "epoch": 1.5905413814561293, + "grad_norm": 2.935518503189087, + "learning_rate": 2.6452952029520295e-06, + "loss": 1.4265, + "step": 1278 + }, + { + "epoch": 1.5917859365276914, + "grad_norm": 2.9750072956085205, + "learning_rate": 2.642988929889299e-06, + "loss": 1.4501, + "step": 1279 + }, + { + "epoch": 1.5930304915992533, + "grad_norm": 2.8086845874786377, + "learning_rate": 2.640682656826569e-06, + "loss": 1.4557, + "step": 1280 + }, + { + "epoch": 1.5930304915992533, + "eval_loss": 1.6302106380462646, + "eval_runtime": 52.3962, + "eval_samples_per_second": 19.085, + "eval_steps_per_second": 0.802, + "step": 1280 + }, + { + "epoch": 1.5942750466708153, + "grad_norm": 2.741849422454834, + "learning_rate": 2.638376383763838e-06, + "loss": 1.3918, + "step": 1281 + }, + { + "epoch": 1.5955196017423772, + "grad_norm": 2.9414806365966797, + "learning_rate": 2.6360701107011072e-06, + "loss": 1.3882, + "step": 1282 + }, + { + "epoch": 1.596764156813939, + "grad_norm": 2.901514768600464, + "learning_rate": 2.6337638376383766e-06, + "loss": 1.4331, + "step": 1283 + }, + { + "epoch": 1.5980087118855009, + "grad_norm": 2.8159122467041016, + "learning_rate": 2.6314575645756456e-06, + "loss": 1.4064, + "step": 1284 + }, + { + "epoch": 1.5992532669570627, + "grad_norm": 2.900561571121216, + "learning_rate": 2.629151291512915e-06, + "loss": 1.3731, + "step": 1285 + }, + { + "epoch": 1.6004978220286248, + "grad_norm": 3.019540548324585, + "learning_rate": 2.626845018450185e-06, + "loss": 1.4506, + "step": 1286 + }, + { + "epoch": 1.6017423771001866, + "grad_norm": 2.9263620376586914, + "learning_rate": 2.6245387453874543e-06, + "loss": 1.4754, + "step": 1287 + }, + { + "epoch": 1.6029869321717487, + "grad_norm": 2.8681881427764893, + "learning_rate": 2.6222324723247233e-06, + "loss": 1.4124, + "step": 1288 + }, + { + "epoch": 1.6042314872433105, + "grad_norm": 2.922945261001587, + "learning_rate": 2.6199261992619928e-06, + "loss": 1.5101, + "step": 1289 + }, + { + "epoch": 1.6054760423148724, + "grad_norm": 2.9548606872558594, + "learning_rate": 2.6176199261992626e-06, + "loss": 1.41, + "step": 1290 + }, + { + "epoch": 1.6054760423148724, + "eval_loss": 1.6357502937316895, + "eval_runtime": 46.9547, + "eval_samples_per_second": 21.297, + "eval_steps_per_second": 0.894, + "step": 1290 + }, + { + "epoch": 1.6067205973864342, + "grad_norm": 2.837184190750122, + "learning_rate": 2.6153136531365316e-06, + "loss": 1.4354, + "step": 1291 + }, + { + "epoch": 1.6079651524579963, + "grad_norm": 2.818990468978882, + "learning_rate": 2.613007380073801e-06, + "loss": 1.4474, + "step": 1292 + }, + { + "epoch": 1.6092097075295582, + "grad_norm": 2.7880146503448486, + "learning_rate": 2.6107011070110704e-06, + "loss": 1.4156, + "step": 1293 + }, + { + "epoch": 1.6104542626011202, + "grad_norm": 2.790971040725708, + "learning_rate": 2.6083948339483394e-06, + "loss": 1.4671, + "step": 1294 + }, + { + "epoch": 1.611698817672682, + "grad_norm": 2.943129777908325, + "learning_rate": 2.606088560885609e-06, + "loss": 1.4801, + "step": 1295 + }, + { + "epoch": 1.612943372744244, + "grad_norm": 2.8589110374450684, + "learning_rate": 2.6037822878228787e-06, + "loss": 1.4894, + "step": 1296 + }, + { + "epoch": 1.6141879278158058, + "grad_norm": 2.8929262161254883, + "learning_rate": 2.6014760147601477e-06, + "loss": 1.4022, + "step": 1297 + }, + { + "epoch": 1.6154324828873676, + "grad_norm": 2.970099925994873, + "learning_rate": 2.599169741697417e-06, + "loss": 1.5136, + "step": 1298 + }, + { + "epoch": 1.6166770379589297, + "grad_norm": 2.928466796875, + "learning_rate": 2.5968634686346866e-06, + "loss": 1.4743, + "step": 1299 + }, + { + "epoch": 1.6179215930304915, + "grad_norm": 2.941580057144165, + "learning_rate": 2.594557195571956e-06, + "loss": 1.4397, + "step": 1300 + }, + { + "epoch": 1.6179215930304915, + "eval_loss": 1.6348339319229126, + "eval_runtime": 50.433, + "eval_samples_per_second": 19.828, + "eval_steps_per_second": 0.833, + "step": 1300 + } + ], + "logging_steps": 1, + "max_steps": 2409, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 100, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 4.052502232170496e+17, + "train_batch_size": 3, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/training_args.bin b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..6cdf93817e8f6d23b60d79888b226cc66ed2e88b --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a50e4918f3c49370326e14a7ed2f73b48b05e521f4b3399b53438fac6abee926 +size 6011 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/zero_to_fp32.py b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..24cc342e78d1a006c782b3a4cd68d9ce786d8fd8 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/zero_to_fp32.py @@ -0,0 +1,604 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: python zero_to_fp32.py . pytorch_model.bin + +import argparse +import torch +import glob +import math +import os +import re +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + + total_files = len(files) + state_dicts = [] + for f in files: + state_dict = torch.load(f, map_location=device) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + if zero_stage <= 2: + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + elif zero_stage == 3: + # if there is more than one param group, there will be multiple flattened tensors - one + # flattened tensor per group - for simplicity merge them into a single tensor + # + # XXX: could make the script more memory efficient for when there are multiple groups - it + # will require matching the sub-lists of param_shapes for each param group flattened tensor + + fp32_flat_groups = [ + torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts)) + ] + + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = fp32_flat_groups[0].numel() * world_size + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + for name, shape in param_shapes.items(): + + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # XXX: memory usage doubles here + state_dict[name] = torch.cat( + tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)), + 0).narrow(0, 0, unpartitioned_numel).view(shape) + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + + Returns: + - pytorch ``state_dict`` + + Note: this approach may not work if your application doesn't have sufficient free CPU memory and + you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters) + print(f"Saving fp32 state dict to {output_file}") + torch.save(state_dict, output_file) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info(f"Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info(f"Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument( + "output_file", + type=str, + help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_file, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fb0f9a53a507ffcdb60410deb71d60ef801bf350 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/config.json @@ -0,0 +1,36 @@ +{ + "_name_or_path": "meta-llama/Llama-3.2-3B", + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "eos_token_id": 128001, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 3072, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 24, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "torch_dtype": "float16", + "transformers_version": "4.45.1", + "use_cache": true, + "vocab_size": 128256 +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/generation_config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9c389b841a9a29ddd904e02d1eb0e08dcce82ad9 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": 128001, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.45.1" +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/latest b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/latest new file mode 100644 index 0000000000000000000000000000000000000000..c2a0b5b17b996514a9bea0debe9403a641844bce --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/latest @@ -0,0 +1 @@ +global_step1400 \ No newline at end of file diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/model.safetensors.index.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..ed64de846d720b9a7859dc20575fea8e8ca51940 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/model.safetensors.index.json @@ -0,0 +1,262 @@ +{ + "metadata": { + "total_size": 7213504512 + }, + "weight_map": { + "lm_head.weight": "model-00002-of-00002.safetensors", + "model.embed_tokens.weight": "model-00001-of-00002.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.norm.weight": "model-00002-of-00002.safetensors" + } +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/rng_state_0.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..466c494b53a5cf3030ac585b29d1563c49b789d4 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5075060d7b0d18a80b592cad740c3383144e4b5ba0bc7e7b9ff8ebb2afad8e5 +size 16567 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/rng_state_1.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..262d37671147896b5dfa4ab77f0e9578ea0d968c --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46766e2820279a98976091f452887a7ad876d6c74fa04b8df6451b77352f4c07 +size 16567 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/rng_state_2.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..b31f6e0c1e981c4c1dee9a4626e2a6c6ae5abb8e --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:deab7068d9411eb964839c7901679767e090930bd8dfb3faa9e266fdf4d5aed9 +size 16567 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/scheduler.pt b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..ebaa878c7fbdf50f50d94d0f589fb9d9c78a45b7 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:444ee29b4620487bb6a50bae5508a70cfe811b44ca4f82431f50157295bb1f92 +size 627 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/special_tokens_map.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..6949c5975ee0e961ef61cf31010dce04df0a03f8 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "[PAD]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/tokenizer.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f28ecaeab53ae07feed29ccf8624d2b0a8344df9 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19fb2e1e3cdd6f7433d89fd6d62c82042599dd4984f342efe7fec6e159e6a8f6 +size 17210734 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/tokenizer_config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8991b49e9c2a43fc527dab9e09ad8171f0cc5943 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/tokenizer_config.json @@ -0,0 +1,2086 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "[PAD]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128257": { + "content": "🅂", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "128258": { + "content": "🄿", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "[PAD]", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/trainer_state.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..cb1e3e55e43bd797bf80eb47eb850c8989eadd22 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/trainer_state.json @@ -0,0 +1,10953 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.7423771001866832, + "eval_steps": 10, + "global_step": 1400, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0012445550715619166, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8368, + "step": 1 + }, + { + "epoch": 0.002489110143123833, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.86, + "step": 2 + }, + { + "epoch": 0.00373366521468575, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8265, + "step": 3 + }, + { + "epoch": 0.004978220286247666, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8271, + "step": 4 + }, + { + "epoch": 0.006222775357809583, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8768, + "step": 5 + }, + { + "epoch": 0.0074673304293715, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8402, + "step": 6 + }, + { + "epoch": 0.008711885500933417, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.785, + "step": 7 + }, + { + "epoch": 0.009956440572495333, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8466, + "step": 8 + }, + { + "epoch": 0.01120099564405725, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.7673, + "step": 9 + }, + { + "epoch": 0.012445550715619166, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8074, + "step": 10 + }, + { + "epoch": 0.012445550715619166, + "eval_loss": 2.8535053730010986, + "eval_runtime": 42.446, + "eval_samples_per_second": 23.559, + "eval_steps_per_second": 0.989, + "step": 10 + }, + { + "epoch": 0.013690105787181083, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8592, + "step": 11 + }, + { + "epoch": 0.014934660858743, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.804, + "step": 12 + }, + { + "epoch": 0.016179215930304917, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8076, + "step": 13 + }, + { + "epoch": 0.017423771001866834, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8219, + "step": 14 + }, + { + "epoch": 0.018668326073428748, + "grad_norm": 5.105710506439209, + "learning_rate": 2.0746887966804982e-08, + "loss": 2.7995, + "step": 15 + }, + { + "epoch": 0.019912881144990666, + "grad_norm": 5.156588077545166, + "learning_rate": 4.1493775933609963e-08, + "loss": 2.8577, + "step": 16 + }, + { + "epoch": 0.021157436216552583, + "grad_norm": 5.709373950958252, + "learning_rate": 6.224066390041494e-08, + "loss": 2.8433, + "step": 17 + }, + { + "epoch": 0.0224019912881145, + "grad_norm": 5.64658784866333, + "learning_rate": 8.298755186721993e-08, + "loss": 2.8428, + "step": 18 + }, + { + "epoch": 0.023646546359676415, + "grad_norm": 5.541937351226807, + "learning_rate": 1.037344398340249e-07, + "loss": 2.7785, + "step": 19 + }, + { + "epoch": 0.024891101431238332, + "grad_norm": 5.233374118804932, + "learning_rate": 1.2448132780082988e-07, + "loss": 2.8599, + "step": 20 + }, + { + "epoch": 0.024891101431238332, + "eval_loss": 2.8521182537078857, + "eval_runtime": 43.5664, + "eval_samples_per_second": 22.953, + "eval_steps_per_second": 0.964, + "step": 20 + }, + { + "epoch": 0.02613565650280025, + "grad_norm": 5.407598972320557, + "learning_rate": 1.4522821576763488e-07, + "loss": 2.8081, + "step": 21 + }, + { + "epoch": 0.027380211574362167, + "grad_norm": 5.0483317375183105, + "learning_rate": 1.6597510373443985e-07, + "loss": 2.8489, + "step": 22 + }, + { + "epoch": 0.02862476664592408, + "grad_norm": 4.985588550567627, + "learning_rate": 1.8672199170124483e-07, + "loss": 2.8375, + "step": 23 + }, + { + "epoch": 0.029869321717486, + "grad_norm": 5.201700687408447, + "learning_rate": 2.074688796680498e-07, + "loss": 2.8355, + "step": 24 + }, + { + "epoch": 0.031113876789047916, + "grad_norm": 5.009235382080078, + "learning_rate": 2.2821576763485478e-07, + "loss": 2.7774, + "step": 25 + }, + { + "epoch": 0.03235843186060983, + "grad_norm": 4.509210109710693, + "learning_rate": 2.4896265560165975e-07, + "loss": 2.7992, + "step": 26 + }, + { + "epoch": 0.03360298693217175, + "grad_norm": 4.529262065887451, + "learning_rate": 2.6970954356846476e-07, + "loss": 2.836, + "step": 27 + }, + { + "epoch": 0.03484754200373367, + "grad_norm": 4.526787281036377, + "learning_rate": 2.9045643153526976e-07, + "loss": 2.8128, + "step": 28 + }, + { + "epoch": 0.03609209707529558, + "grad_norm": 4.016609191894531, + "learning_rate": 3.112033195020747e-07, + "loss": 2.7947, + "step": 29 + }, + { + "epoch": 0.037336652146857496, + "grad_norm": 3.4784562587738037, + "learning_rate": 3.319502074688797e-07, + "loss": 2.7854, + "step": 30 + }, + { + "epoch": 0.037336652146857496, + "eval_loss": 2.8020100593566895, + "eval_runtime": 44.8464, + "eval_samples_per_second": 22.298, + "eval_steps_per_second": 0.937, + "step": 30 + }, + { + "epoch": 0.038581207218419414, + "grad_norm": 3.4887945652008057, + "learning_rate": 3.5269709543568466e-07, + "loss": 2.7908, + "step": 31 + }, + { + "epoch": 0.03982576228998133, + "grad_norm": 3.4533374309539795, + "learning_rate": 3.7344398340248966e-07, + "loss": 2.7358, + "step": 32 + }, + { + "epoch": 0.04107031736154325, + "grad_norm": 3.270954132080078, + "learning_rate": 3.941908713692946e-07, + "loss": 2.8013, + "step": 33 + }, + { + "epoch": 0.042314872433105166, + "grad_norm": 3.61322283744812, + "learning_rate": 4.149377593360996e-07, + "loss": 2.7521, + "step": 34 + }, + { + "epoch": 0.043559427504667084, + "grad_norm": 3.158719062805176, + "learning_rate": 4.3568464730290456e-07, + "loss": 2.7573, + "step": 35 + }, + { + "epoch": 0.044803982576229, + "grad_norm": 2.888746500015259, + "learning_rate": 4.5643153526970956e-07, + "loss": 2.7795, + "step": 36 + }, + { + "epoch": 0.04604853764779091, + "grad_norm": 3.022629499435425, + "learning_rate": 4.771784232365145e-07, + "loss": 2.7891, + "step": 37 + }, + { + "epoch": 0.04729309271935283, + "grad_norm": 3.038097381591797, + "learning_rate": 4.979253112033195e-07, + "loss": 2.7376, + "step": 38 + }, + { + "epoch": 0.04853764779091475, + "grad_norm": 2.9992239475250244, + "learning_rate": 5.186721991701245e-07, + "loss": 2.7279, + "step": 39 + }, + { + "epoch": 0.049782202862476664, + "grad_norm": 2.908842086791992, + "learning_rate": 5.394190871369295e-07, + "loss": 2.6582, + "step": 40 + }, + { + "epoch": 0.049782202862476664, + "eval_loss": 2.725968837738037, + "eval_runtime": 44.9498, + "eval_samples_per_second": 22.247, + "eval_steps_per_second": 0.934, + "step": 40 + }, + { + "epoch": 0.05102675793403858, + "grad_norm": 2.8025174140930176, + "learning_rate": 5.601659751037345e-07, + "loss": 2.6809, + "step": 41 + }, + { + "epoch": 0.0522713130056005, + "grad_norm": 2.7555699348449707, + "learning_rate": 5.809128630705395e-07, + "loss": 2.6954, + "step": 42 + }, + { + "epoch": 0.053515868077162417, + "grad_norm": 2.4732933044433594, + "learning_rate": 6.016597510373444e-07, + "loss": 2.7307, + "step": 43 + }, + { + "epoch": 0.054760423148724334, + "grad_norm": 2.451366424560547, + "learning_rate": 6.224066390041494e-07, + "loss": 2.7064, + "step": 44 + }, + { + "epoch": 0.056004978220286245, + "grad_norm": 2.5768678188323975, + "learning_rate": 6.431535269709543e-07, + "loss": 2.6466, + "step": 45 + }, + { + "epoch": 0.05724953329184816, + "grad_norm": 2.579332113265991, + "learning_rate": 6.639004149377594e-07, + "loss": 2.6127, + "step": 46 + }, + { + "epoch": 0.05849408836341008, + "grad_norm": 2.231207847595215, + "learning_rate": 6.846473029045644e-07, + "loss": 2.6429, + "step": 47 + }, + { + "epoch": 0.059738643434972, + "grad_norm": 2.134375810623169, + "learning_rate": 7.053941908713693e-07, + "loss": 2.6423, + "step": 48 + }, + { + "epoch": 0.060983198506533914, + "grad_norm": 2.0485494136810303, + "learning_rate": 7.261410788381744e-07, + "loss": 2.581, + "step": 49 + }, + { + "epoch": 0.06222775357809583, + "grad_norm": 2.02909517288208, + "learning_rate": 7.468879668049793e-07, + "loss": 2.6357, + "step": 50 + }, + { + "epoch": 0.06222775357809583, + "eval_loss": 2.632822036743164, + "eval_runtime": 47.0104, + "eval_samples_per_second": 21.272, + "eval_steps_per_second": 0.893, + "step": 50 + }, + { + "epoch": 0.06347230864965775, + "grad_norm": 1.892500877380371, + "learning_rate": 7.676348547717843e-07, + "loss": 2.5945, + "step": 51 + }, + { + "epoch": 0.06471686372121967, + "grad_norm": 2.107921600341797, + "learning_rate": 7.883817427385892e-07, + "loss": 2.5867, + "step": 52 + }, + { + "epoch": 0.06596141879278158, + "grad_norm": 1.8287527561187744, + "learning_rate": 8.091286307053943e-07, + "loss": 2.5862, + "step": 53 + }, + { + "epoch": 0.0672059738643435, + "grad_norm": 1.7342065572738647, + "learning_rate": 8.298755186721992e-07, + "loss": 2.5735, + "step": 54 + }, + { + "epoch": 0.06845052893590542, + "grad_norm": 1.7905038595199585, + "learning_rate": 8.506224066390042e-07, + "loss": 2.572, + "step": 55 + }, + { + "epoch": 0.06969508400746734, + "grad_norm": 1.9715144634246826, + "learning_rate": 8.713692946058091e-07, + "loss": 2.5797, + "step": 56 + }, + { + "epoch": 0.07093963907902924, + "grad_norm": 1.788966417312622, + "learning_rate": 8.921161825726142e-07, + "loss": 2.5567, + "step": 57 + }, + { + "epoch": 0.07218419415059116, + "grad_norm": 1.8978915214538574, + "learning_rate": 9.128630705394191e-07, + "loss": 2.5776, + "step": 58 + }, + { + "epoch": 0.07342874922215308, + "grad_norm": 1.6401689052581787, + "learning_rate": 9.336099585062241e-07, + "loss": 2.5662, + "step": 59 + }, + { + "epoch": 0.07467330429371499, + "grad_norm": 1.8200798034667969, + "learning_rate": 9.54356846473029e-07, + "loss": 2.5001, + "step": 60 + }, + { + "epoch": 0.07467330429371499, + "eval_loss": 2.569110870361328, + "eval_runtime": 47.3292, + "eval_samples_per_second": 21.129, + "eval_steps_per_second": 0.887, + "step": 60 + }, + { + "epoch": 0.07591785936527691, + "grad_norm": 1.5468674898147583, + "learning_rate": 9.751037344398341e-07, + "loss": 2.5308, + "step": 61 + }, + { + "epoch": 0.07716241443683883, + "grad_norm": 1.723684310913086, + "learning_rate": 9.95850622406639e-07, + "loss": 2.4907, + "step": 62 + }, + { + "epoch": 0.07840696950840075, + "grad_norm": 1.6346293687820435, + "learning_rate": 1.0165975103734441e-06, + "loss": 2.514, + "step": 63 + }, + { + "epoch": 0.07965152457996266, + "grad_norm": 1.8902088403701782, + "learning_rate": 1.037344398340249e-06, + "loss": 2.4859, + "step": 64 + }, + { + "epoch": 0.08089607965152458, + "grad_norm": 2.092611312866211, + "learning_rate": 1.058091286307054e-06, + "loss": 2.4888, + "step": 65 + }, + { + "epoch": 0.0821406347230865, + "grad_norm": 1.758482575416565, + "learning_rate": 1.078838174273859e-06, + "loss": 2.4537, + "step": 66 + }, + { + "epoch": 0.08338518979464841, + "grad_norm": 1.8941714763641357, + "learning_rate": 1.099585062240664e-06, + "loss": 2.5416, + "step": 67 + }, + { + "epoch": 0.08462974486621033, + "grad_norm": 1.7030234336853027, + "learning_rate": 1.120331950207469e-06, + "loss": 2.4791, + "step": 68 + }, + { + "epoch": 0.08587429993777225, + "grad_norm": 1.7602269649505615, + "learning_rate": 1.141078838174274e-06, + "loss": 2.4316, + "step": 69 + }, + { + "epoch": 0.08711885500933417, + "grad_norm": 1.7635431289672852, + "learning_rate": 1.161825726141079e-06, + "loss": 2.4674, + "step": 70 + }, + { + "epoch": 0.08711885500933417, + "eval_loss": 2.4922080039978027, + "eval_runtime": 45.3028, + "eval_samples_per_second": 22.074, + "eval_steps_per_second": 0.927, + "step": 70 + }, + { + "epoch": 0.08836341008089608, + "grad_norm": 2.0913803577423096, + "learning_rate": 1.182572614107884e-06, + "loss": 2.4341, + "step": 71 + }, + { + "epoch": 0.089607965152458, + "grad_norm": 1.8292183876037598, + "learning_rate": 1.2033195020746888e-06, + "loss": 2.4291, + "step": 72 + }, + { + "epoch": 0.09085252022401992, + "grad_norm": 2.206770181655884, + "learning_rate": 1.224066390041494e-06, + "loss": 2.3764, + "step": 73 + }, + { + "epoch": 0.09209707529558182, + "grad_norm": 1.8263559341430664, + "learning_rate": 1.2448132780082988e-06, + "loss": 2.4082, + "step": 74 + }, + { + "epoch": 0.09334163036714374, + "grad_norm": 1.9405455589294434, + "learning_rate": 1.2655601659751037e-06, + "loss": 2.4673, + "step": 75 + }, + { + "epoch": 0.09458618543870566, + "grad_norm": 2.138108015060425, + "learning_rate": 1.2863070539419086e-06, + "loss": 2.3879, + "step": 76 + }, + { + "epoch": 0.09583074051026758, + "grad_norm": 1.9024745225906372, + "learning_rate": 1.307053941908714e-06, + "loss": 2.4344, + "step": 77 + }, + { + "epoch": 0.0970752955818295, + "grad_norm": 1.9136689901351929, + "learning_rate": 1.3278008298755188e-06, + "loss": 2.474, + "step": 78 + }, + { + "epoch": 0.09831985065339141, + "grad_norm": 2.4752795696258545, + "learning_rate": 1.3485477178423237e-06, + "loss": 2.3412, + "step": 79 + }, + { + "epoch": 0.09956440572495333, + "grad_norm": 1.8720006942749023, + "learning_rate": 1.3692946058091288e-06, + "loss": 2.3438, + "step": 80 + }, + { + "epoch": 0.09956440572495333, + "eval_loss": 2.3980512619018555, + "eval_runtime": 48.2788, + "eval_samples_per_second": 20.713, + "eval_steps_per_second": 0.87, + "step": 80 + }, + { + "epoch": 0.10080896079651525, + "grad_norm": 2.671691656112671, + "learning_rate": 1.3900414937759337e-06, + "loss": 2.3336, + "step": 81 + }, + { + "epoch": 0.10205351586807716, + "grad_norm": 2.2953391075134277, + "learning_rate": 1.4107883817427386e-06, + "loss": 2.377, + "step": 82 + }, + { + "epoch": 0.10329807093963908, + "grad_norm": 3.009018898010254, + "learning_rate": 1.4315352697095435e-06, + "loss": 2.2977, + "step": 83 + }, + { + "epoch": 0.104542626011201, + "grad_norm": 2.664454936981201, + "learning_rate": 1.4522821576763488e-06, + "loss": 2.3271, + "step": 84 + }, + { + "epoch": 0.10578718108276292, + "grad_norm": 3.017303705215454, + "learning_rate": 1.4730290456431537e-06, + "loss": 2.3251, + "step": 85 + }, + { + "epoch": 0.10703173615432483, + "grad_norm": 2.634716510772705, + "learning_rate": 1.4937759336099586e-06, + "loss": 2.332, + "step": 86 + }, + { + "epoch": 0.10827629122588675, + "grad_norm": 3.059644937515259, + "learning_rate": 1.5145228215767635e-06, + "loss": 2.3478, + "step": 87 + }, + { + "epoch": 0.10952084629744867, + "grad_norm": 2.6962637901306152, + "learning_rate": 1.5352697095435686e-06, + "loss": 2.2792, + "step": 88 + }, + { + "epoch": 0.11076540136901059, + "grad_norm": 3.419729709625244, + "learning_rate": 1.5560165975103735e-06, + "loss": 2.2571, + "step": 89 + }, + { + "epoch": 0.11200995644057249, + "grad_norm": 2.740781545639038, + "learning_rate": 1.5767634854771784e-06, + "loss": 2.2875, + "step": 90 + }, + { + "epoch": 0.11200995644057249, + "eval_loss": 2.30843186378479, + "eval_runtime": 49.9264, + "eval_samples_per_second": 20.029, + "eval_steps_per_second": 0.841, + "step": 90 + }, + { + "epoch": 0.1132545115121344, + "grad_norm": 2.5608789920806885, + "learning_rate": 1.5975103734439833e-06, + "loss": 2.2592, + "step": 91 + }, + { + "epoch": 0.11449906658369632, + "grad_norm": 2.687999963760376, + "learning_rate": 1.6182572614107886e-06, + "loss": 2.2546, + "step": 92 + }, + { + "epoch": 0.11574362165525824, + "grad_norm": 2.695909023284912, + "learning_rate": 1.6390041493775935e-06, + "loss": 2.2525, + "step": 93 + }, + { + "epoch": 0.11698817672682016, + "grad_norm": 2.818357467651367, + "learning_rate": 1.6597510373443984e-06, + "loss": 2.216, + "step": 94 + }, + { + "epoch": 0.11823273179838208, + "grad_norm": 2.884119987487793, + "learning_rate": 1.6804979253112035e-06, + "loss": 2.2321, + "step": 95 + }, + { + "epoch": 0.119477286869944, + "grad_norm": 2.52104115486145, + "learning_rate": 1.7012448132780084e-06, + "loss": 2.199, + "step": 96 + }, + { + "epoch": 0.12072184194150591, + "grad_norm": 2.420313596725464, + "learning_rate": 1.7219917012448133e-06, + "loss": 2.1862, + "step": 97 + }, + { + "epoch": 0.12196639701306783, + "grad_norm": 2.8047542572021484, + "learning_rate": 1.7427385892116182e-06, + "loss": 2.1793, + "step": 98 + }, + { + "epoch": 0.12321095208462975, + "grad_norm": 2.836482286453247, + "learning_rate": 1.7634854771784235e-06, + "loss": 2.2271, + "step": 99 + }, + { + "epoch": 0.12445550715619166, + "grad_norm": 2.5282301902770996, + "learning_rate": 1.7842323651452284e-06, + "loss": 2.1768, + "step": 100 + }, + { + "epoch": 0.12445550715619166, + "eval_loss": 2.2312686443328857, + "eval_runtime": 49.2382, + "eval_samples_per_second": 20.309, + "eval_steps_per_second": 0.853, + "step": 100 + }, + { + "epoch": 0.12570006222775357, + "grad_norm": 3.3407280445098877, + "learning_rate": 1.8049792531120333e-06, + "loss": 2.1666, + "step": 101 + }, + { + "epoch": 0.1269446172993155, + "grad_norm": 2.4754133224487305, + "learning_rate": 1.8257261410788382e-06, + "loss": 2.1768, + "step": 102 + }, + { + "epoch": 0.1281891723708774, + "grad_norm": 3.430889129638672, + "learning_rate": 1.8464730290456433e-06, + "loss": 2.1953, + "step": 103 + }, + { + "epoch": 0.12943372744243933, + "grad_norm": 2.835294246673584, + "learning_rate": 1.8672199170124482e-06, + "loss": 2.146, + "step": 104 + }, + { + "epoch": 0.13067828251400124, + "grad_norm": 3.1532323360443115, + "learning_rate": 1.8879668049792531e-06, + "loss": 2.1729, + "step": 105 + }, + { + "epoch": 0.13192283758556317, + "grad_norm": 3.2278342247009277, + "learning_rate": 1.908713692946058e-06, + "loss": 2.1336, + "step": 106 + }, + { + "epoch": 0.13316739265712507, + "grad_norm": 2.7892515659332275, + "learning_rate": 1.929460580912863e-06, + "loss": 2.141, + "step": 107 + }, + { + "epoch": 0.134411947728687, + "grad_norm": 3.0179977416992188, + "learning_rate": 1.9502074688796682e-06, + "loss": 2.1255, + "step": 108 + }, + { + "epoch": 0.1356565028002489, + "grad_norm": 2.977935552597046, + "learning_rate": 1.970954356846473e-06, + "loss": 2.0986, + "step": 109 + }, + { + "epoch": 0.13690105787181084, + "grad_norm": 3.3042492866516113, + "learning_rate": 1.991701244813278e-06, + "loss": 2.0992, + "step": 110 + }, + { + "epoch": 0.13690105787181084, + "eval_loss": 2.171299695968628, + "eval_runtime": 42.9511, + "eval_samples_per_second": 23.282, + "eval_steps_per_second": 0.978, + "step": 110 + }, + { + "epoch": 0.13814561294337274, + "grad_norm": 2.9750890731811523, + "learning_rate": 2.012448132780083e-06, + "loss": 2.1116, + "step": 111 + }, + { + "epoch": 0.13939016801493467, + "grad_norm": 2.8199286460876465, + "learning_rate": 2.0331950207468883e-06, + "loss": 2.0822, + "step": 112 + }, + { + "epoch": 0.14063472308649658, + "grad_norm": 2.930532217025757, + "learning_rate": 2.053941908713693e-06, + "loss": 2.161, + "step": 113 + }, + { + "epoch": 0.14187927815805848, + "grad_norm": 3.3400321006774902, + "learning_rate": 2.074688796680498e-06, + "loss": 2.0953, + "step": 114 + }, + { + "epoch": 0.1431238332296204, + "grad_norm": 2.716564655303955, + "learning_rate": 2.095435684647303e-06, + "loss": 2.0757, + "step": 115 + }, + { + "epoch": 0.14436838830118232, + "grad_norm": 2.7586236000061035, + "learning_rate": 2.116182572614108e-06, + "loss": 2.0979, + "step": 116 + }, + { + "epoch": 0.14561294337274425, + "grad_norm": 2.622126340866089, + "learning_rate": 2.136929460580913e-06, + "loss": 2.0748, + "step": 117 + }, + { + "epoch": 0.14685749844430615, + "grad_norm": 2.646477460861206, + "learning_rate": 2.157676348547718e-06, + "loss": 2.0605, + "step": 118 + }, + { + "epoch": 0.14810205351586808, + "grad_norm": 2.8882410526275635, + "learning_rate": 2.178423236514523e-06, + "loss": 2.0501, + "step": 119 + }, + { + "epoch": 0.14934660858742999, + "grad_norm": 2.8302247524261475, + "learning_rate": 2.199170124481328e-06, + "loss": 2.0592, + "step": 120 + }, + { + "epoch": 0.14934660858742999, + "eval_loss": 2.1291966438293457, + "eval_runtime": 47.5399, + "eval_samples_per_second": 21.035, + "eval_steps_per_second": 0.883, + "step": 120 + }, + { + "epoch": 0.15059116365899192, + "grad_norm": 3.26979660987854, + "learning_rate": 2.219917012448133e-06, + "loss": 2.0422, + "step": 121 + }, + { + "epoch": 0.15183571873055382, + "grad_norm": 2.7483913898468018, + "learning_rate": 2.240663900414938e-06, + "loss": 2.0872, + "step": 122 + }, + { + "epoch": 0.15308027380211575, + "grad_norm": 3.1252024173736572, + "learning_rate": 2.2614107883817427e-06, + "loss": 2.0684, + "step": 123 + }, + { + "epoch": 0.15432482887367766, + "grad_norm": 2.9175283908843994, + "learning_rate": 2.282157676348548e-06, + "loss": 2.0522, + "step": 124 + }, + { + "epoch": 0.1555693839452396, + "grad_norm": 2.7950754165649414, + "learning_rate": 2.302904564315353e-06, + "loss": 2.065, + "step": 125 + }, + { + "epoch": 0.1568139390168015, + "grad_norm": 2.84049654006958, + "learning_rate": 2.323651452282158e-06, + "loss": 2.0781, + "step": 126 + }, + { + "epoch": 0.15805849408836342, + "grad_norm": 2.7396647930145264, + "learning_rate": 2.3443983402489627e-06, + "loss": 2.1758, + "step": 127 + }, + { + "epoch": 0.15930304915992533, + "grad_norm": 3.0765926837921143, + "learning_rate": 2.365145228215768e-06, + "loss": 2.0172, + "step": 128 + }, + { + "epoch": 0.16054760423148726, + "grad_norm": 3.217189073562622, + "learning_rate": 2.385892116182573e-06, + "loss": 2.0286, + "step": 129 + }, + { + "epoch": 0.16179215930304916, + "grad_norm": 3.141545295715332, + "learning_rate": 2.4066390041493776e-06, + "loss": 2.0583, + "step": 130 + }, + { + "epoch": 0.16179215930304916, + "eval_loss": 2.0946149826049805, + "eval_runtime": 54.92, + "eval_samples_per_second": 18.208, + "eval_steps_per_second": 0.765, + "step": 130 + }, + { + "epoch": 0.16303671437461106, + "grad_norm": 2.850052833557129, + "learning_rate": 2.4273858921161828e-06, + "loss": 2.0746, + "step": 131 + }, + { + "epoch": 0.164281269446173, + "grad_norm": 3.28913950920105, + "learning_rate": 2.448132780082988e-06, + "loss": 2.0539, + "step": 132 + }, + { + "epoch": 0.1655258245177349, + "grad_norm": 2.8819124698638916, + "learning_rate": 2.468879668049793e-06, + "loss": 1.9843, + "step": 133 + }, + { + "epoch": 0.16677037958929683, + "grad_norm": 3.6254632472991943, + "learning_rate": 2.4896265560165977e-06, + "loss": 2.0233, + "step": 134 + }, + { + "epoch": 0.16801493466085873, + "grad_norm": 2.7385146617889404, + "learning_rate": 2.5103734439834028e-06, + "loss": 1.9667, + "step": 135 + }, + { + "epoch": 0.16925948973242066, + "grad_norm": 2.9722647666931152, + "learning_rate": 2.5311203319502074e-06, + "loss": 1.9527, + "step": 136 + }, + { + "epoch": 0.17050404480398257, + "grad_norm": 3.139526605606079, + "learning_rate": 2.5518672199170125e-06, + "loss": 1.9967, + "step": 137 + }, + { + "epoch": 0.1717485998755445, + "grad_norm": 3.399920701980591, + "learning_rate": 2.5726141078838172e-06, + "loss": 1.9798, + "step": 138 + }, + { + "epoch": 0.1729931549471064, + "grad_norm": 2.8017327785491943, + "learning_rate": 2.5933609958506228e-06, + "loss": 1.983, + "step": 139 + }, + { + "epoch": 0.17423771001866833, + "grad_norm": 3.9967198371887207, + "learning_rate": 2.614107883817428e-06, + "loss": 1.977, + "step": 140 + }, + { + "epoch": 0.17423771001866833, + "eval_loss": 2.059037685394287, + "eval_runtime": 52.6232, + "eval_samples_per_second": 19.003, + "eval_steps_per_second": 0.798, + "step": 140 + }, + { + "epoch": 0.17548226509023024, + "grad_norm": 2.893092393875122, + "learning_rate": 2.6348547717842326e-06, + "loss": 1.9888, + "step": 141 + }, + { + "epoch": 0.17672682016179217, + "grad_norm": 3.634352207183838, + "learning_rate": 2.6556016597510377e-06, + "loss": 2.0726, + "step": 142 + }, + { + "epoch": 0.17797137523335407, + "grad_norm": 3.3651444911956787, + "learning_rate": 2.6763485477178423e-06, + "loss": 1.973, + "step": 143 + }, + { + "epoch": 0.179215930304916, + "grad_norm": 3.968986988067627, + "learning_rate": 2.6970954356846475e-06, + "loss": 1.9948, + "step": 144 + }, + { + "epoch": 0.1804604853764779, + "grad_norm": 3.20105242729187, + "learning_rate": 2.717842323651452e-06, + "loss": 1.9238, + "step": 145 + }, + { + "epoch": 0.18170504044803984, + "grad_norm": 3.648339033126831, + "learning_rate": 2.7385892116182577e-06, + "loss": 1.9923, + "step": 146 + }, + { + "epoch": 0.18294959551960174, + "grad_norm": 3.0856316089630127, + "learning_rate": 2.7593360995850628e-06, + "loss": 1.9749, + "step": 147 + }, + { + "epoch": 0.18419415059116365, + "grad_norm": 3.2681071758270264, + "learning_rate": 2.7800829875518675e-06, + "loss": 1.9304, + "step": 148 + }, + { + "epoch": 0.18543870566272558, + "grad_norm": 2.634958267211914, + "learning_rate": 2.8008298755186726e-06, + "loss": 1.9237, + "step": 149 + }, + { + "epoch": 0.18668326073428748, + "grad_norm": 2.769491672515869, + "learning_rate": 2.8215767634854773e-06, + "loss": 1.8963, + "step": 150 + }, + { + "epoch": 0.18668326073428748, + "eval_loss": 2.0394654273986816, + "eval_runtime": 55.2136, + "eval_samples_per_second": 18.111, + "eval_steps_per_second": 0.761, + "step": 150 + }, + { + "epoch": 0.1879278158058494, + "grad_norm": 3.4345781803131104, + "learning_rate": 2.8423236514522824e-06, + "loss": 1.9836, + "step": 151 + }, + { + "epoch": 0.18917237087741132, + "grad_norm": 3.115727424621582, + "learning_rate": 2.863070539419087e-06, + "loss": 1.9352, + "step": 152 + }, + { + "epoch": 0.19041692594897325, + "grad_norm": 3.050652503967285, + "learning_rate": 2.883817427385892e-06, + "loss": 1.9031, + "step": 153 + }, + { + "epoch": 0.19166148102053515, + "grad_norm": 2.9404428005218506, + "learning_rate": 2.9045643153526977e-06, + "loss": 1.9196, + "step": 154 + }, + { + "epoch": 0.19290603609209708, + "grad_norm": 3.080810785293579, + "learning_rate": 2.9253112033195024e-06, + "loss": 1.9405, + "step": 155 + }, + { + "epoch": 0.194150591163659, + "grad_norm": 3.058558702468872, + "learning_rate": 2.9460580912863075e-06, + "loss": 1.9052, + "step": 156 + }, + { + "epoch": 0.19539514623522092, + "grad_norm": 3.307955026626587, + "learning_rate": 2.966804979253112e-06, + "loss": 1.9245, + "step": 157 + }, + { + "epoch": 0.19663970130678282, + "grad_norm": 2.845506191253662, + "learning_rate": 2.9875518672199173e-06, + "loss": 1.9461, + "step": 158 + }, + { + "epoch": 0.19788425637834475, + "grad_norm": 2.758654832839966, + "learning_rate": 3.008298755186722e-06, + "loss": 1.8885, + "step": 159 + }, + { + "epoch": 0.19912881144990666, + "grad_norm": 3.161252737045288, + "learning_rate": 3.029045643153527e-06, + "loss": 1.8535, + "step": 160 + }, + { + "epoch": 0.19912881144990666, + "eval_loss": 2.018101453781128, + "eval_runtime": 51.0744, + "eval_samples_per_second": 19.579, + "eval_steps_per_second": 0.822, + "step": 160 + }, + { + "epoch": 0.2003733665214686, + "grad_norm": 3.196847677230835, + "learning_rate": 3.0497925311203326e-06, + "loss": 1.8853, + "step": 161 + }, + { + "epoch": 0.2016179215930305, + "grad_norm": 2.8364663124084473, + "learning_rate": 3.0705394190871373e-06, + "loss": 1.9463, + "step": 162 + }, + { + "epoch": 0.2028624766645924, + "grad_norm": 3.0874054431915283, + "learning_rate": 3.0912863070539424e-06, + "loss": 1.9254, + "step": 163 + }, + { + "epoch": 0.20410703173615433, + "grad_norm": 2.7914493083953857, + "learning_rate": 3.112033195020747e-06, + "loss": 1.9213, + "step": 164 + }, + { + "epoch": 0.20535158680771623, + "grad_norm": 3.3871428966522217, + "learning_rate": 3.132780082987552e-06, + "loss": 1.8993, + "step": 165 + }, + { + "epoch": 0.20659614187927816, + "grad_norm": 3.096653461456299, + "learning_rate": 3.153526970954357e-06, + "loss": 1.8827, + "step": 166 + }, + { + "epoch": 0.20784069695084006, + "grad_norm": 2.8276076316833496, + "learning_rate": 3.174273858921162e-06, + "loss": 1.912, + "step": 167 + }, + { + "epoch": 0.209085252022402, + "grad_norm": 3.3058435916900635, + "learning_rate": 3.1950207468879666e-06, + "loss": 1.8381, + "step": 168 + }, + { + "epoch": 0.2103298070939639, + "grad_norm": 3.5017333030700684, + "learning_rate": 3.215767634854772e-06, + "loss": 1.9544, + "step": 169 + }, + { + "epoch": 0.21157436216552583, + "grad_norm": 3.457296133041382, + "learning_rate": 3.2365145228215773e-06, + "loss": 1.922, + "step": 170 + }, + { + "epoch": 0.21157436216552583, + "eval_loss": 1.988455891609192, + "eval_runtime": 56.2099, + "eval_samples_per_second": 17.79, + "eval_steps_per_second": 0.747, + "step": 170 + }, + { + "epoch": 0.21281891723708773, + "grad_norm": 3.1964566707611084, + "learning_rate": 3.257261410788382e-06, + "loss": 1.8925, + "step": 171 + }, + { + "epoch": 0.21406347230864967, + "grad_norm": 3.234652042388916, + "learning_rate": 3.278008298755187e-06, + "loss": 1.884, + "step": 172 + }, + { + "epoch": 0.21530802738021157, + "grad_norm": 3.14414119720459, + "learning_rate": 3.2987551867219918e-06, + "loss": 1.8905, + "step": 173 + }, + { + "epoch": 0.2165525824517735, + "grad_norm": 3.606379508972168, + "learning_rate": 3.319502074688797e-06, + "loss": 1.946, + "step": 174 + }, + { + "epoch": 0.2177971375233354, + "grad_norm": 2.986646890640259, + "learning_rate": 3.3402489626556016e-06, + "loss": 1.8691, + "step": 175 + }, + { + "epoch": 0.21904169259489734, + "grad_norm": 3.7298269271850586, + "learning_rate": 3.360995850622407e-06, + "loss": 1.8717, + "step": 176 + }, + { + "epoch": 0.22028624766645924, + "grad_norm": 3.422295093536377, + "learning_rate": 3.381742738589212e-06, + "loss": 1.8568, + "step": 177 + }, + { + "epoch": 0.22153080273802117, + "grad_norm": 3.2293782234191895, + "learning_rate": 3.402489626556017e-06, + "loss": 1.8471, + "step": 178 + }, + { + "epoch": 0.22277535780958307, + "grad_norm": 3.2293782234191895, + "learning_rate": 3.402489626556017e-06, + "loss": 1.9303, + "step": 179 + }, + { + "epoch": 0.22401991288114498, + "grad_norm": 3.506223440170288, + "learning_rate": 3.423236514522822e-06, + "loss": 1.8237, + "step": 180 + }, + { + "epoch": 0.22401991288114498, + "eval_loss": 1.9734643697738647, + "eval_runtime": 52.2815, + "eval_samples_per_second": 19.127, + "eval_steps_per_second": 0.803, + "step": 180 + }, + { + "epoch": 0.2252644679527069, + "grad_norm": 3.0156619548797607, + "learning_rate": 3.4439834024896267e-06, + "loss": 1.908, + "step": 181 + }, + { + "epoch": 0.2265090230242688, + "grad_norm": 3.55517315864563, + "learning_rate": 3.4647302904564318e-06, + "loss": 1.9104, + "step": 182 + }, + { + "epoch": 0.22775357809583074, + "grad_norm": 3.144984006881714, + "learning_rate": 3.4854771784232365e-06, + "loss": 1.8198, + "step": 183 + }, + { + "epoch": 0.22899813316739265, + "grad_norm": 4.302074432373047, + "learning_rate": 3.5062240663900416e-06, + "loss": 1.8237, + "step": 184 + }, + { + "epoch": 0.23024268823895458, + "grad_norm": 3.0522446632385254, + "learning_rate": 3.526970954356847e-06, + "loss": 1.8409, + "step": 185 + }, + { + "epoch": 0.23148724331051648, + "grad_norm": 3.8607394695281982, + "learning_rate": 3.5477178423236518e-06, + "loss": 1.8807, + "step": 186 + }, + { + "epoch": 0.23273179838207841, + "grad_norm": 2.9236302375793457, + "learning_rate": 3.568464730290457e-06, + "loss": 1.7873, + "step": 187 + }, + { + "epoch": 0.23397635345364032, + "grad_norm": 4.013780117034912, + "learning_rate": 3.5892116182572616e-06, + "loss": 1.7909, + "step": 188 + }, + { + "epoch": 0.23522090852520225, + "grad_norm": 3.0933122634887695, + "learning_rate": 3.6099585062240667e-06, + "loss": 1.8469, + "step": 189 + }, + { + "epoch": 0.23646546359676415, + "grad_norm": 3.487816095352173, + "learning_rate": 3.6307053941908714e-06, + "loss": 1.8469, + "step": 190 + }, + { + "epoch": 0.23646546359676415, + "eval_loss": 1.9604240655899048, + "eval_runtime": 53.4604, + "eval_samples_per_second": 18.705, + "eval_steps_per_second": 0.786, + "step": 190 + }, + { + "epoch": 0.23771001866832608, + "grad_norm": 2.829219341278076, + "learning_rate": 3.6514522821576765e-06, + "loss": 1.8639, + "step": 191 + }, + { + "epoch": 0.238954573739888, + "grad_norm": 3.595534324645996, + "learning_rate": 3.672199170124482e-06, + "loss": 1.8988, + "step": 192 + }, + { + "epoch": 0.24019912881144992, + "grad_norm": 2.9022483825683594, + "learning_rate": 3.6929460580912867e-06, + "loss": 1.8972, + "step": 193 + }, + { + "epoch": 0.24144368388301182, + "grad_norm": 3.0667994022369385, + "learning_rate": 3.713692946058092e-06, + "loss": 1.8499, + "step": 194 + }, + { + "epoch": 0.24268823895457373, + "grad_norm": 3.0498485565185547, + "learning_rate": 3.7344398340248965e-06, + "loss": 1.8341, + "step": 195 + }, + { + "epoch": 0.24393279402613566, + "grad_norm": 4.058084964752197, + "learning_rate": 3.7551867219917016e-06, + "loss": 1.8641, + "step": 196 + }, + { + "epoch": 0.24517734909769756, + "grad_norm": 2.9995028972625732, + "learning_rate": 3.7759336099585063e-06, + "loss": 1.7885, + "step": 197 + }, + { + "epoch": 0.2464219041692595, + "grad_norm": 3.254704475402832, + "learning_rate": 3.7966804979253114e-06, + "loss": 1.7544, + "step": 198 + }, + { + "epoch": 0.2476664592408214, + "grad_norm": 2.9126744270324707, + "learning_rate": 3.817427385892116e-06, + "loss": 1.8461, + "step": 199 + }, + { + "epoch": 0.24891101431238333, + "grad_norm": 3.475247621536255, + "learning_rate": 3.838174273858922e-06, + "loss": 1.8095, + "step": 200 + }, + { + "epoch": 0.24891101431238333, + "eval_loss": 1.9373760223388672, + "eval_runtime": 51.8733, + "eval_samples_per_second": 19.278, + "eval_steps_per_second": 0.81, + "step": 200 + }, + { + "epoch": 0.25015556938394523, + "grad_norm": 3.2663731575012207, + "learning_rate": 3.858921161825726e-06, + "loss": 1.8076, + "step": 201 + }, + { + "epoch": 0.25140012445550713, + "grad_norm": 3.1151726245880127, + "learning_rate": 3.879668049792531e-06, + "loss": 1.8143, + "step": 202 + }, + { + "epoch": 0.2526446795270691, + "grad_norm": 3.102038860321045, + "learning_rate": 3.9004149377593365e-06, + "loss": 1.8279, + "step": 203 + }, + { + "epoch": 0.253889234598631, + "grad_norm": 3.370642900466919, + "learning_rate": 3.921161825726142e-06, + "loss": 1.8522, + "step": 204 + }, + { + "epoch": 0.2551337896701929, + "grad_norm": 3.116128921508789, + "learning_rate": 3.941908713692946e-06, + "loss": 1.8104, + "step": 205 + }, + { + "epoch": 0.2563783447417548, + "grad_norm": 3.12028431892395, + "learning_rate": 3.962655601659751e-06, + "loss": 1.7723, + "step": 206 + }, + { + "epoch": 0.25762289981331676, + "grad_norm": 3.1251418590545654, + "learning_rate": 3.983402489626556e-06, + "loss": 1.863, + "step": 207 + }, + { + "epoch": 0.25886745488487867, + "grad_norm": 3.073702812194824, + "learning_rate": 4.004149377593361e-06, + "loss": 1.805, + "step": 208 + }, + { + "epoch": 0.26011200995644057, + "grad_norm": 2.9772469997406006, + "learning_rate": 4.024896265560166e-06, + "loss": 1.8251, + "step": 209 + }, + { + "epoch": 0.2613565650280025, + "grad_norm": 2.9970037937164307, + "learning_rate": 4.045643153526971e-06, + "loss": 1.8386, + "step": 210 + }, + { + "epoch": 0.2613565650280025, + "eval_loss": 1.9167065620422363, + "eval_runtime": 42.3328, + "eval_samples_per_second": 23.622, + "eval_steps_per_second": 0.992, + "step": 210 + }, + { + "epoch": 0.26260112009956443, + "grad_norm": 3.1391561031341553, + "learning_rate": 4.0663900414937765e-06, + "loss": 1.8109, + "step": 211 + }, + { + "epoch": 0.26384567517112634, + "grad_norm": 3.161538600921631, + "learning_rate": 4.087136929460581e-06, + "loss": 1.8441, + "step": 212 + }, + { + "epoch": 0.26509023024268824, + "grad_norm": 3.12412166595459, + "learning_rate": 4.107883817427386e-06, + "loss": 1.7736, + "step": 213 + }, + { + "epoch": 0.26633478531425014, + "grad_norm": 3.3241145610809326, + "learning_rate": 4.128630705394191e-06, + "loss": 1.8067, + "step": 214 + }, + { + "epoch": 0.26757934038581205, + "grad_norm": 3.503307819366455, + "learning_rate": 4.149377593360996e-06, + "loss": 1.7858, + "step": 215 + }, + { + "epoch": 0.268823895457374, + "grad_norm": 3.1009578704833984, + "learning_rate": 4.170124481327801e-06, + "loss": 1.7542, + "step": 216 + }, + { + "epoch": 0.2700684505289359, + "grad_norm": 3.2236011028289795, + "learning_rate": 4.190871369294606e-06, + "loss": 1.8006, + "step": 217 + }, + { + "epoch": 0.2713130056004978, + "grad_norm": 3.048935651779175, + "learning_rate": 4.211618257261411e-06, + "loss": 1.7706, + "step": 218 + }, + { + "epoch": 0.2725575606720597, + "grad_norm": 3.3416147232055664, + "learning_rate": 4.232365145228216e-06, + "loss": 1.8108, + "step": 219 + }, + { + "epoch": 0.2738021157436217, + "grad_norm": 3.2010738849639893, + "learning_rate": 4.253112033195021e-06, + "loss": 1.8561, + "step": 220 + }, + { + "epoch": 0.2738021157436217, + "eval_loss": 1.9111930131912231, + "eval_runtime": 50.4077, + "eval_samples_per_second": 19.838, + "eval_steps_per_second": 0.833, + "step": 220 + }, + { + "epoch": 0.2750466708151836, + "grad_norm": 3.4033374786376953, + "learning_rate": 4.273858921161826e-06, + "loss": 1.8218, + "step": 221 + }, + { + "epoch": 0.2762912258867455, + "grad_norm": 3.2086002826690674, + "learning_rate": 4.294605809128631e-06, + "loss": 1.8213, + "step": 222 + }, + { + "epoch": 0.2775357809583074, + "grad_norm": 4.090956211090088, + "learning_rate": 4.315352697095436e-06, + "loss": 1.8455, + "step": 223 + }, + { + "epoch": 0.27878033602986935, + "grad_norm": 3.1142985820770264, + "learning_rate": 4.336099585062241e-06, + "loss": 1.7737, + "step": 224 + }, + { + "epoch": 0.28002489110143125, + "grad_norm": 3.369669198989868, + "learning_rate": 4.356846473029046e-06, + "loss": 1.824, + "step": 225 + }, + { + "epoch": 0.28126944617299315, + "grad_norm": 3.165672779083252, + "learning_rate": 4.3775933609958506e-06, + "loss": 1.7768, + "step": 226 + }, + { + "epoch": 0.28251400124455506, + "grad_norm": 3.3343470096588135, + "learning_rate": 4.398340248962656e-06, + "loss": 1.7278, + "step": 227 + }, + { + "epoch": 0.28375855631611696, + "grad_norm": 3.2574994564056396, + "learning_rate": 4.419087136929461e-06, + "loss": 1.8439, + "step": 228 + }, + { + "epoch": 0.2850031113876789, + "grad_norm": 3.043928623199463, + "learning_rate": 4.439834024896266e-06, + "loss": 1.818, + "step": 229 + }, + { + "epoch": 0.2862476664592408, + "grad_norm": 2.9703125953674316, + "learning_rate": 4.460580912863071e-06, + "loss": 1.773, + "step": 230 + }, + { + "epoch": 0.2862476664592408, + "eval_loss": 1.898109793663025, + "eval_runtime": 45.6023, + "eval_samples_per_second": 21.929, + "eval_steps_per_second": 0.921, + "step": 230 + }, + { + "epoch": 0.2874922215308027, + "grad_norm": 2.9997220039367676, + "learning_rate": 4.481327800829876e-06, + "loss": 1.7727, + "step": 231 + }, + { + "epoch": 0.28873677660236463, + "grad_norm": 3.1358482837677, + "learning_rate": 4.502074688796681e-06, + "loss": 1.7764, + "step": 232 + }, + { + "epoch": 0.2899813316739266, + "grad_norm": 3.395747661590576, + "learning_rate": 4.5228215767634855e-06, + "loss": 1.7826, + "step": 233 + }, + { + "epoch": 0.2912258867454885, + "grad_norm": 3.7494754791259766, + "learning_rate": 4.543568464730291e-06, + "loss": 1.7181, + "step": 234 + }, + { + "epoch": 0.2924704418170504, + "grad_norm": 3.42293381690979, + "learning_rate": 4.564315352697096e-06, + "loss": 1.7773, + "step": 235 + }, + { + "epoch": 0.2937149968886123, + "grad_norm": 3.2524514198303223, + "learning_rate": 4.585062240663901e-06, + "loss": 1.7294, + "step": 236 + }, + { + "epoch": 0.29495955196017426, + "grad_norm": 3.657869577407837, + "learning_rate": 4.605809128630706e-06, + "loss": 1.762, + "step": 237 + }, + { + "epoch": 0.29620410703173616, + "grad_norm": 3.127372980117798, + "learning_rate": 4.626556016597511e-06, + "loss": 1.7623, + "step": 238 + }, + { + "epoch": 0.29744866210329807, + "grad_norm": 4.166962146759033, + "learning_rate": 4.647302904564316e-06, + "loss": 1.6995, + "step": 239 + }, + { + "epoch": 0.29869321717485997, + "grad_norm": 3.094264030456543, + "learning_rate": 4.66804979253112e-06, + "loss": 1.7469, + "step": 240 + }, + { + "epoch": 0.29869321717485997, + "eval_loss": 1.8908178806304932, + "eval_runtime": 44.6332, + "eval_samples_per_second": 22.405, + "eval_steps_per_second": 0.941, + "step": 240 + }, + { + "epoch": 0.29993777224642193, + "grad_norm": 3.8644745349884033, + "learning_rate": 4.6887966804979255e-06, + "loss": 1.7644, + "step": 241 + }, + { + "epoch": 0.30118232731798383, + "grad_norm": 3.1488852500915527, + "learning_rate": 4.709543568464731e-06, + "loss": 1.772, + "step": 242 + }, + { + "epoch": 0.30242688238954574, + "grad_norm": 3.3179638385772705, + "learning_rate": 4.730290456431536e-06, + "loss": 1.7588, + "step": 243 + }, + { + "epoch": 0.30367143746110764, + "grad_norm": 3.328355550765991, + "learning_rate": 4.751037344398341e-06, + "loss": 1.7417, + "step": 244 + }, + { + "epoch": 0.30491599253266954, + "grad_norm": 3.2690482139587402, + "learning_rate": 4.771784232365146e-06, + "loss": 1.7562, + "step": 245 + }, + { + "epoch": 0.3061605476042315, + "grad_norm": 3.2759454250335693, + "learning_rate": 4.792531120331951e-06, + "loss": 1.7536, + "step": 246 + }, + { + "epoch": 0.3074051026757934, + "grad_norm": 2.937964916229248, + "learning_rate": 4.813278008298755e-06, + "loss": 1.7518, + "step": 247 + }, + { + "epoch": 0.3086496577473553, + "grad_norm": 3.0995302200317383, + "learning_rate": 4.83402489626556e-06, + "loss": 1.7593, + "step": 248 + }, + { + "epoch": 0.3098942128189172, + "grad_norm": 3.230459213256836, + "learning_rate": 4.8547717842323655e-06, + "loss": 1.6962, + "step": 249 + }, + { + "epoch": 0.3111387678904792, + "grad_norm": 3.242576837539673, + "learning_rate": 4.875518672199171e-06, + "loss": 1.7307, + "step": 250 + }, + { + "epoch": 0.3111387678904792, + "eval_loss": 1.8805371522903442, + "eval_runtime": 45.9886, + "eval_samples_per_second": 21.745, + "eval_steps_per_second": 0.913, + "step": 250 + }, + { + "epoch": 0.3123833229620411, + "grad_norm": 3.097045421600342, + "learning_rate": 4.896265560165976e-06, + "loss": 1.7438, + "step": 251 + }, + { + "epoch": 0.313627878033603, + "grad_norm": 3.2428948879241943, + "learning_rate": 4.91701244813278e-06, + "loss": 1.7179, + "step": 252 + }, + { + "epoch": 0.3148724331051649, + "grad_norm": 3.196274518966675, + "learning_rate": 4.937759336099586e-06, + "loss": 1.7855, + "step": 253 + }, + { + "epoch": 0.31611698817672684, + "grad_norm": 2.978203535079956, + "learning_rate": 4.95850622406639e-06, + "loss": 1.7144, + "step": 254 + }, + { + "epoch": 0.31736154324828875, + "grad_norm": 3.2641701698303223, + "learning_rate": 4.979253112033195e-06, + "loss": 1.7427, + "step": 255 + }, + { + "epoch": 0.31860609831985065, + "grad_norm": 2.7441232204437256, + "learning_rate": 5e-06, + "loss": 1.7619, + "step": 256 + }, + { + "epoch": 0.31985065339141255, + "grad_norm": 3.0723495483398438, + "learning_rate": 4.99769372693727e-06, + "loss": 1.8017, + "step": 257 + }, + { + "epoch": 0.3210952084629745, + "grad_norm": 3.104752540588379, + "learning_rate": 4.995387453874539e-06, + "loss": 1.7761, + "step": 258 + }, + { + "epoch": 0.3223397635345364, + "grad_norm": 3.138627052307129, + "learning_rate": 4.993081180811809e-06, + "loss": 1.7879, + "step": 259 + }, + { + "epoch": 0.3235843186060983, + "grad_norm": 2.8567333221435547, + "learning_rate": 4.990774907749078e-06, + "loss": 1.7561, + "step": 260 + }, + { + "epoch": 0.3235843186060983, + "eval_loss": 1.8606494665145874, + "eval_runtime": 46.453, + "eval_samples_per_second": 21.527, + "eval_steps_per_second": 0.904, + "step": 260 + }, + { + "epoch": 0.3248288736776602, + "grad_norm": 3.2696540355682373, + "learning_rate": 4.988468634686347e-06, + "loss": 1.7201, + "step": 261 + }, + { + "epoch": 0.3260734287492221, + "grad_norm": 2.6692731380462646, + "learning_rate": 4.986162361623617e-06, + "loss": 1.7264, + "step": 262 + }, + { + "epoch": 0.3273179838207841, + "grad_norm": 3.047549247741699, + "learning_rate": 4.983856088560886e-06, + "loss": 1.7362, + "step": 263 + }, + { + "epoch": 0.328562538892346, + "grad_norm": 3.0667457580566406, + "learning_rate": 4.981549815498156e-06, + "loss": 1.7321, + "step": 264 + }, + { + "epoch": 0.3298070939639079, + "grad_norm": 2.560047149658203, + "learning_rate": 4.979243542435424e-06, + "loss": 1.7508, + "step": 265 + }, + { + "epoch": 0.3310516490354698, + "grad_norm": 3.26595401763916, + "learning_rate": 4.976937269372694e-06, + "loss": 1.7248, + "step": 266 + }, + { + "epoch": 0.33229620410703176, + "grad_norm": 2.929210662841797, + "learning_rate": 4.974630996309964e-06, + "loss": 1.697, + "step": 267 + }, + { + "epoch": 0.33354075917859366, + "grad_norm": 3.1526286602020264, + "learning_rate": 4.972324723247233e-06, + "loss": 1.7385, + "step": 268 + }, + { + "epoch": 0.33478531425015556, + "grad_norm": 3.001619577407837, + "learning_rate": 4.970018450184502e-06, + "loss": 1.7072, + "step": 269 + }, + { + "epoch": 0.33602986932171747, + "grad_norm": 3.2464189529418945, + "learning_rate": 4.9677121771217715e-06, + "loss": 1.7361, + "step": 270 + }, + { + "epoch": 0.33602986932171747, + "eval_loss": 1.860226035118103, + "eval_runtime": 46.6881, + "eval_samples_per_second": 21.419, + "eval_steps_per_second": 0.9, + "step": 270 + }, + { + "epoch": 0.3372744243932794, + "grad_norm": 3.2104530334472656, + "learning_rate": 4.965405904059041e-06, + "loss": 1.7171, + "step": 271 + }, + { + "epoch": 0.33851897946484133, + "grad_norm": 3.146847724914551, + "learning_rate": 4.96309963099631e-06, + "loss": 1.7311, + "step": 272 + }, + { + "epoch": 0.33976353453640323, + "grad_norm": 3.1431286334991455, + "learning_rate": 4.96079335793358e-06, + "loss": 1.7345, + "step": 273 + }, + { + "epoch": 0.34100808960796514, + "grad_norm": 2.847163677215576, + "learning_rate": 4.958487084870849e-06, + "loss": 1.7262, + "step": 274 + }, + { + "epoch": 0.3422526446795271, + "grad_norm": 3.1987810134887695, + "learning_rate": 4.956180811808119e-06, + "loss": 1.7945, + "step": 275 + }, + { + "epoch": 0.343497199751089, + "grad_norm": 2.7475385665893555, + "learning_rate": 4.953874538745388e-06, + "loss": 1.76, + "step": 276 + }, + { + "epoch": 0.3447417548226509, + "grad_norm": 3.0427663326263428, + "learning_rate": 4.9515682656826574e-06, + "loss": 1.7324, + "step": 277 + }, + { + "epoch": 0.3459863098942128, + "grad_norm": 3.605212688446045, + "learning_rate": 4.949261992619927e-06, + "loss": 1.7712, + "step": 278 + }, + { + "epoch": 0.3472308649657747, + "grad_norm": 3.0564935207366943, + "learning_rate": 4.946955719557196e-06, + "loss": 1.6792, + "step": 279 + }, + { + "epoch": 0.34847542003733667, + "grad_norm": 3.1069741249084473, + "learning_rate": 4.944649446494466e-06, + "loss": 1.7145, + "step": 280 + }, + { + "epoch": 0.34847542003733667, + "eval_loss": 1.8537051677703857, + "eval_runtime": 47.4813, + "eval_samples_per_second": 21.061, + "eval_steps_per_second": 0.885, + "step": 280 + }, + { + "epoch": 0.3497199751088986, + "grad_norm": 2.8021512031555176, + "learning_rate": 4.942343173431734e-06, + "loss": 1.7145, + "step": 281 + }, + { + "epoch": 0.3509645301804605, + "grad_norm": 3.5549023151397705, + "learning_rate": 4.940036900369004e-06, + "loss": 1.7642, + "step": 282 + }, + { + "epoch": 0.3522090852520224, + "grad_norm": 2.7648985385894775, + "learning_rate": 4.937730627306274e-06, + "loss": 1.7255, + "step": 283 + }, + { + "epoch": 0.35345364032358434, + "grad_norm": 3.0815863609313965, + "learning_rate": 4.9354243542435426e-06, + "loss": 1.7055, + "step": 284 + }, + { + "epoch": 0.35469819539514624, + "grad_norm": 2.9009227752685547, + "learning_rate": 4.933118081180812e-06, + "loss": 1.7019, + "step": 285 + }, + { + "epoch": 0.35594275046670815, + "grad_norm": 2.618429660797119, + "learning_rate": 4.930811808118081e-06, + "loss": 1.6318, + "step": 286 + }, + { + "epoch": 0.35718730553827005, + "grad_norm": 3.368230104446411, + "learning_rate": 4.928505535055351e-06, + "loss": 1.7304, + "step": 287 + }, + { + "epoch": 0.358431860609832, + "grad_norm": 3.256889820098877, + "learning_rate": 4.92619926199262e-06, + "loss": 1.7019, + "step": 288 + }, + { + "epoch": 0.3596764156813939, + "grad_norm": 2.9366304874420166, + "learning_rate": 4.92389298892989e-06, + "loss": 1.734, + "step": 289 + }, + { + "epoch": 0.3609209707529558, + "grad_norm": 3.0393142700195312, + "learning_rate": 4.921586715867159e-06, + "loss": 1.7504, + "step": 290 + }, + { + "epoch": 0.3609209707529558, + "eval_loss": 1.8359886407852173, + "eval_runtime": 49.7622, + "eval_samples_per_second": 20.096, + "eval_steps_per_second": 0.844, + "step": 290 + }, + { + "epoch": 0.3621655258245177, + "grad_norm": 2.7288589477539062, + "learning_rate": 4.9192804428044285e-06, + "loss": 1.7004, + "step": 291 + }, + { + "epoch": 0.3634100808960797, + "grad_norm": 3.2793378829956055, + "learning_rate": 4.916974169741698e-06, + "loss": 1.7028, + "step": 292 + }, + { + "epoch": 0.3646546359676416, + "grad_norm": 2.960880756378174, + "learning_rate": 4.914667896678967e-06, + "loss": 1.6759, + "step": 293 + }, + { + "epoch": 0.3658991910392035, + "grad_norm": 2.836421012878418, + "learning_rate": 4.912361623616237e-06, + "loss": 1.6475, + "step": 294 + }, + { + "epoch": 0.3671437461107654, + "grad_norm": 2.9495465755462646, + "learning_rate": 4.910055350553506e-06, + "loss": 1.7076, + "step": 295 + }, + { + "epoch": 0.3683883011823273, + "grad_norm": 2.954730272293091, + "learning_rate": 4.907749077490776e-06, + "loss": 1.7261, + "step": 296 + }, + { + "epoch": 0.36963285625388925, + "grad_norm": 2.8167543411254883, + "learning_rate": 4.905442804428044e-06, + "loss": 1.6939, + "step": 297 + }, + { + "epoch": 0.37087741132545116, + "grad_norm": 2.857316017150879, + "learning_rate": 4.903136531365314e-06, + "loss": 1.7362, + "step": 298 + }, + { + "epoch": 0.37212196639701306, + "grad_norm": 2.8612918853759766, + "learning_rate": 4.900830258302584e-06, + "loss": 1.6489, + "step": 299 + }, + { + "epoch": 0.37336652146857496, + "grad_norm": 2.9618914127349854, + "learning_rate": 4.898523985239853e-06, + "loss": 1.718, + "step": 300 + }, + { + "epoch": 0.37336652146857496, + "eval_loss": 1.827571988105774, + "eval_runtime": 46.4552, + "eval_samples_per_second": 21.526, + "eval_steps_per_second": 0.904, + "step": 300 + }, + { + "epoch": 0.3746110765401369, + "grad_norm": 2.8706953525543213, + "learning_rate": 4.896217712177122e-06, + "loss": 1.6925, + "step": 301 + }, + { + "epoch": 0.3758556316116988, + "grad_norm": 3.2565090656280518, + "learning_rate": 4.893911439114391e-06, + "loss": 1.7369, + "step": 302 + }, + { + "epoch": 0.37710018668326073, + "grad_norm": 2.928858995437622, + "learning_rate": 4.891605166051661e-06, + "loss": 1.6882, + "step": 303 + }, + { + "epoch": 0.37834474175482263, + "grad_norm": 3.042314052581787, + "learning_rate": 4.88929889298893e-06, + "loss": 1.6466, + "step": 304 + }, + { + "epoch": 0.3795892968263846, + "grad_norm": 2.7509143352508545, + "learning_rate": 4.8869926199262e-06, + "loss": 1.685, + "step": 305 + }, + { + "epoch": 0.3808338518979465, + "grad_norm": 2.9601588249206543, + "learning_rate": 4.884686346863469e-06, + "loss": 1.6772, + "step": 306 + }, + { + "epoch": 0.3820784069695084, + "grad_norm": 2.618608236312866, + "learning_rate": 4.8823800738007384e-06, + "loss": 1.7127, + "step": 307 + }, + { + "epoch": 0.3833229620410703, + "grad_norm": 3.047581195831299, + "learning_rate": 4.880073800738008e-06, + "loss": 1.7174, + "step": 308 + }, + { + "epoch": 0.3845675171126322, + "grad_norm": 2.87735652923584, + "learning_rate": 4.877767527675277e-06, + "loss": 1.6677, + "step": 309 + }, + { + "epoch": 0.38581207218419417, + "grad_norm": 2.7169721126556396, + "learning_rate": 4.875461254612546e-06, + "loss": 1.7486, + "step": 310 + }, + { + "epoch": 0.38581207218419417, + "eval_loss": 1.8221794366836548, + "eval_runtime": 43.1681, + "eval_samples_per_second": 23.165, + "eval_steps_per_second": 0.973, + "step": 310 + }, + { + "epoch": 0.38705662725575607, + "grad_norm": 2.7094991207122803, + "learning_rate": 4.873154981549816e-06, + "loss": 1.7348, + "step": 311 + }, + { + "epoch": 0.388301182327318, + "grad_norm": 2.8989078998565674, + "learning_rate": 4.8708487084870856e-06, + "loss": 1.7352, + "step": 312 + }, + { + "epoch": 0.3895457373988799, + "grad_norm": 2.9565601348876953, + "learning_rate": 4.868542435424355e-06, + "loss": 1.7063, + "step": 313 + }, + { + "epoch": 0.39079029247044184, + "grad_norm": 2.8354082107543945, + "learning_rate": 4.8662361623616235e-06, + "loss": 1.6808, + "step": 314 + }, + { + "epoch": 0.39203484754200374, + "grad_norm": 2.8287479877471924, + "learning_rate": 4.863929889298894e-06, + "loss": 1.6731, + "step": 315 + }, + { + "epoch": 0.39327940261356564, + "grad_norm": 2.6537821292877197, + "learning_rate": 4.861623616236163e-06, + "loss": 1.695, + "step": 316 + }, + { + "epoch": 0.39452395768512755, + "grad_norm": 2.7135190963745117, + "learning_rate": 4.859317343173432e-06, + "loss": 1.6897, + "step": 317 + }, + { + "epoch": 0.3957685127566895, + "grad_norm": 3.1019129753112793, + "learning_rate": 4.857011070110701e-06, + "loss": 1.7219, + "step": 318 + }, + { + "epoch": 0.3970130678282514, + "grad_norm": 2.945737838745117, + "learning_rate": 4.854704797047971e-06, + "loss": 1.7316, + "step": 319 + }, + { + "epoch": 0.3982576228998133, + "grad_norm": 2.7668251991271973, + "learning_rate": 4.85239852398524e-06, + "loss": 1.6461, + "step": 320 + }, + { + "epoch": 0.3982576228998133, + "eval_loss": 1.8210570812225342, + "eval_runtime": 46.2552, + "eval_samples_per_second": 21.619, + "eval_steps_per_second": 0.908, + "step": 320 + }, + { + "epoch": 0.3995021779713752, + "grad_norm": 2.738109588623047, + "learning_rate": 4.8500922509225095e-06, + "loss": 1.6907, + "step": 321 + }, + { + "epoch": 0.4007467330429372, + "grad_norm": 3.0195062160491943, + "learning_rate": 4.847785977859779e-06, + "loss": 1.7099, + "step": 322 + }, + { + "epoch": 0.4019912881144991, + "grad_norm": 2.8672404289245605, + "learning_rate": 4.845479704797048e-06, + "loss": 1.6509, + "step": 323 + }, + { + "epoch": 0.403235843186061, + "grad_norm": 2.9041008949279785, + "learning_rate": 4.843173431734318e-06, + "loss": 1.5818, + "step": 324 + }, + { + "epoch": 0.4044803982576229, + "grad_norm": 2.9105985164642334, + "learning_rate": 4.840867158671587e-06, + "loss": 1.7002, + "step": 325 + }, + { + "epoch": 0.4057249533291848, + "grad_norm": 2.749009847640991, + "learning_rate": 4.838560885608857e-06, + "loss": 1.7021, + "step": 326 + }, + { + "epoch": 0.40696950840074675, + "grad_norm": 2.6998822689056396, + "learning_rate": 4.836254612546126e-06, + "loss": 1.6853, + "step": 327 + }, + { + "epoch": 0.40821406347230865, + "grad_norm": 2.678201675415039, + "learning_rate": 4.8339483394833955e-06, + "loss": 1.6934, + "step": 328 + }, + { + "epoch": 0.40945861854387056, + "grad_norm": 2.774845600128174, + "learning_rate": 4.831642066420665e-06, + "loss": 1.657, + "step": 329 + }, + { + "epoch": 0.41070317361543246, + "grad_norm": 2.6363000869750977, + "learning_rate": 4.8293357933579335e-06, + "loss": 1.6555, + "step": 330 + }, + { + "epoch": 0.41070317361543246, + "eval_loss": 1.8126581907272339, + "eval_runtime": 52.7717, + "eval_samples_per_second": 18.95, + "eval_steps_per_second": 0.796, + "step": 330 + }, + { + "epoch": 0.4119477286869944, + "grad_norm": 3.011659622192383, + "learning_rate": 4.827029520295204e-06, + "loss": 1.7202, + "step": 331 + }, + { + "epoch": 0.4131922837585563, + "grad_norm": 2.8322594165802, + "learning_rate": 4.824723247232473e-06, + "loss": 1.6736, + "step": 332 + }, + { + "epoch": 0.4144368388301182, + "grad_norm": 2.9360146522521973, + "learning_rate": 4.822416974169742e-06, + "loss": 1.7213, + "step": 333 + }, + { + "epoch": 0.41568139390168013, + "grad_norm": 2.810206413269043, + "learning_rate": 4.820110701107011e-06, + "loss": 1.6867, + "step": 334 + }, + { + "epoch": 0.4169259489732421, + "grad_norm": 2.6864097118377686, + "learning_rate": 4.817804428044281e-06, + "loss": 1.6854, + "step": 335 + }, + { + "epoch": 0.418170504044804, + "grad_norm": 2.7396061420440674, + "learning_rate": 4.81549815498155e-06, + "loss": 1.68, + "step": 336 + }, + { + "epoch": 0.4194150591163659, + "grad_norm": 2.602771520614624, + "learning_rate": 4.8131918819188194e-06, + "loss": 1.6269, + "step": 337 + }, + { + "epoch": 0.4206596141879278, + "grad_norm": 2.8174757957458496, + "learning_rate": 4.810885608856089e-06, + "loss": 1.6981, + "step": 338 + }, + { + "epoch": 0.42190416925948976, + "grad_norm": 3.2398130893707275, + "learning_rate": 4.808579335793358e-06, + "loss": 1.7061, + "step": 339 + }, + { + "epoch": 0.42314872433105166, + "grad_norm": 2.7119736671447754, + "learning_rate": 4.806273062730628e-06, + "loss": 1.6875, + "step": 340 + }, + { + "epoch": 0.42314872433105166, + "eval_loss": 1.7973068952560425, + "eval_runtime": 53.5791, + "eval_samples_per_second": 18.664, + "eval_steps_per_second": 0.784, + "step": 340 + }, + { + "epoch": 0.42439327940261357, + "grad_norm": 2.868533134460449, + "learning_rate": 4.803966789667897e-06, + "loss": 1.6967, + "step": 341 + }, + { + "epoch": 0.42563783447417547, + "grad_norm": 2.528083562850952, + "learning_rate": 4.8016605166051665e-06, + "loss": 1.6195, + "step": 342 + }, + { + "epoch": 0.4268823895457374, + "grad_norm": 2.7885115146636963, + "learning_rate": 4.799354243542436e-06, + "loss": 1.6197, + "step": 343 + }, + { + "epoch": 0.42812694461729933, + "grad_norm": 2.768247604370117, + "learning_rate": 4.797047970479705e-06, + "loss": 1.6271, + "step": 344 + }, + { + "epoch": 0.42937149968886124, + "grad_norm": 2.7710275650024414, + "learning_rate": 4.794741697416975e-06, + "loss": 1.6768, + "step": 345 + }, + { + "epoch": 0.43061605476042314, + "grad_norm": 2.968337297439575, + "learning_rate": 4.792435424354243e-06, + "loss": 1.6433, + "step": 346 + }, + { + "epoch": 0.43186060983198504, + "grad_norm": 2.5739808082580566, + "learning_rate": 4.790129151291514e-06, + "loss": 1.6327, + "step": 347 + }, + { + "epoch": 0.433105164903547, + "grad_norm": 2.7676408290863037, + "learning_rate": 4.787822878228783e-06, + "loss": 1.6178, + "step": 348 + }, + { + "epoch": 0.4343497199751089, + "grad_norm": 2.959059476852417, + "learning_rate": 4.7855166051660525e-06, + "loss": 1.6809, + "step": 349 + }, + { + "epoch": 0.4355942750466708, + "grad_norm": 2.559967279434204, + "learning_rate": 4.783210332103321e-06, + "loss": 1.6704, + "step": 350 + }, + { + "epoch": 0.4355942750466708, + "eval_loss": 1.8013949394226074, + "eval_runtime": 50.4649, + "eval_samples_per_second": 19.816, + "eval_steps_per_second": 0.832, + "step": 350 + }, + { + "epoch": 0.4368388301182327, + "grad_norm": 2.817901611328125, + "learning_rate": 4.7809040590405905e-06, + "loss": 1.6296, + "step": 351 + }, + { + "epoch": 0.43808338518979467, + "grad_norm": 2.7116055488586426, + "learning_rate": 4.778597785977861e-06, + "loss": 1.6652, + "step": 352 + }, + { + "epoch": 0.4393279402613566, + "grad_norm": 2.544750452041626, + "learning_rate": 4.776291512915129e-06, + "loss": 1.5668, + "step": 353 + }, + { + "epoch": 0.4405724953329185, + "grad_norm": 2.720534086227417, + "learning_rate": 4.773985239852399e-06, + "loss": 1.6152, + "step": 354 + }, + { + "epoch": 0.4418170504044804, + "grad_norm": 2.7270126342773438, + "learning_rate": 4.771678966789668e-06, + "loss": 1.5624, + "step": 355 + }, + { + "epoch": 0.44306160547604234, + "grad_norm": 2.7986624240875244, + "learning_rate": 4.769372693726938e-06, + "loss": 1.6621, + "step": 356 + }, + { + "epoch": 0.44430616054760425, + "grad_norm": 2.943107843399048, + "learning_rate": 4.767066420664207e-06, + "loss": 1.6713, + "step": 357 + }, + { + "epoch": 0.44555071561916615, + "grad_norm": 2.739898681640625, + "learning_rate": 4.7647601476014765e-06, + "loss": 1.6363, + "step": 358 + }, + { + "epoch": 0.44679527069072805, + "grad_norm": 2.8729329109191895, + "learning_rate": 4.762453874538746e-06, + "loss": 1.6605, + "step": 359 + }, + { + "epoch": 0.44803982576228996, + "grad_norm": 2.909029483795166, + "learning_rate": 4.760147601476015e-06, + "loss": 1.6277, + "step": 360 + }, + { + "epoch": 0.44803982576228996, + "eval_loss": 1.8110179901123047, + "eval_runtime": 47.924, + "eval_samples_per_second": 20.866, + "eval_steps_per_second": 0.876, + "step": 360 + }, + { + "epoch": 0.4492843808338519, + "grad_norm": 3.3826682567596436, + "learning_rate": 4.757841328413285e-06, + "loss": 1.7155, + "step": 361 + }, + { + "epoch": 0.4505289359054138, + "grad_norm": 3.0586729049682617, + "learning_rate": 4.755535055350554e-06, + "loss": 1.5991, + "step": 362 + }, + { + "epoch": 0.4517734909769757, + "grad_norm": 2.645576238632202, + "learning_rate": 4.753228782287823e-06, + "loss": 1.6504, + "step": 363 + }, + { + "epoch": 0.4530180460485376, + "grad_norm": 2.705967664718628, + "learning_rate": 4.750922509225093e-06, + "loss": 1.6761, + "step": 364 + }, + { + "epoch": 0.4542626011200996, + "grad_norm": 2.7651922702789307, + "learning_rate": 4.748616236162362e-06, + "loss": 1.61, + "step": 365 + }, + { + "epoch": 0.4555071561916615, + "grad_norm": 2.5654966831207275, + "learning_rate": 4.746309963099631e-06, + "loss": 1.6335, + "step": 366 + }, + { + "epoch": 0.4567517112632234, + "grad_norm": 2.6606173515319824, + "learning_rate": 4.7440036900369e-06, + "loss": 1.6207, + "step": 367 + }, + { + "epoch": 0.4579962663347853, + "grad_norm": 2.724755048751831, + "learning_rate": 4.741697416974171e-06, + "loss": 1.6329, + "step": 368 + }, + { + "epoch": 0.45924082140634725, + "grad_norm": 2.7019615173339844, + "learning_rate": 4.739391143911439e-06, + "loss": 1.6763, + "step": 369 + }, + { + "epoch": 0.46048537647790916, + "grad_norm": 2.6594254970550537, + "learning_rate": 4.737084870848709e-06, + "loss": 1.6174, + "step": 370 + }, + { + "epoch": 0.46048537647790916, + "eval_loss": 1.7924141883850098, + "eval_runtime": 53.0349, + "eval_samples_per_second": 18.855, + "eval_steps_per_second": 0.792, + "step": 370 + }, + { + "epoch": 0.46172993154947106, + "grad_norm": 2.668421745300293, + "learning_rate": 4.734778597785978e-06, + "loss": 1.6421, + "step": 371 + }, + { + "epoch": 0.46297448662103297, + "grad_norm": 2.7148995399475098, + "learning_rate": 4.7324723247232475e-06, + "loss": 1.6635, + "step": 372 + }, + { + "epoch": 0.4642190416925949, + "grad_norm": 3.156646490097046, + "learning_rate": 4.730166051660517e-06, + "loss": 1.6679, + "step": 373 + }, + { + "epoch": 0.46546359676415683, + "grad_norm": 2.8170645236968994, + "learning_rate": 4.727859778597786e-06, + "loss": 1.6492, + "step": 374 + }, + { + "epoch": 0.46670815183571873, + "grad_norm": 2.9796457290649414, + "learning_rate": 4.725553505535056e-06, + "loss": 1.6025, + "step": 375 + }, + { + "epoch": 0.46795270690728064, + "grad_norm": 2.6493406295776367, + "learning_rate": 4.723247232472325e-06, + "loss": 1.6398, + "step": 376 + }, + { + "epoch": 0.46919726197884254, + "grad_norm": 3.2643542289733887, + "learning_rate": 4.720940959409595e-06, + "loss": 1.6413, + "step": 377 + }, + { + "epoch": 0.4704418170504045, + "grad_norm": 2.7613179683685303, + "learning_rate": 4.718634686346864e-06, + "loss": 1.6251, + "step": 378 + }, + { + "epoch": 0.4716863721219664, + "grad_norm": 3.1216518878936768, + "learning_rate": 4.716328413284133e-06, + "loss": 1.6268, + "step": 379 + }, + { + "epoch": 0.4729309271935283, + "grad_norm": 2.571647882461548, + "learning_rate": 4.714022140221403e-06, + "loss": 1.6245, + "step": 380 + }, + { + "epoch": 0.4729309271935283, + "eval_loss": 1.7875893115997314, + "eval_runtime": 49.3058, + "eval_samples_per_second": 20.282, + "eval_steps_per_second": 0.852, + "step": 380 + }, + { + "epoch": 0.4741754822650902, + "grad_norm": 3.443906545639038, + "learning_rate": 4.711715867158672e-06, + "loss": 1.6799, + "step": 381 + }, + { + "epoch": 0.47542003733665217, + "grad_norm": 2.7887957096099854, + "learning_rate": 4.709409594095941e-06, + "loss": 1.6589, + "step": 382 + }, + { + "epoch": 0.47666459240821407, + "grad_norm": 3.2833151817321777, + "learning_rate": 4.70710332103321e-06, + "loss": 1.6064, + "step": 383 + }, + { + "epoch": 0.477909147479776, + "grad_norm": 2.6975057125091553, + "learning_rate": 4.704797047970481e-06, + "loss": 1.6029, + "step": 384 + }, + { + "epoch": 0.4791537025513379, + "grad_norm": 2.745246171951294, + "learning_rate": 4.702490774907749e-06, + "loss": 1.7127, + "step": 385 + }, + { + "epoch": 0.48039825762289984, + "grad_norm": 3.0522775650024414, + "learning_rate": 4.700184501845019e-06, + "loss": 1.6508, + "step": 386 + }, + { + "epoch": 0.48164281269446174, + "grad_norm": 2.689218044281006, + "learning_rate": 4.697878228782288e-06, + "loss": 1.6971, + "step": 387 + }, + { + "epoch": 0.48288736776602365, + "grad_norm": 3.060361862182617, + "learning_rate": 4.6955719557195575e-06, + "loss": 1.6464, + "step": 388 + }, + { + "epoch": 0.48413192283758555, + "grad_norm": 2.872262477874756, + "learning_rate": 4.693265682656827e-06, + "loss": 1.6293, + "step": 389 + }, + { + "epoch": 0.48537647790914745, + "grad_norm": 2.980050563812256, + "learning_rate": 4.690959409594096e-06, + "loss": 1.6435, + "step": 390 + }, + { + "epoch": 0.48537647790914745, + "eval_loss": 1.7791496515274048, + "eval_runtime": 50.0299, + "eval_samples_per_second": 19.988, + "eval_steps_per_second": 0.839, + "step": 390 + }, + { + "epoch": 0.4866210329807094, + "grad_norm": 2.764827251434326, + "learning_rate": 4.688653136531366e-06, + "loss": 1.6279, + "step": 391 + }, + { + "epoch": 0.4878655880522713, + "grad_norm": 2.6912333965301514, + "learning_rate": 4.686346863468635e-06, + "loss": 1.7276, + "step": 392 + }, + { + "epoch": 0.4891101431238332, + "grad_norm": 2.8358330726623535, + "learning_rate": 4.6840405904059046e-06, + "loss": 1.6482, + "step": 393 + }, + { + "epoch": 0.4903546981953951, + "grad_norm": 2.6581575870513916, + "learning_rate": 4.681734317343174e-06, + "loss": 1.6853, + "step": 394 + }, + { + "epoch": 0.4915992532669571, + "grad_norm": 2.941425323486328, + "learning_rate": 4.6794280442804426e-06, + "loss": 1.6075, + "step": 395 + }, + { + "epoch": 0.492843808338519, + "grad_norm": 3.2334814071655273, + "learning_rate": 4.677121771217713e-06, + "loss": 1.6157, + "step": 396 + }, + { + "epoch": 0.4940883634100809, + "grad_norm": 2.984410524368286, + "learning_rate": 4.674815498154982e-06, + "loss": 1.6257, + "step": 397 + }, + { + "epoch": 0.4953329184816428, + "grad_norm": 3.0650789737701416, + "learning_rate": 4.672509225092252e-06, + "loss": 1.6573, + "step": 398 + }, + { + "epoch": 0.49657747355320475, + "grad_norm": 2.798664093017578, + "learning_rate": 4.67020295202952e-06, + "loss": 1.638, + "step": 399 + }, + { + "epoch": 0.49782202862476665, + "grad_norm": 2.8266706466674805, + "learning_rate": 4.66789667896679e-06, + "loss": 1.6484, + "step": 400 + }, + { + "epoch": 0.49782202862476665, + "eval_loss": 1.7714862823486328, + "eval_runtime": 48.0657, + "eval_samples_per_second": 20.805, + "eval_steps_per_second": 0.874, + "step": 400 + }, + { + "epoch": 0.49906658369632856, + "grad_norm": 2.9928388595581055, + "learning_rate": 4.66559040590406e-06, + "loss": 1.6721, + "step": 401 + }, + { + "epoch": 0.5003111387678905, + "grad_norm": 2.573514461517334, + "learning_rate": 4.6632841328413285e-06, + "loss": 1.679, + "step": 402 + }, + { + "epoch": 0.5015556938394524, + "grad_norm": 3.1736996173858643, + "learning_rate": 4.660977859778598e-06, + "loss": 1.7124, + "step": 403 + }, + { + "epoch": 0.5028002489110143, + "grad_norm": 2.858849287033081, + "learning_rate": 4.658671586715867e-06, + "loss": 1.6622, + "step": 404 + }, + { + "epoch": 0.5040448039825762, + "grad_norm": 2.825698137283325, + "learning_rate": 4.656365313653137e-06, + "loss": 1.6464, + "step": 405 + }, + { + "epoch": 0.5052893590541382, + "grad_norm": 3.783891439437866, + "learning_rate": 4.654059040590406e-06, + "loss": 1.6665, + "step": 406 + }, + { + "epoch": 0.5065339141257, + "grad_norm": 2.789813756942749, + "learning_rate": 4.651752767527676e-06, + "loss": 1.6064, + "step": 407 + }, + { + "epoch": 0.507778469197262, + "grad_norm": 3.3106348514556885, + "learning_rate": 4.649446494464945e-06, + "loss": 1.6542, + "step": 408 + }, + { + "epoch": 0.5090230242688238, + "grad_norm": 2.8805112838745117, + "learning_rate": 4.6471402214022145e-06, + "loss": 1.6058, + "step": 409 + }, + { + "epoch": 0.5102675793403858, + "grad_norm": 3.30898118019104, + "learning_rate": 4.644833948339484e-06, + "loss": 1.5933, + "step": 410 + }, + { + "epoch": 0.5102675793403858, + "eval_loss": 1.7787818908691406, + "eval_runtime": 44.0876, + "eval_samples_per_second": 22.682, + "eval_steps_per_second": 0.953, + "step": 410 + }, + { + "epoch": 0.5115121344119478, + "grad_norm": 3.3563625812530518, + "learning_rate": 4.642527675276753e-06, + "loss": 1.6751, + "step": 411 + }, + { + "epoch": 0.5127566894835096, + "grad_norm": 2.968120574951172, + "learning_rate": 4.640221402214023e-06, + "loss": 1.663, + "step": 412 + }, + { + "epoch": 0.5140012445550716, + "grad_norm": 3.310248374938965, + "learning_rate": 4.637915129151292e-06, + "loss": 1.658, + "step": 413 + }, + { + "epoch": 0.5152457996266335, + "grad_norm": 2.7011170387268066, + "learning_rate": 4.635608856088562e-06, + "loss": 1.6894, + "step": 414 + }, + { + "epoch": 0.5164903546981954, + "grad_norm": 2.804901361465454, + "learning_rate": 4.63330258302583e-06, + "loss": 1.602, + "step": 415 + }, + { + "epoch": 0.5177349097697573, + "grad_norm": 3.2412750720977783, + "learning_rate": 4.6309963099631e-06, + "loss": 1.6158, + "step": 416 + }, + { + "epoch": 0.5189794648413192, + "grad_norm": 2.699909210205078, + "learning_rate": 4.62869003690037e-06, + "loss": 1.5842, + "step": 417 + }, + { + "epoch": 0.5202240199128811, + "grad_norm": 3.065929412841797, + "learning_rate": 4.6263837638376384e-06, + "loss": 1.6434, + "step": 418 + }, + { + "epoch": 0.5214685749844431, + "grad_norm": 2.895305633544922, + "learning_rate": 4.624077490774908e-06, + "loss": 1.5969, + "step": 419 + }, + { + "epoch": 0.522713130056005, + "grad_norm": 3.1590664386749268, + "learning_rate": 4.621771217712177e-06, + "loss": 1.6211, + "step": 420 + }, + { + "epoch": 0.522713130056005, + "eval_loss": 1.7674189805984497, + "eval_runtime": 53.328, + "eval_samples_per_second": 18.752, + "eval_steps_per_second": 0.788, + "step": 420 + }, + { + "epoch": 0.5239576851275669, + "grad_norm": 2.7126312255859375, + "learning_rate": 4.619464944649447e-06, + "loss": 1.6246, + "step": 421 + }, + { + "epoch": 0.5252022401991289, + "grad_norm": 2.918508768081665, + "learning_rate": 4.617158671586716e-06, + "loss": 1.6937, + "step": 422 + }, + { + "epoch": 0.5264467952706907, + "grad_norm": 3.008610725402832, + "learning_rate": 4.6148523985239856e-06, + "loss": 1.6864, + "step": 423 + }, + { + "epoch": 0.5276913503422527, + "grad_norm": 2.6612043380737305, + "learning_rate": 4.612546125461255e-06, + "loss": 1.5785, + "step": 424 + }, + { + "epoch": 0.5289359054138145, + "grad_norm": 2.850679874420166, + "learning_rate": 4.610239852398524e-06, + "loss": 1.5899, + "step": 425 + }, + { + "epoch": 0.5301804604853765, + "grad_norm": 2.9205031394958496, + "learning_rate": 4.607933579335794e-06, + "loss": 1.5867, + "step": 426 + }, + { + "epoch": 0.5314250155569384, + "grad_norm": 2.929532527923584, + "learning_rate": 4.605627306273063e-06, + "loss": 1.6182, + "step": 427 + }, + { + "epoch": 0.5326695706285003, + "grad_norm": 2.874936103820801, + "learning_rate": 4.603321033210333e-06, + "loss": 1.5789, + "step": 428 + }, + { + "epoch": 0.5339141257000622, + "grad_norm": 2.8703525066375732, + "learning_rate": 4.601014760147602e-06, + "loss": 1.5859, + "step": 429 + }, + { + "epoch": 0.5351586807716241, + "grad_norm": 2.7076902389526367, + "learning_rate": 4.5987084870848715e-06, + "loss": 1.644, + "step": 430 + }, + { + "epoch": 0.5351586807716241, + "eval_loss": 1.7592095136642456, + "eval_runtime": 47.9438, + "eval_samples_per_second": 20.858, + "eval_steps_per_second": 0.876, + "step": 430 + }, + { + "epoch": 0.536403235843186, + "grad_norm": 2.839160680770874, + "learning_rate": 4.59640221402214e-06, + "loss": 1.6357, + "step": 431 + }, + { + "epoch": 0.537647790914748, + "grad_norm": 2.674025535583496, + "learning_rate": 4.5940959409594095e-06, + "loss": 1.676, + "step": 432 + }, + { + "epoch": 0.5388923459863099, + "grad_norm": 3.0619537830352783, + "learning_rate": 4.59178966789668e-06, + "loss": 1.5617, + "step": 433 + }, + { + "epoch": 0.5401369010578718, + "grad_norm": 3.058418035507202, + "learning_rate": 4.589483394833948e-06, + "loss": 1.5951, + "step": 434 + }, + { + "epoch": 0.5413814561294338, + "grad_norm": 3.0046396255493164, + "learning_rate": 4.587177121771218e-06, + "loss": 1.6473, + "step": 435 + }, + { + "epoch": 0.5426260112009956, + "grad_norm": 2.7736752033233643, + "learning_rate": 4.584870848708487e-06, + "loss": 1.6262, + "step": 436 + }, + { + "epoch": 0.5438705662725576, + "grad_norm": 2.7804994583129883, + "learning_rate": 4.5825645756457575e-06, + "loss": 1.6886, + "step": 437 + }, + { + "epoch": 0.5451151213441194, + "grad_norm": 3.0717954635620117, + "learning_rate": 4.580258302583026e-06, + "loss": 1.6287, + "step": 438 + }, + { + "epoch": 0.5463596764156814, + "grad_norm": 2.800243854522705, + "learning_rate": 4.5779520295202955e-06, + "loss": 1.6169, + "step": 439 + }, + { + "epoch": 0.5476042314872434, + "grad_norm": 2.9581644535064697, + "learning_rate": 4.575645756457565e-06, + "loss": 1.6617, + "step": 440 + }, + { + "epoch": 0.5476042314872434, + "eval_loss": 1.7642868757247925, + "eval_runtime": 51.2646, + "eval_samples_per_second": 19.507, + "eval_steps_per_second": 0.819, + "step": 440 + }, + { + "epoch": 0.5488487865588052, + "grad_norm": 3.0067460536956787, + "learning_rate": 4.573339483394834e-06, + "loss": 1.6482, + "step": 441 + }, + { + "epoch": 0.5500933416303672, + "grad_norm": 2.9075019359588623, + "learning_rate": 4.571033210332104e-06, + "loss": 1.6595, + "step": 442 + }, + { + "epoch": 0.551337896701929, + "grad_norm": 2.7788755893707275, + "learning_rate": 4.568726937269373e-06, + "loss": 1.5733, + "step": 443 + }, + { + "epoch": 0.552582451773491, + "grad_norm": 2.7775425910949707, + "learning_rate": 4.566420664206643e-06, + "loss": 1.6925, + "step": 444 + }, + { + "epoch": 0.5538270068450529, + "grad_norm": 2.6949567794799805, + "learning_rate": 4.564114391143912e-06, + "loss": 1.5738, + "step": 445 + }, + { + "epoch": 0.5550715619166148, + "grad_norm": 2.7093007564544678, + "learning_rate": 4.5618081180811814e-06, + "loss": 1.5848, + "step": 446 + }, + { + "epoch": 0.5563161169881767, + "grad_norm": 2.6920173168182373, + "learning_rate": 4.559501845018451e-06, + "loss": 1.6092, + "step": 447 + }, + { + "epoch": 0.5575606720597387, + "grad_norm": 2.733731985092163, + "learning_rate": 4.5571955719557194e-06, + "loss": 1.6056, + "step": 448 + }, + { + "epoch": 0.5588052271313005, + "grad_norm": 2.9156274795532227, + "learning_rate": 4.55488929889299e-06, + "loss": 1.6436, + "step": 449 + }, + { + "epoch": 0.5600497822028625, + "grad_norm": 2.8802616596221924, + "learning_rate": 4.552583025830259e-06, + "loss": 1.6309, + "step": 450 + }, + { + "epoch": 0.5600497822028625, + "eval_loss": 1.7486340999603271, + "eval_runtime": 47.2219, + "eval_samples_per_second": 21.177, + "eval_steps_per_second": 0.889, + "step": 450 + }, + { + "epoch": 0.5612943372744243, + "grad_norm": 2.729743242263794, + "learning_rate": 4.550276752767528e-06, + "loss": 1.7171, + "step": 451 + }, + { + "epoch": 0.5625388923459863, + "grad_norm": 2.800049066543579, + "learning_rate": 4.547970479704797e-06, + "loss": 1.6098, + "step": 452 + }, + { + "epoch": 0.5637834474175483, + "grad_norm": 2.9659311771392822, + "learning_rate": 4.5456642066420666e-06, + "loss": 1.6646, + "step": 453 + }, + { + "epoch": 0.5650280024891101, + "grad_norm": 2.5417754650115967, + "learning_rate": 4.543357933579336e-06, + "loss": 1.5847, + "step": 454 + }, + { + "epoch": 0.5662725575606721, + "grad_norm": 2.8471338748931885, + "learning_rate": 4.541051660516605e-06, + "loss": 1.6235, + "step": 455 + }, + { + "epoch": 0.5675171126322339, + "grad_norm": 2.7039637565612793, + "learning_rate": 4.538745387453875e-06, + "loss": 1.5613, + "step": 456 + }, + { + "epoch": 0.5687616677037959, + "grad_norm": 3.0383341312408447, + "learning_rate": 4.536439114391144e-06, + "loss": 1.6485, + "step": 457 + }, + { + "epoch": 0.5700062227753578, + "grad_norm": 2.528388261795044, + "learning_rate": 4.534132841328414e-06, + "loss": 1.6117, + "step": 458 + }, + { + "epoch": 0.5712507778469197, + "grad_norm": 2.7623119354248047, + "learning_rate": 4.531826568265683e-06, + "loss": 1.6017, + "step": 459 + }, + { + "epoch": 0.5724953329184816, + "grad_norm": 2.9213945865631104, + "learning_rate": 4.5295202952029525e-06, + "loss": 1.643, + "step": 460 + }, + { + "epoch": 0.5724953329184816, + "eval_loss": 1.747575044631958, + "eval_runtime": 45.2844, + "eval_samples_per_second": 22.083, + "eval_steps_per_second": 0.927, + "step": 460 + }, + { + "epoch": 0.5737398879900436, + "grad_norm": 2.664275884628296, + "learning_rate": 4.527214022140222e-06, + "loss": 1.6154, + "step": 461 + }, + { + "epoch": 0.5749844430616055, + "grad_norm": 2.5891788005828857, + "learning_rate": 4.524907749077491e-06, + "loss": 1.5975, + "step": 462 + }, + { + "epoch": 0.5762289981331674, + "grad_norm": 2.8126487731933594, + "learning_rate": 4.522601476014761e-06, + "loss": 1.6306, + "step": 463 + }, + { + "epoch": 0.5774735532047293, + "grad_norm": 2.799391031265259, + "learning_rate": 4.520295202952029e-06, + "loss": 1.6139, + "step": 464 + }, + { + "epoch": 0.5787181082762912, + "grad_norm": 2.7323718070983887, + "learning_rate": 4.5179889298893e-06, + "loss": 1.632, + "step": 465 + }, + { + "epoch": 0.5799626633478532, + "grad_norm": 2.601386785507202, + "learning_rate": 4.515682656826569e-06, + "loss": 1.6139, + "step": 466 + }, + { + "epoch": 0.581207218419415, + "grad_norm": 2.562162399291992, + "learning_rate": 4.513376383763838e-06, + "loss": 1.5168, + "step": 467 + }, + { + "epoch": 0.582451773490977, + "grad_norm": 2.6912002563476562, + "learning_rate": 4.511070110701107e-06, + "loss": 1.6249, + "step": 468 + }, + { + "epoch": 0.583696328562539, + "grad_norm": 2.6971495151519775, + "learning_rate": 4.5087638376383765e-06, + "loss": 1.616, + "step": 469 + }, + { + "epoch": 0.5849408836341008, + "grad_norm": 2.6898910999298096, + "learning_rate": 4.506457564575646e-06, + "loss": 1.614, + "step": 470 + }, + { + "epoch": 0.5849408836341008, + "eval_loss": 1.7460769414901733, + "eval_runtime": 45.3055, + "eval_samples_per_second": 22.072, + "eval_steps_per_second": 0.927, + "step": 470 + }, + { + "epoch": 0.5861854387056628, + "grad_norm": 2.698180675506592, + "learning_rate": 4.504151291512915e-06, + "loss": 1.5993, + "step": 471 + }, + { + "epoch": 0.5874299937772246, + "grad_norm": 2.7125210762023926, + "learning_rate": 4.501845018450185e-06, + "loss": 1.6113, + "step": 472 + }, + { + "epoch": 0.5886745488487866, + "grad_norm": 2.754445791244507, + "learning_rate": 4.499538745387454e-06, + "loss": 1.6719, + "step": 473 + }, + { + "epoch": 0.5899191039203485, + "grad_norm": 2.6979362964630127, + "learning_rate": 4.497232472324724e-06, + "loss": 1.608, + "step": 474 + }, + { + "epoch": 0.5911636589919104, + "grad_norm": 2.7694528102874756, + "learning_rate": 4.494926199261993e-06, + "loss": 1.5695, + "step": 475 + }, + { + "epoch": 0.5924082140634723, + "grad_norm": 2.653353214263916, + "learning_rate": 4.4926199261992624e-06, + "loss": 1.5523, + "step": 476 + }, + { + "epoch": 0.5936527691350342, + "grad_norm": 2.6632070541381836, + "learning_rate": 4.490313653136532e-06, + "loss": 1.5675, + "step": 477 + }, + { + "epoch": 0.5948973242065961, + "grad_norm": 3.038543939590454, + "learning_rate": 4.488007380073801e-06, + "loss": 1.5692, + "step": 478 + }, + { + "epoch": 0.5961418792781581, + "grad_norm": 2.8123953342437744, + "learning_rate": 4.485701107011071e-06, + "loss": 1.6223, + "step": 479 + }, + { + "epoch": 0.5973864343497199, + "grad_norm": 2.5845773220062256, + "learning_rate": 4.483394833948339e-06, + "loss": 1.5773, + "step": 480 + }, + { + "epoch": 0.5973864343497199, + "eval_loss": 1.7419319152832031, + "eval_runtime": 44.6842, + "eval_samples_per_second": 22.379, + "eval_steps_per_second": 0.94, + "step": 480 + }, + { + "epoch": 0.5986309894212819, + "grad_norm": 3.069936752319336, + "learning_rate": 4.4810885608856096e-06, + "loss": 1.6323, + "step": 481 + }, + { + "epoch": 0.5998755444928439, + "grad_norm": 2.9085781574249268, + "learning_rate": 4.478782287822879e-06, + "loss": 1.5845, + "step": 482 + }, + { + "epoch": 0.6011200995644057, + "grad_norm": 2.8568010330200195, + "learning_rate": 4.4764760147601476e-06, + "loss": 1.5898, + "step": 483 + }, + { + "epoch": 0.6023646546359677, + "grad_norm": 3.089081048965454, + "learning_rate": 4.474169741697417e-06, + "loss": 1.6622, + "step": 484 + }, + { + "epoch": 0.6036092097075295, + "grad_norm": 2.8515470027923584, + "learning_rate": 4.471863468634686e-06, + "loss": 1.5859, + "step": 485 + }, + { + "epoch": 0.6048537647790915, + "grad_norm": 2.647765874862671, + "learning_rate": 4.469557195571957e-06, + "loss": 1.5926, + "step": 486 + }, + { + "epoch": 0.6060983198506534, + "grad_norm": 2.61676025390625, + "learning_rate": 4.467250922509225e-06, + "loss": 1.5684, + "step": 487 + }, + { + "epoch": 0.6073428749222153, + "grad_norm": 2.7127039432525635, + "learning_rate": 4.464944649446495e-06, + "loss": 1.5702, + "step": 488 + }, + { + "epoch": 0.6085874299937772, + "grad_norm": 2.9932055473327637, + "learning_rate": 4.462638376383764e-06, + "loss": 1.5976, + "step": 489 + }, + { + "epoch": 0.6098319850653391, + "grad_norm": 2.6508774757385254, + "learning_rate": 4.4603321033210335e-06, + "loss": 1.586, + "step": 490 + }, + { + "epoch": 0.6098319850653391, + "eval_loss": 1.7357326745986938, + "eval_runtime": 45.393, + "eval_samples_per_second": 22.03, + "eval_steps_per_second": 0.925, + "step": 490 + }, + { + "epoch": 0.611076540136901, + "grad_norm": 2.7626044750213623, + "learning_rate": 4.458025830258303e-06, + "loss": 1.5773, + "step": 491 + }, + { + "epoch": 0.612321095208463, + "grad_norm": 2.595003604888916, + "learning_rate": 4.455719557195572e-06, + "loss": 1.601, + "step": 492 + }, + { + "epoch": 0.6135656502800249, + "grad_norm": 2.595767021179199, + "learning_rate": 4.453413284132842e-06, + "loss": 1.6287, + "step": 493 + }, + { + "epoch": 0.6148102053515868, + "grad_norm": 2.755845308303833, + "learning_rate": 4.451107011070111e-06, + "loss": 1.593, + "step": 494 + }, + { + "epoch": 0.6160547604231488, + "grad_norm": 2.7302653789520264, + "learning_rate": 4.448800738007381e-06, + "loss": 1.5768, + "step": 495 + }, + { + "epoch": 0.6172993154947106, + "grad_norm": 2.6493024826049805, + "learning_rate": 4.446494464944649e-06, + "loss": 1.6059, + "step": 496 + }, + { + "epoch": 0.6185438705662726, + "grad_norm": 2.8633735179901123, + "learning_rate": 4.4441881918819195e-06, + "loss": 1.598, + "step": 497 + }, + { + "epoch": 0.6197884256378344, + "grad_norm": 2.65639066696167, + "learning_rate": 4.441881918819189e-06, + "loss": 1.6036, + "step": 498 + }, + { + "epoch": 0.6210329807093964, + "grad_norm": 2.6867759227752686, + "learning_rate": 4.439575645756458e-06, + "loss": 1.5676, + "step": 499 + }, + { + "epoch": 0.6222775357809583, + "grad_norm": 2.7986443042755127, + "learning_rate": 4.437269372693727e-06, + "loss": 1.524, + "step": 500 + }, + { + "epoch": 0.6222775357809583, + "eval_loss": 1.7449125051498413, + "eval_runtime": 43.9329, + "eval_samples_per_second": 22.762, + "eval_steps_per_second": 0.956, + "step": 500 + }, + { + "epoch": 0.6235220908525202, + "grad_norm": 2.665905714035034, + "learning_rate": 4.434963099630996e-06, + "loss": 1.5635, + "step": 501 + }, + { + "epoch": 0.6247666459240822, + "grad_norm": 2.902435541152954, + "learning_rate": 4.432656826568267e-06, + "loss": 1.6377, + "step": 502 + }, + { + "epoch": 0.6260112009956441, + "grad_norm": 2.6872262954711914, + "learning_rate": 4.430350553505535e-06, + "loss": 1.5553, + "step": 503 + }, + { + "epoch": 0.627255756067206, + "grad_norm": 2.676621913909912, + "learning_rate": 4.428044280442805e-06, + "loss": 1.6101, + "step": 504 + }, + { + "epoch": 0.6285003111387679, + "grad_norm": 2.756347179412842, + "learning_rate": 4.425738007380074e-06, + "loss": 1.5884, + "step": 505 + }, + { + "epoch": 0.6297448662103298, + "grad_norm": 2.856882333755493, + "learning_rate": 4.4234317343173434e-06, + "loss": 1.5635, + "step": 506 + }, + { + "epoch": 0.6309894212818917, + "grad_norm": 2.7708330154418945, + "learning_rate": 4.421125461254613e-06, + "loss": 1.6028, + "step": 507 + }, + { + "epoch": 0.6322339763534537, + "grad_norm": 2.8167600631713867, + "learning_rate": 4.418819188191882e-06, + "loss": 1.6026, + "step": 508 + }, + { + "epoch": 0.6334785314250155, + "grad_norm": 2.54194974899292, + "learning_rate": 4.416512915129152e-06, + "loss": 1.6001, + "step": 509 + }, + { + "epoch": 0.6347230864965775, + "grad_norm": 2.683037519454956, + "learning_rate": 4.414206642066421e-06, + "loss": 1.6084, + "step": 510 + }, + { + "epoch": 0.6347230864965775, + "eval_loss": 1.7376186847686768, + "eval_runtime": 42.3026, + "eval_samples_per_second": 23.639, + "eval_steps_per_second": 0.993, + "step": 510 + }, + { + "epoch": 0.6359676415681393, + "grad_norm": 2.82627010345459, + "learning_rate": 4.4119003690036905e-06, + "loss": 1.6258, + "step": 511 + }, + { + "epoch": 0.6372121966397013, + "grad_norm": 2.6139848232269287, + "learning_rate": 4.40959409594096e-06, + "loss": 1.6029, + "step": 512 + }, + { + "epoch": 0.6384567517112633, + "grad_norm": 2.646712303161621, + "learning_rate": 4.407287822878229e-06, + "loss": 1.5602, + "step": 513 + }, + { + "epoch": 0.6397013067828251, + "grad_norm": 2.910935163497925, + "learning_rate": 4.404981549815499e-06, + "loss": 1.5757, + "step": 514 + }, + { + "epoch": 0.6409458618543871, + "grad_norm": 2.6001148223876953, + "learning_rate": 4.402675276752768e-06, + "loss": 1.5857, + "step": 515 + }, + { + "epoch": 0.642190416925949, + "grad_norm": 2.8525235652923584, + "learning_rate": 4.400369003690037e-06, + "loss": 1.6092, + "step": 516 + }, + { + "epoch": 0.6434349719975109, + "grad_norm": 2.5335488319396973, + "learning_rate": 4.398062730627306e-06, + "loss": 1.5849, + "step": 517 + }, + { + "epoch": 0.6446795270690728, + "grad_norm": 2.5787103176116943, + "learning_rate": 4.3957564575645765e-06, + "loss": 1.634, + "step": 518 + }, + { + "epoch": 0.6459240821406347, + "grad_norm": 2.6188197135925293, + "learning_rate": 4.393450184501845e-06, + "loss": 1.5477, + "step": 519 + }, + { + "epoch": 0.6471686372121966, + "grad_norm": 2.6548666954040527, + "learning_rate": 4.3911439114391145e-06, + "loss": 1.6133, + "step": 520 + }, + { + "epoch": 0.6471686372121966, + "eval_loss": 1.7393039464950562, + "eval_runtime": 45.9234, + "eval_samples_per_second": 21.775, + "eval_steps_per_second": 0.915, + "step": 520 + }, + { + "epoch": 0.6484131922837586, + "grad_norm": 2.907928228378296, + "learning_rate": 4.388837638376384e-06, + "loss": 1.587, + "step": 521 + }, + { + "epoch": 0.6496577473553204, + "grad_norm": 2.7899692058563232, + "learning_rate": 4.386531365313653e-06, + "loss": 1.6126, + "step": 522 + }, + { + "epoch": 0.6509023024268824, + "grad_norm": 2.680147886276245, + "learning_rate": 4.384225092250923e-06, + "loss": 1.565, + "step": 523 + }, + { + "epoch": 0.6521468574984443, + "grad_norm": 2.6590754985809326, + "learning_rate": 4.381918819188192e-06, + "loss": 1.6127, + "step": 524 + }, + { + "epoch": 0.6533914125700062, + "grad_norm": 2.6638906002044678, + "learning_rate": 4.379612546125462e-06, + "loss": 1.5615, + "step": 525 + }, + { + "epoch": 0.6546359676415682, + "grad_norm": 2.6518194675445557, + "learning_rate": 4.377306273062731e-06, + "loss": 1.6202, + "step": 526 + }, + { + "epoch": 0.65588052271313, + "grad_norm": 2.5323619842529297, + "learning_rate": 4.3750000000000005e-06, + "loss": 1.5305, + "step": 527 + }, + { + "epoch": 0.657125077784692, + "grad_norm": 2.6871442794799805, + "learning_rate": 4.37269372693727e-06, + "loss": 1.6043, + "step": 528 + }, + { + "epoch": 0.6583696328562539, + "grad_norm": 2.760746717453003, + "learning_rate": 4.370387453874539e-06, + "loss": 1.6059, + "step": 529 + }, + { + "epoch": 0.6596141879278158, + "grad_norm": 2.780482769012451, + "learning_rate": 4.368081180811809e-06, + "loss": 1.6082, + "step": 530 + }, + { + "epoch": 0.6596141879278158, + "eval_loss": 1.7219713926315308, + "eval_runtime": 51.8478, + "eval_samples_per_second": 19.287, + "eval_steps_per_second": 0.81, + "step": 530 + }, + { + "epoch": 0.6608587429993777, + "grad_norm": 2.605890989303589, + "learning_rate": 4.365774907749078e-06, + "loss": 1.5554, + "step": 531 + }, + { + "epoch": 0.6621032980709396, + "grad_norm": 2.731555461883545, + "learning_rate": 4.363468634686347e-06, + "loss": 1.606, + "step": 532 + }, + { + "epoch": 0.6633478531425016, + "grad_norm": 2.8356943130493164, + "learning_rate": 4.361162361623616e-06, + "loss": 1.5601, + "step": 533 + }, + { + "epoch": 0.6645924082140635, + "grad_norm": 2.7196593284606934, + "learning_rate": 4.3588560885608864e-06, + "loss": 1.5722, + "step": 534 + }, + { + "epoch": 0.6658369632856254, + "grad_norm": 2.621371269226074, + "learning_rate": 4.356549815498156e-06, + "loss": 1.6194, + "step": 535 + }, + { + "epoch": 0.6670815183571873, + "grad_norm": 2.653916120529175, + "learning_rate": 4.354243542435424e-06, + "loss": 1.5592, + "step": 536 + }, + { + "epoch": 0.6683260734287492, + "grad_norm": 2.88431453704834, + "learning_rate": 4.351937269372694e-06, + "loss": 1.6296, + "step": 537 + }, + { + "epoch": 0.6695706285003111, + "grad_norm": 2.667130470275879, + "learning_rate": 4.349630996309963e-06, + "loss": 1.5624, + "step": 538 + }, + { + "epoch": 0.6708151835718731, + "grad_norm": 2.6453566551208496, + "learning_rate": 4.347324723247233e-06, + "loss": 1.6025, + "step": 539 + }, + { + "epoch": 0.6720597386434349, + "grad_norm": 3.032271146774292, + "learning_rate": 4.345018450184502e-06, + "loss": 1.6771, + "step": 540 + }, + { + "epoch": 0.6720597386434349, + "eval_loss": 1.721895456314087, + "eval_runtime": 50.62, + "eval_samples_per_second": 19.755, + "eval_steps_per_second": 0.83, + "step": 540 + }, + { + "epoch": 0.6733042937149969, + "grad_norm": 2.8476362228393555, + "learning_rate": 4.3427121771217715e-06, + "loss": 1.5663, + "step": 541 + }, + { + "epoch": 0.6745488487865589, + "grad_norm": 2.615602970123291, + "learning_rate": 4.340405904059041e-06, + "loss": 1.5815, + "step": 542 + }, + { + "epoch": 0.6757934038581207, + "grad_norm": 3.221571683883667, + "learning_rate": 4.33809963099631e-06, + "loss": 1.5955, + "step": 543 + }, + { + "epoch": 0.6770379589296827, + "grad_norm": 2.602994203567505, + "learning_rate": 4.33579335793358e-06, + "loss": 1.5667, + "step": 544 + }, + { + "epoch": 0.6782825140012445, + "grad_norm": 2.9380433559417725, + "learning_rate": 4.333487084870848e-06, + "loss": 1.5651, + "step": 545 + }, + { + "epoch": 0.6795270690728065, + "grad_norm": 3.0689425468444824, + "learning_rate": 4.331180811808119e-06, + "loss": 1.6003, + "step": 546 + }, + { + "epoch": 0.6807716241443684, + "grad_norm": 2.770317316055298, + "learning_rate": 4.328874538745388e-06, + "loss": 1.5576, + "step": 547 + }, + { + "epoch": 0.6820161792159303, + "grad_norm": 2.8631815910339355, + "learning_rate": 4.3265682656826575e-06, + "loss": 1.5308, + "step": 548 + }, + { + "epoch": 0.6832607342874922, + "grad_norm": 2.785576820373535, + "learning_rate": 4.324261992619926e-06, + "loss": 1.6113, + "step": 549 + }, + { + "epoch": 0.6845052893590542, + "grad_norm": 2.723919630050659, + "learning_rate": 4.321955719557196e-06, + "loss": 1.5606, + "step": 550 + }, + { + "epoch": 0.6845052893590542, + "eval_loss": 1.721803069114685, + "eval_runtime": 51.6036, + "eval_samples_per_second": 19.378, + "eval_steps_per_second": 0.814, + "step": 550 + }, + { + "epoch": 0.685749844430616, + "grad_norm": 3.081198215484619, + "learning_rate": 4.319649446494466e-06, + "loss": 1.5643, + "step": 551 + }, + { + "epoch": 0.686994399502178, + "grad_norm": 2.757392168045044, + "learning_rate": 4.317343173431734e-06, + "loss": 1.5909, + "step": 552 + }, + { + "epoch": 0.6882389545737398, + "grad_norm": 3.038753032684326, + "learning_rate": 4.315036900369004e-06, + "loss": 1.6023, + "step": 553 + }, + { + "epoch": 0.6894835096453018, + "grad_norm": 2.8246452808380127, + "learning_rate": 4.312730627306273e-06, + "loss": 1.5548, + "step": 554 + }, + { + "epoch": 0.6907280647168638, + "grad_norm": 2.589320182800293, + "learning_rate": 4.310424354243543e-06, + "loss": 1.5424, + "step": 555 + }, + { + "epoch": 0.6919726197884256, + "grad_norm": 2.6413373947143555, + "learning_rate": 4.308118081180812e-06, + "loss": 1.6243, + "step": 556 + }, + { + "epoch": 0.6932171748599876, + "grad_norm": 2.764784336090088, + "learning_rate": 4.3058118081180815e-06, + "loss": 1.589, + "step": 557 + }, + { + "epoch": 0.6944617299315494, + "grad_norm": 2.806821823120117, + "learning_rate": 4.303505535055351e-06, + "loss": 1.621, + "step": 558 + }, + { + "epoch": 0.6957062850031114, + "grad_norm": 2.718017578125, + "learning_rate": 4.30119926199262e-06, + "loss": 1.5773, + "step": 559 + }, + { + "epoch": 0.6969508400746733, + "grad_norm": 2.7620160579681396, + "learning_rate": 4.29889298892989e-06, + "loss": 1.5868, + "step": 560 + }, + { + "epoch": 0.6969508400746733, + "eval_loss": 1.7152249813079834, + "eval_runtime": 52.0463, + "eval_samples_per_second": 19.214, + "eval_steps_per_second": 0.807, + "step": 560 + }, + { + "epoch": 0.6981953951462352, + "grad_norm": 2.66686749458313, + "learning_rate": 4.296586715867159e-06, + "loss": 1.534, + "step": 561 + }, + { + "epoch": 0.6994399502177971, + "grad_norm": 2.812577247619629, + "learning_rate": 4.2942804428044286e-06, + "loss": 1.5501, + "step": 562 + }, + { + "epoch": 0.7006845052893591, + "grad_norm": 2.578508138656616, + "learning_rate": 4.291974169741698e-06, + "loss": 1.6286, + "step": 563 + }, + { + "epoch": 0.701929060360921, + "grad_norm": 2.524249315261841, + "learning_rate": 4.289667896678967e-06, + "loss": 1.5719, + "step": 564 + }, + { + "epoch": 0.7031736154324829, + "grad_norm": 2.827235460281372, + "learning_rate": 4.287361623616236e-06, + "loss": 1.5333, + "step": 565 + }, + { + "epoch": 0.7044181705040448, + "grad_norm": 2.6359963417053223, + "learning_rate": 4.285055350553506e-06, + "loss": 1.5849, + "step": 566 + }, + { + "epoch": 0.7056627255756067, + "grad_norm": 2.930530071258545, + "learning_rate": 4.282749077490776e-06, + "loss": 1.5672, + "step": 567 + }, + { + "epoch": 0.7069072806471687, + "grad_norm": 2.750102996826172, + "learning_rate": 4.280442804428044e-06, + "loss": 1.5696, + "step": 568 + }, + { + "epoch": 0.7081518357187305, + "grad_norm": 2.869690418243408, + "learning_rate": 4.278136531365314e-06, + "loss": 1.5689, + "step": 569 + }, + { + "epoch": 0.7093963907902925, + "grad_norm": 2.954852819442749, + "learning_rate": 4.275830258302583e-06, + "loss": 1.5931, + "step": 570 + }, + { + "epoch": 0.7093963907902925, + "eval_loss": 1.7143300771713257, + "eval_runtime": 52.7293, + "eval_samples_per_second": 18.965, + "eval_steps_per_second": 0.797, + "step": 570 + }, + { + "epoch": 0.7106409458618543, + "grad_norm": 2.702223539352417, + "learning_rate": 4.273523985239853e-06, + "loss": 1.5635, + "step": 571 + }, + { + "epoch": 0.7118855009334163, + "grad_norm": 2.689995050430298, + "learning_rate": 4.271217712177122e-06, + "loss": 1.5545, + "step": 572 + }, + { + "epoch": 0.7131300560049783, + "grad_norm": 2.68979811668396, + "learning_rate": 4.268911439114391e-06, + "loss": 1.5404, + "step": 573 + }, + { + "epoch": 0.7143746110765401, + "grad_norm": 2.7477986812591553, + "learning_rate": 4.266605166051661e-06, + "loss": 1.5719, + "step": 574 + }, + { + "epoch": 0.7156191661481021, + "grad_norm": 2.975778341293335, + "learning_rate": 4.26429889298893e-06, + "loss": 1.5587, + "step": 575 + }, + { + "epoch": 0.716863721219664, + "grad_norm": 2.658170700073242, + "learning_rate": 4.2619926199262e-06, + "loss": 1.5746, + "step": 576 + }, + { + "epoch": 0.7181082762912259, + "grad_norm": 2.8825011253356934, + "learning_rate": 4.259686346863469e-06, + "loss": 1.5246, + "step": 577 + }, + { + "epoch": 0.7193528313627878, + "grad_norm": 2.845280170440674, + "learning_rate": 4.2573800738007385e-06, + "loss": 1.5703, + "step": 578 + }, + { + "epoch": 0.7205973864343497, + "grad_norm": 2.660616636276245, + "learning_rate": 4.255073800738008e-06, + "loss": 1.5373, + "step": 579 + }, + { + "epoch": 0.7218419415059116, + "grad_norm": 2.749447822570801, + "learning_rate": 4.252767527675277e-06, + "loss": 1.59, + "step": 580 + }, + { + "epoch": 0.7218419415059116, + "eval_loss": 1.7149444818496704, + "eval_runtime": 52.2099, + "eval_samples_per_second": 19.153, + "eval_steps_per_second": 0.804, + "step": 580 + }, + { + "epoch": 0.7230864965774736, + "grad_norm": 2.813328742980957, + "learning_rate": 4.250461254612546e-06, + "loss": 1.5694, + "step": 581 + }, + { + "epoch": 0.7243310516490354, + "grad_norm": 2.823866844177246, + "learning_rate": 4.248154981549816e-06, + "loss": 1.5143, + "step": 582 + }, + { + "epoch": 0.7255756067205974, + "grad_norm": 2.71337890625, + "learning_rate": 4.245848708487086e-06, + "loss": 1.5915, + "step": 583 + }, + { + "epoch": 0.7268201617921594, + "grad_norm": 2.718085765838623, + "learning_rate": 4.243542435424355e-06, + "loss": 1.604, + "step": 584 + }, + { + "epoch": 0.7280647168637212, + "grad_norm": 2.5741796493530273, + "learning_rate": 4.241236162361624e-06, + "loss": 1.5398, + "step": 585 + }, + { + "epoch": 0.7293092719352832, + "grad_norm": 2.727114200592041, + "learning_rate": 4.238929889298893e-06, + "loss": 1.5886, + "step": 586 + }, + { + "epoch": 0.730553827006845, + "grad_norm": 2.6703338623046875, + "learning_rate": 4.236623616236163e-06, + "loss": 1.5959, + "step": 587 + }, + { + "epoch": 0.731798382078407, + "grad_norm": 2.750326633453369, + "learning_rate": 4.234317343173432e-06, + "loss": 1.5916, + "step": 588 + }, + { + "epoch": 0.7330429371499689, + "grad_norm": 2.6680715084075928, + "learning_rate": 4.232011070110701e-06, + "loss": 1.562, + "step": 589 + }, + { + "epoch": 0.7342874922215308, + "grad_norm": 2.6603832244873047, + "learning_rate": 4.229704797047971e-06, + "loss": 1.6121, + "step": 590 + }, + { + "epoch": 0.7342874922215308, + "eval_loss": 1.7144734859466553, + "eval_runtime": 50.5317, + "eval_samples_per_second": 19.79, + "eval_steps_per_second": 0.831, + "step": 590 + }, + { + "epoch": 0.7355320472930927, + "grad_norm": 2.7868523597717285, + "learning_rate": 4.22739852398524e-06, + "loss": 1.5488, + "step": 591 + }, + { + "epoch": 0.7367766023646546, + "grad_norm": 2.755270004272461, + "learning_rate": 4.2250922509225096e-06, + "loss": 1.5801, + "step": 592 + }, + { + "epoch": 0.7380211574362165, + "grad_norm": 2.7614126205444336, + "learning_rate": 4.222785977859779e-06, + "loss": 1.5507, + "step": 593 + }, + { + "epoch": 0.7392657125077785, + "grad_norm": 2.8674862384796143, + "learning_rate": 4.220479704797048e-06, + "loss": 1.5807, + "step": 594 + }, + { + "epoch": 0.7405102675793404, + "grad_norm": 2.7366859912872314, + "learning_rate": 4.218173431734318e-06, + "loss": 1.6594, + "step": 595 + }, + { + "epoch": 0.7417548226509023, + "grad_norm": 2.6231045722961426, + "learning_rate": 4.215867158671587e-06, + "loss": 1.5609, + "step": 596 + }, + { + "epoch": 0.7429993777224643, + "grad_norm": 2.7694146633148193, + "learning_rate": 4.213560885608857e-06, + "loss": 1.5889, + "step": 597 + }, + { + "epoch": 0.7442439327940261, + "grad_norm": 2.6275386810302734, + "learning_rate": 4.211254612546125e-06, + "loss": 1.575, + "step": 598 + }, + { + "epoch": 0.7454884878655881, + "grad_norm": 2.9413866996765137, + "learning_rate": 4.2089483394833955e-06, + "loss": 1.589, + "step": 599 + }, + { + "epoch": 0.7467330429371499, + "grad_norm": 2.768606185913086, + "learning_rate": 4.206642066420665e-06, + "loss": 1.5738, + "step": 600 + }, + { + "epoch": 0.7467330429371499, + "eval_loss": 1.7148027420043945, + "eval_runtime": 54.7031, + "eval_samples_per_second": 18.28, + "eval_steps_per_second": 0.768, + "step": 600 + }, + { + "epoch": 0.7479775980087119, + "grad_norm": 2.7917706966400146, + "learning_rate": 4.2043357933579335e-06, + "loss": 1.499, + "step": 601 + }, + { + "epoch": 0.7492221530802738, + "grad_norm": 2.5982208251953125, + "learning_rate": 4.202029520295203e-06, + "loss": 1.5113, + "step": 602 + }, + { + "epoch": 0.7504667081518357, + "grad_norm": 2.692032814025879, + "learning_rate": 4.199723247232473e-06, + "loss": 1.4721, + "step": 603 + }, + { + "epoch": 0.7517112632233977, + "grad_norm": 2.975860834121704, + "learning_rate": 4.197416974169742e-06, + "loss": 1.5147, + "step": 604 + }, + { + "epoch": 0.7529558182949595, + "grad_norm": 2.758610248565674, + "learning_rate": 4.195110701107011e-06, + "loss": 1.5674, + "step": 605 + }, + { + "epoch": 0.7542003733665215, + "grad_norm": 2.7620184421539307, + "learning_rate": 4.192804428044281e-06, + "loss": 1.5669, + "step": 606 + }, + { + "epoch": 0.7554449284380834, + "grad_norm": 3.043940305709839, + "learning_rate": 4.19049815498155e-06, + "loss": 1.587, + "step": 607 + }, + { + "epoch": 0.7566894835096453, + "grad_norm": 2.7482151985168457, + "learning_rate": 4.1881918819188195e-06, + "loss": 1.5451, + "step": 608 + }, + { + "epoch": 0.7579340385812072, + "grad_norm": 2.7259294986724854, + "learning_rate": 4.185885608856089e-06, + "loss": 1.5295, + "step": 609 + }, + { + "epoch": 0.7591785936527692, + "grad_norm": 2.6340832710266113, + "learning_rate": 4.183579335793358e-06, + "loss": 1.5335, + "step": 610 + }, + { + "epoch": 0.7591785936527692, + "eval_loss": 1.7038393020629883, + "eval_runtime": 44.0792, + "eval_samples_per_second": 22.686, + "eval_steps_per_second": 0.953, + "step": 610 + }, + { + "epoch": 0.760423148724331, + "grad_norm": 2.6144909858703613, + "learning_rate": 4.181273062730628e-06, + "loss": 1.5907, + "step": 611 + }, + { + "epoch": 0.761667703795893, + "grad_norm": 2.7754175662994385, + "learning_rate": 4.178966789667897e-06, + "loss": 1.5773, + "step": 612 + }, + { + "epoch": 0.7629122588674548, + "grad_norm": 2.6313252449035645, + "learning_rate": 4.176660516605167e-06, + "loss": 1.5675, + "step": 613 + }, + { + "epoch": 0.7641568139390168, + "grad_norm": 2.549074649810791, + "learning_rate": 4.174354243542435e-06, + "loss": 1.595, + "step": 614 + }, + { + "epoch": 0.7654013690105788, + "grad_norm": 2.611804246902466, + "learning_rate": 4.1720479704797054e-06, + "loss": 1.5416, + "step": 615 + }, + { + "epoch": 0.7666459240821406, + "grad_norm": 2.6322927474975586, + "learning_rate": 4.169741697416975e-06, + "loss": 1.604, + "step": 616 + }, + { + "epoch": 0.7678904791537026, + "grad_norm": 2.5792219638824463, + "learning_rate": 4.1674354243542434e-06, + "loss": 1.5976, + "step": 617 + }, + { + "epoch": 0.7691350342252644, + "grad_norm": 2.845416307449341, + "learning_rate": 4.165129151291513e-06, + "loss": 1.5945, + "step": 618 + }, + { + "epoch": 0.7703795892968264, + "grad_norm": 2.8647871017456055, + "learning_rate": 4.162822878228783e-06, + "loss": 1.5637, + "step": 619 + }, + { + "epoch": 0.7716241443683883, + "grad_norm": 2.590719699859619, + "learning_rate": 4.1605166051660526e-06, + "loss": 1.556, + "step": 620 + }, + { + "epoch": 0.7716241443683883, + "eval_loss": 1.7115222215652466, + "eval_runtime": 50.3905, + "eval_samples_per_second": 19.845, + "eval_steps_per_second": 0.833, + "step": 620 + }, + { + "epoch": 0.7728686994399502, + "grad_norm": 2.626723527908325, + "learning_rate": 4.158210332103321e-06, + "loss": 1.5486, + "step": 621 + }, + { + "epoch": 0.7741132545115121, + "grad_norm": 2.713069438934326, + "learning_rate": 4.1559040590405906e-06, + "loss": 1.5474, + "step": 622 + }, + { + "epoch": 0.7753578095830741, + "grad_norm": 2.685563087463379, + "learning_rate": 4.15359778597786e-06, + "loss": 1.5529, + "step": 623 + }, + { + "epoch": 0.776602364654636, + "grad_norm": 2.690586805343628, + "learning_rate": 4.151291512915129e-06, + "loss": 1.5293, + "step": 624 + }, + { + "epoch": 0.7778469197261979, + "grad_norm": 2.865345001220703, + "learning_rate": 4.148985239852399e-06, + "loss": 1.5095, + "step": 625 + }, + { + "epoch": 0.7790914747977598, + "grad_norm": 2.92393159866333, + "learning_rate": 4.146678966789668e-06, + "loss": 1.5561, + "step": 626 + }, + { + "epoch": 0.7803360298693217, + "grad_norm": 2.554480791091919, + "learning_rate": 4.144372693726938e-06, + "loss": 1.4876, + "step": 627 + }, + { + "epoch": 0.7815805849408837, + "grad_norm": 2.6128833293914795, + "learning_rate": 4.142066420664207e-06, + "loss": 1.5378, + "step": 628 + }, + { + "epoch": 0.7828251400124455, + "grad_norm": 3.0110888481140137, + "learning_rate": 4.1397601476014765e-06, + "loss": 1.5859, + "step": 629 + }, + { + "epoch": 0.7840696950840075, + "grad_norm": 2.8152706623077393, + "learning_rate": 4.137453874538745e-06, + "loss": 1.5841, + "step": 630 + }, + { + "epoch": 0.7840696950840075, + "eval_loss": 1.702181339263916, + "eval_runtime": 53.6926, + "eval_samples_per_second": 18.625, + "eval_steps_per_second": 0.782, + "step": 630 + }, + { + "epoch": 0.7853142501555694, + "grad_norm": 2.8634233474731445, + "learning_rate": 4.135147601476015e-06, + "loss": 1.564, + "step": 631 + }, + { + "epoch": 0.7865588052271313, + "grad_norm": 2.8939666748046875, + "learning_rate": 4.132841328413285e-06, + "loss": 1.5859, + "step": 632 + }, + { + "epoch": 0.7878033602986932, + "grad_norm": 2.6560444831848145, + "learning_rate": 4.130535055350554e-06, + "loss": 1.5199, + "step": 633 + }, + { + "epoch": 0.7890479153702551, + "grad_norm": 2.964721202850342, + "learning_rate": 4.128228782287823e-06, + "loss": 1.5888, + "step": 634 + }, + { + "epoch": 0.790292470441817, + "grad_norm": 2.74668288230896, + "learning_rate": 4.125922509225092e-06, + "loss": 1.5847, + "step": 635 + }, + { + "epoch": 0.791537025513379, + "grad_norm": 2.723123550415039, + "learning_rate": 4.1236162361623625e-06, + "loss": 1.5166, + "step": 636 + }, + { + "epoch": 0.7927815805849409, + "grad_norm": 2.7265713214874268, + "learning_rate": 4.121309963099631e-06, + "loss": 1.5954, + "step": 637 + }, + { + "epoch": 0.7940261356565028, + "grad_norm": 2.979126214981079, + "learning_rate": 4.1190036900369005e-06, + "loss": 1.5583, + "step": 638 + }, + { + "epoch": 0.7952706907280647, + "grad_norm": 2.844376802444458, + "learning_rate": 4.11669741697417e-06, + "loss": 1.5564, + "step": 639 + }, + { + "epoch": 0.7965152457996266, + "grad_norm": 2.7583703994750977, + "learning_rate": 4.114391143911439e-06, + "loss": 1.5094, + "step": 640 + }, + { + "epoch": 0.7965152457996266, + "eval_loss": 1.6986565589904785, + "eval_runtime": 47.3549, + "eval_samples_per_second": 21.117, + "eval_steps_per_second": 0.887, + "step": 640 + }, + { + "epoch": 0.7977598008711886, + "grad_norm": 2.8942224979400635, + "learning_rate": 4.112084870848709e-06, + "loss": 1.5504, + "step": 641 + }, + { + "epoch": 0.7990043559427504, + "grad_norm": 2.6590495109558105, + "learning_rate": 4.109778597785978e-06, + "loss": 1.5227, + "step": 642 + }, + { + "epoch": 0.8002489110143124, + "grad_norm": 2.5988378524780273, + "learning_rate": 4.107472324723248e-06, + "loss": 1.5054, + "step": 643 + }, + { + "epoch": 0.8014934660858744, + "grad_norm": 2.787335157394409, + "learning_rate": 4.105166051660517e-06, + "loss": 1.5844, + "step": 644 + }, + { + "epoch": 0.8027380211574362, + "grad_norm": 2.8202896118164062, + "learning_rate": 4.1028597785977864e-06, + "loss": 1.5596, + "step": 645 + }, + { + "epoch": 0.8039825762289982, + "grad_norm": 2.65376615524292, + "learning_rate": 4.100553505535056e-06, + "loss": 1.5893, + "step": 646 + }, + { + "epoch": 0.80522713130056, + "grad_norm": 2.5933308601379395, + "learning_rate": 4.098247232472325e-06, + "loss": 1.5403, + "step": 647 + }, + { + "epoch": 0.806471686372122, + "grad_norm": 2.777070999145508, + "learning_rate": 4.095940959409595e-06, + "loss": 1.5231, + "step": 648 + }, + { + "epoch": 0.8077162414436839, + "grad_norm": 2.6427664756774902, + "learning_rate": 4.093634686346864e-06, + "loss": 1.5937, + "step": 649 + }, + { + "epoch": 0.8089607965152458, + "grad_norm": 2.651561737060547, + "learning_rate": 4.091328413284133e-06, + "loss": 1.4847, + "step": 650 + }, + { + "epoch": 0.8089607965152458, + "eval_loss": 1.7089996337890625, + "eval_runtime": 47.4346, + "eval_samples_per_second": 21.082, + "eval_steps_per_second": 0.885, + "step": 650 + }, + { + "epoch": 0.8102053515868077, + "grad_norm": 2.844067335128784, + "learning_rate": 4.089022140221402e-06, + "loss": 1.488, + "step": 651 + }, + { + "epoch": 0.8114499066583696, + "grad_norm": 2.603480577468872, + "learning_rate": 4.086715867158672e-06, + "loss": 1.5248, + "step": 652 + }, + { + "epoch": 0.8126944617299315, + "grad_norm": 2.789322853088379, + "learning_rate": 4.084409594095941e-06, + "loss": 1.5119, + "step": 653 + }, + { + "epoch": 0.8139390168014935, + "grad_norm": 2.7183773517608643, + "learning_rate": 4.08210332103321e-06, + "loss": 1.5123, + "step": 654 + }, + { + "epoch": 0.8151835718730553, + "grad_norm": 2.80613374710083, + "learning_rate": 4.07979704797048e-06, + "loss": 1.5526, + "step": 655 + }, + { + "epoch": 0.8164281269446173, + "grad_norm": 2.641035795211792, + "learning_rate": 4.077490774907749e-06, + "loss": 1.6127, + "step": 656 + }, + { + "epoch": 0.8176726820161793, + "grad_norm": 2.773684024810791, + "learning_rate": 4.075184501845019e-06, + "loss": 1.5404, + "step": 657 + }, + { + "epoch": 0.8189172370877411, + "grad_norm": 2.6951382160186768, + "learning_rate": 4.072878228782288e-06, + "loss": 1.4928, + "step": 658 + }, + { + "epoch": 0.8201617921593031, + "grad_norm": 2.931771755218506, + "learning_rate": 4.0705719557195575e-06, + "loss": 1.5949, + "step": 659 + }, + { + "epoch": 0.8214063472308649, + "grad_norm": 2.8786075115203857, + "learning_rate": 4.068265682656827e-06, + "loss": 1.5582, + "step": 660 + }, + { + "epoch": 0.8214063472308649, + "eval_loss": 1.698564052581787, + "eval_runtime": 46.6299, + "eval_samples_per_second": 21.445, + "eval_steps_per_second": 0.901, + "step": 660 + }, + { + "epoch": 0.8226509023024269, + "grad_norm": 2.7402451038360596, + "learning_rate": 4.065959409594096e-06, + "loss": 1.5562, + "step": 661 + }, + { + "epoch": 0.8238954573739888, + "grad_norm": 2.7968692779541016, + "learning_rate": 4.063653136531366e-06, + "loss": 1.556, + "step": 662 + }, + { + "epoch": 0.8251400124455507, + "grad_norm": 2.8233132362365723, + "learning_rate": 4.061346863468635e-06, + "loss": 1.5527, + "step": 663 + }, + { + "epoch": 0.8263845675171126, + "grad_norm": 2.713390827178955, + "learning_rate": 4.059040590405905e-06, + "loss": 1.6012, + "step": 664 + }, + { + "epoch": 0.8276291225886746, + "grad_norm": 2.7783989906311035, + "learning_rate": 4.056734317343174e-06, + "loss": 1.5247, + "step": 665 + }, + { + "epoch": 0.8288736776602365, + "grad_norm": 2.808554172515869, + "learning_rate": 4.054428044280443e-06, + "loss": 1.5412, + "step": 666 + }, + { + "epoch": 0.8301182327317984, + "grad_norm": 2.8476831912994385, + "learning_rate": 4.052121771217712e-06, + "loss": 1.5307, + "step": 667 + }, + { + "epoch": 0.8313627878033603, + "grad_norm": 2.8287601470947266, + "learning_rate": 4.049815498154982e-06, + "loss": 1.6021, + "step": 668 + }, + { + "epoch": 0.8326073428749222, + "grad_norm": 2.975510835647583, + "learning_rate": 4.047509225092252e-06, + "loss": 1.5474, + "step": 669 + }, + { + "epoch": 0.8338518979464842, + "grad_norm": 2.723236560821533, + "learning_rate": 4.04520295202952e-06, + "loss": 1.5242, + "step": 670 + }, + { + "epoch": 0.8338518979464842, + "eval_loss": 1.7005239725112915, + "eval_runtime": 49.4504, + "eval_samples_per_second": 20.222, + "eval_steps_per_second": 0.849, + "step": 670 + }, + { + "epoch": 0.835096453018046, + "grad_norm": 2.7521748542785645, + "learning_rate": 4.04289667896679e-06, + "loss": 1.5467, + "step": 671 + }, + { + "epoch": 0.836341008089608, + "grad_norm": 2.7770044803619385, + "learning_rate": 4.04059040590406e-06, + "loss": 1.5407, + "step": 672 + }, + { + "epoch": 0.8375855631611698, + "grad_norm": 2.744323492050171, + "learning_rate": 4.038284132841329e-06, + "loss": 1.5422, + "step": 673 + }, + { + "epoch": 0.8388301182327318, + "grad_norm": 2.6699817180633545, + "learning_rate": 4.035977859778598e-06, + "loss": 1.5508, + "step": 674 + }, + { + "epoch": 0.8400746733042938, + "grad_norm": 2.705273151397705, + "learning_rate": 4.0336715867158674e-06, + "loss": 1.5099, + "step": 675 + }, + { + "epoch": 0.8413192283758556, + "grad_norm": 2.9485747814178467, + "learning_rate": 4.031365313653137e-06, + "loss": 1.5529, + "step": 676 + }, + { + "epoch": 0.8425637834474176, + "grad_norm": 3.1750423908233643, + "learning_rate": 4.029059040590406e-06, + "loss": 1.5894, + "step": 677 + }, + { + "epoch": 0.8438083385189795, + "grad_norm": 2.76448655128479, + "learning_rate": 4.026752767527676e-06, + "loss": 1.5752, + "step": 678 + }, + { + "epoch": 0.8450528935905414, + "grad_norm": 2.676708698272705, + "learning_rate": 4.024446494464945e-06, + "loss": 1.5487, + "step": 679 + }, + { + "epoch": 0.8462974486621033, + "grad_norm": 2.6832051277160645, + "learning_rate": 4.0221402214022145e-06, + "loss": 1.5055, + "step": 680 + }, + { + "epoch": 0.8462974486621033, + "eval_loss": 1.6945017576217651, + "eval_runtime": 43.9369, + "eval_samples_per_second": 22.76, + "eval_steps_per_second": 0.956, + "step": 680 + }, + { + "epoch": 0.8475420037336652, + "grad_norm": 2.6812336444854736, + "learning_rate": 4.019833948339484e-06, + "loss": 1.5315, + "step": 681 + }, + { + "epoch": 0.8487865588052271, + "grad_norm": 2.9394326210021973, + "learning_rate": 4.017527675276753e-06, + "loss": 1.5636, + "step": 682 + }, + { + "epoch": 0.8500311138767891, + "grad_norm": 2.944952964782715, + "learning_rate": 4.015221402214022e-06, + "loss": 1.56, + "step": 683 + }, + { + "epoch": 0.8512756689483509, + "grad_norm": 2.7475314140319824, + "learning_rate": 4.012915129151292e-06, + "loss": 1.5442, + "step": 684 + }, + { + "epoch": 0.8525202240199129, + "grad_norm": 3.0572879314422607, + "learning_rate": 4.010608856088562e-06, + "loss": 1.5023, + "step": 685 + }, + { + "epoch": 0.8537647790914747, + "grad_norm": 2.75365948677063, + "learning_rate": 4.00830258302583e-06, + "loss": 1.5414, + "step": 686 + }, + { + "epoch": 0.8550093341630367, + "grad_norm": 2.5714449882507324, + "learning_rate": 4.0059963099631e-06, + "loss": 1.5027, + "step": 687 + }, + { + "epoch": 0.8562538892345987, + "grad_norm": 2.755167007446289, + "learning_rate": 4.003690036900369e-06, + "loss": 1.5788, + "step": 688 + }, + { + "epoch": 0.8574984443061605, + "grad_norm": 2.798967123031616, + "learning_rate": 4.0013837638376385e-06, + "loss": 1.5162, + "step": 689 + }, + { + "epoch": 0.8587429993777225, + "grad_norm": 2.803614854812622, + "learning_rate": 3.999077490774908e-06, + "loss": 1.5079, + "step": 690 + }, + { + "epoch": 0.8587429993777225, + "eval_loss": 1.6924962997436523, + "eval_runtime": 44.47, + "eval_samples_per_second": 22.487, + "eval_steps_per_second": 0.944, + "step": 690 + }, + { + "epoch": 0.8599875544492844, + "grad_norm": 2.6898066997528076, + "learning_rate": 3.996771217712177e-06, + "loss": 1.5478, + "step": 691 + }, + { + "epoch": 0.8612321095208463, + "grad_norm": 2.6376144886016846, + "learning_rate": 3.994464944649447e-06, + "loss": 1.5362, + "step": 692 + }, + { + "epoch": 0.8624766645924082, + "grad_norm": 2.7240405082702637, + "learning_rate": 3.992158671586716e-06, + "loss": 1.5502, + "step": 693 + }, + { + "epoch": 0.8637212196639701, + "grad_norm": 2.929445266723633, + "learning_rate": 3.989852398523986e-06, + "loss": 1.5799, + "step": 694 + }, + { + "epoch": 0.864965774735532, + "grad_norm": 2.593223810195923, + "learning_rate": 3.987546125461255e-06, + "loss": 1.5352, + "step": 695 + }, + { + "epoch": 0.866210329807094, + "grad_norm": 2.7710154056549072, + "learning_rate": 3.9852398523985245e-06, + "loss": 1.5048, + "step": 696 + }, + { + "epoch": 0.8674548848786559, + "grad_norm": 2.7370848655700684, + "learning_rate": 3.982933579335794e-06, + "loss": 1.5233, + "step": 697 + }, + { + "epoch": 0.8686994399502178, + "grad_norm": 2.8632168769836426, + "learning_rate": 3.980627306273063e-06, + "loss": 1.5435, + "step": 698 + }, + { + "epoch": 0.8699439950217797, + "grad_norm": 2.752298593521118, + "learning_rate": 3.978321033210332e-06, + "loss": 1.5728, + "step": 699 + }, + { + "epoch": 0.8711885500933416, + "grad_norm": 2.8345625400543213, + "learning_rate": 3.976014760147602e-06, + "loss": 1.5727, + "step": 700 + }, + { + "epoch": 0.8711885500933416, + "eval_loss": 1.6855305433273315, + "eval_runtime": 43.3683, + "eval_samples_per_second": 23.058, + "eval_steps_per_second": 0.968, + "step": 700 + }, + { + "epoch": 0.8724331051649036, + "grad_norm": 2.8135335445404053, + "learning_rate": 3.973708487084872e-06, + "loss": 1.5086, + "step": 701 + }, + { + "epoch": 0.8736776602364654, + "grad_norm": 2.7416603565216064, + "learning_rate": 3.97140221402214e-06, + "loss": 1.545, + "step": 702 + }, + { + "epoch": 0.8749222153080274, + "grad_norm": 2.6902289390563965, + "learning_rate": 3.96909594095941e-06, + "loss": 1.5509, + "step": 703 + }, + { + "epoch": 0.8761667703795893, + "grad_norm": 2.6537084579467773, + "learning_rate": 3.966789667896679e-06, + "loss": 1.562, + "step": 704 + }, + { + "epoch": 0.8774113254511512, + "grad_norm": 2.6979284286499023, + "learning_rate": 3.964483394833948e-06, + "loss": 1.4829, + "step": 705 + }, + { + "epoch": 0.8786558805227132, + "grad_norm": 2.5795764923095703, + "learning_rate": 3.962177121771218e-06, + "loss": 1.4677, + "step": 706 + }, + { + "epoch": 0.879900435594275, + "grad_norm": 3.0185086727142334, + "learning_rate": 3.959870848708487e-06, + "loss": 1.5608, + "step": 707 + }, + { + "epoch": 0.881144990665837, + "grad_norm": 2.6458113193511963, + "learning_rate": 3.957564575645757e-06, + "loss": 1.5342, + "step": 708 + }, + { + "epoch": 0.8823895457373989, + "grad_norm": 2.7985665798187256, + "learning_rate": 3.955258302583026e-06, + "loss": 1.527, + "step": 709 + }, + { + "epoch": 0.8836341008089608, + "grad_norm": 2.9069082736968994, + "learning_rate": 3.9529520295202955e-06, + "loss": 1.5156, + "step": 710 + }, + { + "epoch": 0.8836341008089608, + "eval_loss": 1.6898822784423828, + "eval_runtime": 42.8749, + "eval_samples_per_second": 23.324, + "eval_steps_per_second": 0.98, + "step": 710 + }, + { + "epoch": 0.8848786558805227, + "grad_norm": 2.760143995285034, + "learning_rate": 3.950645756457565e-06, + "loss": 1.5124, + "step": 711 + }, + { + "epoch": 0.8861232109520847, + "grad_norm": 2.8085529804229736, + "learning_rate": 3.948339483394834e-06, + "loss": 1.5862, + "step": 712 + }, + { + "epoch": 0.8873677660236465, + "grad_norm": 2.909905195236206, + "learning_rate": 3.946033210332104e-06, + "loss": 1.5557, + "step": 713 + }, + { + "epoch": 0.8886123210952085, + "grad_norm": 2.826899290084839, + "learning_rate": 3.943726937269373e-06, + "loss": 1.5277, + "step": 714 + }, + { + "epoch": 0.8898568761667703, + "grad_norm": 2.6269052028656006, + "learning_rate": 3.941420664206642e-06, + "loss": 1.4819, + "step": 715 + }, + { + "epoch": 0.8911014312383323, + "grad_norm": 2.6687698364257812, + "learning_rate": 3.939114391143912e-06, + "loss": 1.5156, + "step": 716 + }, + { + "epoch": 0.8923459863098943, + "grad_norm": 2.9605629444122314, + "learning_rate": 3.9368081180811815e-06, + "loss": 1.5358, + "step": 717 + }, + { + "epoch": 0.8935905413814561, + "grad_norm": 3.0367166996002197, + "learning_rate": 3.934501845018451e-06, + "loss": 1.5847, + "step": 718 + }, + { + "epoch": 0.8948350964530181, + "grad_norm": 2.528796434402466, + "learning_rate": 3.9321955719557195e-06, + "loss": 1.4851, + "step": 719 + }, + { + "epoch": 0.8960796515245799, + "grad_norm": 2.7116613388061523, + "learning_rate": 3.929889298892989e-06, + "loss": 1.4706, + "step": 720 + }, + { + "epoch": 0.8960796515245799, + "eval_loss": 1.6845883131027222, + "eval_runtime": 44.4734, + "eval_samples_per_second": 22.485, + "eval_steps_per_second": 0.944, + "step": 720 + }, + { + "epoch": 0.8973242065961419, + "grad_norm": 2.771864414215088, + "learning_rate": 3.927583025830259e-06, + "loss": 1.5444, + "step": 721 + }, + { + "epoch": 0.8985687616677038, + "grad_norm": 2.773746967315674, + "learning_rate": 3.925276752767528e-06, + "loss": 1.564, + "step": 722 + }, + { + "epoch": 0.8998133167392657, + "grad_norm": 2.7069854736328125, + "learning_rate": 3.922970479704797e-06, + "loss": 1.5601, + "step": 723 + }, + { + "epoch": 0.9010578718108276, + "grad_norm": 2.840261697769165, + "learning_rate": 3.920664206642067e-06, + "loss": 1.5733, + "step": 724 + }, + { + "epoch": 0.9023024268823896, + "grad_norm": 2.709897041320801, + "learning_rate": 3.918357933579336e-06, + "loss": 1.528, + "step": 725 + }, + { + "epoch": 0.9035469819539514, + "grad_norm": 2.662367343902588, + "learning_rate": 3.9160516605166055e-06, + "loss": 1.5532, + "step": 726 + }, + { + "epoch": 0.9047915370255134, + "grad_norm": 2.6775717735290527, + "learning_rate": 3.913745387453875e-06, + "loss": 1.5464, + "step": 727 + }, + { + "epoch": 0.9060360920970753, + "grad_norm": 2.617842674255371, + "learning_rate": 3.911439114391144e-06, + "loss": 1.4539, + "step": 728 + }, + { + "epoch": 0.9072806471686372, + "grad_norm": 2.668649435043335, + "learning_rate": 3.909132841328414e-06, + "loss": 1.5627, + "step": 729 + }, + { + "epoch": 0.9085252022401992, + "grad_norm": 2.5331950187683105, + "learning_rate": 3.906826568265683e-06, + "loss": 1.5165, + "step": 730 + }, + { + "epoch": 0.9085252022401992, + "eval_loss": 1.6860331296920776, + "eval_runtime": 46.094, + "eval_samples_per_second": 21.695, + "eval_steps_per_second": 0.911, + "step": 730 + }, + { + "epoch": 0.909769757311761, + "grad_norm": 2.602492332458496, + "learning_rate": 3.9045202952029526e-06, + "loss": 1.5145, + "step": 731 + }, + { + "epoch": 0.911014312383323, + "grad_norm": 2.632782459259033, + "learning_rate": 3.902214022140222e-06, + "loss": 1.4413, + "step": 732 + }, + { + "epoch": 0.9122588674548848, + "grad_norm": 2.6025912761688232, + "learning_rate": 3.899907749077491e-06, + "loss": 1.5263, + "step": 733 + }, + { + "epoch": 0.9135034225264468, + "grad_norm": 2.770116090774536, + "learning_rate": 3.897601476014761e-06, + "loss": 1.5514, + "step": 734 + }, + { + "epoch": 0.9147479775980087, + "grad_norm": 2.7822961807250977, + "learning_rate": 3.895295202952029e-06, + "loss": 1.5296, + "step": 735 + }, + { + "epoch": 0.9159925326695706, + "grad_norm": 2.6689720153808594, + "learning_rate": 3.892988929889299e-06, + "loss": 1.4552, + "step": 736 + }, + { + "epoch": 0.9172370877411326, + "grad_norm": 2.8021798133850098, + "learning_rate": 3.890682656826569e-06, + "loss": 1.5308, + "step": 737 + }, + { + "epoch": 0.9184816428126945, + "grad_norm": 2.575313091278076, + "learning_rate": 3.888376383763838e-06, + "loss": 1.4807, + "step": 738 + }, + { + "epoch": 0.9197261978842564, + "grad_norm": 2.5955779552459717, + "learning_rate": 3.886070110701107e-06, + "loss": 1.4222, + "step": 739 + }, + { + "epoch": 0.9209707529558183, + "grad_norm": 2.647939920425415, + "learning_rate": 3.8837638376383765e-06, + "loss": 1.5285, + "step": 740 + }, + { + "epoch": 0.9209707529558183, + "eval_loss": 1.6907480955123901, + "eval_runtime": 45.7678, + "eval_samples_per_second": 21.849, + "eval_steps_per_second": 0.918, + "step": 740 + }, + { + "epoch": 0.9222153080273802, + "grad_norm": 2.7664523124694824, + "learning_rate": 3.881457564575646e-06, + "loss": 1.5349, + "step": 741 + }, + { + "epoch": 0.9234598630989421, + "grad_norm": 2.5770998001098633, + "learning_rate": 3.879151291512915e-06, + "loss": 1.5071, + "step": 742 + }, + { + "epoch": 0.9247044181705041, + "grad_norm": 2.502567768096924, + "learning_rate": 3.876845018450185e-06, + "loss": 1.492, + "step": 743 + }, + { + "epoch": 0.9259489732420659, + "grad_norm": 2.6531119346618652, + "learning_rate": 3.874538745387454e-06, + "loss": 1.5045, + "step": 744 + }, + { + "epoch": 0.9271935283136279, + "grad_norm": 2.7343876361846924, + "learning_rate": 3.872232472324724e-06, + "loss": 1.5194, + "step": 745 + }, + { + "epoch": 0.9284380833851898, + "grad_norm": 2.842313528060913, + "learning_rate": 3.869926199261993e-06, + "loss": 1.5588, + "step": 746 + }, + { + "epoch": 0.9296826384567517, + "grad_norm": 2.718245267868042, + "learning_rate": 3.8676199261992625e-06, + "loss": 1.5252, + "step": 747 + }, + { + "epoch": 0.9309271935283137, + "grad_norm": 2.695392370223999, + "learning_rate": 3.865313653136532e-06, + "loss": 1.5596, + "step": 748 + }, + { + "epoch": 0.9321717485998755, + "grad_norm": 2.7859959602355957, + "learning_rate": 3.863007380073801e-06, + "loss": 1.5151, + "step": 749 + }, + { + "epoch": 0.9334163036714375, + "grad_norm": 2.5199174880981445, + "learning_rate": 3.860701107011071e-06, + "loss": 1.4948, + "step": 750 + }, + { + "epoch": 0.9334163036714375, + "eval_loss": 1.6868674755096436, + "eval_runtime": 46.4115, + "eval_samples_per_second": 21.546, + "eval_steps_per_second": 0.905, + "step": 750 + }, + { + "epoch": 0.9346608587429994, + "grad_norm": 2.8194046020507812, + "learning_rate": 3.858394833948339e-06, + "loss": 1.5144, + "step": 751 + }, + { + "epoch": 0.9359054138145613, + "grad_norm": 2.810380697250366, + "learning_rate": 3.856088560885609e-06, + "loss": 1.4998, + "step": 752 + }, + { + "epoch": 0.9371499688861232, + "grad_norm": 2.6906750202178955, + "learning_rate": 3.853782287822879e-06, + "loss": 1.4748, + "step": 753 + }, + { + "epoch": 0.9383945239576851, + "grad_norm": 2.9030277729034424, + "learning_rate": 3.851476014760148e-06, + "loss": 1.615, + "step": 754 + }, + { + "epoch": 0.939639079029247, + "grad_norm": 2.8690176010131836, + "learning_rate": 3.849169741697417e-06, + "loss": 1.5083, + "step": 755 + }, + { + "epoch": 0.940883634100809, + "grad_norm": 2.669646978378296, + "learning_rate": 3.8468634686346865e-06, + "loss": 1.5572, + "step": 756 + }, + { + "epoch": 0.9421281891723708, + "grad_norm": 2.6517696380615234, + "learning_rate": 3.844557195571956e-06, + "loss": 1.5142, + "step": 757 + }, + { + "epoch": 0.9433727442439328, + "grad_norm": 2.5524444580078125, + "learning_rate": 3.842250922509225e-06, + "loss": 1.4924, + "step": 758 + }, + { + "epoch": 0.9446172993154948, + "grad_norm": 2.6532633304595947, + "learning_rate": 3.839944649446495e-06, + "loss": 1.484, + "step": 759 + }, + { + "epoch": 0.9458618543870566, + "grad_norm": 2.7779057025909424, + "learning_rate": 3.837638376383764e-06, + "loss": 1.5106, + "step": 760 + }, + { + "epoch": 0.9458618543870566, + "eval_loss": 1.6845752000808716, + "eval_runtime": 50.3415, + "eval_samples_per_second": 19.864, + "eval_steps_per_second": 0.834, + "step": 760 + }, + { + "epoch": 0.9471064094586186, + "grad_norm": 2.6192541122436523, + "learning_rate": 3.8353321033210336e-06, + "loss": 1.5454, + "step": 761 + }, + { + "epoch": 0.9483509645301804, + "grad_norm": 2.561861991882324, + "learning_rate": 3.833025830258303e-06, + "loss": 1.5141, + "step": 762 + }, + { + "epoch": 0.9495955196017424, + "grad_norm": 2.661829948425293, + "learning_rate": 3.830719557195572e-06, + "loss": 1.4997, + "step": 763 + }, + { + "epoch": 0.9508400746733043, + "grad_norm": 2.725275754928589, + "learning_rate": 3.828413284132842e-06, + "loss": 1.5224, + "step": 764 + }, + { + "epoch": 0.9520846297448662, + "grad_norm": 2.5664193630218506, + "learning_rate": 3.826107011070111e-06, + "loss": 1.5073, + "step": 765 + }, + { + "epoch": 0.9533291848164281, + "grad_norm": 2.595189332962036, + "learning_rate": 3.823800738007381e-06, + "loss": 1.5379, + "step": 766 + }, + { + "epoch": 0.95457373988799, + "grad_norm": 2.738060235977173, + "learning_rate": 3.821494464944649e-06, + "loss": 1.4814, + "step": 767 + }, + { + "epoch": 0.955818294959552, + "grad_norm": 2.601071357727051, + "learning_rate": 3.819188191881919e-06, + "loss": 1.4812, + "step": 768 + }, + { + "epoch": 0.9570628500311139, + "grad_norm": 2.657212257385254, + "learning_rate": 3.816881918819189e-06, + "loss": 1.4748, + "step": 769 + }, + { + "epoch": 0.9583074051026758, + "grad_norm": 2.7673351764678955, + "learning_rate": 3.814575645756458e-06, + "loss": 1.5535, + "step": 770 + }, + { + "epoch": 0.9583074051026758, + "eval_loss": 1.677243947982788, + "eval_runtime": 51.5537, + "eval_samples_per_second": 19.397, + "eval_steps_per_second": 0.815, + "step": 770 + }, + { + "epoch": 0.9595519601742377, + "grad_norm": 2.760890007019043, + "learning_rate": 3.812269372693727e-06, + "loss": 1.5654, + "step": 771 + }, + { + "epoch": 0.9607965152457997, + "grad_norm": 2.6934309005737305, + "learning_rate": 3.809963099630997e-06, + "loss": 1.5149, + "step": 772 + }, + { + "epoch": 0.9620410703173615, + "grad_norm": 2.729950428009033, + "learning_rate": 3.8076568265682662e-06, + "loss": 1.5579, + "step": 773 + }, + { + "epoch": 0.9632856253889235, + "grad_norm": 2.732926607131958, + "learning_rate": 3.8053505535055352e-06, + "loss": 1.5179, + "step": 774 + }, + { + "epoch": 0.9645301804604853, + "grad_norm": 2.745391368865967, + "learning_rate": 3.8030442804428046e-06, + "loss": 1.5124, + "step": 775 + }, + { + "epoch": 0.9657747355320473, + "grad_norm": 2.8050146102905273, + "learning_rate": 3.800738007380074e-06, + "loss": 1.519, + "step": 776 + }, + { + "epoch": 0.9670192906036092, + "grad_norm": 2.697171449661255, + "learning_rate": 3.798431734317343e-06, + "loss": 1.5219, + "step": 777 + }, + { + "epoch": 0.9682638456751711, + "grad_norm": 2.6375980377197266, + "learning_rate": 3.796125461254613e-06, + "loss": 1.5345, + "step": 778 + }, + { + "epoch": 0.9695084007467331, + "grad_norm": 2.586636781692505, + "learning_rate": 3.7938191881918823e-06, + "loss": 1.5182, + "step": 779 + }, + { + "epoch": 0.9707529558182949, + "grad_norm": 2.626453399658203, + "learning_rate": 3.7915129151291518e-06, + "loss": 1.4817, + "step": 780 + }, + { + "epoch": 0.9707529558182949, + "eval_loss": 1.6734713315963745, + "eval_runtime": 48.1436, + "eval_samples_per_second": 20.771, + "eval_steps_per_second": 0.872, + "step": 780 + }, + { + "epoch": 0.9719975108898569, + "grad_norm": 2.579970121383667, + "learning_rate": 3.7892066420664208e-06, + "loss": 1.4981, + "step": 781 + }, + { + "epoch": 0.9732420659614188, + "grad_norm": 2.6834843158721924, + "learning_rate": 3.7869003690036906e-06, + "loss": 1.5044, + "step": 782 + }, + { + "epoch": 0.9744866210329807, + "grad_norm": 2.586982488632202, + "learning_rate": 3.78459409594096e-06, + "loss": 1.5079, + "step": 783 + }, + { + "epoch": 0.9757311761045426, + "grad_norm": 2.910027503967285, + "learning_rate": 3.782287822878229e-06, + "loss": 1.5695, + "step": 784 + }, + { + "epoch": 0.9769757311761046, + "grad_norm": 2.7088494300842285, + "learning_rate": 3.7799815498154984e-06, + "loss": 1.4986, + "step": 785 + }, + { + "epoch": 0.9782202862476664, + "grad_norm": 2.581325054168701, + "learning_rate": 3.777675276752768e-06, + "loss": 1.5137, + "step": 786 + }, + { + "epoch": 0.9794648413192284, + "grad_norm": 2.700709819793701, + "learning_rate": 3.775369003690037e-06, + "loss": 1.4718, + "step": 787 + }, + { + "epoch": 0.9807093963907902, + "grad_norm": 2.6712708473205566, + "learning_rate": 3.7730627306273067e-06, + "loss": 1.5208, + "step": 788 + }, + { + "epoch": 0.9819539514623522, + "grad_norm": 2.6799817085266113, + "learning_rate": 3.770756457564576e-06, + "loss": 1.5374, + "step": 789 + }, + { + "epoch": 0.9831985065339142, + "grad_norm": 2.618988037109375, + "learning_rate": 3.768450184501845e-06, + "loss": 1.4997, + "step": 790 + }, + { + "epoch": 0.9831985065339142, + "eval_loss": 1.6860820055007935, + "eval_runtime": 51.7654, + "eval_samples_per_second": 19.318, + "eval_steps_per_second": 0.811, + "step": 790 + }, + { + "epoch": 0.984443061605476, + "grad_norm": 2.6899526119232178, + "learning_rate": 3.7661439114391146e-06, + "loss": 1.5216, + "step": 791 + }, + { + "epoch": 0.985687616677038, + "grad_norm": 2.8003487586975098, + "learning_rate": 3.763837638376384e-06, + "loss": 1.557, + "step": 792 + }, + { + "epoch": 0.9869321717485999, + "grad_norm": 2.744536876678467, + "learning_rate": 3.761531365313654e-06, + "loss": 1.5355, + "step": 793 + }, + { + "epoch": 0.9881767268201618, + "grad_norm": 2.587250232696533, + "learning_rate": 3.759225092250923e-06, + "loss": 1.5259, + "step": 794 + }, + { + "epoch": 0.9894212818917237, + "grad_norm": 2.616291046142578, + "learning_rate": 3.7569188191881922e-06, + "loss": 1.5176, + "step": 795 + }, + { + "epoch": 0.9906658369632856, + "grad_norm": 2.6410577297210693, + "learning_rate": 3.7546125461254617e-06, + "loss": 1.511, + "step": 796 + }, + { + "epoch": 0.9919103920348475, + "grad_norm": 2.577373504638672, + "learning_rate": 3.7523062730627307e-06, + "loss": 1.4715, + "step": 797 + }, + { + "epoch": 0.9931549471064095, + "grad_norm": 2.67305326461792, + "learning_rate": 3.7500000000000005e-06, + "loss": 1.4828, + "step": 798 + }, + { + "epoch": 0.9943995021779714, + "grad_norm": 3.1631500720977783, + "learning_rate": 3.74769372693727e-06, + "loss": 1.5602, + "step": 799 + }, + { + "epoch": 0.9956440572495333, + "grad_norm": 2.9222350120544434, + "learning_rate": 3.745387453874539e-06, + "loss": 1.4814, + "step": 800 + }, + { + "epoch": 0.9956440572495333, + "eval_loss": 1.6810544729232788, + "eval_runtime": 51.1262, + "eval_samples_per_second": 19.559, + "eval_steps_per_second": 0.821, + "step": 800 + }, + { + "epoch": 0.9968886123210952, + "grad_norm": 2.884312629699707, + "learning_rate": 3.7430811808118084e-06, + "loss": 1.4553, + "step": 801 + }, + { + "epoch": 0.9981331673926571, + "grad_norm": 2.712655782699585, + "learning_rate": 3.7407749077490778e-06, + "loss": 1.5275, + "step": 802 + }, + { + "epoch": 0.9993777224642191, + "grad_norm": 2.8700571060180664, + "learning_rate": 3.7384686346863468e-06, + "loss": 1.4923, + "step": 803 + }, + { + "epoch": 1.000622277535781, + "grad_norm": 2.7779245376586914, + "learning_rate": 3.7361623616236166e-06, + "loss": 1.5224, + "step": 804 + }, + { + "epoch": 1.0018668326073428, + "grad_norm": 2.7679810523986816, + "learning_rate": 3.733856088560886e-06, + "loss": 1.4977, + "step": 805 + }, + { + "epoch": 1.0031113876789048, + "grad_norm": 2.797023296356201, + "learning_rate": 3.7315498154981555e-06, + "loss": 1.4396, + "step": 806 + }, + { + "epoch": 1.0043559427504667, + "grad_norm": 3.007962465286255, + "learning_rate": 3.7292435424354245e-06, + "loss": 1.5015, + "step": 807 + }, + { + "epoch": 1.0056004978220285, + "grad_norm": 3.127639055252075, + "learning_rate": 3.726937269372694e-06, + "loss": 1.5176, + "step": 808 + }, + { + "epoch": 1.0068450528935906, + "grad_norm": 2.9001357555389404, + "learning_rate": 3.7246309963099637e-06, + "loss": 1.4735, + "step": 809 + }, + { + "epoch": 1.0080896079651525, + "grad_norm": 2.8245413303375244, + "learning_rate": 3.7223247232472327e-06, + "loss": 1.486, + "step": 810 + }, + { + "epoch": 1.0080896079651525, + "eval_loss": 1.6835130453109741, + "eval_runtime": 44.5682, + "eval_samples_per_second": 22.438, + "eval_steps_per_second": 0.942, + "step": 810 + }, + { + "epoch": 1.0093341630367143, + "grad_norm": 2.780172824859619, + "learning_rate": 3.720018450184502e-06, + "loss": 1.4437, + "step": 811 + }, + { + "epoch": 1.0105787181082764, + "grad_norm": 2.6013996601104736, + "learning_rate": 3.7177121771217716e-06, + "loss": 1.4529, + "step": 812 + }, + { + "epoch": 1.0118232731798382, + "grad_norm": 2.831015110015869, + "learning_rate": 3.7154059040590406e-06, + "loss": 1.4825, + "step": 813 + }, + { + "epoch": 1.0130678282514, + "grad_norm": 2.8341829776763916, + "learning_rate": 3.71309963099631e-06, + "loss": 1.4736, + "step": 814 + }, + { + "epoch": 1.0143123833229621, + "grad_norm": 2.769540309906006, + "learning_rate": 3.71079335793358e-06, + "loss": 1.4724, + "step": 815 + }, + { + "epoch": 1.015556938394524, + "grad_norm": 3.0083069801330566, + "learning_rate": 3.708487084870849e-06, + "loss": 1.4985, + "step": 816 + }, + { + "epoch": 1.0168014934660858, + "grad_norm": 3.052168607711792, + "learning_rate": 3.7061808118081183e-06, + "loss": 1.5193, + "step": 817 + }, + { + "epoch": 1.0180460485376477, + "grad_norm": 2.924877405166626, + "learning_rate": 3.7038745387453877e-06, + "loss": 1.5219, + "step": 818 + }, + { + "epoch": 1.0192906036092098, + "grad_norm": 3.150123357772827, + "learning_rate": 3.7015682656826576e-06, + "loss": 1.5026, + "step": 819 + }, + { + "epoch": 1.0205351586807716, + "grad_norm": 2.881655216217041, + "learning_rate": 3.6992619926199266e-06, + "loss": 1.5011, + "step": 820 + }, + { + "epoch": 1.0205351586807716, + "eval_loss": 1.6745303869247437, + "eval_runtime": 50.6791, + "eval_samples_per_second": 19.732, + "eval_steps_per_second": 0.829, + "step": 820 + }, + { + "epoch": 1.0217797137523335, + "grad_norm": 2.824249505996704, + "learning_rate": 3.696955719557196e-06, + "loss": 1.4642, + "step": 821 + }, + { + "epoch": 1.0230242688238955, + "grad_norm": 2.859144926071167, + "learning_rate": 3.6946494464944654e-06, + "loss": 1.4681, + "step": 822 + }, + { + "epoch": 1.0242688238954574, + "grad_norm": 2.90312123298645, + "learning_rate": 3.6923431734317344e-06, + "loss": 1.4926, + "step": 823 + }, + { + "epoch": 1.0255133789670192, + "grad_norm": 2.8340659141540527, + "learning_rate": 3.690036900369004e-06, + "loss": 1.4834, + "step": 824 + }, + { + "epoch": 1.0267579340385813, + "grad_norm": 2.9743151664733887, + "learning_rate": 3.6877306273062737e-06, + "loss": 1.4574, + "step": 825 + }, + { + "epoch": 1.0280024891101431, + "grad_norm": 3.032179594039917, + "learning_rate": 3.6854243542435427e-06, + "loss": 1.5571, + "step": 826 + }, + { + "epoch": 1.029247044181705, + "grad_norm": 2.816826343536377, + "learning_rate": 3.683118081180812e-06, + "loss": 1.455, + "step": 827 + }, + { + "epoch": 1.030491599253267, + "grad_norm": 3.046696186065674, + "learning_rate": 3.6808118081180815e-06, + "loss": 1.4871, + "step": 828 + }, + { + "epoch": 1.031736154324829, + "grad_norm": 2.8517391681671143, + "learning_rate": 3.678505535055351e-06, + "loss": 1.5292, + "step": 829 + }, + { + "epoch": 1.0329807093963908, + "grad_norm": 2.752811908721924, + "learning_rate": 3.67619926199262e-06, + "loss": 1.5091, + "step": 830 + }, + { + "epoch": 1.0329807093963908, + "eval_loss": 1.6692287921905518, + "eval_runtime": 50.8347, + "eval_samples_per_second": 19.672, + "eval_steps_per_second": 0.826, + "step": 830 + }, + { + "epoch": 1.0342252644679526, + "grad_norm": 2.9220356941223145, + "learning_rate": 3.6738929889298898e-06, + "loss": 1.4247, + "step": 831 + }, + { + "epoch": 1.0354698195395147, + "grad_norm": 2.8903002738952637, + "learning_rate": 3.671586715867159e-06, + "loss": 1.5595, + "step": 832 + }, + { + "epoch": 1.0367143746110765, + "grad_norm": 2.6495542526245117, + "learning_rate": 3.669280442804428e-06, + "loss": 1.4076, + "step": 833 + }, + { + "epoch": 1.0379589296826384, + "grad_norm": 2.880809783935547, + "learning_rate": 3.6669741697416976e-06, + "loss": 1.5285, + "step": 834 + }, + { + "epoch": 1.0392034847542004, + "grad_norm": 2.919142007827759, + "learning_rate": 3.6646678966789675e-06, + "loss": 1.5165, + "step": 835 + }, + { + "epoch": 1.0404480398257623, + "grad_norm": 3.1180522441864014, + "learning_rate": 3.6623616236162365e-06, + "loss": 1.5224, + "step": 836 + }, + { + "epoch": 1.0416925948973241, + "grad_norm": 2.767709255218506, + "learning_rate": 3.660055350553506e-06, + "loss": 1.4859, + "step": 837 + }, + { + "epoch": 1.0429371499688862, + "grad_norm": 2.708828926086426, + "learning_rate": 3.6577490774907753e-06, + "loss": 1.4803, + "step": 838 + }, + { + "epoch": 1.044181705040448, + "grad_norm": 2.688359260559082, + "learning_rate": 3.6554428044280443e-06, + "loss": 1.4571, + "step": 839 + }, + { + "epoch": 1.04542626011201, + "grad_norm": 2.691812753677368, + "learning_rate": 3.6531365313653137e-06, + "loss": 1.5043, + "step": 840 + }, + { + "epoch": 1.04542626011201, + "eval_loss": 1.6769559383392334, + "eval_runtime": 49.6233, + "eval_samples_per_second": 20.152, + "eval_steps_per_second": 0.846, + "step": 840 + }, + { + "epoch": 1.046670815183572, + "grad_norm": 2.686372995376587, + "learning_rate": 3.6508302583025836e-06, + "loss": 1.5221, + "step": 841 + }, + { + "epoch": 1.0479153702551338, + "grad_norm": 2.742859125137329, + "learning_rate": 3.648523985239853e-06, + "loss": 1.4685, + "step": 842 + }, + { + "epoch": 1.0491599253266957, + "grad_norm": 2.874018907546997, + "learning_rate": 3.646217712177122e-06, + "loss": 1.4813, + "step": 843 + }, + { + "epoch": 1.0504044803982575, + "grad_norm": 2.8159990310668945, + "learning_rate": 3.6439114391143914e-06, + "loss": 1.4421, + "step": 844 + }, + { + "epoch": 1.0516490354698196, + "grad_norm": 2.714787006378174, + "learning_rate": 3.641605166051661e-06, + "loss": 1.4879, + "step": 845 + }, + { + "epoch": 1.0528935905413814, + "grad_norm": 2.885148525238037, + "learning_rate": 3.63929889298893e-06, + "loss": 1.4725, + "step": 846 + }, + { + "epoch": 1.0541381456129433, + "grad_norm": 2.7272610664367676, + "learning_rate": 3.6369926199261997e-06, + "loss": 1.4276, + "step": 847 + }, + { + "epoch": 1.0553827006845053, + "grad_norm": 2.9114978313446045, + "learning_rate": 3.634686346863469e-06, + "loss": 1.4194, + "step": 848 + }, + { + "epoch": 1.0566272557560672, + "grad_norm": 3.019313335418701, + "learning_rate": 3.632380073800738e-06, + "loss": 1.5266, + "step": 849 + }, + { + "epoch": 1.057871810827629, + "grad_norm": 2.714751958847046, + "learning_rate": 3.6300738007380075e-06, + "loss": 1.4593, + "step": 850 + }, + { + "epoch": 1.057871810827629, + "eval_loss": 1.6760298013687134, + "eval_runtime": 52.1681, + "eval_samples_per_second": 19.169, + "eval_steps_per_second": 0.805, + "step": 850 + }, + { + "epoch": 1.0591163658991911, + "grad_norm": 3.0507936477661133, + "learning_rate": 3.627767527675277e-06, + "loss": 1.5064, + "step": 851 + }, + { + "epoch": 1.060360920970753, + "grad_norm": 2.8116295337677, + "learning_rate": 3.625461254612546e-06, + "loss": 1.5218, + "step": 852 + }, + { + "epoch": 1.0616054760423148, + "grad_norm": 2.8801465034484863, + "learning_rate": 3.623154981549816e-06, + "loss": 1.4641, + "step": 853 + }, + { + "epoch": 1.0628500311138769, + "grad_norm": 3.0581862926483154, + "learning_rate": 3.6208487084870852e-06, + "loss": 1.483, + "step": 854 + }, + { + "epoch": 1.0640945861854387, + "grad_norm": 2.898780584335327, + "learning_rate": 3.6185424354243547e-06, + "loss": 1.4623, + "step": 855 + }, + { + "epoch": 1.0653391412570006, + "grad_norm": 2.971820116043091, + "learning_rate": 3.6162361623616237e-06, + "loss": 1.4455, + "step": 856 + }, + { + "epoch": 1.0665836963285624, + "grad_norm": 3.0021913051605225, + "learning_rate": 3.6139298892988935e-06, + "loss": 1.4878, + "step": 857 + }, + { + "epoch": 1.0678282514001245, + "grad_norm": 3.0697457790374756, + "learning_rate": 3.611623616236163e-06, + "loss": 1.5008, + "step": 858 + }, + { + "epoch": 1.0690728064716863, + "grad_norm": 3.0787923336029053, + "learning_rate": 3.609317343173432e-06, + "loss": 1.4612, + "step": 859 + }, + { + "epoch": 1.0703173615432482, + "grad_norm": 2.789339780807495, + "learning_rate": 3.6070110701107014e-06, + "loss": 1.4287, + "step": 860 + }, + { + "epoch": 1.0703173615432482, + "eval_loss": 1.6747506856918335, + "eval_runtime": 51.4912, + "eval_samples_per_second": 19.421, + "eval_steps_per_second": 0.816, + "step": 860 + }, + { + "epoch": 1.0715619166148103, + "grad_norm": 2.9622788429260254, + "learning_rate": 3.6047047970479708e-06, + "loss": 1.4905, + "step": 861 + }, + { + "epoch": 1.072806471686372, + "grad_norm": 3.0675268173217773, + "learning_rate": 3.6023985239852398e-06, + "loss": 1.461, + "step": 862 + }, + { + "epoch": 1.074051026757934, + "grad_norm": 2.820996046066284, + "learning_rate": 3.6000922509225096e-06, + "loss": 1.4948, + "step": 863 + }, + { + "epoch": 1.075295581829496, + "grad_norm": 2.833761215209961, + "learning_rate": 3.597785977859779e-06, + "loss": 1.4683, + "step": 864 + }, + { + "epoch": 1.0765401369010579, + "grad_norm": 2.917313814163208, + "learning_rate": 3.595479704797048e-06, + "loss": 1.5191, + "step": 865 + }, + { + "epoch": 1.0777846919726197, + "grad_norm": 2.876166820526123, + "learning_rate": 3.5931734317343175e-06, + "loss": 1.4854, + "step": 866 + }, + { + "epoch": 1.0790292470441818, + "grad_norm": 3.139049768447876, + "learning_rate": 3.590867158671587e-06, + "loss": 1.5134, + "step": 867 + }, + { + "epoch": 1.0802738021157436, + "grad_norm": 2.9956753253936768, + "learning_rate": 3.5885608856088567e-06, + "loss": 1.5003, + "step": 868 + }, + { + "epoch": 1.0815183571873055, + "grad_norm": 2.957181930541992, + "learning_rate": 3.5862546125461257e-06, + "loss": 1.4588, + "step": 869 + }, + { + "epoch": 1.0827629122588673, + "grad_norm": 2.8576438426971436, + "learning_rate": 3.583948339483395e-06, + "loss": 1.48, + "step": 870 + }, + { + "epoch": 1.0827629122588673, + "eval_loss": 1.6724848747253418, + "eval_runtime": 50.9669, + "eval_samples_per_second": 19.621, + "eval_steps_per_second": 0.824, + "step": 870 + }, + { + "epoch": 1.0840074673304294, + "grad_norm": 2.8183560371398926, + "learning_rate": 3.5816420664206646e-06, + "loss": 1.4818, + "step": 871 + }, + { + "epoch": 1.0852520224019913, + "grad_norm": 2.7531797885894775, + "learning_rate": 3.5793357933579336e-06, + "loss": 1.4712, + "step": 872 + }, + { + "epoch": 1.086496577473553, + "grad_norm": 2.923962354660034, + "learning_rate": 3.5770295202952034e-06, + "loss": 1.4864, + "step": 873 + }, + { + "epoch": 1.0877411325451152, + "grad_norm": 2.851724147796631, + "learning_rate": 3.574723247232473e-06, + "loss": 1.4771, + "step": 874 + }, + { + "epoch": 1.088985687616677, + "grad_norm": 3.024634838104248, + "learning_rate": 3.572416974169742e-06, + "loss": 1.4643, + "step": 875 + }, + { + "epoch": 1.0902302426882389, + "grad_norm": 2.7100844383239746, + "learning_rate": 3.5701107011070113e-06, + "loss": 1.4773, + "step": 876 + }, + { + "epoch": 1.091474797759801, + "grad_norm": 2.730502128601074, + "learning_rate": 3.5678044280442807e-06, + "loss": 1.5051, + "step": 877 + }, + { + "epoch": 1.0927193528313628, + "grad_norm": 2.747082233428955, + "learning_rate": 3.5654981549815497e-06, + "loss": 1.5207, + "step": 878 + }, + { + "epoch": 1.0939639079029246, + "grad_norm": 2.718358039855957, + "learning_rate": 3.5631918819188195e-06, + "loss": 1.5067, + "step": 879 + }, + { + "epoch": 1.0952084629744867, + "grad_norm": 2.7794573307037354, + "learning_rate": 3.560885608856089e-06, + "loss": 1.4616, + "step": 880 + }, + { + "epoch": 1.0952084629744867, + "eval_loss": 1.6676132678985596, + "eval_runtime": 51.6157, + "eval_samples_per_second": 19.374, + "eval_steps_per_second": 0.814, + "step": 880 + }, + { + "epoch": 1.0964530180460486, + "grad_norm": 2.727416753768921, + "learning_rate": 3.5585793357933584e-06, + "loss": 1.4043, + "step": 881 + }, + { + "epoch": 1.0976975731176104, + "grad_norm": 2.856900930404663, + "learning_rate": 3.5562730627306274e-06, + "loss": 1.5198, + "step": 882 + }, + { + "epoch": 1.0989421281891725, + "grad_norm": 2.872823715209961, + "learning_rate": 3.553966789667897e-06, + "loss": 1.4981, + "step": 883 + }, + { + "epoch": 1.1001866832607343, + "grad_norm": 2.7724361419677734, + "learning_rate": 3.5516605166051667e-06, + "loss": 1.4819, + "step": 884 + }, + { + "epoch": 1.1014312383322962, + "grad_norm": 2.884434461593628, + "learning_rate": 3.5493542435424357e-06, + "loss": 1.433, + "step": 885 + }, + { + "epoch": 1.102675793403858, + "grad_norm": 2.887873411178589, + "learning_rate": 3.547047970479705e-06, + "loss": 1.4677, + "step": 886 + }, + { + "epoch": 1.10392034847542, + "grad_norm": 2.946845054626465, + "learning_rate": 3.5447416974169745e-06, + "loss": 1.4983, + "step": 887 + }, + { + "epoch": 1.105164903546982, + "grad_norm": 2.752552032470703, + "learning_rate": 3.5424354243542435e-06, + "loss": 1.4766, + "step": 888 + }, + { + "epoch": 1.1064094586185438, + "grad_norm": 2.788634777069092, + "learning_rate": 3.5401291512915133e-06, + "loss": 1.4747, + "step": 889 + }, + { + "epoch": 1.1076540136901059, + "grad_norm": 2.782792568206787, + "learning_rate": 3.5378228782287828e-06, + "loss": 1.5503, + "step": 890 + }, + { + "epoch": 1.1076540136901059, + "eval_loss": 1.6698333024978638, + "eval_runtime": 49.1373, + "eval_samples_per_second": 20.351, + "eval_steps_per_second": 0.855, + "step": 890 + }, + { + "epoch": 1.1088985687616677, + "grad_norm": 2.8351166248321533, + "learning_rate": 3.535516605166052e-06, + "loss": 1.4589, + "step": 891 + }, + { + "epoch": 1.1101431238332296, + "grad_norm": 2.798872709274292, + "learning_rate": 3.533210332103321e-06, + "loss": 1.4496, + "step": 892 + }, + { + "epoch": 1.1113876789047916, + "grad_norm": 3.0863239765167236, + "learning_rate": 3.5309040590405906e-06, + "loss": 1.4698, + "step": 893 + }, + { + "epoch": 1.1126322339763535, + "grad_norm": 2.853862762451172, + "learning_rate": 3.5285977859778605e-06, + "loss": 1.4429, + "step": 894 + }, + { + "epoch": 1.1138767890479153, + "grad_norm": 2.8946170806884766, + "learning_rate": 3.5262915129151295e-06, + "loss": 1.4697, + "step": 895 + }, + { + "epoch": 1.1151213441194774, + "grad_norm": 3.043823003768921, + "learning_rate": 3.523985239852399e-06, + "loss": 1.4666, + "step": 896 + }, + { + "epoch": 1.1163658991910392, + "grad_norm": 2.7822980880737305, + "learning_rate": 3.5216789667896683e-06, + "loss": 1.4523, + "step": 897 + }, + { + "epoch": 1.117610454262601, + "grad_norm": 2.9454171657562256, + "learning_rate": 3.5193726937269373e-06, + "loss": 1.5009, + "step": 898 + }, + { + "epoch": 1.1188550093341632, + "grad_norm": 2.8533740043640137, + "learning_rate": 3.5170664206642067e-06, + "loss": 1.4265, + "step": 899 + }, + { + "epoch": 1.120099564405725, + "grad_norm": 2.894937038421631, + "learning_rate": 3.5147601476014766e-06, + "loss": 1.4895, + "step": 900 + }, + { + "epoch": 1.120099564405725, + "eval_loss": 1.6701407432556152, + "eval_runtime": 45.477, + "eval_samples_per_second": 21.989, + "eval_steps_per_second": 0.924, + "step": 900 + }, + { + "epoch": 1.1213441194772868, + "grad_norm": 2.933821201324463, + "learning_rate": 3.5124538745387456e-06, + "loss": 1.468, + "step": 901 + }, + { + "epoch": 1.1225886745488487, + "grad_norm": 2.834265947341919, + "learning_rate": 3.510147601476015e-06, + "loss": 1.4621, + "step": 902 + }, + { + "epoch": 1.1238332296204108, + "grad_norm": 2.9064722061157227, + "learning_rate": 3.5078413284132844e-06, + "loss": 1.504, + "step": 903 + }, + { + "epoch": 1.1250777846919726, + "grad_norm": 2.8753886222839355, + "learning_rate": 3.505535055350554e-06, + "loss": 1.4643, + "step": 904 + }, + { + "epoch": 1.1263223397635345, + "grad_norm": 2.894659996032715, + "learning_rate": 3.503228782287823e-06, + "loss": 1.4909, + "step": 905 + }, + { + "epoch": 1.1275668948350965, + "grad_norm": 3.085977792739868, + "learning_rate": 3.5009225092250927e-06, + "loss": 1.459, + "step": 906 + }, + { + "epoch": 1.1288114499066584, + "grad_norm": 2.9958064556121826, + "learning_rate": 3.498616236162362e-06, + "loss": 1.4152, + "step": 907 + }, + { + "epoch": 1.1300560049782202, + "grad_norm": 2.9136762619018555, + "learning_rate": 3.496309963099631e-06, + "loss": 1.473, + "step": 908 + }, + { + "epoch": 1.1313005600497823, + "grad_norm": 2.956437826156616, + "learning_rate": 3.4940036900369005e-06, + "loss": 1.5167, + "step": 909 + }, + { + "epoch": 1.1325451151213441, + "grad_norm": 2.7354865074157715, + "learning_rate": 3.4916974169741704e-06, + "loss": 1.4413, + "step": 910 + }, + { + "epoch": 1.1325451151213441, + "eval_loss": 1.6629912853240967, + "eval_runtime": 41.5478, + "eval_samples_per_second": 24.069, + "eval_steps_per_second": 1.011, + "step": 910 + }, + { + "epoch": 1.133789670192906, + "grad_norm": 2.7443008422851562, + "learning_rate": 3.4893911439114394e-06, + "loss": 1.4475, + "step": 911 + }, + { + "epoch": 1.135034225264468, + "grad_norm": 2.7811238765716553, + "learning_rate": 3.487084870848709e-06, + "loss": 1.4257, + "step": 912 + }, + { + "epoch": 1.13627878033603, + "grad_norm": 2.7916479110717773, + "learning_rate": 3.4847785977859782e-06, + "loss": 1.4048, + "step": 913 + }, + { + "epoch": 1.1375233354075918, + "grad_norm": 2.7835633754730225, + "learning_rate": 3.4824723247232472e-06, + "loss": 1.3996, + "step": 914 + }, + { + "epoch": 1.1387678904791536, + "grad_norm": 2.8188705444335938, + "learning_rate": 3.4801660516605166e-06, + "loss": 1.4541, + "step": 915 + }, + { + "epoch": 1.1400124455507157, + "grad_norm": 3.0134100914001465, + "learning_rate": 3.4778597785977865e-06, + "loss": 1.4353, + "step": 916 + }, + { + "epoch": 1.1412570006222775, + "grad_norm": 2.6965606212615967, + "learning_rate": 3.475553505535056e-06, + "loss": 1.4308, + "step": 917 + }, + { + "epoch": 1.1425015556938394, + "grad_norm": 2.8179919719696045, + "learning_rate": 3.473247232472325e-06, + "loss": 1.5204, + "step": 918 + }, + { + "epoch": 1.1437461107654014, + "grad_norm": 2.7213902473449707, + "learning_rate": 3.4709409594095943e-06, + "loss": 1.4344, + "step": 919 + }, + { + "epoch": 1.1449906658369633, + "grad_norm": 2.850193738937378, + "learning_rate": 3.4686346863468638e-06, + "loss": 1.473, + "step": 920 + }, + { + "epoch": 1.1449906658369633, + "eval_loss": 1.663345217704773, + "eval_runtime": 43.8397, + "eval_samples_per_second": 22.81, + "eval_steps_per_second": 0.958, + "step": 920 + }, + { + "epoch": 1.1462352209085251, + "grad_norm": 2.780198574066162, + "learning_rate": 3.4663284132841328e-06, + "loss": 1.4472, + "step": 921 + }, + { + "epoch": 1.1474797759800872, + "grad_norm": 2.9373831748962402, + "learning_rate": 3.4640221402214026e-06, + "loss": 1.426, + "step": 922 + }, + { + "epoch": 1.148724331051649, + "grad_norm": 2.6986873149871826, + "learning_rate": 3.461715867158672e-06, + "loss": 1.3953, + "step": 923 + }, + { + "epoch": 1.149968886123211, + "grad_norm": 2.927441120147705, + "learning_rate": 3.459409594095941e-06, + "loss": 1.4578, + "step": 924 + }, + { + "epoch": 1.151213441194773, + "grad_norm": 2.8469648361206055, + "learning_rate": 3.4571033210332105e-06, + "loss": 1.4313, + "step": 925 + }, + { + "epoch": 1.1524579962663348, + "grad_norm": 2.8114013671875, + "learning_rate": 3.4547970479704803e-06, + "loss": 1.4267, + "step": 926 + }, + { + "epoch": 1.1537025513378967, + "grad_norm": 2.977605104446411, + "learning_rate": 3.4524907749077493e-06, + "loss": 1.4747, + "step": 927 + }, + { + "epoch": 1.1549471064094585, + "grad_norm": 2.9327852725982666, + "learning_rate": 3.4501845018450187e-06, + "loss": 1.4219, + "step": 928 + }, + { + "epoch": 1.1561916614810206, + "grad_norm": 2.844038963317871, + "learning_rate": 3.447878228782288e-06, + "loss": 1.5146, + "step": 929 + }, + { + "epoch": 1.1574362165525824, + "grad_norm": 2.9402201175689697, + "learning_rate": 3.4455719557195576e-06, + "loss": 1.45, + "step": 930 + }, + { + "epoch": 1.1574362165525824, + "eval_loss": 1.6620122194290161, + "eval_runtime": 46.1748, + "eval_samples_per_second": 21.657, + "eval_steps_per_second": 0.91, + "step": 930 + }, + { + "epoch": 1.1586807716241443, + "grad_norm": 2.842651128768921, + "learning_rate": 3.4432656826568266e-06, + "loss": 1.4415, + "step": 931 + }, + { + "epoch": 1.1599253266957064, + "grad_norm": 2.8909685611724854, + "learning_rate": 3.4409594095940964e-06, + "loss": 1.4529, + "step": 932 + }, + { + "epoch": 1.1611698817672682, + "grad_norm": 2.8025808334350586, + "learning_rate": 3.438653136531366e-06, + "loss": 1.4552, + "step": 933 + }, + { + "epoch": 1.16241443683883, + "grad_norm": 2.6995794773101807, + "learning_rate": 3.436346863468635e-06, + "loss": 1.4435, + "step": 934 + }, + { + "epoch": 1.1636589919103921, + "grad_norm": 2.7307121753692627, + "learning_rate": 3.4340405904059043e-06, + "loss": 1.4447, + "step": 935 + }, + { + "epoch": 1.164903546981954, + "grad_norm": 2.9532501697540283, + "learning_rate": 3.4317343173431737e-06, + "loss": 1.4747, + "step": 936 + }, + { + "epoch": 1.1661481020535158, + "grad_norm": 3.0135788917541504, + "learning_rate": 3.4294280442804427e-06, + "loss": 1.525, + "step": 937 + }, + { + "epoch": 1.167392657125078, + "grad_norm": 2.796355962753296, + "learning_rate": 3.4271217712177125e-06, + "loss": 1.4539, + "step": 938 + }, + { + "epoch": 1.1686372121966397, + "grad_norm": 2.9422309398651123, + "learning_rate": 3.424815498154982e-06, + "loss": 1.4664, + "step": 939 + }, + { + "epoch": 1.1698817672682016, + "grad_norm": 2.7680039405822754, + "learning_rate": 3.4225092250922514e-06, + "loss": 1.4837, + "step": 940 + }, + { + "epoch": 1.1698817672682016, + "eval_loss": 1.660908818244934, + "eval_runtime": 45.6998, + "eval_samples_per_second": 21.882, + "eval_steps_per_second": 0.919, + "step": 940 + }, + { + "epoch": 1.1711263223397634, + "grad_norm": 2.8888845443725586, + "learning_rate": 3.4202029520295204e-06, + "loss": 1.5315, + "step": 941 + }, + { + "epoch": 1.1723708774113255, + "grad_norm": 2.7124130725860596, + "learning_rate": 3.41789667896679e-06, + "loss": 1.4188, + "step": 942 + }, + { + "epoch": 1.1736154324828874, + "grad_norm": 2.75203013420105, + "learning_rate": 3.4155904059040596e-06, + "loss": 1.4533, + "step": 943 + }, + { + "epoch": 1.1748599875544492, + "grad_norm": 2.865480422973633, + "learning_rate": 3.4132841328413286e-06, + "loss": 1.4532, + "step": 944 + }, + { + "epoch": 1.1761045426260113, + "grad_norm": 2.8618321418762207, + "learning_rate": 3.410977859778598e-06, + "loss": 1.4386, + "step": 945 + }, + { + "epoch": 1.1773490976975731, + "grad_norm": 2.7656145095825195, + "learning_rate": 3.4086715867158675e-06, + "loss": 1.3996, + "step": 946 + }, + { + "epoch": 1.178593652769135, + "grad_norm": 2.76213002204895, + "learning_rate": 3.4063653136531365e-06, + "loss": 1.4979, + "step": 947 + }, + { + "epoch": 1.179838207840697, + "grad_norm": 2.8581957817077637, + "learning_rate": 3.4040590405904063e-06, + "loss": 1.4673, + "step": 948 + }, + { + "epoch": 1.1810827629122589, + "grad_norm": 2.7825586795806885, + "learning_rate": 3.4017527675276758e-06, + "loss": 1.4453, + "step": 949 + }, + { + "epoch": 1.1823273179838207, + "grad_norm": 2.9226131439208984, + "learning_rate": 3.3994464944649448e-06, + "loss": 1.4879, + "step": 950 + }, + { + "epoch": 1.1823273179838207, + "eval_loss": 1.6621263027191162, + "eval_runtime": 45.1222, + "eval_samples_per_second": 22.162, + "eval_steps_per_second": 0.931, + "step": 950 + }, + { + "epoch": 1.1835718730553828, + "grad_norm": 2.9997975826263428, + "learning_rate": 3.397140221402214e-06, + "loss": 1.505, + "step": 951 + }, + { + "epoch": 1.1848164281269447, + "grad_norm": 2.7225453853607178, + "learning_rate": 3.3948339483394836e-06, + "loss": 1.4695, + "step": 952 + }, + { + "epoch": 1.1860609831985065, + "grad_norm": 2.916473150253296, + "learning_rate": 3.3925276752767534e-06, + "loss": 1.4901, + "step": 953 + }, + { + "epoch": 1.1873055382700684, + "grad_norm": 2.8371644020080566, + "learning_rate": 3.3902214022140224e-06, + "loss": 1.4944, + "step": 954 + }, + { + "epoch": 1.1885500933416304, + "grad_norm": 2.813662528991699, + "learning_rate": 3.387915129151292e-06, + "loss": 1.4603, + "step": 955 + }, + { + "epoch": 1.1897946484131923, + "grad_norm": 2.991931676864624, + "learning_rate": 3.3856088560885613e-06, + "loss": 1.5496, + "step": 956 + }, + { + "epoch": 1.1910392034847541, + "grad_norm": 2.6252501010894775, + "learning_rate": 3.3833025830258303e-06, + "loss": 1.4405, + "step": 957 + }, + { + "epoch": 1.1922837585563162, + "grad_norm": 2.747725009918213, + "learning_rate": 3.3809963099630997e-06, + "loss": 1.4689, + "step": 958 + }, + { + "epoch": 1.193528313627878, + "grad_norm": 2.887763261795044, + "learning_rate": 3.3786900369003696e-06, + "loss": 1.4501, + "step": 959 + }, + { + "epoch": 1.1947728686994399, + "grad_norm": 2.9954354763031006, + "learning_rate": 3.3763837638376386e-06, + "loss": 1.4978, + "step": 960 + }, + { + "epoch": 1.1947728686994399, + "eval_loss": 1.664442539215088, + "eval_runtime": 49.1266, + "eval_samples_per_second": 20.356, + "eval_steps_per_second": 0.855, + "step": 960 + }, + { + "epoch": 1.196017423771002, + "grad_norm": 3.0174152851104736, + "learning_rate": 3.374077490774908e-06, + "loss": 1.4353, + "step": 961 + }, + { + "epoch": 1.1972619788425638, + "grad_norm": 2.9082882404327393, + "learning_rate": 3.3717712177121774e-06, + "loss": 1.4878, + "step": 962 + }, + { + "epoch": 1.1985065339141256, + "grad_norm": 2.897993326187134, + "learning_rate": 3.3694649446494464e-06, + "loss": 1.4975, + "step": 963 + }, + { + "epoch": 1.1997510889856877, + "grad_norm": 2.739470958709717, + "learning_rate": 3.3671586715867163e-06, + "loss": 1.4202, + "step": 964 + }, + { + "epoch": 1.2009956440572496, + "grad_norm": 2.817786693572998, + "learning_rate": 3.3648523985239857e-06, + "loss": 1.5195, + "step": 965 + }, + { + "epoch": 1.2022401991288114, + "grad_norm": 2.8476386070251465, + "learning_rate": 3.362546125461255e-06, + "loss": 1.444, + "step": 966 + }, + { + "epoch": 1.2034847542003733, + "grad_norm": 2.804047107696533, + "learning_rate": 3.360239852398524e-06, + "loss": 1.4442, + "step": 967 + }, + { + "epoch": 1.2047293092719353, + "grad_norm": 2.9285430908203125, + "learning_rate": 3.3579335793357935e-06, + "loss": 1.4877, + "step": 968 + }, + { + "epoch": 1.2059738643434972, + "grad_norm": 2.8814878463745117, + "learning_rate": 3.3556273062730634e-06, + "loss": 1.5078, + "step": 969 + }, + { + "epoch": 1.207218419415059, + "grad_norm": 2.970728635787964, + "learning_rate": 3.3533210332103324e-06, + "loss": 1.4166, + "step": 970 + }, + { + "epoch": 1.207218419415059, + "eval_loss": 1.6635315418243408, + "eval_runtime": 47.2321, + "eval_samples_per_second": 21.172, + "eval_steps_per_second": 0.889, + "step": 970 + }, + { + "epoch": 1.208462974486621, + "grad_norm": 2.9836292266845703, + "learning_rate": 3.3510147601476018e-06, + "loss": 1.4777, + "step": 971 + }, + { + "epoch": 1.209707529558183, + "grad_norm": 2.8495216369628906, + "learning_rate": 3.348708487084871e-06, + "loss": 1.4829, + "step": 972 + }, + { + "epoch": 1.2109520846297448, + "grad_norm": 2.7709312438964844, + "learning_rate": 3.34640221402214e-06, + "loss": 1.4242, + "step": 973 + }, + { + "epoch": 1.2121966397013069, + "grad_norm": 2.999462366104126, + "learning_rate": 3.3440959409594096e-06, + "loss": 1.4148, + "step": 974 + }, + { + "epoch": 1.2134411947728687, + "grad_norm": 2.8719074726104736, + "learning_rate": 3.3417896678966795e-06, + "loss": 1.4196, + "step": 975 + }, + { + "epoch": 1.2146857498444306, + "grad_norm": 2.8469338417053223, + "learning_rate": 3.3394833948339485e-06, + "loss": 1.4941, + "step": 976 + }, + { + "epoch": 1.2159303049159926, + "grad_norm": 2.9780938625335693, + "learning_rate": 3.337177121771218e-06, + "loss": 1.513, + "step": 977 + }, + { + "epoch": 1.2171748599875545, + "grad_norm": 2.814760684967041, + "learning_rate": 3.3348708487084873e-06, + "loss": 1.4204, + "step": 978 + }, + { + "epoch": 1.2184194150591163, + "grad_norm": 3.016261339187622, + "learning_rate": 3.332564575645757e-06, + "loss": 1.4389, + "step": 979 + }, + { + "epoch": 1.2196639701306782, + "grad_norm": 2.9385855197906494, + "learning_rate": 3.330258302583026e-06, + "loss": 1.4885, + "step": 980 + }, + { + "epoch": 1.2196639701306782, + "eval_loss": 1.6688873767852783, + "eval_runtime": 46.2942, + "eval_samples_per_second": 21.601, + "eval_steps_per_second": 0.907, + "step": 980 + }, + { + "epoch": 1.2209085252022402, + "grad_norm": 3.1434757709503174, + "learning_rate": 3.3279520295202956e-06, + "loss": 1.5157, + "step": 981 + }, + { + "epoch": 1.222153080273802, + "grad_norm": 3.0104827880859375, + "learning_rate": 3.325645756457565e-06, + "loss": 1.49, + "step": 982 + }, + { + "epoch": 1.223397635345364, + "grad_norm": 3.1913645267486572, + "learning_rate": 3.323339483394834e-06, + "loss": 1.4937, + "step": 983 + }, + { + "epoch": 1.224642190416926, + "grad_norm": 3.19496750831604, + "learning_rate": 3.3210332103321034e-06, + "loss": 1.5197, + "step": 984 + }, + { + "epoch": 1.2258867454884879, + "grad_norm": 3.046232223510742, + "learning_rate": 3.3187269372693733e-06, + "loss": 1.4497, + "step": 985 + }, + { + "epoch": 1.2271313005600497, + "grad_norm": 2.850675344467163, + "learning_rate": 3.3164206642066423e-06, + "loss": 1.4595, + "step": 986 + }, + { + "epoch": 1.2283758556316118, + "grad_norm": 2.7744576930999756, + "learning_rate": 3.3141143911439117e-06, + "loss": 1.4726, + "step": 987 + }, + { + "epoch": 1.2296204107031736, + "grad_norm": 3.017153024673462, + "learning_rate": 3.311808118081181e-06, + "loss": 1.493, + "step": 988 + }, + { + "epoch": 1.2308649657747355, + "grad_norm": 2.8821120262145996, + "learning_rate": 3.3095018450184506e-06, + "loss": 1.4734, + "step": 989 + }, + { + "epoch": 1.2321095208462975, + "grad_norm": 2.8805840015411377, + "learning_rate": 3.3071955719557196e-06, + "loss": 1.4562, + "step": 990 + }, + { + "epoch": 1.2321095208462975, + "eval_loss": 1.669345736503601, + "eval_runtime": 47.0893, + "eval_samples_per_second": 21.236, + "eval_steps_per_second": 0.892, + "step": 990 + }, + { + "epoch": 1.2333540759178594, + "grad_norm": 2.7599008083343506, + "learning_rate": 3.3048892988929894e-06, + "loss": 1.4218, + "step": 991 + }, + { + "epoch": 1.2345986309894212, + "grad_norm": 2.9835240840911865, + "learning_rate": 3.302583025830259e-06, + "loss": 1.4596, + "step": 992 + }, + { + "epoch": 1.235843186060983, + "grad_norm": 2.843358039855957, + "learning_rate": 3.300276752767528e-06, + "loss": 1.4561, + "step": 993 + }, + { + "epoch": 1.2370877411325452, + "grad_norm": 2.7386093139648438, + "learning_rate": 3.2979704797047972e-06, + "loss": 1.4475, + "step": 994 + }, + { + "epoch": 1.238332296204107, + "grad_norm": 2.668161392211914, + "learning_rate": 3.2956642066420667e-06, + "loss": 1.4661, + "step": 995 + }, + { + "epoch": 1.2395768512756689, + "grad_norm": 2.947235107421875, + "learning_rate": 3.2933579335793357e-06, + "loss": 1.4358, + "step": 996 + }, + { + "epoch": 1.240821406347231, + "grad_norm": 2.8159878253936768, + "learning_rate": 3.2910516605166055e-06, + "loss": 1.4432, + "step": 997 + }, + { + "epoch": 1.2420659614187928, + "grad_norm": 2.896597385406494, + "learning_rate": 3.288745387453875e-06, + "loss": 1.5585, + "step": 998 + }, + { + "epoch": 1.2433105164903546, + "grad_norm": 2.80241322517395, + "learning_rate": 3.286439114391144e-06, + "loss": 1.5076, + "step": 999 + }, + { + "epoch": 1.2445550715619167, + "grad_norm": 3.008939027786255, + "learning_rate": 3.2841328413284134e-06, + "loss": 1.4234, + "step": 1000 + }, + { + "epoch": 1.2445550715619167, + "eval_loss": 1.6621588468551636, + "eval_runtime": 50.4926, + "eval_samples_per_second": 19.805, + "eval_steps_per_second": 0.832, + "step": 1000 + }, + { + "epoch": 1.2457996266334785, + "grad_norm": 2.9872381687164307, + "learning_rate": 3.281826568265683e-06, + "loss": 1.453, + "step": 1001 + }, + { + "epoch": 1.2470441817050404, + "grad_norm": 2.704676628112793, + "learning_rate": 3.2795202952029526e-06, + "loss": 1.4711, + "step": 1002 + }, + { + "epoch": 1.2482887367766025, + "grad_norm": 2.8100757598876953, + "learning_rate": 3.2772140221402216e-06, + "loss": 1.3884, + "step": 1003 + }, + { + "epoch": 1.2495332918481643, + "grad_norm": 2.902953624725342, + "learning_rate": 3.274907749077491e-06, + "loss": 1.4986, + "step": 1004 + }, + { + "epoch": 1.2507778469197262, + "grad_norm": 2.980163097381592, + "learning_rate": 3.2726014760147605e-06, + "loss": 1.5067, + "step": 1005 + }, + { + "epoch": 1.252022401991288, + "grad_norm": 2.8820345401763916, + "learning_rate": 3.2702952029520295e-06, + "loss": 1.4995, + "step": 1006 + }, + { + "epoch": 1.25326695706285, + "grad_norm": 2.8219635486602783, + "learning_rate": 3.2679889298892993e-06, + "loss": 1.4881, + "step": 1007 + }, + { + "epoch": 1.254511512134412, + "grad_norm": 2.8593199253082275, + "learning_rate": 3.2656826568265687e-06, + "loss": 1.4469, + "step": 1008 + }, + { + "epoch": 1.255756067205974, + "grad_norm": 2.888357639312744, + "learning_rate": 3.2633763837638377e-06, + "loss": 1.4869, + "step": 1009 + }, + { + "epoch": 1.2570006222775358, + "grad_norm": 2.9985013008117676, + "learning_rate": 3.261070110701107e-06, + "loss": 1.4724, + "step": 1010 + }, + { + "epoch": 1.2570006222775358, + "eval_loss": 1.6492141485214233, + "eval_runtime": 45.2197, + "eval_samples_per_second": 22.114, + "eval_steps_per_second": 0.929, + "step": 1010 + }, + { + "epoch": 1.2582451773490977, + "grad_norm": 2.860780715942383, + "learning_rate": 3.2587638376383766e-06, + "loss": 1.4618, + "step": 1011 + }, + { + "epoch": 1.2594897324206595, + "grad_norm": 2.766479730606079, + "learning_rate": 3.2564575645756456e-06, + "loss": 1.4642, + "step": 1012 + }, + { + "epoch": 1.2607342874922216, + "grad_norm": 2.711759328842163, + "learning_rate": 3.2541512915129154e-06, + "loss": 1.4283, + "step": 1013 + }, + { + "epoch": 1.2619788425637835, + "grad_norm": 2.8365769386291504, + "learning_rate": 3.251845018450185e-06, + "loss": 1.4248, + "step": 1014 + }, + { + "epoch": 1.2632233976353453, + "grad_norm": 2.957620143890381, + "learning_rate": 3.2495387453874543e-06, + "loss": 1.4662, + "step": 1015 + }, + { + "epoch": 1.2644679527069074, + "grad_norm": 2.8066208362579346, + "learning_rate": 3.2472324723247233e-06, + "loss": 1.4447, + "step": 1016 + }, + { + "epoch": 1.2657125077784692, + "grad_norm": 2.902040958404541, + "learning_rate": 3.244926199261993e-06, + "loss": 1.4743, + "step": 1017 + }, + { + "epoch": 1.266957062850031, + "grad_norm": 2.7636687755584717, + "learning_rate": 3.2426199261992625e-06, + "loss": 1.4465, + "step": 1018 + }, + { + "epoch": 1.268201617921593, + "grad_norm": 2.95261549949646, + "learning_rate": 3.2403136531365315e-06, + "loss": 1.4769, + "step": 1019 + }, + { + "epoch": 1.269446172993155, + "grad_norm": 2.911128520965576, + "learning_rate": 3.238007380073801e-06, + "loss": 1.3843, + "step": 1020 + }, + { + "epoch": 1.269446172993155, + "eval_loss": 1.6526964902877808, + "eval_runtime": 48.2126, + "eval_samples_per_second": 20.741, + "eval_steps_per_second": 0.871, + "step": 1020 + }, + { + "epoch": 1.2706907280647168, + "grad_norm": 2.769951820373535, + "learning_rate": 3.2357011070110704e-06, + "loss": 1.4847, + "step": 1021 + }, + { + "epoch": 1.271935283136279, + "grad_norm": 3.0451674461364746, + "learning_rate": 3.2333948339483394e-06, + "loss": 1.4453, + "step": 1022 + }, + { + "epoch": 1.2731798382078408, + "grad_norm": 2.9543521404266357, + "learning_rate": 3.2310885608856092e-06, + "loss": 1.4789, + "step": 1023 + }, + { + "epoch": 1.2744243932794026, + "grad_norm": 2.9174394607543945, + "learning_rate": 3.2287822878228787e-06, + "loss": 1.4807, + "step": 1024 + }, + { + "epoch": 1.2756689483509644, + "grad_norm": 2.888258934020996, + "learning_rate": 3.2264760147601477e-06, + "loss": 1.4367, + "step": 1025 + }, + { + "epoch": 1.2769135034225265, + "grad_norm": 2.8346047401428223, + "learning_rate": 3.224169741697417e-06, + "loss": 1.4313, + "step": 1026 + }, + { + "epoch": 1.2781580584940884, + "grad_norm": 2.7531332969665527, + "learning_rate": 3.2218634686346865e-06, + "loss": 1.4018, + "step": 1027 + }, + { + "epoch": 1.2794026135656502, + "grad_norm": 3.136946439743042, + "learning_rate": 3.2195571955719564e-06, + "loss": 1.4417, + "step": 1028 + }, + { + "epoch": 1.2806471686372123, + "grad_norm": 2.9618165493011475, + "learning_rate": 3.2172509225092254e-06, + "loss": 1.5117, + "step": 1029 + }, + { + "epoch": 1.2818917237087741, + "grad_norm": 3.0740747451782227, + "learning_rate": 3.2149446494464948e-06, + "loss": 1.4641, + "step": 1030 + }, + { + "epoch": 1.2818917237087741, + "eval_loss": 1.6572761535644531, + "eval_runtime": 51.6696, + "eval_samples_per_second": 19.354, + "eval_steps_per_second": 0.813, + "step": 1030 + }, + { + "epoch": 1.283136278780336, + "grad_norm": 2.9225106239318848, + "learning_rate": 3.212638376383764e-06, + "loss": 1.4421, + "step": 1031 + }, + { + "epoch": 1.2843808338518978, + "grad_norm": 2.9530649185180664, + "learning_rate": 3.210332103321033e-06, + "loss": 1.5138, + "step": 1032 + }, + { + "epoch": 1.28562538892346, + "grad_norm": 2.8513238430023193, + "learning_rate": 3.208025830258303e-06, + "loss": 1.411, + "step": 1033 + }, + { + "epoch": 1.2868699439950217, + "grad_norm": 2.7440171241760254, + "learning_rate": 3.2057195571955725e-06, + "loss": 1.4182, + "step": 1034 + }, + { + "epoch": 1.2881144990665838, + "grad_norm": 2.8126630783081055, + "learning_rate": 3.2034132841328415e-06, + "loss": 1.4333, + "step": 1035 + }, + { + "epoch": 1.2893590541381457, + "grad_norm": 2.8598744869232178, + "learning_rate": 3.201107011070111e-06, + "loss": 1.439, + "step": 1036 + }, + { + "epoch": 1.2906036092097075, + "grad_norm": 2.901622772216797, + "learning_rate": 3.1988007380073803e-06, + "loss": 1.4461, + "step": 1037 + }, + { + "epoch": 1.2918481642812694, + "grad_norm": 2.7670180797576904, + "learning_rate": 3.1964944649446493e-06, + "loss": 1.4772, + "step": 1038 + }, + { + "epoch": 1.2930927193528314, + "grad_norm": 2.7860350608825684, + "learning_rate": 3.194188191881919e-06, + "loss": 1.4414, + "step": 1039 + }, + { + "epoch": 1.2943372744243933, + "grad_norm": 2.822052478790283, + "learning_rate": 3.1918819188191886e-06, + "loss": 1.4192, + "step": 1040 + }, + { + "epoch": 1.2943372744243933, + "eval_loss": 1.663284420967102, + "eval_runtime": 47.2822, + "eval_samples_per_second": 21.15, + "eval_steps_per_second": 0.888, + "step": 1040 + }, + { + "epoch": 1.2955818294959551, + "grad_norm": 2.909109115600586, + "learning_rate": 3.189575645756458e-06, + "loss": 1.4795, + "step": 1041 + }, + { + "epoch": 1.2968263845675172, + "grad_norm": 2.933673143386841, + "learning_rate": 3.187269372693727e-06, + "loss": 1.5038, + "step": 1042 + }, + { + "epoch": 1.298070939639079, + "grad_norm": 3.032518148422241, + "learning_rate": 3.1849630996309964e-06, + "loss": 1.5133, + "step": 1043 + }, + { + "epoch": 1.299315494710641, + "grad_norm": 2.826040506362915, + "learning_rate": 3.1826568265682663e-06, + "loss": 1.4395, + "step": 1044 + }, + { + "epoch": 1.3005600497822027, + "grad_norm": 2.8152658939361572, + "learning_rate": 3.1803505535055353e-06, + "loss": 1.4787, + "step": 1045 + }, + { + "epoch": 1.3018046048537648, + "grad_norm": 2.9410431385040283, + "learning_rate": 3.1780442804428047e-06, + "loss": 1.4375, + "step": 1046 + }, + { + "epoch": 1.3030491599253267, + "grad_norm": 2.948636531829834, + "learning_rate": 3.175738007380074e-06, + "loss": 1.4847, + "step": 1047 + }, + { + "epoch": 1.3042937149968887, + "grad_norm": 2.6287615299224854, + "learning_rate": 3.173431734317343e-06, + "loss": 1.4186, + "step": 1048 + }, + { + "epoch": 1.3055382700684506, + "grad_norm": 2.836029052734375, + "learning_rate": 3.1711254612546125e-06, + "loss": 1.4251, + "step": 1049 + }, + { + "epoch": 1.3067828251400124, + "grad_norm": 2.982081651687622, + "learning_rate": 3.1688191881918824e-06, + "loss": 1.4951, + "step": 1050 + }, + { + "epoch": 1.3067828251400124, + "eval_loss": 1.6578067541122437, + "eval_runtime": 47.5847, + "eval_samples_per_second": 21.015, + "eval_steps_per_second": 0.883, + "step": 1050 + }, + { + "epoch": 1.3080273802115743, + "grad_norm": 2.9297118186950684, + "learning_rate": 3.166512915129152e-06, + "loss": 1.4699, + "step": 1051 + }, + { + "epoch": 1.3092719352831363, + "grad_norm": 2.906435251235962, + "learning_rate": 3.164206642066421e-06, + "loss": 1.4562, + "step": 1052 + }, + { + "epoch": 1.3105164903546982, + "grad_norm": 2.8655896186828613, + "learning_rate": 3.1619003690036902e-06, + "loss": 1.4351, + "step": 1053 + }, + { + "epoch": 1.31176104542626, + "grad_norm": 2.779244899749756, + "learning_rate": 3.15959409594096e-06, + "loss": 1.4479, + "step": 1054 + }, + { + "epoch": 1.3130056004978221, + "grad_norm": 2.9863739013671875, + "learning_rate": 3.157287822878229e-06, + "loss": 1.4964, + "step": 1055 + }, + { + "epoch": 1.314250155569384, + "grad_norm": 2.821871042251587, + "learning_rate": 3.1549815498154985e-06, + "loss": 1.4173, + "step": 1056 + }, + { + "epoch": 1.3154947106409458, + "grad_norm": 2.8573882579803467, + "learning_rate": 3.152675276752768e-06, + "loss": 1.4877, + "step": 1057 + }, + { + "epoch": 1.3167392657125077, + "grad_norm": 2.9824137687683105, + "learning_rate": 3.150369003690037e-06, + "loss": 1.5124, + "step": 1058 + }, + { + "epoch": 1.3179838207840697, + "grad_norm": 2.881098747253418, + "learning_rate": 3.1480627306273063e-06, + "loss": 1.4346, + "step": 1059 + }, + { + "epoch": 1.3192283758556316, + "grad_norm": 2.918782949447632, + "learning_rate": 3.145756457564576e-06, + "loss": 1.4797, + "step": 1060 + }, + { + "epoch": 1.3192283758556316, + "eval_loss": 1.6510179042816162, + "eval_runtime": 53.6356, + "eval_samples_per_second": 18.644, + "eval_steps_per_second": 0.783, + "step": 1060 + }, + { + "epoch": 1.3204729309271936, + "grad_norm": 3.045776128768921, + "learning_rate": 3.143450184501845e-06, + "loss": 1.5039, + "step": 1061 + }, + { + "epoch": 1.3217174859987555, + "grad_norm": 3.041501760482788, + "learning_rate": 3.1411439114391146e-06, + "loss": 1.4878, + "step": 1062 + }, + { + "epoch": 1.3229620410703173, + "grad_norm": 2.917837142944336, + "learning_rate": 3.138837638376384e-06, + "loss": 1.4202, + "step": 1063 + }, + { + "epoch": 1.3242065961418792, + "grad_norm": 2.708070755004883, + "learning_rate": 3.1365313653136535e-06, + "loss": 1.4071, + "step": 1064 + }, + { + "epoch": 1.3254511512134413, + "grad_norm": 2.9146156311035156, + "learning_rate": 3.1342250922509225e-06, + "loss": 1.4428, + "step": 1065 + }, + { + "epoch": 1.326695706285003, + "grad_norm": 3.220919132232666, + "learning_rate": 3.1319188191881923e-06, + "loss": 1.4733, + "step": 1066 + }, + { + "epoch": 1.327940261356565, + "grad_norm": 2.9127554893493652, + "learning_rate": 3.1296125461254617e-06, + "loss": 1.4554, + "step": 1067 + }, + { + "epoch": 1.329184816428127, + "grad_norm": 3.053191661834717, + "learning_rate": 3.1273062730627307e-06, + "loss": 1.4044, + "step": 1068 + }, + { + "epoch": 1.3304293714996889, + "grad_norm": 3.2937021255493164, + "learning_rate": 3.125e-06, + "loss": 1.4949, + "step": 1069 + }, + { + "epoch": 1.3316739265712507, + "grad_norm": 2.976717948913574, + "learning_rate": 3.12269372693727e-06, + "loss": 1.446, + "step": 1070 + }, + { + "epoch": 1.3316739265712507, + "eval_loss": 1.6503105163574219, + "eval_runtime": 50.7579, + "eval_samples_per_second": 19.701, + "eval_steps_per_second": 0.827, + "step": 1070 + }, + { + "epoch": 1.3329184816428126, + "grad_norm": 3.0860936641693115, + "learning_rate": 3.120387453874539e-06, + "loss": 1.4055, + "step": 1071 + }, + { + "epoch": 1.3341630367143746, + "grad_norm": 3.0401217937469482, + "learning_rate": 3.1180811808118084e-06, + "loss": 1.4688, + "step": 1072 + }, + { + "epoch": 1.3354075917859365, + "grad_norm": 2.953253984451294, + "learning_rate": 3.115774907749078e-06, + "loss": 1.5001, + "step": 1073 + }, + { + "epoch": 1.3366521468574986, + "grad_norm": 3.0665059089660645, + "learning_rate": 3.113468634686347e-06, + "loss": 1.4217, + "step": 1074 + }, + { + "epoch": 1.3378967019290604, + "grad_norm": 3.121600389480591, + "learning_rate": 3.1111623616236163e-06, + "loss": 1.4508, + "step": 1075 + }, + { + "epoch": 1.3391412570006223, + "grad_norm": 2.8057339191436768, + "learning_rate": 3.108856088560886e-06, + "loss": 1.4241, + "step": 1076 + }, + { + "epoch": 1.340385812072184, + "grad_norm": 3.0147411823272705, + "learning_rate": 3.1065498154981555e-06, + "loss": 1.479, + "step": 1077 + }, + { + "epoch": 1.3416303671437462, + "grad_norm": 3.030010223388672, + "learning_rate": 3.1042435424354245e-06, + "loss": 1.4861, + "step": 1078 + }, + { + "epoch": 1.342874922215308, + "grad_norm": 3.027998208999634, + "learning_rate": 3.101937269372694e-06, + "loss": 1.4916, + "step": 1079 + }, + { + "epoch": 1.3441194772868699, + "grad_norm": 2.9252536296844482, + "learning_rate": 3.0996309963099634e-06, + "loss": 1.418, + "step": 1080 + }, + { + "epoch": 1.3441194772868699, + "eval_loss": 1.6490333080291748, + "eval_runtime": 47.4026, + "eval_samples_per_second": 21.096, + "eval_steps_per_second": 0.886, + "step": 1080 + }, + { + "epoch": 1.345364032358432, + "grad_norm": 3.110847234725952, + "learning_rate": 3.0973247232472324e-06, + "loss": 1.4127, + "step": 1081 + }, + { + "epoch": 1.3466085874299938, + "grad_norm": 2.846491575241089, + "learning_rate": 3.0950184501845022e-06, + "loss": 1.444, + "step": 1082 + }, + { + "epoch": 1.3478531425015556, + "grad_norm": 2.950533151626587, + "learning_rate": 3.0927121771217716e-06, + "loss": 1.4456, + "step": 1083 + }, + { + "epoch": 1.3490976975731175, + "grad_norm": 2.875964403152466, + "learning_rate": 3.0904059040590406e-06, + "loss": 1.4485, + "step": 1084 + }, + { + "epoch": 1.3503422526446796, + "grad_norm": 2.9612326622009277, + "learning_rate": 3.08809963099631e-06, + "loss": 1.4446, + "step": 1085 + }, + { + "epoch": 1.3515868077162414, + "grad_norm": 2.925297975540161, + "learning_rate": 3.0857933579335795e-06, + "loss": 1.4847, + "step": 1086 + }, + { + "epoch": 1.3528313627878035, + "grad_norm": 2.991616725921631, + "learning_rate": 3.0834870848708485e-06, + "loss": 1.5151, + "step": 1087 + }, + { + "epoch": 1.3540759178593653, + "grad_norm": 2.96329927444458, + "learning_rate": 3.0811808118081183e-06, + "loss": 1.4574, + "step": 1088 + }, + { + "epoch": 1.3553204729309272, + "grad_norm": 2.948702573776245, + "learning_rate": 3.0788745387453878e-06, + "loss": 1.4585, + "step": 1089 + }, + { + "epoch": 1.356565028002489, + "grad_norm": 3.218463659286499, + "learning_rate": 3.076568265682657e-06, + "loss": 1.4439, + "step": 1090 + }, + { + "epoch": 1.356565028002489, + "eval_loss": 1.645838975906372, + "eval_runtime": 47.048, + "eval_samples_per_second": 21.255, + "eval_steps_per_second": 0.893, + "step": 1090 + }, + { + "epoch": 1.357809583074051, + "grad_norm": 2.9799578189849854, + "learning_rate": 3.074261992619926e-06, + "loss": 1.4883, + "step": 1091 + }, + { + "epoch": 1.359054138145613, + "grad_norm": 2.7660505771636963, + "learning_rate": 3.071955719557196e-06, + "loss": 1.4349, + "step": 1092 + }, + { + "epoch": 1.3602986932171748, + "grad_norm": 3.0201916694641113, + "learning_rate": 3.0696494464944655e-06, + "loss": 1.418, + "step": 1093 + }, + { + "epoch": 1.3615432482887369, + "grad_norm": 2.844207763671875, + "learning_rate": 3.0673431734317345e-06, + "loss": 1.4769, + "step": 1094 + }, + { + "epoch": 1.3627878033602987, + "grad_norm": 2.8306753635406494, + "learning_rate": 3.065036900369004e-06, + "loss": 1.4653, + "step": 1095 + }, + { + "epoch": 1.3640323584318605, + "grad_norm": 2.805023431777954, + "learning_rate": 3.0627306273062733e-06, + "loss": 1.4771, + "step": 1096 + }, + { + "epoch": 1.3652769135034224, + "grad_norm": 3.020883083343506, + "learning_rate": 3.0604243542435423e-06, + "loss": 1.4237, + "step": 1097 + }, + { + "epoch": 1.3665214685749845, + "grad_norm": 2.8395731449127197, + "learning_rate": 3.058118081180812e-06, + "loss": 1.4647, + "step": 1098 + }, + { + "epoch": 1.3677660236465463, + "grad_norm": 2.871156692504883, + "learning_rate": 3.0558118081180816e-06, + "loss": 1.4196, + "step": 1099 + }, + { + "epoch": 1.3690105787181084, + "grad_norm": 2.974905014038086, + "learning_rate": 3.053505535055351e-06, + "loss": 1.4779, + "step": 1100 + }, + { + "epoch": 1.3690105787181084, + "eval_loss": 1.651503562927246, + "eval_runtime": 54.2713, + "eval_samples_per_second": 18.426, + "eval_steps_per_second": 0.774, + "step": 1100 + }, + { + "epoch": 1.3702551337896702, + "grad_norm": 2.7591328620910645, + "learning_rate": 3.05119926199262e-06, + "loss": 1.3899, + "step": 1101 + }, + { + "epoch": 1.371499688861232, + "grad_norm": 2.8575668334960938, + "learning_rate": 3.0488929889298894e-06, + "loss": 1.4482, + "step": 1102 + }, + { + "epoch": 1.372744243932794, + "grad_norm": 2.9049339294433594, + "learning_rate": 3.0465867158671593e-06, + "loss": 1.4785, + "step": 1103 + }, + { + "epoch": 1.373988799004356, + "grad_norm": 2.816453695297241, + "learning_rate": 3.0442804428044283e-06, + "loss": 1.4601, + "step": 1104 + }, + { + "epoch": 1.3752333540759178, + "grad_norm": 2.8844189643859863, + "learning_rate": 3.0419741697416977e-06, + "loss": 1.496, + "step": 1105 + }, + { + "epoch": 1.3764779091474797, + "grad_norm": 3.014744281768799, + "learning_rate": 3.039667896678967e-06, + "loss": 1.4098, + "step": 1106 + }, + { + "epoch": 1.3777224642190418, + "grad_norm": 2.8243560791015625, + "learning_rate": 3.037361623616236e-06, + "loss": 1.4086, + "step": 1107 + }, + { + "epoch": 1.3789670192906036, + "grad_norm": 2.8333253860473633, + "learning_rate": 3.035055350553506e-06, + "loss": 1.3516, + "step": 1108 + }, + { + "epoch": 1.3802115743621655, + "grad_norm": 2.926455020904541, + "learning_rate": 3.0327490774907754e-06, + "loss": 1.3976, + "step": 1109 + }, + { + "epoch": 1.3814561294337273, + "grad_norm": 2.900937080383301, + "learning_rate": 3.0304428044280444e-06, + "loss": 1.4336, + "step": 1110 + }, + { + "epoch": 1.3814561294337273, + "eval_loss": 1.647048830986023, + "eval_runtime": 42.3955, + "eval_samples_per_second": 23.587, + "eval_steps_per_second": 0.991, + "step": 1110 + }, + { + "epoch": 1.3827006845052894, + "grad_norm": 3.0756969451904297, + "learning_rate": 3.028136531365314e-06, + "loss": 1.4674, + "step": 1111 + }, + { + "epoch": 1.3839452395768512, + "grad_norm": 2.7754578590393066, + "learning_rate": 3.0258302583025832e-06, + "loss": 1.4435, + "step": 1112 + }, + { + "epoch": 1.3851897946484133, + "grad_norm": 2.8011393547058105, + "learning_rate": 3.023523985239853e-06, + "loss": 1.4212, + "step": 1113 + }, + { + "epoch": 1.3864343497199751, + "grad_norm": 2.7063710689544678, + "learning_rate": 3.021217712177122e-06, + "loss": 1.3769, + "step": 1114 + }, + { + "epoch": 1.387678904791537, + "grad_norm": 2.830411434173584, + "learning_rate": 3.0189114391143915e-06, + "loss": 1.4832, + "step": 1115 + }, + { + "epoch": 1.3889234598630988, + "grad_norm": 2.9075911045074463, + "learning_rate": 3.016605166051661e-06, + "loss": 1.486, + "step": 1116 + }, + { + "epoch": 1.390168014934661, + "grad_norm": 2.847822666168213, + "learning_rate": 3.01429889298893e-06, + "loss": 1.424, + "step": 1117 + }, + { + "epoch": 1.3914125700062228, + "grad_norm": 2.8822033405303955, + "learning_rate": 3.0119926199261993e-06, + "loss": 1.4218, + "step": 1118 + }, + { + "epoch": 1.3926571250777848, + "grad_norm": 2.8944740295410156, + "learning_rate": 3.009686346863469e-06, + "loss": 1.4283, + "step": 1119 + }, + { + "epoch": 1.3939016801493467, + "grad_norm": 2.765190839767456, + "learning_rate": 3.007380073800738e-06, + "loss": 1.4622, + "step": 1120 + }, + { + "epoch": 1.3939016801493467, + "eval_loss": 1.6451724767684937, + "eval_runtime": 43.7449, + "eval_samples_per_second": 22.86, + "eval_steps_per_second": 0.96, + "step": 1120 + }, + { + "epoch": 1.3951462352209085, + "grad_norm": 2.877002716064453, + "learning_rate": 3.0050738007380076e-06, + "loss": 1.4145, + "step": 1121 + }, + { + "epoch": 1.3963907902924704, + "grad_norm": 2.870389699935913, + "learning_rate": 3.002767527675277e-06, + "loss": 1.4401, + "step": 1122 + }, + { + "epoch": 1.3976353453640322, + "grad_norm": 2.7788920402526855, + "learning_rate": 3.000461254612546e-06, + "loss": 1.4514, + "step": 1123 + }, + { + "epoch": 1.3988799004355943, + "grad_norm": 2.9269332885742188, + "learning_rate": 2.998154981549816e-06, + "loss": 1.3961, + "step": 1124 + }, + { + "epoch": 1.4001244555071561, + "grad_norm": 3.013291835784912, + "learning_rate": 2.9958487084870853e-06, + "loss": 1.4597, + "step": 1125 + }, + { + "epoch": 1.4013690105787182, + "grad_norm": 2.9063880443573, + "learning_rate": 2.9935424354243547e-06, + "loss": 1.4668, + "step": 1126 + }, + { + "epoch": 1.40261356565028, + "grad_norm": 2.8926970958709717, + "learning_rate": 2.9912361623616237e-06, + "loss": 1.4332, + "step": 1127 + }, + { + "epoch": 1.403858120721842, + "grad_norm": 2.8563647270202637, + "learning_rate": 2.988929889298893e-06, + "loss": 1.407, + "step": 1128 + }, + { + "epoch": 1.4051026757934038, + "grad_norm": 2.8725359439849854, + "learning_rate": 2.986623616236163e-06, + "loss": 1.4434, + "step": 1129 + }, + { + "epoch": 1.4063472308649658, + "grad_norm": 2.906898260116577, + "learning_rate": 2.984317343173432e-06, + "loss": 1.4745, + "step": 1130 + }, + { + "epoch": 1.4063472308649658, + "eval_loss": 1.647884726524353, + "eval_runtime": 44.9221, + "eval_samples_per_second": 22.261, + "eval_steps_per_second": 0.935, + "step": 1130 + }, + { + "epoch": 1.4075917859365277, + "grad_norm": 2.8297719955444336, + "learning_rate": 2.9820110701107014e-06, + "loss": 1.4542, + "step": 1131 + }, + { + "epoch": 1.4088363410080897, + "grad_norm": 2.841031551361084, + "learning_rate": 2.979704797047971e-06, + "loss": 1.4105, + "step": 1132 + }, + { + "epoch": 1.4100808960796516, + "grad_norm": 2.867791175842285, + "learning_rate": 2.97739852398524e-06, + "loss": 1.4409, + "step": 1133 + }, + { + "epoch": 1.4113254511512134, + "grad_norm": 2.8156039714813232, + "learning_rate": 2.9750922509225093e-06, + "loss": 1.4284, + "step": 1134 + }, + { + "epoch": 1.4125700062227753, + "grad_norm": 2.825985908508301, + "learning_rate": 2.972785977859779e-06, + "loss": 1.4594, + "step": 1135 + }, + { + "epoch": 1.4138145612943374, + "grad_norm": 3.0134711265563965, + "learning_rate": 2.970479704797048e-06, + "loss": 1.4743, + "step": 1136 + }, + { + "epoch": 1.4150591163658992, + "grad_norm": 2.9829142093658447, + "learning_rate": 2.9681734317343175e-06, + "loss": 1.4935, + "step": 1137 + }, + { + "epoch": 1.416303671437461, + "grad_norm": 3.067620277404785, + "learning_rate": 2.965867158671587e-06, + "loss": 1.4266, + "step": 1138 + }, + { + "epoch": 1.4175482265090231, + "grad_norm": 2.8969919681549072, + "learning_rate": 2.9635608856088564e-06, + "loss": 1.451, + "step": 1139 + }, + { + "epoch": 1.418792781580585, + "grad_norm": 2.8706412315368652, + "learning_rate": 2.9612546125461254e-06, + "loss": 1.4834, + "step": 1140 + }, + { + "epoch": 1.418792781580585, + "eval_loss": 1.643480658531189, + "eval_runtime": 44.743, + "eval_samples_per_second": 22.35, + "eval_steps_per_second": 0.939, + "step": 1140 + }, + { + "epoch": 1.4200373366521468, + "grad_norm": 2.8801848888397217, + "learning_rate": 2.958948339483395e-06, + "loss": 1.4258, + "step": 1141 + }, + { + "epoch": 1.4212818917237087, + "grad_norm": 2.774635076522827, + "learning_rate": 2.9566420664206646e-06, + "loss": 1.416, + "step": 1142 + }, + { + "epoch": 1.4225264467952707, + "grad_norm": 2.8460140228271484, + "learning_rate": 2.9543357933579336e-06, + "loss": 1.4213, + "step": 1143 + }, + { + "epoch": 1.4237710018668326, + "grad_norm": 2.907888889312744, + "learning_rate": 2.952029520295203e-06, + "loss": 1.4507, + "step": 1144 + }, + { + "epoch": 1.4250155569383947, + "grad_norm": 2.789041757583618, + "learning_rate": 2.949723247232473e-06, + "loss": 1.4257, + "step": 1145 + }, + { + "epoch": 1.4262601120099565, + "grad_norm": 2.78859806060791, + "learning_rate": 2.947416974169742e-06, + "loss": 1.4377, + "step": 1146 + }, + { + "epoch": 1.4275046670815184, + "grad_norm": 2.840303421020508, + "learning_rate": 2.9451107011070113e-06, + "loss": 1.4684, + "step": 1147 + }, + { + "epoch": 1.4287492221530802, + "grad_norm": 2.8800530433654785, + "learning_rate": 2.9428044280442807e-06, + "loss": 1.4912, + "step": 1148 + }, + { + "epoch": 1.4299937772246423, + "grad_norm": 2.841092824935913, + "learning_rate": 2.9404981549815497e-06, + "loss": 1.4234, + "step": 1149 + }, + { + "epoch": 1.4312383322962041, + "grad_norm": 2.8226537704467773, + "learning_rate": 2.938191881918819e-06, + "loss": 1.4578, + "step": 1150 + }, + { + "epoch": 1.4312383322962041, + "eval_loss": 1.643249273300171, + "eval_runtime": 43.5586, + "eval_samples_per_second": 22.958, + "eval_steps_per_second": 0.964, + "step": 1150 + }, + { + "epoch": 1.432482887367766, + "grad_norm": 2.869736909866333, + "learning_rate": 2.935885608856089e-06, + "loss": 1.4701, + "step": 1151 + }, + { + "epoch": 1.433727442439328, + "grad_norm": 2.689211130142212, + "learning_rate": 2.9335793357933584e-06, + "loss": 1.4483, + "step": 1152 + }, + { + "epoch": 1.4349719975108899, + "grad_norm": 2.9267077445983887, + "learning_rate": 2.9312730627306274e-06, + "loss": 1.4446, + "step": 1153 + }, + { + "epoch": 1.4362165525824517, + "grad_norm": 2.906862497329712, + "learning_rate": 2.928966789667897e-06, + "loss": 1.4524, + "step": 1154 + }, + { + "epoch": 1.4374611076540136, + "grad_norm": 2.8249800205230713, + "learning_rate": 2.9266605166051663e-06, + "loss": 1.4208, + "step": 1155 + }, + { + "epoch": 1.4387056627255757, + "grad_norm": 2.9836032390594482, + "learning_rate": 2.9243542435424353e-06, + "loss": 1.4581, + "step": 1156 + }, + { + "epoch": 1.4399502177971375, + "grad_norm": 2.9595284461975098, + "learning_rate": 2.922047970479705e-06, + "loss": 1.4141, + "step": 1157 + }, + { + "epoch": 1.4411947728686996, + "grad_norm": 3.0363423824310303, + "learning_rate": 2.9197416974169746e-06, + "loss": 1.4859, + "step": 1158 + }, + { + "epoch": 1.4424393279402614, + "grad_norm": 2.911201000213623, + "learning_rate": 2.9174354243542436e-06, + "loss": 1.4346, + "step": 1159 + }, + { + "epoch": 1.4436838830118233, + "grad_norm": 2.8617193698883057, + "learning_rate": 2.915129151291513e-06, + "loss": 1.433, + "step": 1160 + }, + { + "epoch": 1.4436838830118233, + "eval_loss": 1.6437232494354248, + "eval_runtime": 43.961, + "eval_samples_per_second": 22.747, + "eval_steps_per_second": 0.955, + "step": 1160 + }, + { + "epoch": 1.4449284380833851, + "grad_norm": 2.9584295749664307, + "learning_rate": 2.912822878228783e-06, + "loss": 1.4847, + "step": 1161 + }, + { + "epoch": 1.4461729931549472, + "grad_norm": 2.8209400177001953, + "learning_rate": 2.9105166051660522e-06, + "loss": 1.421, + "step": 1162 + }, + { + "epoch": 1.447417548226509, + "grad_norm": 2.847637414932251, + "learning_rate": 2.9082103321033212e-06, + "loss": 1.4326, + "step": 1163 + }, + { + "epoch": 1.4486621032980709, + "grad_norm": 2.836228132247925, + "learning_rate": 2.9059040590405907e-06, + "loss": 1.4459, + "step": 1164 + }, + { + "epoch": 1.449906658369633, + "grad_norm": 2.8327207565307617, + "learning_rate": 2.90359778597786e-06, + "loss": 1.4189, + "step": 1165 + }, + { + "epoch": 1.4511512134411948, + "grad_norm": 2.880643606185913, + "learning_rate": 2.901291512915129e-06, + "loss": 1.4214, + "step": 1166 + }, + { + "epoch": 1.4523957685127566, + "grad_norm": 2.9637348651885986, + "learning_rate": 2.898985239852399e-06, + "loss": 1.4471, + "step": 1167 + }, + { + "epoch": 1.4536403235843185, + "grad_norm": 3.006145477294922, + "learning_rate": 2.8966789667896684e-06, + "loss": 1.4228, + "step": 1168 + }, + { + "epoch": 1.4548848786558806, + "grad_norm": 2.8547627925872803, + "learning_rate": 2.8943726937269374e-06, + "loss": 1.4707, + "step": 1169 + }, + { + "epoch": 1.4561294337274424, + "grad_norm": 2.973092794418335, + "learning_rate": 2.8920664206642068e-06, + "loss": 1.4804, + "step": 1170 + }, + { + "epoch": 1.4561294337274424, + "eval_loss": 1.6340677738189697, + "eval_runtime": 44.6094, + "eval_samples_per_second": 22.417, + "eval_steps_per_second": 0.942, + "step": 1170 + }, + { + "epoch": 1.4573739887990045, + "grad_norm": 2.9433603286743164, + "learning_rate": 2.889760147601476e-06, + "loss": 1.4354, + "step": 1171 + }, + { + "epoch": 1.4586185438705663, + "grad_norm": 2.872236728668213, + "learning_rate": 2.887453874538745e-06, + "loss": 1.418, + "step": 1172 + }, + { + "epoch": 1.4598630989421282, + "grad_norm": 2.847557783126831, + "learning_rate": 2.885147601476015e-06, + "loss": 1.4306, + "step": 1173 + }, + { + "epoch": 1.46110765401369, + "grad_norm": 2.7554969787597656, + "learning_rate": 2.8828413284132845e-06, + "loss": 1.4295, + "step": 1174 + }, + { + "epoch": 1.462352209085252, + "grad_norm": 2.8754842281341553, + "learning_rate": 2.880535055350554e-06, + "loss": 1.4482, + "step": 1175 + }, + { + "epoch": 1.463596764156814, + "grad_norm": 3.0115292072296143, + "learning_rate": 2.878228782287823e-06, + "loss": 1.4428, + "step": 1176 + }, + { + "epoch": 1.4648413192283758, + "grad_norm": 2.8976168632507324, + "learning_rate": 2.8759225092250923e-06, + "loss": 1.4707, + "step": 1177 + }, + { + "epoch": 1.4660858742999379, + "grad_norm": 2.7981812953948975, + "learning_rate": 2.873616236162362e-06, + "loss": 1.4693, + "step": 1178 + }, + { + "epoch": 1.4673304293714997, + "grad_norm": 2.8562371730804443, + "learning_rate": 2.871309963099631e-06, + "loss": 1.4242, + "step": 1179 + }, + { + "epoch": 1.4685749844430616, + "grad_norm": 2.8705570697784424, + "learning_rate": 2.8690036900369006e-06, + "loss": 1.4725, + "step": 1180 + }, + { + "epoch": 1.4685749844430616, + "eval_loss": 1.6338286399841309, + "eval_runtime": 44.7171, + "eval_samples_per_second": 22.363, + "eval_steps_per_second": 0.939, + "step": 1180 + }, + { + "epoch": 1.4698195395146234, + "grad_norm": 2.8025710582733154, + "learning_rate": 2.86669741697417e-06, + "loss": 1.427, + "step": 1181 + }, + { + "epoch": 1.4710640945861855, + "grad_norm": 2.845151901245117, + "learning_rate": 2.864391143911439e-06, + "loss": 1.4402, + "step": 1182 + }, + { + "epoch": 1.4723086496577473, + "grad_norm": 2.943373680114746, + "learning_rate": 2.862084870848709e-06, + "loss": 1.5194, + "step": 1183 + }, + { + "epoch": 1.4735532047293094, + "grad_norm": 2.896955966949463, + "learning_rate": 2.8597785977859783e-06, + "loss": 1.4242, + "step": 1184 + }, + { + "epoch": 1.4747977598008712, + "grad_norm": 3.131305456161499, + "learning_rate": 2.8574723247232473e-06, + "loss": 1.5008, + "step": 1185 + }, + { + "epoch": 1.476042314872433, + "grad_norm": 2.8605735301971436, + "learning_rate": 2.8551660516605167e-06, + "loss": 1.3533, + "step": 1186 + }, + { + "epoch": 1.477286869943995, + "grad_norm": 2.9158811569213867, + "learning_rate": 2.852859778597786e-06, + "loss": 1.4404, + "step": 1187 + }, + { + "epoch": 1.478531425015557, + "grad_norm": 2.7795321941375732, + "learning_rate": 2.850553505535056e-06, + "loss": 1.3977, + "step": 1188 + }, + { + "epoch": 1.4797759800871189, + "grad_norm": 2.8194212913513184, + "learning_rate": 2.848247232472325e-06, + "loss": 1.4111, + "step": 1189 + }, + { + "epoch": 1.4810205351586807, + "grad_norm": 2.983750104904175, + "learning_rate": 2.8459409594095944e-06, + "loss": 1.4061, + "step": 1190 + }, + { + "epoch": 1.4810205351586807, + "eval_loss": 1.6375409364700317, + "eval_runtime": 45.4026, + "eval_samples_per_second": 22.025, + "eval_steps_per_second": 0.925, + "step": 1190 + }, + { + "epoch": 1.4822650902302428, + "grad_norm": 3.0246665477752686, + "learning_rate": 2.843634686346864e-06, + "loss": 1.4532, + "step": 1191 + }, + { + "epoch": 1.4835096453018046, + "grad_norm": 2.8856449127197266, + "learning_rate": 2.841328413284133e-06, + "loss": 1.4528, + "step": 1192 + }, + { + "epoch": 1.4847542003733665, + "grad_norm": 2.848987102508545, + "learning_rate": 2.8390221402214022e-06, + "loss": 1.4369, + "step": 1193 + }, + { + "epoch": 1.4859987554449283, + "grad_norm": 2.970419406890869, + "learning_rate": 2.836715867158672e-06, + "loss": 1.4605, + "step": 1194 + }, + { + "epoch": 1.4872433105164904, + "grad_norm": 3.156613826751709, + "learning_rate": 2.834409594095941e-06, + "loss": 1.4489, + "step": 1195 + }, + { + "epoch": 1.4884878655880522, + "grad_norm": 3.095349073410034, + "learning_rate": 2.8321033210332105e-06, + "loss": 1.4604, + "step": 1196 + }, + { + "epoch": 1.4897324206596143, + "grad_norm": 2.870742082595825, + "learning_rate": 2.82979704797048e-06, + "loss": 1.4284, + "step": 1197 + }, + { + "epoch": 1.4909769757311762, + "grad_norm": 2.9415056705474854, + "learning_rate": 2.827490774907749e-06, + "loss": 1.4715, + "step": 1198 + }, + { + "epoch": 1.492221530802738, + "grad_norm": 2.8297767639160156, + "learning_rate": 2.8251845018450188e-06, + "loss": 1.4508, + "step": 1199 + }, + { + "epoch": 1.4934660858742999, + "grad_norm": 2.872086524963379, + "learning_rate": 2.822878228782288e-06, + "loss": 1.5098, + "step": 1200 + }, + { + "epoch": 1.4934660858742999, + "eval_loss": 1.6357526779174805, + "eval_runtime": 51.422, + "eval_samples_per_second": 19.447, + "eval_steps_per_second": 0.817, + "step": 1200 + }, + { + "epoch": 1.494710640945862, + "grad_norm": 2.7152082920074463, + "learning_rate": 2.8205719557195576e-06, + "loss": 1.4329, + "step": 1201 + }, + { + "epoch": 1.4959551960174238, + "grad_norm": 2.866170883178711, + "learning_rate": 2.8182656826568266e-06, + "loss": 1.4455, + "step": 1202 + }, + { + "epoch": 1.4971997510889856, + "grad_norm": 2.9010684490203857, + "learning_rate": 2.815959409594096e-06, + "loss": 1.423, + "step": 1203 + }, + { + "epoch": 1.4984443061605477, + "grad_norm": 2.8588502407073975, + "learning_rate": 2.813653136531366e-06, + "loss": 1.4041, + "step": 1204 + }, + { + "epoch": 1.4996888612321095, + "grad_norm": 2.932544231414795, + "learning_rate": 2.811346863468635e-06, + "loss": 1.4559, + "step": 1205 + }, + { + "epoch": 1.5009334163036714, + "grad_norm": 2.7486014366149902, + "learning_rate": 2.8090405904059043e-06, + "loss": 1.4337, + "step": 1206 + }, + { + "epoch": 1.5021779713752332, + "grad_norm": 3.0866589546203613, + "learning_rate": 2.8067343173431737e-06, + "loss": 1.4186, + "step": 1207 + }, + { + "epoch": 1.5034225264467953, + "grad_norm": 2.9710276126861572, + "learning_rate": 2.8044280442804427e-06, + "loss": 1.419, + "step": 1208 + }, + { + "epoch": 1.5046670815183572, + "grad_norm": 2.8379950523376465, + "learning_rate": 2.802121771217712e-06, + "loss": 1.4922, + "step": 1209 + }, + { + "epoch": 1.5059116365899192, + "grad_norm": 2.939629554748535, + "learning_rate": 2.799815498154982e-06, + "loss": 1.4402, + "step": 1210 + }, + { + "epoch": 1.5059116365899192, + "eval_loss": 1.6419442892074585, + "eval_runtime": 43.1202, + "eval_samples_per_second": 23.191, + "eval_steps_per_second": 0.974, + "step": 1210 + }, + { + "epoch": 1.507156191661481, + "grad_norm": 3.041389226913452, + "learning_rate": 2.7975092250922514e-06, + "loss": 1.4873, + "step": 1211 + }, + { + "epoch": 1.508400746733043, + "grad_norm": 2.9776995182037354, + "learning_rate": 2.7952029520295204e-06, + "loss": 1.4422, + "step": 1212 + }, + { + "epoch": 1.5096453018046048, + "grad_norm": 2.8798792362213135, + "learning_rate": 2.79289667896679e-06, + "loss": 1.4535, + "step": 1213 + }, + { + "epoch": 1.5108898568761666, + "grad_norm": 2.9016385078430176, + "learning_rate": 2.7905904059040597e-06, + "loss": 1.4462, + "step": 1214 + }, + { + "epoch": 1.5121344119477287, + "grad_norm": 2.8987581729888916, + "learning_rate": 2.7882841328413287e-06, + "loss": 1.4477, + "step": 1215 + }, + { + "epoch": 1.5133789670192908, + "grad_norm": 2.960266351699829, + "learning_rate": 2.785977859778598e-06, + "loss": 1.4811, + "step": 1216 + }, + { + "epoch": 1.5146235220908526, + "grad_norm": 2.8965611457824707, + "learning_rate": 2.7836715867158675e-06, + "loss": 1.4366, + "step": 1217 + }, + { + "epoch": 1.5158680771624145, + "grad_norm": 3.0667364597320557, + "learning_rate": 2.7813653136531365e-06, + "loss": 1.3828, + "step": 1218 + }, + { + "epoch": 1.5171126322339763, + "grad_norm": 2.952362537384033, + "learning_rate": 2.779059040590406e-06, + "loss": 1.3903, + "step": 1219 + }, + { + "epoch": 1.5183571873055381, + "grad_norm": 2.9899795055389404, + "learning_rate": 2.776752767527676e-06, + "loss": 1.4312, + "step": 1220 + }, + { + "epoch": 1.5183571873055381, + "eval_loss": 1.640383005142212, + "eval_runtime": 45.4128, + "eval_samples_per_second": 22.02, + "eval_steps_per_second": 0.925, + "step": 1220 + }, + { + "epoch": 1.5196017423771002, + "grad_norm": 3.2202725410461426, + "learning_rate": 2.774446494464945e-06, + "loss": 1.4344, + "step": 1221 + }, + { + "epoch": 1.520846297448662, + "grad_norm": 3.2202725410461426, + "learning_rate": 2.774446494464945e-06, + "loss": 1.4386, + "step": 1222 + }, + { + "epoch": 1.5220908525202241, + "grad_norm": 2.883223295211792, + "learning_rate": 2.7721402214022142e-06, + "loss": 1.4401, + "step": 1223 + }, + { + "epoch": 1.523335407591786, + "grad_norm": 2.8346424102783203, + "learning_rate": 2.7698339483394837e-06, + "loss": 1.3926, + "step": 1224 + }, + { + "epoch": 1.5245799626633478, + "grad_norm": 2.8898870944976807, + "learning_rate": 2.767527675276753e-06, + "loss": 1.4128, + "step": 1225 + }, + { + "epoch": 1.5258245177349097, + "grad_norm": 2.828594923019409, + "learning_rate": 2.765221402214022e-06, + "loss": 1.4103, + "step": 1226 + }, + { + "epoch": 1.5270690728064715, + "grad_norm": 2.882558822631836, + "learning_rate": 2.762915129151292e-06, + "loss": 1.4186, + "step": 1227 + }, + { + "epoch": 1.5283136278780336, + "grad_norm": 2.955760955810547, + "learning_rate": 2.7606088560885613e-06, + "loss": 1.3705, + "step": 1228 + }, + { + "epoch": 1.5295581829495957, + "grad_norm": 2.862934112548828, + "learning_rate": 2.7583025830258303e-06, + "loss": 1.5012, + "step": 1229 + }, + { + "epoch": 1.5308027380211575, + "grad_norm": 2.926501989364624, + "learning_rate": 2.7559963099630998e-06, + "loss": 1.3875, + "step": 1230 + }, + { + "epoch": 1.5308027380211575, + "eval_loss": 1.6376254558563232, + "eval_runtime": 47.9535, + "eval_samples_per_second": 20.854, + "eval_steps_per_second": 0.876, + "step": 1230 + }, + { + "epoch": 1.5320472930927194, + "grad_norm": 2.7648823261260986, + "learning_rate": 2.753690036900369e-06, + "loss": 1.3781, + "step": 1231 + }, + { + "epoch": 1.5332918481642812, + "grad_norm": 2.927103281021118, + "learning_rate": 2.751383763837638e-06, + "loss": 1.4091, + "step": 1232 + }, + { + "epoch": 1.534536403235843, + "grad_norm": 2.964823007583618, + "learning_rate": 2.749077490774908e-06, + "loss": 1.5075, + "step": 1233 + }, + { + "epoch": 1.5357809583074051, + "grad_norm": 2.7944016456604004, + "learning_rate": 2.7467712177121775e-06, + "loss": 1.3888, + "step": 1234 + }, + { + "epoch": 1.537025513378967, + "grad_norm": 2.891592025756836, + "learning_rate": 2.7444649446494465e-06, + "loss": 1.4825, + "step": 1235 + }, + { + "epoch": 1.538270068450529, + "grad_norm": 2.8120031356811523, + "learning_rate": 2.742158671586716e-06, + "loss": 1.4246, + "step": 1236 + }, + { + "epoch": 1.539514623522091, + "grad_norm": 2.9384255409240723, + "learning_rate": 2.7398523985239857e-06, + "loss": 1.4657, + "step": 1237 + }, + { + "epoch": 1.5407591785936527, + "grad_norm": 2.73111891746521, + "learning_rate": 2.737546125461255e-06, + "loss": 1.4689, + "step": 1238 + }, + { + "epoch": 1.5420037336652146, + "grad_norm": 2.858963966369629, + "learning_rate": 2.735239852398524e-06, + "loss": 1.4626, + "step": 1239 + }, + { + "epoch": 1.5432482887367764, + "grad_norm": 2.9148507118225098, + "learning_rate": 2.7329335793357936e-06, + "loss": 1.4787, + "step": 1240 + }, + { + "epoch": 1.5432482887367764, + "eval_loss": 1.6436412334442139, + "eval_runtime": 47.3139, + "eval_samples_per_second": 21.135, + "eval_steps_per_second": 0.888, + "step": 1240 + }, + { + "epoch": 1.5444928438083385, + "grad_norm": 2.8731327056884766, + "learning_rate": 2.730627306273063e-06, + "loss": 1.4748, + "step": 1241 + }, + { + "epoch": 1.5457373988799006, + "grad_norm": 2.990265369415283, + "learning_rate": 2.728321033210332e-06, + "loss": 1.4375, + "step": 1242 + }, + { + "epoch": 1.5469819539514624, + "grad_norm": 2.874321460723877, + "learning_rate": 2.726014760147602e-06, + "loss": 1.4154, + "step": 1243 + }, + { + "epoch": 1.5482265090230243, + "grad_norm": 2.9316225051879883, + "learning_rate": 2.7237084870848713e-06, + "loss": 1.4285, + "step": 1244 + }, + { + "epoch": 1.5494710640945861, + "grad_norm": 2.783966302871704, + "learning_rate": 2.7214022140221403e-06, + "loss": 1.409, + "step": 1245 + }, + { + "epoch": 1.550715619166148, + "grad_norm": 2.9578707218170166, + "learning_rate": 2.7190959409594097e-06, + "loss": 1.4325, + "step": 1246 + }, + { + "epoch": 1.55196017423771, + "grad_norm": 2.869893789291382, + "learning_rate": 2.716789667896679e-06, + "loss": 1.3989, + "step": 1247 + }, + { + "epoch": 1.553204729309272, + "grad_norm": 2.962944269180298, + "learning_rate": 2.714483394833948e-06, + "loss": 1.4302, + "step": 1248 + }, + { + "epoch": 1.554449284380834, + "grad_norm": 2.974494695663452, + "learning_rate": 2.712177121771218e-06, + "loss": 1.46, + "step": 1249 + }, + { + "epoch": 1.5556938394523958, + "grad_norm": 2.9154603481292725, + "learning_rate": 2.7098708487084874e-06, + "loss": 1.4388, + "step": 1250 + }, + { + "epoch": 1.5556938394523958, + "eval_loss": 1.6357773542404175, + "eval_runtime": 47.6085, + "eval_samples_per_second": 21.005, + "eval_steps_per_second": 0.882, + "step": 1250 + }, + { + "epoch": 1.5569383945239577, + "grad_norm": 3.0333547592163086, + "learning_rate": 2.707564575645757e-06, + "loss": 1.4846, + "step": 1251 + }, + { + "epoch": 1.5581829495955195, + "grad_norm": 3.06097674369812, + "learning_rate": 2.705258302583026e-06, + "loss": 1.4355, + "step": 1252 + }, + { + "epoch": 1.5594275046670814, + "grad_norm": 2.872680425643921, + "learning_rate": 2.7029520295202956e-06, + "loss": 1.4219, + "step": 1253 + }, + { + "epoch": 1.5606720597386434, + "grad_norm": 2.9066731929779053, + "learning_rate": 2.700645756457565e-06, + "loss": 1.384, + "step": 1254 + }, + { + "epoch": 1.5619166148102055, + "grad_norm": 3.0570194721221924, + "learning_rate": 2.698339483394834e-06, + "loss": 1.4687, + "step": 1255 + }, + { + "epoch": 1.5631611698817673, + "grad_norm": 3.1955857276916504, + "learning_rate": 2.6960332103321035e-06, + "loss": 1.4736, + "step": 1256 + }, + { + "epoch": 1.5644057249533292, + "grad_norm": 2.9686198234558105, + "learning_rate": 2.693726937269373e-06, + "loss": 1.4176, + "step": 1257 + }, + { + "epoch": 1.565650280024891, + "grad_norm": 3.1161272525787354, + "learning_rate": 2.691420664206642e-06, + "loss": 1.443, + "step": 1258 + }, + { + "epoch": 1.5668948350964529, + "grad_norm": 2.9031193256378174, + "learning_rate": 2.6891143911439118e-06, + "loss": 1.3871, + "step": 1259 + }, + { + "epoch": 1.568139390168015, + "grad_norm": 2.9120476245880127, + "learning_rate": 2.686808118081181e-06, + "loss": 1.4406, + "step": 1260 + }, + { + "epoch": 1.568139390168015, + "eval_loss": 1.6291446685791016, + "eval_runtime": 49.7006, + "eval_samples_per_second": 20.12, + "eval_steps_per_second": 0.845, + "step": 1260 + }, + { + "epoch": 1.5693839452395768, + "grad_norm": 3.3181824684143066, + "learning_rate": 2.6845018450184506e-06, + "loss": 1.5278, + "step": 1261 + }, + { + "epoch": 1.5706285003111389, + "grad_norm": 2.9628148078918457, + "learning_rate": 2.6821955719557196e-06, + "loss": 1.4206, + "step": 1262 + }, + { + "epoch": 1.5718730553827007, + "grad_norm": 2.9828474521636963, + "learning_rate": 2.679889298892989e-06, + "loss": 1.5478, + "step": 1263 + }, + { + "epoch": 1.5731176104542626, + "grad_norm": 3.018711805343628, + "learning_rate": 2.677583025830259e-06, + "loss": 1.4442, + "step": 1264 + }, + { + "epoch": 1.5743621655258244, + "grad_norm": 2.8596792221069336, + "learning_rate": 2.675276752767528e-06, + "loss": 1.4325, + "step": 1265 + }, + { + "epoch": 1.5756067205973865, + "grad_norm": 2.9592769145965576, + "learning_rate": 2.6729704797047973e-06, + "loss": 1.4059, + "step": 1266 + }, + { + "epoch": 1.5768512756689483, + "grad_norm": 2.773589611053467, + "learning_rate": 2.6706642066420667e-06, + "loss": 1.3608, + "step": 1267 + }, + { + "epoch": 1.5780958307405104, + "grad_norm": 3.313255548477173, + "learning_rate": 2.6683579335793357e-06, + "loss": 1.4552, + "step": 1268 + }, + { + "epoch": 1.5793403858120723, + "grad_norm": 2.977649211883545, + "learning_rate": 2.6660516605166056e-06, + "loss": 1.4221, + "step": 1269 + }, + { + "epoch": 1.580584940883634, + "grad_norm": 3.0672051906585693, + "learning_rate": 2.663745387453875e-06, + "loss": 1.4568, + "step": 1270 + }, + { + "epoch": 1.580584940883634, + "eval_loss": 1.6317907571792603, + "eval_runtime": 47.2354, + "eval_samples_per_second": 21.171, + "eval_steps_per_second": 0.889, + "step": 1270 + }, + { + "epoch": 1.581829495955196, + "grad_norm": 3.029320240020752, + "learning_rate": 2.661439114391144e-06, + "loss": 1.4432, + "step": 1271 + }, + { + "epoch": 1.5830740510267578, + "grad_norm": 3.0183846950531006, + "learning_rate": 2.6591328413284134e-06, + "loss": 1.4332, + "step": 1272 + }, + { + "epoch": 1.5843186060983199, + "grad_norm": 2.856551170349121, + "learning_rate": 2.656826568265683e-06, + "loss": 1.4291, + "step": 1273 + }, + { + "epoch": 1.5855631611698817, + "grad_norm": 3.124971389770508, + "learning_rate": 2.6545202952029527e-06, + "loss": 1.4363, + "step": 1274 + }, + { + "epoch": 1.5868077162414438, + "grad_norm": 2.9470791816711426, + "learning_rate": 2.6522140221402217e-06, + "loss": 1.4502, + "step": 1275 + }, + { + "epoch": 1.5880522713130056, + "grad_norm": 2.9116358757019043, + "learning_rate": 2.649907749077491e-06, + "loss": 1.4314, + "step": 1276 + }, + { + "epoch": 1.5892968263845675, + "grad_norm": 3.2575323581695557, + "learning_rate": 2.6476014760147605e-06, + "loss": 1.4749, + "step": 1277 + }, + { + "epoch": 1.5905413814561293, + "grad_norm": 2.935518503189087, + "learning_rate": 2.6452952029520295e-06, + "loss": 1.4265, + "step": 1278 + }, + { + "epoch": 1.5917859365276914, + "grad_norm": 2.9750072956085205, + "learning_rate": 2.642988929889299e-06, + "loss": 1.4501, + "step": 1279 + }, + { + "epoch": 1.5930304915992533, + "grad_norm": 2.8086845874786377, + "learning_rate": 2.640682656826569e-06, + "loss": 1.4557, + "step": 1280 + }, + { + "epoch": 1.5930304915992533, + "eval_loss": 1.6302106380462646, + "eval_runtime": 52.3962, + "eval_samples_per_second": 19.085, + "eval_steps_per_second": 0.802, + "step": 1280 + }, + { + "epoch": 1.5942750466708153, + "grad_norm": 2.741849422454834, + "learning_rate": 2.638376383763838e-06, + "loss": 1.3918, + "step": 1281 + }, + { + "epoch": 1.5955196017423772, + "grad_norm": 2.9414806365966797, + "learning_rate": 2.6360701107011072e-06, + "loss": 1.3882, + "step": 1282 + }, + { + "epoch": 1.596764156813939, + "grad_norm": 2.901514768600464, + "learning_rate": 2.6337638376383766e-06, + "loss": 1.4331, + "step": 1283 + }, + { + "epoch": 1.5980087118855009, + "grad_norm": 2.8159122467041016, + "learning_rate": 2.6314575645756456e-06, + "loss": 1.4064, + "step": 1284 + }, + { + "epoch": 1.5992532669570627, + "grad_norm": 2.900561571121216, + "learning_rate": 2.629151291512915e-06, + "loss": 1.3731, + "step": 1285 + }, + { + "epoch": 1.6004978220286248, + "grad_norm": 3.019540548324585, + "learning_rate": 2.626845018450185e-06, + "loss": 1.4506, + "step": 1286 + }, + { + "epoch": 1.6017423771001866, + "grad_norm": 2.9263620376586914, + "learning_rate": 2.6245387453874543e-06, + "loss": 1.4754, + "step": 1287 + }, + { + "epoch": 1.6029869321717487, + "grad_norm": 2.8681881427764893, + "learning_rate": 2.6222324723247233e-06, + "loss": 1.4124, + "step": 1288 + }, + { + "epoch": 1.6042314872433105, + "grad_norm": 2.922945261001587, + "learning_rate": 2.6199261992619928e-06, + "loss": 1.5101, + "step": 1289 + }, + { + "epoch": 1.6054760423148724, + "grad_norm": 2.9548606872558594, + "learning_rate": 2.6176199261992626e-06, + "loss": 1.41, + "step": 1290 + }, + { + "epoch": 1.6054760423148724, + "eval_loss": 1.6357502937316895, + "eval_runtime": 46.9547, + "eval_samples_per_second": 21.297, + "eval_steps_per_second": 0.894, + "step": 1290 + }, + { + "epoch": 1.6067205973864342, + "grad_norm": 2.837184190750122, + "learning_rate": 2.6153136531365316e-06, + "loss": 1.4354, + "step": 1291 + }, + { + "epoch": 1.6079651524579963, + "grad_norm": 2.818990468978882, + "learning_rate": 2.613007380073801e-06, + "loss": 1.4474, + "step": 1292 + }, + { + "epoch": 1.6092097075295582, + "grad_norm": 2.7880146503448486, + "learning_rate": 2.6107011070110704e-06, + "loss": 1.4156, + "step": 1293 + }, + { + "epoch": 1.6104542626011202, + "grad_norm": 2.790971040725708, + "learning_rate": 2.6083948339483394e-06, + "loss": 1.4671, + "step": 1294 + }, + { + "epoch": 1.611698817672682, + "grad_norm": 2.943129777908325, + "learning_rate": 2.606088560885609e-06, + "loss": 1.4801, + "step": 1295 + }, + { + "epoch": 1.612943372744244, + "grad_norm": 2.8589110374450684, + "learning_rate": 2.6037822878228787e-06, + "loss": 1.4894, + "step": 1296 + }, + { + "epoch": 1.6141879278158058, + "grad_norm": 2.8929262161254883, + "learning_rate": 2.6014760147601477e-06, + "loss": 1.4022, + "step": 1297 + }, + { + "epoch": 1.6154324828873676, + "grad_norm": 2.970099925994873, + "learning_rate": 2.599169741697417e-06, + "loss": 1.5136, + "step": 1298 + }, + { + "epoch": 1.6166770379589297, + "grad_norm": 2.928466796875, + "learning_rate": 2.5968634686346866e-06, + "loss": 1.4743, + "step": 1299 + }, + { + "epoch": 1.6179215930304915, + "grad_norm": 2.941580057144165, + "learning_rate": 2.594557195571956e-06, + "loss": 1.4397, + "step": 1300 + }, + { + "epoch": 1.6179215930304915, + "eval_loss": 1.6348339319229126, + "eval_runtime": 50.433, + "eval_samples_per_second": 19.828, + "eval_steps_per_second": 0.833, + "step": 1300 + }, + { + "epoch": 1.6191661481020536, + "grad_norm": 2.903212308883667, + "learning_rate": 2.592250922509225e-06, + "loss": 1.4221, + "step": 1301 + }, + { + "epoch": 1.6204107031736155, + "grad_norm": 2.948651075363159, + "learning_rate": 2.589944649446495e-06, + "loss": 1.4363, + "step": 1302 + }, + { + "epoch": 1.6216552582451773, + "grad_norm": 2.8940505981445312, + "learning_rate": 2.5876383763837643e-06, + "loss": 1.398, + "step": 1303 + }, + { + "epoch": 1.6228998133167392, + "grad_norm": 2.950056552886963, + "learning_rate": 2.5853321033210333e-06, + "loss": 1.4425, + "step": 1304 + }, + { + "epoch": 1.6241443683883012, + "grad_norm": 2.9674088954925537, + "learning_rate": 2.5830258302583027e-06, + "loss": 1.4473, + "step": 1305 + }, + { + "epoch": 1.625388923459863, + "grad_norm": 2.7541685104370117, + "learning_rate": 2.5807195571955725e-06, + "loss": 1.4483, + "step": 1306 + }, + { + "epoch": 1.6266334785314251, + "grad_norm": 2.8897035121917725, + "learning_rate": 2.5784132841328415e-06, + "loss": 1.4572, + "step": 1307 + }, + { + "epoch": 1.627878033602987, + "grad_norm": 3.021559238433838, + "learning_rate": 2.576107011070111e-06, + "loss": 1.5164, + "step": 1308 + }, + { + "epoch": 1.6291225886745488, + "grad_norm": 2.884009838104248, + "learning_rate": 2.5738007380073804e-06, + "loss": 1.4196, + "step": 1309 + }, + { + "epoch": 1.6303671437461107, + "grad_norm": 2.749497890472412, + "learning_rate": 2.5714944649446494e-06, + "loss": 1.4324, + "step": 1310 + }, + { + "epoch": 1.6303671437461107, + "eval_loss": 1.6314265727996826, + "eval_runtime": 42.4647, + "eval_samples_per_second": 23.549, + "eval_steps_per_second": 0.989, + "step": 1310 + }, + { + "epoch": 1.6316116988176725, + "grad_norm": 2.8908045291900635, + "learning_rate": 2.5691881918819188e-06, + "loss": 1.4388, + "step": 1311 + }, + { + "epoch": 1.6328562538892346, + "grad_norm": 2.832808256149292, + "learning_rate": 2.5668819188191886e-06, + "loss": 1.4368, + "step": 1312 + }, + { + "epoch": 1.6341008089607967, + "grad_norm": 2.9047906398773193, + "learning_rate": 2.564575645756458e-06, + "loss": 1.4186, + "step": 1313 + }, + { + "epoch": 1.6353453640323585, + "grad_norm": 2.9271419048309326, + "learning_rate": 2.562269372693727e-06, + "loss": 1.3776, + "step": 1314 + }, + { + "epoch": 1.6365899191039204, + "grad_norm": 2.760314702987671, + "learning_rate": 2.5599630996309965e-06, + "loss": 1.386, + "step": 1315 + }, + { + "epoch": 1.6378344741754822, + "grad_norm": 2.9694864749908447, + "learning_rate": 2.557656826568266e-06, + "loss": 1.4542, + "step": 1316 + }, + { + "epoch": 1.639079029247044, + "grad_norm": 2.9950435161590576, + "learning_rate": 2.555350553505535e-06, + "loss": 1.4242, + "step": 1317 + }, + { + "epoch": 1.6403235843186061, + "grad_norm": 2.9291114807128906, + "learning_rate": 2.5530442804428047e-06, + "loss": 1.4289, + "step": 1318 + }, + { + "epoch": 1.641568139390168, + "grad_norm": 2.981961488723755, + "learning_rate": 2.550738007380074e-06, + "loss": 1.4283, + "step": 1319 + }, + { + "epoch": 1.64281269446173, + "grad_norm": 2.734123706817627, + "learning_rate": 2.548431734317343e-06, + "loss": 1.4144, + "step": 1320 + }, + { + "epoch": 1.64281269446173, + "eval_loss": 1.6249239444732666, + "eval_runtime": 48.8116, + "eval_samples_per_second": 20.487, + "eval_steps_per_second": 0.86, + "step": 1320 + }, + { + "epoch": 1.644057249533292, + "grad_norm": 2.9752919673919678, + "learning_rate": 2.5461254612546126e-06, + "loss": 1.3641, + "step": 1321 + }, + { + "epoch": 1.6453018046048538, + "grad_norm": 2.972463607788086, + "learning_rate": 2.543819188191882e-06, + "loss": 1.3797, + "step": 1322 + }, + { + "epoch": 1.6465463596764156, + "grad_norm": 2.905499219894409, + "learning_rate": 2.541512915129152e-06, + "loss": 1.4221, + "step": 1323 + }, + { + "epoch": 1.6477909147479775, + "grad_norm": 2.7921793460845947, + "learning_rate": 2.539206642066421e-06, + "loss": 1.4118, + "step": 1324 + }, + { + "epoch": 1.6490354698195395, + "grad_norm": 3.064558267593384, + "learning_rate": 2.5369003690036903e-06, + "loss": 1.4611, + "step": 1325 + }, + { + "epoch": 1.6502800248911016, + "grad_norm": 2.9690096378326416, + "learning_rate": 2.5345940959409597e-06, + "loss": 1.461, + "step": 1326 + }, + { + "epoch": 1.6515245799626634, + "grad_norm": 3.0379040241241455, + "learning_rate": 2.5322878228782287e-06, + "loss": 1.451, + "step": 1327 + }, + { + "epoch": 1.6527691350342253, + "grad_norm": 2.9426493644714355, + "learning_rate": 2.5299815498154986e-06, + "loss": 1.4671, + "step": 1328 + }, + { + "epoch": 1.6540136901057871, + "grad_norm": 2.915076732635498, + "learning_rate": 2.527675276752768e-06, + "loss": 1.4159, + "step": 1329 + }, + { + "epoch": 1.655258245177349, + "grad_norm": 2.8588879108428955, + "learning_rate": 2.525369003690037e-06, + "loss": 1.4978, + "step": 1330 + }, + { + "epoch": 1.655258245177349, + "eval_loss": 1.624009132385254, + "eval_runtime": 47.0461, + "eval_samples_per_second": 21.256, + "eval_steps_per_second": 0.893, + "step": 1330 + }, + { + "epoch": 1.656502800248911, + "grad_norm": 2.9942209720611572, + "learning_rate": 2.5230627306273064e-06, + "loss": 1.3872, + "step": 1331 + }, + { + "epoch": 1.657747355320473, + "grad_norm": 2.8424410820007324, + "learning_rate": 2.520756457564576e-06, + "loss": 1.4181, + "step": 1332 + }, + { + "epoch": 1.658991910392035, + "grad_norm": 2.9234843254089355, + "learning_rate": 2.518450184501845e-06, + "loss": 1.4516, + "step": 1333 + }, + { + "epoch": 1.6602364654635968, + "grad_norm": 2.8367815017700195, + "learning_rate": 2.5161439114391147e-06, + "loss": 1.4214, + "step": 1334 + }, + { + "epoch": 1.6614810205351587, + "grad_norm": 2.819149971008301, + "learning_rate": 2.513837638376384e-06, + "loss": 1.3883, + "step": 1335 + }, + { + "epoch": 1.6627255756067205, + "grad_norm": 3.024693250656128, + "learning_rate": 2.5115313653136535e-06, + "loss": 1.3902, + "step": 1336 + }, + { + "epoch": 1.6639701306782824, + "grad_norm": 2.903599262237549, + "learning_rate": 2.5092250922509225e-06, + "loss": 1.4238, + "step": 1337 + }, + { + "epoch": 1.6652146857498444, + "grad_norm": 2.8296589851379395, + "learning_rate": 2.506918819188192e-06, + "loss": 1.3993, + "step": 1338 + }, + { + "epoch": 1.6664592408214065, + "grad_norm": 2.873272657394409, + "learning_rate": 2.5046125461254618e-06, + "loss": 1.4186, + "step": 1339 + }, + { + "epoch": 1.6677037958929684, + "grad_norm": 2.9268789291381836, + "learning_rate": 2.5023062730627308e-06, + "loss": 1.4242, + "step": 1340 + }, + { + "epoch": 1.6677037958929684, + "eval_loss": 1.6260672807693481, + "eval_runtime": 47.5895, + "eval_samples_per_second": 21.013, + "eval_steps_per_second": 0.883, + "step": 1340 + }, + { + "epoch": 1.6689483509645302, + "grad_norm": 3.057659387588501, + "learning_rate": 2.5e-06, + "loss": 1.4077, + "step": 1341 + }, + { + "epoch": 1.670192906036092, + "grad_norm": 2.875095844268799, + "learning_rate": 2.4976937269372696e-06, + "loss": 1.394, + "step": 1342 + }, + { + "epoch": 1.671437461107654, + "grad_norm": 2.7579872608184814, + "learning_rate": 2.495387453874539e-06, + "loss": 1.4029, + "step": 1343 + }, + { + "epoch": 1.672682016179216, + "grad_norm": 2.8981001377105713, + "learning_rate": 2.4930811808118085e-06, + "loss": 1.4494, + "step": 1344 + }, + { + "epoch": 1.6739265712507778, + "grad_norm": 2.9696900844573975, + "learning_rate": 2.490774907749078e-06, + "loss": 1.4692, + "step": 1345 + }, + { + "epoch": 1.6751711263223399, + "grad_norm": 2.8094892501831055, + "learning_rate": 2.488468634686347e-06, + "loss": 1.441, + "step": 1346 + }, + { + "epoch": 1.6764156813939017, + "grad_norm": 2.8619563579559326, + "learning_rate": 2.4861623616236163e-06, + "loss": 1.4885, + "step": 1347 + }, + { + "epoch": 1.6776602364654636, + "grad_norm": 2.854151487350464, + "learning_rate": 2.4838560885608857e-06, + "loss": 1.4338, + "step": 1348 + }, + { + "epoch": 1.6789047915370254, + "grad_norm": 2.9707868099212646, + "learning_rate": 2.481549815498155e-06, + "loss": 1.4529, + "step": 1349 + }, + { + "epoch": 1.6801493466085873, + "grad_norm": 3.0289969444274902, + "learning_rate": 2.4792435424354246e-06, + "loss": 1.4855, + "step": 1350 + }, + { + "epoch": 1.6801493466085873, + "eval_loss": 1.6256393194198608, + "eval_runtime": 45.9524, + "eval_samples_per_second": 21.762, + "eval_steps_per_second": 0.914, + "step": 1350 + }, + { + "epoch": 1.6813939016801493, + "grad_norm": 2.8407845497131348, + "learning_rate": 2.476937269372694e-06, + "loss": 1.3876, + "step": 1351 + }, + { + "epoch": 1.6826384567517114, + "grad_norm": 2.9876599311828613, + "learning_rate": 2.4746309963099634e-06, + "loss": 1.3931, + "step": 1352 + }, + { + "epoch": 1.6838830118232733, + "grad_norm": 3.008683919906616, + "learning_rate": 2.472324723247233e-06, + "loss": 1.4482, + "step": 1353 + }, + { + "epoch": 1.6851275668948351, + "grad_norm": 2.8110079765319824, + "learning_rate": 2.470018450184502e-06, + "loss": 1.3851, + "step": 1354 + }, + { + "epoch": 1.686372121966397, + "grad_norm": 2.9402263164520264, + "learning_rate": 2.4677121771217713e-06, + "loss": 1.4009, + "step": 1355 + }, + { + "epoch": 1.6876166770379588, + "grad_norm": 2.9846489429473877, + "learning_rate": 2.4654059040590407e-06, + "loss": 1.3975, + "step": 1356 + }, + { + "epoch": 1.6888612321095209, + "grad_norm": 2.835801362991333, + "learning_rate": 2.46309963099631e-06, + "loss": 1.3421, + "step": 1357 + }, + { + "epoch": 1.6901057871810827, + "grad_norm": 2.9438889026641846, + "learning_rate": 2.4607933579335795e-06, + "loss": 1.4422, + "step": 1358 + }, + { + "epoch": 1.6913503422526448, + "grad_norm": 3.168829917907715, + "learning_rate": 2.458487084870849e-06, + "loss": 1.4596, + "step": 1359 + }, + { + "epoch": 1.6925948973242066, + "grad_norm": 2.8412039279937744, + "learning_rate": 2.4561808118081184e-06, + "loss": 1.4263, + "step": 1360 + }, + { + "epoch": 1.6925948973242066, + "eval_loss": 1.6265885829925537, + "eval_runtime": 50.3402, + "eval_samples_per_second": 19.865, + "eval_steps_per_second": 0.834, + "step": 1360 + }, + { + "epoch": 1.6938394523957685, + "grad_norm": 3.0540060997009277, + "learning_rate": 2.453874538745388e-06, + "loss": 1.4123, + "step": 1361 + }, + { + "epoch": 1.6950840074673303, + "grad_norm": 3.0544989109039307, + "learning_rate": 2.451568265682657e-06, + "loss": 1.412, + "step": 1362 + }, + { + "epoch": 1.6963285625388922, + "grad_norm": 3.114126205444336, + "learning_rate": 2.4492619926199267e-06, + "loss": 1.4952, + "step": 1363 + }, + { + "epoch": 1.6975731176104543, + "grad_norm": 2.9366016387939453, + "learning_rate": 2.4469557195571957e-06, + "loss": 1.5155, + "step": 1364 + }, + { + "epoch": 1.6988176726820163, + "grad_norm": 2.910677671432495, + "learning_rate": 2.444649446494465e-06, + "loss": 1.4393, + "step": 1365 + }, + { + "epoch": 1.7000622277535782, + "grad_norm": 2.8373727798461914, + "learning_rate": 2.4423431734317345e-06, + "loss": 1.4223, + "step": 1366 + }, + { + "epoch": 1.70130678282514, + "grad_norm": 2.865161895751953, + "learning_rate": 2.440036900369004e-06, + "loss": 1.4605, + "step": 1367 + }, + { + "epoch": 1.7025513378967019, + "grad_norm": 2.7929177284240723, + "learning_rate": 2.437730627306273e-06, + "loss": 1.4695, + "step": 1368 + }, + { + "epoch": 1.7037958929682637, + "grad_norm": 2.867953300476074, + "learning_rate": 2.4354243542435428e-06, + "loss": 1.3524, + "step": 1369 + }, + { + "epoch": 1.7050404480398258, + "grad_norm": 2.8980207443237305, + "learning_rate": 2.4331180811808118e-06, + "loss": 1.4568, + "step": 1370 + }, + { + "epoch": 1.7050404480398258, + "eval_loss": 1.6297556161880493, + "eval_runtime": 51.447, + "eval_samples_per_second": 19.437, + "eval_steps_per_second": 0.816, + "step": 1370 + }, + { + "epoch": 1.7062850031113876, + "grad_norm": 2.954972505569458, + "learning_rate": 2.4308118081180816e-06, + "loss": 1.4, + "step": 1371 + }, + { + "epoch": 1.7075295581829497, + "grad_norm": 2.973191738128662, + "learning_rate": 2.4285055350553506e-06, + "loss": 1.4062, + "step": 1372 + }, + { + "epoch": 1.7087741132545116, + "grad_norm": 2.951444387435913, + "learning_rate": 2.42619926199262e-06, + "loss": 1.4523, + "step": 1373 + }, + { + "epoch": 1.7100186683260734, + "grad_norm": 2.850445508956909, + "learning_rate": 2.4238929889298895e-06, + "loss": 1.4282, + "step": 1374 + }, + { + "epoch": 1.7112632233976353, + "grad_norm": 2.7643561363220215, + "learning_rate": 2.421586715867159e-06, + "loss": 1.3807, + "step": 1375 + }, + { + "epoch": 1.712507778469197, + "grad_norm": 2.8872134685516357, + "learning_rate": 2.4192804428044283e-06, + "loss": 1.4801, + "step": 1376 + }, + { + "epoch": 1.7137523335407592, + "grad_norm": 2.8748483657836914, + "learning_rate": 2.4169741697416977e-06, + "loss": 1.381, + "step": 1377 + }, + { + "epoch": 1.7149968886123212, + "grad_norm": 2.7833831310272217, + "learning_rate": 2.4146678966789667e-06, + "loss": 1.389, + "step": 1378 + }, + { + "epoch": 1.716241443683883, + "grad_norm": 2.994715690612793, + "learning_rate": 2.4123616236162366e-06, + "loss": 1.3917, + "step": 1379 + }, + { + "epoch": 1.717485998755445, + "grad_norm": 2.887026786804199, + "learning_rate": 2.4100553505535056e-06, + "loss": 1.3725, + "step": 1380 + }, + { + "epoch": 1.717485998755445, + "eval_loss": 1.6283434629440308, + "eval_runtime": 50.596, + "eval_samples_per_second": 19.764, + "eval_steps_per_second": 0.83, + "step": 1380 + }, + { + "epoch": 1.7187305538270068, + "grad_norm": 3.2046546936035156, + "learning_rate": 2.407749077490775e-06, + "loss": 1.4422, + "step": 1381 + }, + { + "epoch": 1.7199751088985686, + "grad_norm": 3.0105698108673096, + "learning_rate": 2.4054428044280444e-06, + "loss": 1.4494, + "step": 1382 + }, + { + "epoch": 1.7212196639701307, + "grad_norm": 2.9917800426483154, + "learning_rate": 2.403136531365314e-06, + "loss": 1.4407, + "step": 1383 + }, + { + "epoch": 1.7224642190416926, + "grad_norm": 2.844325065612793, + "learning_rate": 2.4008302583025833e-06, + "loss": 1.4067, + "step": 1384 + }, + { + "epoch": 1.7237087741132546, + "grad_norm": 2.919576406478882, + "learning_rate": 2.3985239852398527e-06, + "loss": 1.4352, + "step": 1385 + }, + { + "epoch": 1.7249533291848165, + "grad_norm": 2.8357744216918945, + "learning_rate": 2.3962177121771217e-06, + "loss": 1.3978, + "step": 1386 + }, + { + "epoch": 1.7261978842563783, + "grad_norm": 2.82425856590271, + "learning_rate": 2.3939114391143915e-06, + "loss": 1.4439, + "step": 1387 + }, + { + "epoch": 1.7274424393279402, + "grad_norm": 2.9155638217926025, + "learning_rate": 2.3916051660516605e-06, + "loss": 1.4326, + "step": 1388 + }, + { + "epoch": 1.728686994399502, + "grad_norm": 2.862804651260376, + "learning_rate": 2.3892988929889304e-06, + "loss": 1.3766, + "step": 1389 + }, + { + "epoch": 1.729931549471064, + "grad_norm": 3.012230634689331, + "learning_rate": 2.3869926199261994e-06, + "loss": 1.4569, + "step": 1390 + }, + { + "epoch": 1.729931549471064, + "eval_loss": 1.6268510818481445, + "eval_runtime": 52.5438, + "eval_samples_per_second": 19.032, + "eval_steps_per_second": 0.799, + "step": 1390 + }, + { + "epoch": 1.7311761045426262, + "grad_norm": 2.9922425746917725, + "learning_rate": 2.384686346863469e-06, + "loss": 1.4066, + "step": 1391 + }, + { + "epoch": 1.732420659614188, + "grad_norm": 2.960145950317383, + "learning_rate": 2.3823800738007382e-06, + "loss": 1.4514, + "step": 1392 + }, + { + "epoch": 1.7336652146857499, + "grad_norm": 2.834998846054077, + "learning_rate": 2.3800738007380077e-06, + "loss": 1.4714, + "step": 1393 + }, + { + "epoch": 1.7349097697573117, + "grad_norm": 2.9646077156066895, + "learning_rate": 2.377767527675277e-06, + "loss": 1.4368, + "step": 1394 + }, + { + "epoch": 1.7361543248288736, + "grad_norm": 2.8233699798583984, + "learning_rate": 2.3754612546125465e-06, + "loss": 1.4598, + "step": 1395 + }, + { + "epoch": 1.7373988799004356, + "grad_norm": 2.895472764968872, + "learning_rate": 2.3731549815498155e-06, + "loss": 1.4551, + "step": 1396 + }, + { + "epoch": 1.7386434349719975, + "grad_norm": 2.7917752265930176, + "learning_rate": 2.3708487084870853e-06, + "loss": 1.4759, + "step": 1397 + }, + { + "epoch": 1.7398879900435595, + "grad_norm": 2.820672035217285, + "learning_rate": 2.3685424354243543e-06, + "loss": 1.3711, + "step": 1398 + }, + { + "epoch": 1.7411325451151214, + "grad_norm": 2.804952621459961, + "learning_rate": 2.3662361623616238e-06, + "loss": 1.4227, + "step": 1399 + }, + { + "epoch": 1.7423771001866832, + "grad_norm": 2.981553316116333, + "learning_rate": 2.363929889298893e-06, + "loss": 1.4639, + "step": 1400 + }, + { + "epoch": 1.7423771001866832, + "eval_loss": 1.6229957342147827, + "eval_runtime": 49.1507, + "eval_samples_per_second": 20.346, + "eval_steps_per_second": 0.855, + "step": 1400 + } + ], + "logging_steps": 1, + "max_steps": 2409, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 100, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 4.364233173106688e+17, + "train_batch_size": 3, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/training_args.bin b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..6cdf93817e8f6d23b60d79888b226cc66ed2e88b --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a50e4918f3c49370326e14a7ed2f73b48b05e521f4b3399b53438fac6abee926 +size 6011 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/zero_to_fp32.py b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..24cc342e78d1a006c782b3a4cd68d9ce786d8fd8 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/zero_to_fp32.py @@ -0,0 +1,604 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: python zero_to_fp32.py . pytorch_model.bin + +import argparse +import torch +import glob +import math +import os +import re +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + + total_files = len(files) + state_dicts = [] + for f in files: + state_dict = torch.load(f, map_location=device) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + if zero_stage <= 2: + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + elif zero_stage == 3: + # if there is more than one param group, there will be multiple flattened tensors - one + # flattened tensor per group - for simplicity merge them into a single tensor + # + # XXX: could make the script more memory efficient for when there are multiple groups - it + # will require matching the sub-lists of param_shapes for each param group flattened tensor + + fp32_flat_groups = [ + torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts)) + ] + + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = fp32_flat_groups[0].numel() * world_size + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + for name, shape in param_shapes.items(): + + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # XXX: memory usage doubles here + state_dict[name] = torch.cat( + tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)), + 0).narrow(0, 0, unpartitioned_numel).view(shape) + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + + Returns: + - pytorch ``state_dict`` + + Note: this approach may not work if your application doesn't have sufficient free CPU memory and + you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters) + print(f"Saving fp32 state dict to {output_file}") + torch.save(state_dict, output_file) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info(f"Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info(f"Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument( + "output_file", + type=str, + help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_file, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fb0f9a53a507ffcdb60410deb71d60ef801bf350 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/config.json @@ -0,0 +1,36 @@ +{ + "_name_or_path": "meta-llama/Llama-3.2-3B", + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "eos_token_id": 128001, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 3072, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 24, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "torch_dtype": "float16", + "transformers_version": "4.45.1", + "use_cache": true, + "vocab_size": 128256 +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/generation_config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9c389b841a9a29ddd904e02d1eb0e08dcce82ad9 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": 128001, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.45.1" +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/latest b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/latest new file mode 100644 index 0000000000000000000000000000000000000000..c56ff7708f44fb7928fea2f70d6d7342ce0d5b67 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/latest @@ -0,0 +1 @@ +global_step1500 \ No newline at end of file diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/model.safetensors.index.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..ed64de846d720b9a7859dc20575fea8e8ca51940 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/model.safetensors.index.json @@ -0,0 +1,262 @@ +{ + "metadata": { + "total_size": 7213504512 + }, + "weight_map": { + "lm_head.weight": "model-00002-of-00002.safetensors", + "model.embed_tokens.weight": "model-00001-of-00002.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.norm.weight": "model-00002-of-00002.safetensors" + } +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/rng_state_0.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..491d7aa6873ac345ad7d2d8238b3751df06748aa --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc77a0f3d732b76deec0b9f9ee77f2ad22c4c8a301af0167ff41a26d22d5beb3 +size 16567 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/rng_state_1.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..0c1653c04de59e53b4d05673f04ed4b2c7039c73 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a2973146fd35e5e1a515f6332b8a2b33728c4360853ce2c03e1a7a90b79c2cc +size 16567 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/rng_state_2.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..4cf5bfa682e62dbe5c2e496536a84208dc8433d0 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d1658c107e9585fdf02f4eb4b846349794600f9e1c2f5151d959824c6021559 +size 16567 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/scheduler.pt b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..a94b34e1aa5087ed1129ca130601f722e8d7959d --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d470a1566fade71dbfe3de9e5fe45feb7e321a52bff45349224c90061e1661d9 +size 627 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/special_tokens_map.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..6949c5975ee0e961ef61cf31010dce04df0a03f8 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "[PAD]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/tokenizer.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f28ecaeab53ae07feed29ccf8624d2b0a8344df9 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19fb2e1e3cdd6f7433d89fd6d62c82042599dd4984f342efe7fec6e159e6a8f6 +size 17210734 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/tokenizer_config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8991b49e9c2a43fc527dab9e09ad8171f0cc5943 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/tokenizer_config.json @@ -0,0 +1,2086 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "[PAD]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128257": { + "content": "🅂", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "128258": { + "content": "🄿", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "[PAD]", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/trainer_state.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..8b236addf53b8f141289b7bff8e23da249ce28d9 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/trainer_state.json @@ -0,0 +1,11733 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.866832607342875, + "eval_steps": 10, + "global_step": 1500, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0012445550715619166, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8368, + "step": 1 + }, + { + "epoch": 0.002489110143123833, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.86, + "step": 2 + }, + { + "epoch": 0.00373366521468575, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8265, + "step": 3 + }, + { + "epoch": 0.004978220286247666, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8271, + "step": 4 + }, + { + "epoch": 0.006222775357809583, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8768, + "step": 5 + }, + { + "epoch": 0.0074673304293715, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8402, + "step": 6 + }, + { + "epoch": 0.008711885500933417, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.785, + "step": 7 + }, + { + "epoch": 0.009956440572495333, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8466, + "step": 8 + }, + { + "epoch": 0.01120099564405725, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.7673, + "step": 9 + }, + { + "epoch": 0.012445550715619166, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8074, + "step": 10 + }, + { + "epoch": 0.012445550715619166, + "eval_loss": 2.8535053730010986, + "eval_runtime": 42.446, + "eval_samples_per_second": 23.559, + "eval_steps_per_second": 0.989, + "step": 10 + }, + { + "epoch": 0.013690105787181083, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8592, + "step": 11 + }, + { + "epoch": 0.014934660858743, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.804, + "step": 12 + }, + { + "epoch": 0.016179215930304917, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8076, + "step": 13 + }, + { + "epoch": 0.017423771001866834, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8219, + "step": 14 + }, + { + "epoch": 0.018668326073428748, + "grad_norm": 5.105710506439209, + "learning_rate": 2.0746887966804982e-08, + "loss": 2.7995, + "step": 15 + }, + { + "epoch": 0.019912881144990666, + "grad_norm": 5.156588077545166, + "learning_rate": 4.1493775933609963e-08, + "loss": 2.8577, + "step": 16 + }, + { + "epoch": 0.021157436216552583, + "grad_norm": 5.709373950958252, + "learning_rate": 6.224066390041494e-08, + "loss": 2.8433, + "step": 17 + }, + { + "epoch": 0.0224019912881145, + "grad_norm": 5.64658784866333, + "learning_rate": 8.298755186721993e-08, + "loss": 2.8428, + "step": 18 + }, + { + "epoch": 0.023646546359676415, + "grad_norm": 5.541937351226807, + "learning_rate": 1.037344398340249e-07, + "loss": 2.7785, + "step": 19 + }, + { + "epoch": 0.024891101431238332, + "grad_norm": 5.233374118804932, + "learning_rate": 1.2448132780082988e-07, + "loss": 2.8599, + "step": 20 + }, + { + "epoch": 0.024891101431238332, + "eval_loss": 2.8521182537078857, + "eval_runtime": 43.5664, + "eval_samples_per_second": 22.953, + "eval_steps_per_second": 0.964, + "step": 20 + }, + { + "epoch": 0.02613565650280025, + "grad_norm": 5.407598972320557, + "learning_rate": 1.4522821576763488e-07, + "loss": 2.8081, + "step": 21 + }, + { + "epoch": 0.027380211574362167, + "grad_norm": 5.0483317375183105, + "learning_rate": 1.6597510373443985e-07, + "loss": 2.8489, + "step": 22 + }, + { + "epoch": 0.02862476664592408, + "grad_norm": 4.985588550567627, + "learning_rate": 1.8672199170124483e-07, + "loss": 2.8375, + "step": 23 + }, + { + "epoch": 0.029869321717486, + "grad_norm": 5.201700687408447, + "learning_rate": 2.074688796680498e-07, + "loss": 2.8355, + "step": 24 + }, + { + "epoch": 0.031113876789047916, + "grad_norm": 5.009235382080078, + "learning_rate": 2.2821576763485478e-07, + "loss": 2.7774, + "step": 25 + }, + { + "epoch": 0.03235843186060983, + "grad_norm": 4.509210109710693, + "learning_rate": 2.4896265560165975e-07, + "loss": 2.7992, + "step": 26 + }, + { + "epoch": 0.03360298693217175, + "grad_norm": 4.529262065887451, + "learning_rate": 2.6970954356846476e-07, + "loss": 2.836, + "step": 27 + }, + { + "epoch": 0.03484754200373367, + "grad_norm": 4.526787281036377, + "learning_rate": 2.9045643153526976e-07, + "loss": 2.8128, + "step": 28 + }, + { + "epoch": 0.03609209707529558, + "grad_norm": 4.016609191894531, + "learning_rate": 3.112033195020747e-07, + "loss": 2.7947, + "step": 29 + }, + { + "epoch": 0.037336652146857496, + "grad_norm": 3.4784562587738037, + "learning_rate": 3.319502074688797e-07, + "loss": 2.7854, + "step": 30 + }, + { + "epoch": 0.037336652146857496, + "eval_loss": 2.8020100593566895, + "eval_runtime": 44.8464, + "eval_samples_per_second": 22.298, + "eval_steps_per_second": 0.937, + "step": 30 + }, + { + "epoch": 0.038581207218419414, + "grad_norm": 3.4887945652008057, + "learning_rate": 3.5269709543568466e-07, + "loss": 2.7908, + "step": 31 + }, + { + "epoch": 0.03982576228998133, + "grad_norm": 3.4533374309539795, + "learning_rate": 3.7344398340248966e-07, + "loss": 2.7358, + "step": 32 + }, + { + "epoch": 0.04107031736154325, + "grad_norm": 3.270954132080078, + "learning_rate": 3.941908713692946e-07, + "loss": 2.8013, + "step": 33 + }, + { + "epoch": 0.042314872433105166, + "grad_norm": 3.61322283744812, + "learning_rate": 4.149377593360996e-07, + "loss": 2.7521, + "step": 34 + }, + { + "epoch": 0.043559427504667084, + "grad_norm": 3.158719062805176, + "learning_rate": 4.3568464730290456e-07, + "loss": 2.7573, + "step": 35 + }, + { + "epoch": 0.044803982576229, + "grad_norm": 2.888746500015259, + "learning_rate": 4.5643153526970956e-07, + "loss": 2.7795, + "step": 36 + }, + { + "epoch": 0.04604853764779091, + "grad_norm": 3.022629499435425, + "learning_rate": 4.771784232365145e-07, + "loss": 2.7891, + "step": 37 + }, + { + "epoch": 0.04729309271935283, + "grad_norm": 3.038097381591797, + "learning_rate": 4.979253112033195e-07, + "loss": 2.7376, + "step": 38 + }, + { + "epoch": 0.04853764779091475, + "grad_norm": 2.9992239475250244, + "learning_rate": 5.186721991701245e-07, + "loss": 2.7279, + "step": 39 + }, + { + "epoch": 0.049782202862476664, + "grad_norm": 2.908842086791992, + "learning_rate": 5.394190871369295e-07, + "loss": 2.6582, + "step": 40 + }, + { + "epoch": 0.049782202862476664, + "eval_loss": 2.725968837738037, + "eval_runtime": 44.9498, + "eval_samples_per_second": 22.247, + "eval_steps_per_second": 0.934, + "step": 40 + }, + { + "epoch": 0.05102675793403858, + "grad_norm": 2.8025174140930176, + "learning_rate": 5.601659751037345e-07, + "loss": 2.6809, + "step": 41 + }, + { + "epoch": 0.0522713130056005, + "grad_norm": 2.7555699348449707, + "learning_rate": 5.809128630705395e-07, + "loss": 2.6954, + "step": 42 + }, + { + "epoch": 0.053515868077162417, + "grad_norm": 2.4732933044433594, + "learning_rate": 6.016597510373444e-07, + "loss": 2.7307, + "step": 43 + }, + { + "epoch": 0.054760423148724334, + "grad_norm": 2.451366424560547, + "learning_rate": 6.224066390041494e-07, + "loss": 2.7064, + "step": 44 + }, + { + "epoch": 0.056004978220286245, + "grad_norm": 2.5768678188323975, + "learning_rate": 6.431535269709543e-07, + "loss": 2.6466, + "step": 45 + }, + { + "epoch": 0.05724953329184816, + "grad_norm": 2.579332113265991, + "learning_rate": 6.639004149377594e-07, + "loss": 2.6127, + "step": 46 + }, + { + "epoch": 0.05849408836341008, + "grad_norm": 2.231207847595215, + "learning_rate": 6.846473029045644e-07, + "loss": 2.6429, + "step": 47 + }, + { + "epoch": 0.059738643434972, + "grad_norm": 2.134375810623169, + "learning_rate": 7.053941908713693e-07, + "loss": 2.6423, + "step": 48 + }, + { + "epoch": 0.060983198506533914, + "grad_norm": 2.0485494136810303, + "learning_rate": 7.261410788381744e-07, + "loss": 2.581, + "step": 49 + }, + { + "epoch": 0.06222775357809583, + "grad_norm": 2.02909517288208, + "learning_rate": 7.468879668049793e-07, + "loss": 2.6357, + "step": 50 + }, + { + "epoch": 0.06222775357809583, + "eval_loss": 2.632822036743164, + "eval_runtime": 47.0104, + "eval_samples_per_second": 21.272, + "eval_steps_per_second": 0.893, + "step": 50 + }, + { + "epoch": 0.06347230864965775, + "grad_norm": 1.892500877380371, + "learning_rate": 7.676348547717843e-07, + "loss": 2.5945, + "step": 51 + }, + { + "epoch": 0.06471686372121967, + "grad_norm": 2.107921600341797, + "learning_rate": 7.883817427385892e-07, + "loss": 2.5867, + "step": 52 + }, + { + "epoch": 0.06596141879278158, + "grad_norm": 1.8287527561187744, + "learning_rate": 8.091286307053943e-07, + "loss": 2.5862, + "step": 53 + }, + { + "epoch": 0.0672059738643435, + "grad_norm": 1.7342065572738647, + "learning_rate": 8.298755186721992e-07, + "loss": 2.5735, + "step": 54 + }, + { + "epoch": 0.06845052893590542, + "grad_norm": 1.7905038595199585, + "learning_rate": 8.506224066390042e-07, + "loss": 2.572, + "step": 55 + }, + { + "epoch": 0.06969508400746734, + "grad_norm": 1.9715144634246826, + "learning_rate": 8.713692946058091e-07, + "loss": 2.5797, + "step": 56 + }, + { + "epoch": 0.07093963907902924, + "grad_norm": 1.788966417312622, + "learning_rate": 8.921161825726142e-07, + "loss": 2.5567, + "step": 57 + }, + { + "epoch": 0.07218419415059116, + "grad_norm": 1.8978915214538574, + "learning_rate": 9.128630705394191e-07, + "loss": 2.5776, + "step": 58 + }, + { + "epoch": 0.07342874922215308, + "grad_norm": 1.6401689052581787, + "learning_rate": 9.336099585062241e-07, + "loss": 2.5662, + "step": 59 + }, + { + "epoch": 0.07467330429371499, + "grad_norm": 1.8200798034667969, + "learning_rate": 9.54356846473029e-07, + "loss": 2.5001, + "step": 60 + }, + { + "epoch": 0.07467330429371499, + "eval_loss": 2.569110870361328, + "eval_runtime": 47.3292, + "eval_samples_per_second": 21.129, + "eval_steps_per_second": 0.887, + "step": 60 + }, + { + "epoch": 0.07591785936527691, + "grad_norm": 1.5468674898147583, + "learning_rate": 9.751037344398341e-07, + "loss": 2.5308, + "step": 61 + }, + { + "epoch": 0.07716241443683883, + "grad_norm": 1.723684310913086, + "learning_rate": 9.95850622406639e-07, + "loss": 2.4907, + "step": 62 + }, + { + "epoch": 0.07840696950840075, + "grad_norm": 1.6346293687820435, + "learning_rate": 1.0165975103734441e-06, + "loss": 2.514, + "step": 63 + }, + { + "epoch": 0.07965152457996266, + "grad_norm": 1.8902088403701782, + "learning_rate": 1.037344398340249e-06, + "loss": 2.4859, + "step": 64 + }, + { + "epoch": 0.08089607965152458, + "grad_norm": 2.092611312866211, + "learning_rate": 1.058091286307054e-06, + "loss": 2.4888, + "step": 65 + }, + { + "epoch": 0.0821406347230865, + "grad_norm": 1.758482575416565, + "learning_rate": 1.078838174273859e-06, + "loss": 2.4537, + "step": 66 + }, + { + "epoch": 0.08338518979464841, + "grad_norm": 1.8941714763641357, + "learning_rate": 1.099585062240664e-06, + "loss": 2.5416, + "step": 67 + }, + { + "epoch": 0.08462974486621033, + "grad_norm": 1.7030234336853027, + "learning_rate": 1.120331950207469e-06, + "loss": 2.4791, + "step": 68 + }, + { + "epoch": 0.08587429993777225, + "grad_norm": 1.7602269649505615, + "learning_rate": 1.141078838174274e-06, + "loss": 2.4316, + "step": 69 + }, + { + "epoch": 0.08711885500933417, + "grad_norm": 1.7635431289672852, + "learning_rate": 1.161825726141079e-06, + "loss": 2.4674, + "step": 70 + }, + { + "epoch": 0.08711885500933417, + "eval_loss": 2.4922080039978027, + "eval_runtime": 45.3028, + "eval_samples_per_second": 22.074, + "eval_steps_per_second": 0.927, + "step": 70 + }, + { + "epoch": 0.08836341008089608, + "grad_norm": 2.0913803577423096, + "learning_rate": 1.182572614107884e-06, + "loss": 2.4341, + "step": 71 + }, + { + "epoch": 0.089607965152458, + "grad_norm": 1.8292183876037598, + "learning_rate": 1.2033195020746888e-06, + "loss": 2.4291, + "step": 72 + }, + { + "epoch": 0.09085252022401992, + "grad_norm": 2.206770181655884, + "learning_rate": 1.224066390041494e-06, + "loss": 2.3764, + "step": 73 + }, + { + "epoch": 0.09209707529558182, + "grad_norm": 1.8263559341430664, + "learning_rate": 1.2448132780082988e-06, + "loss": 2.4082, + "step": 74 + }, + { + "epoch": 0.09334163036714374, + "grad_norm": 1.9405455589294434, + "learning_rate": 1.2655601659751037e-06, + "loss": 2.4673, + "step": 75 + }, + { + "epoch": 0.09458618543870566, + "grad_norm": 2.138108015060425, + "learning_rate": 1.2863070539419086e-06, + "loss": 2.3879, + "step": 76 + }, + { + "epoch": 0.09583074051026758, + "grad_norm": 1.9024745225906372, + "learning_rate": 1.307053941908714e-06, + "loss": 2.4344, + "step": 77 + }, + { + "epoch": 0.0970752955818295, + "grad_norm": 1.9136689901351929, + "learning_rate": 1.3278008298755188e-06, + "loss": 2.474, + "step": 78 + }, + { + "epoch": 0.09831985065339141, + "grad_norm": 2.4752795696258545, + "learning_rate": 1.3485477178423237e-06, + "loss": 2.3412, + "step": 79 + }, + { + "epoch": 0.09956440572495333, + "grad_norm": 1.8720006942749023, + "learning_rate": 1.3692946058091288e-06, + "loss": 2.3438, + "step": 80 + }, + { + "epoch": 0.09956440572495333, + "eval_loss": 2.3980512619018555, + "eval_runtime": 48.2788, + "eval_samples_per_second": 20.713, + "eval_steps_per_second": 0.87, + "step": 80 + }, + { + "epoch": 0.10080896079651525, + "grad_norm": 2.671691656112671, + "learning_rate": 1.3900414937759337e-06, + "loss": 2.3336, + "step": 81 + }, + { + "epoch": 0.10205351586807716, + "grad_norm": 2.2953391075134277, + "learning_rate": 1.4107883817427386e-06, + "loss": 2.377, + "step": 82 + }, + { + "epoch": 0.10329807093963908, + "grad_norm": 3.009018898010254, + "learning_rate": 1.4315352697095435e-06, + "loss": 2.2977, + "step": 83 + }, + { + "epoch": 0.104542626011201, + "grad_norm": 2.664454936981201, + "learning_rate": 1.4522821576763488e-06, + "loss": 2.3271, + "step": 84 + }, + { + "epoch": 0.10578718108276292, + "grad_norm": 3.017303705215454, + "learning_rate": 1.4730290456431537e-06, + "loss": 2.3251, + "step": 85 + }, + { + "epoch": 0.10703173615432483, + "grad_norm": 2.634716510772705, + "learning_rate": 1.4937759336099586e-06, + "loss": 2.332, + "step": 86 + }, + { + "epoch": 0.10827629122588675, + "grad_norm": 3.059644937515259, + "learning_rate": 1.5145228215767635e-06, + "loss": 2.3478, + "step": 87 + }, + { + "epoch": 0.10952084629744867, + "grad_norm": 2.6962637901306152, + "learning_rate": 1.5352697095435686e-06, + "loss": 2.2792, + "step": 88 + }, + { + "epoch": 0.11076540136901059, + "grad_norm": 3.419729709625244, + "learning_rate": 1.5560165975103735e-06, + "loss": 2.2571, + "step": 89 + }, + { + "epoch": 0.11200995644057249, + "grad_norm": 2.740781545639038, + "learning_rate": 1.5767634854771784e-06, + "loss": 2.2875, + "step": 90 + }, + { + "epoch": 0.11200995644057249, + "eval_loss": 2.30843186378479, + "eval_runtime": 49.9264, + "eval_samples_per_second": 20.029, + "eval_steps_per_second": 0.841, + "step": 90 + }, + { + "epoch": 0.1132545115121344, + "grad_norm": 2.5608789920806885, + "learning_rate": 1.5975103734439833e-06, + "loss": 2.2592, + "step": 91 + }, + { + "epoch": 0.11449906658369632, + "grad_norm": 2.687999963760376, + "learning_rate": 1.6182572614107886e-06, + "loss": 2.2546, + "step": 92 + }, + { + "epoch": 0.11574362165525824, + "grad_norm": 2.695909023284912, + "learning_rate": 1.6390041493775935e-06, + "loss": 2.2525, + "step": 93 + }, + { + "epoch": 0.11698817672682016, + "grad_norm": 2.818357467651367, + "learning_rate": 1.6597510373443984e-06, + "loss": 2.216, + "step": 94 + }, + { + "epoch": 0.11823273179838208, + "grad_norm": 2.884119987487793, + "learning_rate": 1.6804979253112035e-06, + "loss": 2.2321, + "step": 95 + }, + { + "epoch": 0.119477286869944, + "grad_norm": 2.52104115486145, + "learning_rate": 1.7012448132780084e-06, + "loss": 2.199, + "step": 96 + }, + { + "epoch": 0.12072184194150591, + "grad_norm": 2.420313596725464, + "learning_rate": 1.7219917012448133e-06, + "loss": 2.1862, + "step": 97 + }, + { + "epoch": 0.12196639701306783, + "grad_norm": 2.8047542572021484, + "learning_rate": 1.7427385892116182e-06, + "loss": 2.1793, + "step": 98 + }, + { + "epoch": 0.12321095208462975, + "grad_norm": 2.836482286453247, + "learning_rate": 1.7634854771784235e-06, + "loss": 2.2271, + "step": 99 + }, + { + "epoch": 0.12445550715619166, + "grad_norm": 2.5282301902770996, + "learning_rate": 1.7842323651452284e-06, + "loss": 2.1768, + "step": 100 + }, + { + "epoch": 0.12445550715619166, + "eval_loss": 2.2312686443328857, + "eval_runtime": 49.2382, + "eval_samples_per_second": 20.309, + "eval_steps_per_second": 0.853, + "step": 100 + }, + { + "epoch": 0.12570006222775357, + "grad_norm": 3.3407280445098877, + "learning_rate": 1.8049792531120333e-06, + "loss": 2.1666, + "step": 101 + }, + { + "epoch": 0.1269446172993155, + "grad_norm": 2.4754133224487305, + "learning_rate": 1.8257261410788382e-06, + "loss": 2.1768, + "step": 102 + }, + { + "epoch": 0.1281891723708774, + "grad_norm": 3.430889129638672, + "learning_rate": 1.8464730290456433e-06, + "loss": 2.1953, + "step": 103 + }, + { + "epoch": 0.12943372744243933, + "grad_norm": 2.835294246673584, + "learning_rate": 1.8672199170124482e-06, + "loss": 2.146, + "step": 104 + }, + { + "epoch": 0.13067828251400124, + "grad_norm": 3.1532323360443115, + "learning_rate": 1.8879668049792531e-06, + "loss": 2.1729, + "step": 105 + }, + { + "epoch": 0.13192283758556317, + "grad_norm": 3.2278342247009277, + "learning_rate": 1.908713692946058e-06, + "loss": 2.1336, + "step": 106 + }, + { + "epoch": 0.13316739265712507, + "grad_norm": 2.7892515659332275, + "learning_rate": 1.929460580912863e-06, + "loss": 2.141, + "step": 107 + }, + { + "epoch": 0.134411947728687, + "grad_norm": 3.0179977416992188, + "learning_rate": 1.9502074688796682e-06, + "loss": 2.1255, + "step": 108 + }, + { + "epoch": 0.1356565028002489, + "grad_norm": 2.977935552597046, + "learning_rate": 1.970954356846473e-06, + "loss": 2.0986, + "step": 109 + }, + { + "epoch": 0.13690105787181084, + "grad_norm": 3.3042492866516113, + "learning_rate": 1.991701244813278e-06, + "loss": 2.0992, + "step": 110 + }, + { + "epoch": 0.13690105787181084, + "eval_loss": 2.171299695968628, + "eval_runtime": 42.9511, + "eval_samples_per_second": 23.282, + "eval_steps_per_second": 0.978, + "step": 110 + }, + { + "epoch": 0.13814561294337274, + "grad_norm": 2.9750890731811523, + "learning_rate": 2.012448132780083e-06, + "loss": 2.1116, + "step": 111 + }, + { + "epoch": 0.13939016801493467, + "grad_norm": 2.8199286460876465, + "learning_rate": 2.0331950207468883e-06, + "loss": 2.0822, + "step": 112 + }, + { + "epoch": 0.14063472308649658, + "grad_norm": 2.930532217025757, + "learning_rate": 2.053941908713693e-06, + "loss": 2.161, + "step": 113 + }, + { + "epoch": 0.14187927815805848, + "grad_norm": 3.3400321006774902, + "learning_rate": 2.074688796680498e-06, + "loss": 2.0953, + "step": 114 + }, + { + "epoch": 0.1431238332296204, + "grad_norm": 2.716564655303955, + "learning_rate": 2.095435684647303e-06, + "loss": 2.0757, + "step": 115 + }, + { + "epoch": 0.14436838830118232, + "grad_norm": 2.7586236000061035, + "learning_rate": 2.116182572614108e-06, + "loss": 2.0979, + "step": 116 + }, + { + "epoch": 0.14561294337274425, + "grad_norm": 2.622126340866089, + "learning_rate": 2.136929460580913e-06, + "loss": 2.0748, + "step": 117 + }, + { + "epoch": 0.14685749844430615, + "grad_norm": 2.646477460861206, + "learning_rate": 2.157676348547718e-06, + "loss": 2.0605, + "step": 118 + }, + { + "epoch": 0.14810205351586808, + "grad_norm": 2.8882410526275635, + "learning_rate": 2.178423236514523e-06, + "loss": 2.0501, + "step": 119 + }, + { + "epoch": 0.14934660858742999, + "grad_norm": 2.8302247524261475, + "learning_rate": 2.199170124481328e-06, + "loss": 2.0592, + "step": 120 + }, + { + "epoch": 0.14934660858742999, + "eval_loss": 2.1291966438293457, + "eval_runtime": 47.5399, + "eval_samples_per_second": 21.035, + "eval_steps_per_second": 0.883, + "step": 120 + }, + { + "epoch": 0.15059116365899192, + "grad_norm": 3.26979660987854, + "learning_rate": 2.219917012448133e-06, + "loss": 2.0422, + "step": 121 + }, + { + "epoch": 0.15183571873055382, + "grad_norm": 2.7483913898468018, + "learning_rate": 2.240663900414938e-06, + "loss": 2.0872, + "step": 122 + }, + { + "epoch": 0.15308027380211575, + "grad_norm": 3.1252024173736572, + "learning_rate": 2.2614107883817427e-06, + "loss": 2.0684, + "step": 123 + }, + { + "epoch": 0.15432482887367766, + "grad_norm": 2.9175283908843994, + "learning_rate": 2.282157676348548e-06, + "loss": 2.0522, + "step": 124 + }, + { + "epoch": 0.1555693839452396, + "grad_norm": 2.7950754165649414, + "learning_rate": 2.302904564315353e-06, + "loss": 2.065, + "step": 125 + }, + { + "epoch": 0.1568139390168015, + "grad_norm": 2.84049654006958, + "learning_rate": 2.323651452282158e-06, + "loss": 2.0781, + "step": 126 + }, + { + "epoch": 0.15805849408836342, + "grad_norm": 2.7396647930145264, + "learning_rate": 2.3443983402489627e-06, + "loss": 2.1758, + "step": 127 + }, + { + "epoch": 0.15930304915992533, + "grad_norm": 3.0765926837921143, + "learning_rate": 2.365145228215768e-06, + "loss": 2.0172, + "step": 128 + }, + { + "epoch": 0.16054760423148726, + "grad_norm": 3.217189073562622, + "learning_rate": 2.385892116182573e-06, + "loss": 2.0286, + "step": 129 + }, + { + "epoch": 0.16179215930304916, + "grad_norm": 3.141545295715332, + "learning_rate": 2.4066390041493776e-06, + "loss": 2.0583, + "step": 130 + }, + { + "epoch": 0.16179215930304916, + "eval_loss": 2.0946149826049805, + "eval_runtime": 54.92, + "eval_samples_per_second": 18.208, + "eval_steps_per_second": 0.765, + "step": 130 + }, + { + "epoch": 0.16303671437461106, + "grad_norm": 2.850052833557129, + "learning_rate": 2.4273858921161828e-06, + "loss": 2.0746, + "step": 131 + }, + { + "epoch": 0.164281269446173, + "grad_norm": 3.28913950920105, + "learning_rate": 2.448132780082988e-06, + "loss": 2.0539, + "step": 132 + }, + { + "epoch": 0.1655258245177349, + "grad_norm": 2.8819124698638916, + "learning_rate": 2.468879668049793e-06, + "loss": 1.9843, + "step": 133 + }, + { + "epoch": 0.16677037958929683, + "grad_norm": 3.6254632472991943, + "learning_rate": 2.4896265560165977e-06, + "loss": 2.0233, + "step": 134 + }, + { + "epoch": 0.16801493466085873, + "grad_norm": 2.7385146617889404, + "learning_rate": 2.5103734439834028e-06, + "loss": 1.9667, + "step": 135 + }, + { + "epoch": 0.16925948973242066, + "grad_norm": 2.9722647666931152, + "learning_rate": 2.5311203319502074e-06, + "loss": 1.9527, + "step": 136 + }, + { + "epoch": 0.17050404480398257, + "grad_norm": 3.139526605606079, + "learning_rate": 2.5518672199170125e-06, + "loss": 1.9967, + "step": 137 + }, + { + "epoch": 0.1717485998755445, + "grad_norm": 3.399920701980591, + "learning_rate": 2.5726141078838172e-06, + "loss": 1.9798, + "step": 138 + }, + { + "epoch": 0.1729931549471064, + "grad_norm": 2.8017327785491943, + "learning_rate": 2.5933609958506228e-06, + "loss": 1.983, + "step": 139 + }, + { + "epoch": 0.17423771001866833, + "grad_norm": 3.9967198371887207, + "learning_rate": 2.614107883817428e-06, + "loss": 1.977, + "step": 140 + }, + { + "epoch": 0.17423771001866833, + "eval_loss": 2.059037685394287, + "eval_runtime": 52.6232, + "eval_samples_per_second": 19.003, + "eval_steps_per_second": 0.798, + "step": 140 + }, + { + "epoch": 0.17548226509023024, + "grad_norm": 2.893092393875122, + "learning_rate": 2.6348547717842326e-06, + "loss": 1.9888, + "step": 141 + }, + { + "epoch": 0.17672682016179217, + "grad_norm": 3.634352207183838, + "learning_rate": 2.6556016597510377e-06, + "loss": 2.0726, + "step": 142 + }, + { + "epoch": 0.17797137523335407, + "grad_norm": 3.3651444911956787, + "learning_rate": 2.6763485477178423e-06, + "loss": 1.973, + "step": 143 + }, + { + "epoch": 0.179215930304916, + "grad_norm": 3.968986988067627, + "learning_rate": 2.6970954356846475e-06, + "loss": 1.9948, + "step": 144 + }, + { + "epoch": 0.1804604853764779, + "grad_norm": 3.20105242729187, + "learning_rate": 2.717842323651452e-06, + "loss": 1.9238, + "step": 145 + }, + { + "epoch": 0.18170504044803984, + "grad_norm": 3.648339033126831, + "learning_rate": 2.7385892116182577e-06, + "loss": 1.9923, + "step": 146 + }, + { + "epoch": 0.18294959551960174, + "grad_norm": 3.0856316089630127, + "learning_rate": 2.7593360995850628e-06, + "loss": 1.9749, + "step": 147 + }, + { + "epoch": 0.18419415059116365, + "grad_norm": 3.2681071758270264, + "learning_rate": 2.7800829875518675e-06, + "loss": 1.9304, + "step": 148 + }, + { + "epoch": 0.18543870566272558, + "grad_norm": 2.634958267211914, + "learning_rate": 2.8008298755186726e-06, + "loss": 1.9237, + "step": 149 + }, + { + "epoch": 0.18668326073428748, + "grad_norm": 2.769491672515869, + "learning_rate": 2.8215767634854773e-06, + "loss": 1.8963, + "step": 150 + }, + { + "epoch": 0.18668326073428748, + "eval_loss": 2.0394654273986816, + "eval_runtime": 55.2136, + "eval_samples_per_second": 18.111, + "eval_steps_per_second": 0.761, + "step": 150 + }, + { + "epoch": 0.1879278158058494, + "grad_norm": 3.4345781803131104, + "learning_rate": 2.8423236514522824e-06, + "loss": 1.9836, + "step": 151 + }, + { + "epoch": 0.18917237087741132, + "grad_norm": 3.115727424621582, + "learning_rate": 2.863070539419087e-06, + "loss": 1.9352, + "step": 152 + }, + { + "epoch": 0.19041692594897325, + "grad_norm": 3.050652503967285, + "learning_rate": 2.883817427385892e-06, + "loss": 1.9031, + "step": 153 + }, + { + "epoch": 0.19166148102053515, + "grad_norm": 2.9404428005218506, + "learning_rate": 2.9045643153526977e-06, + "loss": 1.9196, + "step": 154 + }, + { + "epoch": 0.19290603609209708, + "grad_norm": 3.080810785293579, + "learning_rate": 2.9253112033195024e-06, + "loss": 1.9405, + "step": 155 + }, + { + "epoch": 0.194150591163659, + "grad_norm": 3.058558702468872, + "learning_rate": 2.9460580912863075e-06, + "loss": 1.9052, + "step": 156 + }, + { + "epoch": 0.19539514623522092, + "grad_norm": 3.307955026626587, + "learning_rate": 2.966804979253112e-06, + "loss": 1.9245, + "step": 157 + }, + { + "epoch": 0.19663970130678282, + "grad_norm": 2.845506191253662, + "learning_rate": 2.9875518672199173e-06, + "loss": 1.9461, + "step": 158 + }, + { + "epoch": 0.19788425637834475, + "grad_norm": 2.758654832839966, + "learning_rate": 3.008298755186722e-06, + "loss": 1.8885, + "step": 159 + }, + { + "epoch": 0.19912881144990666, + "grad_norm": 3.161252737045288, + "learning_rate": 3.029045643153527e-06, + "loss": 1.8535, + "step": 160 + }, + { + "epoch": 0.19912881144990666, + "eval_loss": 2.018101453781128, + "eval_runtime": 51.0744, + "eval_samples_per_second": 19.579, + "eval_steps_per_second": 0.822, + "step": 160 + }, + { + "epoch": 0.2003733665214686, + "grad_norm": 3.196847677230835, + "learning_rate": 3.0497925311203326e-06, + "loss": 1.8853, + "step": 161 + }, + { + "epoch": 0.2016179215930305, + "grad_norm": 2.8364663124084473, + "learning_rate": 3.0705394190871373e-06, + "loss": 1.9463, + "step": 162 + }, + { + "epoch": 0.2028624766645924, + "grad_norm": 3.0874054431915283, + "learning_rate": 3.0912863070539424e-06, + "loss": 1.9254, + "step": 163 + }, + { + "epoch": 0.20410703173615433, + "grad_norm": 2.7914493083953857, + "learning_rate": 3.112033195020747e-06, + "loss": 1.9213, + "step": 164 + }, + { + "epoch": 0.20535158680771623, + "grad_norm": 3.3871428966522217, + "learning_rate": 3.132780082987552e-06, + "loss": 1.8993, + "step": 165 + }, + { + "epoch": 0.20659614187927816, + "grad_norm": 3.096653461456299, + "learning_rate": 3.153526970954357e-06, + "loss": 1.8827, + "step": 166 + }, + { + "epoch": 0.20784069695084006, + "grad_norm": 2.8276076316833496, + "learning_rate": 3.174273858921162e-06, + "loss": 1.912, + "step": 167 + }, + { + "epoch": 0.209085252022402, + "grad_norm": 3.3058435916900635, + "learning_rate": 3.1950207468879666e-06, + "loss": 1.8381, + "step": 168 + }, + { + "epoch": 0.2103298070939639, + "grad_norm": 3.5017333030700684, + "learning_rate": 3.215767634854772e-06, + "loss": 1.9544, + "step": 169 + }, + { + "epoch": 0.21157436216552583, + "grad_norm": 3.457296133041382, + "learning_rate": 3.2365145228215773e-06, + "loss": 1.922, + "step": 170 + }, + { + "epoch": 0.21157436216552583, + "eval_loss": 1.988455891609192, + "eval_runtime": 56.2099, + "eval_samples_per_second": 17.79, + "eval_steps_per_second": 0.747, + "step": 170 + }, + { + "epoch": 0.21281891723708773, + "grad_norm": 3.1964566707611084, + "learning_rate": 3.257261410788382e-06, + "loss": 1.8925, + "step": 171 + }, + { + "epoch": 0.21406347230864967, + "grad_norm": 3.234652042388916, + "learning_rate": 3.278008298755187e-06, + "loss": 1.884, + "step": 172 + }, + { + "epoch": 0.21530802738021157, + "grad_norm": 3.14414119720459, + "learning_rate": 3.2987551867219918e-06, + "loss": 1.8905, + "step": 173 + }, + { + "epoch": 0.2165525824517735, + "grad_norm": 3.606379508972168, + "learning_rate": 3.319502074688797e-06, + "loss": 1.946, + "step": 174 + }, + { + "epoch": 0.2177971375233354, + "grad_norm": 2.986646890640259, + "learning_rate": 3.3402489626556016e-06, + "loss": 1.8691, + "step": 175 + }, + { + "epoch": 0.21904169259489734, + "grad_norm": 3.7298269271850586, + "learning_rate": 3.360995850622407e-06, + "loss": 1.8717, + "step": 176 + }, + { + "epoch": 0.22028624766645924, + "grad_norm": 3.422295093536377, + "learning_rate": 3.381742738589212e-06, + "loss": 1.8568, + "step": 177 + }, + { + "epoch": 0.22153080273802117, + "grad_norm": 3.2293782234191895, + "learning_rate": 3.402489626556017e-06, + "loss": 1.8471, + "step": 178 + }, + { + "epoch": 0.22277535780958307, + "grad_norm": 3.2293782234191895, + "learning_rate": 3.402489626556017e-06, + "loss": 1.9303, + "step": 179 + }, + { + "epoch": 0.22401991288114498, + "grad_norm": 3.506223440170288, + "learning_rate": 3.423236514522822e-06, + "loss": 1.8237, + "step": 180 + }, + { + "epoch": 0.22401991288114498, + "eval_loss": 1.9734643697738647, + "eval_runtime": 52.2815, + "eval_samples_per_second": 19.127, + "eval_steps_per_second": 0.803, + "step": 180 + }, + { + "epoch": 0.2252644679527069, + "grad_norm": 3.0156619548797607, + "learning_rate": 3.4439834024896267e-06, + "loss": 1.908, + "step": 181 + }, + { + "epoch": 0.2265090230242688, + "grad_norm": 3.55517315864563, + "learning_rate": 3.4647302904564318e-06, + "loss": 1.9104, + "step": 182 + }, + { + "epoch": 0.22775357809583074, + "grad_norm": 3.144984006881714, + "learning_rate": 3.4854771784232365e-06, + "loss": 1.8198, + "step": 183 + }, + { + "epoch": 0.22899813316739265, + "grad_norm": 4.302074432373047, + "learning_rate": 3.5062240663900416e-06, + "loss": 1.8237, + "step": 184 + }, + { + "epoch": 0.23024268823895458, + "grad_norm": 3.0522446632385254, + "learning_rate": 3.526970954356847e-06, + "loss": 1.8409, + "step": 185 + }, + { + "epoch": 0.23148724331051648, + "grad_norm": 3.8607394695281982, + "learning_rate": 3.5477178423236518e-06, + "loss": 1.8807, + "step": 186 + }, + { + "epoch": 0.23273179838207841, + "grad_norm": 2.9236302375793457, + "learning_rate": 3.568464730290457e-06, + "loss": 1.7873, + "step": 187 + }, + { + "epoch": 0.23397635345364032, + "grad_norm": 4.013780117034912, + "learning_rate": 3.5892116182572616e-06, + "loss": 1.7909, + "step": 188 + }, + { + "epoch": 0.23522090852520225, + "grad_norm": 3.0933122634887695, + "learning_rate": 3.6099585062240667e-06, + "loss": 1.8469, + "step": 189 + }, + { + "epoch": 0.23646546359676415, + "grad_norm": 3.487816095352173, + "learning_rate": 3.6307053941908714e-06, + "loss": 1.8469, + "step": 190 + }, + { + "epoch": 0.23646546359676415, + "eval_loss": 1.9604240655899048, + "eval_runtime": 53.4604, + "eval_samples_per_second": 18.705, + "eval_steps_per_second": 0.786, + "step": 190 + }, + { + "epoch": 0.23771001866832608, + "grad_norm": 2.829219341278076, + "learning_rate": 3.6514522821576765e-06, + "loss": 1.8639, + "step": 191 + }, + { + "epoch": 0.238954573739888, + "grad_norm": 3.595534324645996, + "learning_rate": 3.672199170124482e-06, + "loss": 1.8988, + "step": 192 + }, + { + "epoch": 0.24019912881144992, + "grad_norm": 2.9022483825683594, + "learning_rate": 3.6929460580912867e-06, + "loss": 1.8972, + "step": 193 + }, + { + "epoch": 0.24144368388301182, + "grad_norm": 3.0667994022369385, + "learning_rate": 3.713692946058092e-06, + "loss": 1.8499, + "step": 194 + }, + { + "epoch": 0.24268823895457373, + "grad_norm": 3.0498485565185547, + "learning_rate": 3.7344398340248965e-06, + "loss": 1.8341, + "step": 195 + }, + { + "epoch": 0.24393279402613566, + "grad_norm": 4.058084964752197, + "learning_rate": 3.7551867219917016e-06, + "loss": 1.8641, + "step": 196 + }, + { + "epoch": 0.24517734909769756, + "grad_norm": 2.9995028972625732, + "learning_rate": 3.7759336099585063e-06, + "loss": 1.7885, + "step": 197 + }, + { + "epoch": 0.2464219041692595, + "grad_norm": 3.254704475402832, + "learning_rate": 3.7966804979253114e-06, + "loss": 1.7544, + "step": 198 + }, + { + "epoch": 0.2476664592408214, + "grad_norm": 2.9126744270324707, + "learning_rate": 3.817427385892116e-06, + "loss": 1.8461, + "step": 199 + }, + { + "epoch": 0.24891101431238333, + "grad_norm": 3.475247621536255, + "learning_rate": 3.838174273858922e-06, + "loss": 1.8095, + "step": 200 + }, + { + "epoch": 0.24891101431238333, + "eval_loss": 1.9373760223388672, + "eval_runtime": 51.8733, + "eval_samples_per_second": 19.278, + "eval_steps_per_second": 0.81, + "step": 200 + }, + { + "epoch": 0.25015556938394523, + "grad_norm": 3.2663731575012207, + "learning_rate": 3.858921161825726e-06, + "loss": 1.8076, + "step": 201 + }, + { + "epoch": 0.25140012445550713, + "grad_norm": 3.1151726245880127, + "learning_rate": 3.879668049792531e-06, + "loss": 1.8143, + "step": 202 + }, + { + "epoch": 0.2526446795270691, + "grad_norm": 3.102038860321045, + "learning_rate": 3.9004149377593365e-06, + "loss": 1.8279, + "step": 203 + }, + { + "epoch": 0.253889234598631, + "grad_norm": 3.370642900466919, + "learning_rate": 3.921161825726142e-06, + "loss": 1.8522, + "step": 204 + }, + { + "epoch": 0.2551337896701929, + "grad_norm": 3.116128921508789, + "learning_rate": 3.941908713692946e-06, + "loss": 1.8104, + "step": 205 + }, + { + "epoch": 0.2563783447417548, + "grad_norm": 3.12028431892395, + "learning_rate": 3.962655601659751e-06, + "loss": 1.7723, + "step": 206 + }, + { + "epoch": 0.25762289981331676, + "grad_norm": 3.1251418590545654, + "learning_rate": 3.983402489626556e-06, + "loss": 1.863, + "step": 207 + }, + { + "epoch": 0.25886745488487867, + "grad_norm": 3.073702812194824, + "learning_rate": 4.004149377593361e-06, + "loss": 1.805, + "step": 208 + }, + { + "epoch": 0.26011200995644057, + "grad_norm": 2.9772469997406006, + "learning_rate": 4.024896265560166e-06, + "loss": 1.8251, + "step": 209 + }, + { + "epoch": 0.2613565650280025, + "grad_norm": 2.9970037937164307, + "learning_rate": 4.045643153526971e-06, + "loss": 1.8386, + "step": 210 + }, + { + "epoch": 0.2613565650280025, + "eval_loss": 1.9167065620422363, + "eval_runtime": 42.3328, + "eval_samples_per_second": 23.622, + "eval_steps_per_second": 0.992, + "step": 210 + }, + { + "epoch": 0.26260112009956443, + "grad_norm": 3.1391561031341553, + "learning_rate": 4.0663900414937765e-06, + "loss": 1.8109, + "step": 211 + }, + { + "epoch": 0.26384567517112634, + "grad_norm": 3.161538600921631, + "learning_rate": 4.087136929460581e-06, + "loss": 1.8441, + "step": 212 + }, + { + "epoch": 0.26509023024268824, + "grad_norm": 3.12412166595459, + "learning_rate": 4.107883817427386e-06, + "loss": 1.7736, + "step": 213 + }, + { + "epoch": 0.26633478531425014, + "grad_norm": 3.3241145610809326, + "learning_rate": 4.128630705394191e-06, + "loss": 1.8067, + "step": 214 + }, + { + "epoch": 0.26757934038581205, + "grad_norm": 3.503307819366455, + "learning_rate": 4.149377593360996e-06, + "loss": 1.7858, + "step": 215 + }, + { + "epoch": 0.268823895457374, + "grad_norm": 3.1009578704833984, + "learning_rate": 4.170124481327801e-06, + "loss": 1.7542, + "step": 216 + }, + { + "epoch": 0.2700684505289359, + "grad_norm": 3.2236011028289795, + "learning_rate": 4.190871369294606e-06, + "loss": 1.8006, + "step": 217 + }, + { + "epoch": 0.2713130056004978, + "grad_norm": 3.048935651779175, + "learning_rate": 4.211618257261411e-06, + "loss": 1.7706, + "step": 218 + }, + { + "epoch": 0.2725575606720597, + "grad_norm": 3.3416147232055664, + "learning_rate": 4.232365145228216e-06, + "loss": 1.8108, + "step": 219 + }, + { + "epoch": 0.2738021157436217, + "grad_norm": 3.2010738849639893, + "learning_rate": 4.253112033195021e-06, + "loss": 1.8561, + "step": 220 + }, + { + "epoch": 0.2738021157436217, + "eval_loss": 1.9111930131912231, + "eval_runtime": 50.4077, + "eval_samples_per_second": 19.838, + "eval_steps_per_second": 0.833, + "step": 220 + }, + { + "epoch": 0.2750466708151836, + "grad_norm": 3.4033374786376953, + "learning_rate": 4.273858921161826e-06, + "loss": 1.8218, + "step": 221 + }, + { + "epoch": 0.2762912258867455, + "grad_norm": 3.2086002826690674, + "learning_rate": 4.294605809128631e-06, + "loss": 1.8213, + "step": 222 + }, + { + "epoch": 0.2775357809583074, + "grad_norm": 4.090956211090088, + "learning_rate": 4.315352697095436e-06, + "loss": 1.8455, + "step": 223 + }, + { + "epoch": 0.27878033602986935, + "grad_norm": 3.1142985820770264, + "learning_rate": 4.336099585062241e-06, + "loss": 1.7737, + "step": 224 + }, + { + "epoch": 0.28002489110143125, + "grad_norm": 3.369669198989868, + "learning_rate": 4.356846473029046e-06, + "loss": 1.824, + "step": 225 + }, + { + "epoch": 0.28126944617299315, + "grad_norm": 3.165672779083252, + "learning_rate": 4.3775933609958506e-06, + "loss": 1.7768, + "step": 226 + }, + { + "epoch": 0.28251400124455506, + "grad_norm": 3.3343470096588135, + "learning_rate": 4.398340248962656e-06, + "loss": 1.7278, + "step": 227 + }, + { + "epoch": 0.28375855631611696, + "grad_norm": 3.2574994564056396, + "learning_rate": 4.419087136929461e-06, + "loss": 1.8439, + "step": 228 + }, + { + "epoch": 0.2850031113876789, + "grad_norm": 3.043928623199463, + "learning_rate": 4.439834024896266e-06, + "loss": 1.818, + "step": 229 + }, + { + "epoch": 0.2862476664592408, + "grad_norm": 2.9703125953674316, + "learning_rate": 4.460580912863071e-06, + "loss": 1.773, + "step": 230 + }, + { + "epoch": 0.2862476664592408, + "eval_loss": 1.898109793663025, + "eval_runtime": 45.6023, + "eval_samples_per_second": 21.929, + "eval_steps_per_second": 0.921, + "step": 230 + }, + { + "epoch": 0.2874922215308027, + "grad_norm": 2.9997220039367676, + "learning_rate": 4.481327800829876e-06, + "loss": 1.7727, + "step": 231 + }, + { + "epoch": 0.28873677660236463, + "grad_norm": 3.1358482837677, + "learning_rate": 4.502074688796681e-06, + "loss": 1.7764, + "step": 232 + }, + { + "epoch": 0.2899813316739266, + "grad_norm": 3.395747661590576, + "learning_rate": 4.5228215767634855e-06, + "loss": 1.7826, + "step": 233 + }, + { + "epoch": 0.2912258867454885, + "grad_norm": 3.7494754791259766, + "learning_rate": 4.543568464730291e-06, + "loss": 1.7181, + "step": 234 + }, + { + "epoch": 0.2924704418170504, + "grad_norm": 3.42293381690979, + "learning_rate": 4.564315352697096e-06, + "loss": 1.7773, + "step": 235 + }, + { + "epoch": 0.2937149968886123, + "grad_norm": 3.2524514198303223, + "learning_rate": 4.585062240663901e-06, + "loss": 1.7294, + "step": 236 + }, + { + "epoch": 0.29495955196017426, + "grad_norm": 3.657869577407837, + "learning_rate": 4.605809128630706e-06, + "loss": 1.762, + "step": 237 + }, + { + "epoch": 0.29620410703173616, + "grad_norm": 3.127372980117798, + "learning_rate": 4.626556016597511e-06, + "loss": 1.7623, + "step": 238 + }, + { + "epoch": 0.29744866210329807, + "grad_norm": 4.166962146759033, + "learning_rate": 4.647302904564316e-06, + "loss": 1.6995, + "step": 239 + }, + { + "epoch": 0.29869321717485997, + "grad_norm": 3.094264030456543, + "learning_rate": 4.66804979253112e-06, + "loss": 1.7469, + "step": 240 + }, + { + "epoch": 0.29869321717485997, + "eval_loss": 1.8908178806304932, + "eval_runtime": 44.6332, + "eval_samples_per_second": 22.405, + "eval_steps_per_second": 0.941, + "step": 240 + }, + { + "epoch": 0.29993777224642193, + "grad_norm": 3.8644745349884033, + "learning_rate": 4.6887966804979255e-06, + "loss": 1.7644, + "step": 241 + }, + { + "epoch": 0.30118232731798383, + "grad_norm": 3.1488852500915527, + "learning_rate": 4.709543568464731e-06, + "loss": 1.772, + "step": 242 + }, + { + "epoch": 0.30242688238954574, + "grad_norm": 3.3179638385772705, + "learning_rate": 4.730290456431536e-06, + "loss": 1.7588, + "step": 243 + }, + { + "epoch": 0.30367143746110764, + "grad_norm": 3.328355550765991, + "learning_rate": 4.751037344398341e-06, + "loss": 1.7417, + "step": 244 + }, + { + "epoch": 0.30491599253266954, + "grad_norm": 3.2690482139587402, + "learning_rate": 4.771784232365146e-06, + "loss": 1.7562, + "step": 245 + }, + { + "epoch": 0.3061605476042315, + "grad_norm": 3.2759454250335693, + "learning_rate": 4.792531120331951e-06, + "loss": 1.7536, + "step": 246 + }, + { + "epoch": 0.3074051026757934, + "grad_norm": 2.937964916229248, + "learning_rate": 4.813278008298755e-06, + "loss": 1.7518, + "step": 247 + }, + { + "epoch": 0.3086496577473553, + "grad_norm": 3.0995302200317383, + "learning_rate": 4.83402489626556e-06, + "loss": 1.7593, + "step": 248 + }, + { + "epoch": 0.3098942128189172, + "grad_norm": 3.230459213256836, + "learning_rate": 4.8547717842323655e-06, + "loss": 1.6962, + "step": 249 + }, + { + "epoch": 0.3111387678904792, + "grad_norm": 3.242576837539673, + "learning_rate": 4.875518672199171e-06, + "loss": 1.7307, + "step": 250 + }, + { + "epoch": 0.3111387678904792, + "eval_loss": 1.8805371522903442, + "eval_runtime": 45.9886, + "eval_samples_per_second": 21.745, + "eval_steps_per_second": 0.913, + "step": 250 + }, + { + "epoch": 0.3123833229620411, + "grad_norm": 3.097045421600342, + "learning_rate": 4.896265560165976e-06, + "loss": 1.7438, + "step": 251 + }, + { + "epoch": 0.313627878033603, + "grad_norm": 3.2428948879241943, + "learning_rate": 4.91701244813278e-06, + "loss": 1.7179, + "step": 252 + }, + { + "epoch": 0.3148724331051649, + "grad_norm": 3.196274518966675, + "learning_rate": 4.937759336099586e-06, + "loss": 1.7855, + "step": 253 + }, + { + "epoch": 0.31611698817672684, + "grad_norm": 2.978203535079956, + "learning_rate": 4.95850622406639e-06, + "loss": 1.7144, + "step": 254 + }, + { + "epoch": 0.31736154324828875, + "grad_norm": 3.2641701698303223, + "learning_rate": 4.979253112033195e-06, + "loss": 1.7427, + "step": 255 + }, + { + "epoch": 0.31860609831985065, + "grad_norm": 2.7441232204437256, + "learning_rate": 5e-06, + "loss": 1.7619, + "step": 256 + }, + { + "epoch": 0.31985065339141255, + "grad_norm": 3.0723495483398438, + "learning_rate": 4.99769372693727e-06, + "loss": 1.8017, + "step": 257 + }, + { + "epoch": 0.3210952084629745, + "grad_norm": 3.104752540588379, + "learning_rate": 4.995387453874539e-06, + "loss": 1.7761, + "step": 258 + }, + { + "epoch": 0.3223397635345364, + "grad_norm": 3.138627052307129, + "learning_rate": 4.993081180811809e-06, + "loss": 1.7879, + "step": 259 + }, + { + "epoch": 0.3235843186060983, + "grad_norm": 2.8567333221435547, + "learning_rate": 4.990774907749078e-06, + "loss": 1.7561, + "step": 260 + }, + { + "epoch": 0.3235843186060983, + "eval_loss": 1.8606494665145874, + "eval_runtime": 46.453, + "eval_samples_per_second": 21.527, + "eval_steps_per_second": 0.904, + "step": 260 + }, + { + "epoch": 0.3248288736776602, + "grad_norm": 3.2696540355682373, + "learning_rate": 4.988468634686347e-06, + "loss": 1.7201, + "step": 261 + }, + { + "epoch": 0.3260734287492221, + "grad_norm": 2.6692731380462646, + "learning_rate": 4.986162361623617e-06, + "loss": 1.7264, + "step": 262 + }, + { + "epoch": 0.3273179838207841, + "grad_norm": 3.047549247741699, + "learning_rate": 4.983856088560886e-06, + "loss": 1.7362, + "step": 263 + }, + { + "epoch": 0.328562538892346, + "grad_norm": 3.0667457580566406, + "learning_rate": 4.981549815498156e-06, + "loss": 1.7321, + "step": 264 + }, + { + "epoch": 0.3298070939639079, + "grad_norm": 2.560047149658203, + "learning_rate": 4.979243542435424e-06, + "loss": 1.7508, + "step": 265 + }, + { + "epoch": 0.3310516490354698, + "grad_norm": 3.26595401763916, + "learning_rate": 4.976937269372694e-06, + "loss": 1.7248, + "step": 266 + }, + { + "epoch": 0.33229620410703176, + "grad_norm": 2.929210662841797, + "learning_rate": 4.974630996309964e-06, + "loss": 1.697, + "step": 267 + }, + { + "epoch": 0.33354075917859366, + "grad_norm": 3.1526286602020264, + "learning_rate": 4.972324723247233e-06, + "loss": 1.7385, + "step": 268 + }, + { + "epoch": 0.33478531425015556, + "grad_norm": 3.001619577407837, + "learning_rate": 4.970018450184502e-06, + "loss": 1.7072, + "step": 269 + }, + { + "epoch": 0.33602986932171747, + "grad_norm": 3.2464189529418945, + "learning_rate": 4.9677121771217715e-06, + "loss": 1.7361, + "step": 270 + }, + { + "epoch": 0.33602986932171747, + "eval_loss": 1.860226035118103, + "eval_runtime": 46.6881, + "eval_samples_per_second": 21.419, + "eval_steps_per_second": 0.9, + "step": 270 + }, + { + "epoch": 0.3372744243932794, + "grad_norm": 3.2104530334472656, + "learning_rate": 4.965405904059041e-06, + "loss": 1.7171, + "step": 271 + }, + { + "epoch": 0.33851897946484133, + "grad_norm": 3.146847724914551, + "learning_rate": 4.96309963099631e-06, + "loss": 1.7311, + "step": 272 + }, + { + "epoch": 0.33976353453640323, + "grad_norm": 3.1431286334991455, + "learning_rate": 4.96079335793358e-06, + "loss": 1.7345, + "step": 273 + }, + { + "epoch": 0.34100808960796514, + "grad_norm": 2.847163677215576, + "learning_rate": 4.958487084870849e-06, + "loss": 1.7262, + "step": 274 + }, + { + "epoch": 0.3422526446795271, + "grad_norm": 3.1987810134887695, + "learning_rate": 4.956180811808119e-06, + "loss": 1.7945, + "step": 275 + }, + { + "epoch": 0.343497199751089, + "grad_norm": 2.7475385665893555, + "learning_rate": 4.953874538745388e-06, + "loss": 1.76, + "step": 276 + }, + { + "epoch": 0.3447417548226509, + "grad_norm": 3.0427663326263428, + "learning_rate": 4.9515682656826574e-06, + "loss": 1.7324, + "step": 277 + }, + { + "epoch": 0.3459863098942128, + "grad_norm": 3.605212688446045, + "learning_rate": 4.949261992619927e-06, + "loss": 1.7712, + "step": 278 + }, + { + "epoch": 0.3472308649657747, + "grad_norm": 3.0564935207366943, + "learning_rate": 4.946955719557196e-06, + "loss": 1.6792, + "step": 279 + }, + { + "epoch": 0.34847542003733667, + "grad_norm": 3.1069741249084473, + "learning_rate": 4.944649446494466e-06, + "loss": 1.7145, + "step": 280 + }, + { + "epoch": 0.34847542003733667, + "eval_loss": 1.8537051677703857, + "eval_runtime": 47.4813, + "eval_samples_per_second": 21.061, + "eval_steps_per_second": 0.885, + "step": 280 + }, + { + "epoch": 0.3497199751088986, + "grad_norm": 2.8021512031555176, + "learning_rate": 4.942343173431734e-06, + "loss": 1.7145, + "step": 281 + }, + { + "epoch": 0.3509645301804605, + "grad_norm": 3.5549023151397705, + "learning_rate": 4.940036900369004e-06, + "loss": 1.7642, + "step": 282 + }, + { + "epoch": 0.3522090852520224, + "grad_norm": 2.7648985385894775, + "learning_rate": 4.937730627306274e-06, + "loss": 1.7255, + "step": 283 + }, + { + "epoch": 0.35345364032358434, + "grad_norm": 3.0815863609313965, + "learning_rate": 4.9354243542435426e-06, + "loss": 1.7055, + "step": 284 + }, + { + "epoch": 0.35469819539514624, + "grad_norm": 2.9009227752685547, + "learning_rate": 4.933118081180812e-06, + "loss": 1.7019, + "step": 285 + }, + { + "epoch": 0.35594275046670815, + "grad_norm": 2.618429660797119, + "learning_rate": 4.930811808118081e-06, + "loss": 1.6318, + "step": 286 + }, + { + "epoch": 0.35718730553827005, + "grad_norm": 3.368230104446411, + "learning_rate": 4.928505535055351e-06, + "loss": 1.7304, + "step": 287 + }, + { + "epoch": 0.358431860609832, + "grad_norm": 3.256889820098877, + "learning_rate": 4.92619926199262e-06, + "loss": 1.7019, + "step": 288 + }, + { + "epoch": 0.3596764156813939, + "grad_norm": 2.9366304874420166, + "learning_rate": 4.92389298892989e-06, + "loss": 1.734, + "step": 289 + }, + { + "epoch": 0.3609209707529558, + "grad_norm": 3.0393142700195312, + "learning_rate": 4.921586715867159e-06, + "loss": 1.7504, + "step": 290 + }, + { + "epoch": 0.3609209707529558, + "eval_loss": 1.8359886407852173, + "eval_runtime": 49.7622, + "eval_samples_per_second": 20.096, + "eval_steps_per_second": 0.844, + "step": 290 + }, + { + "epoch": 0.3621655258245177, + "grad_norm": 2.7288589477539062, + "learning_rate": 4.9192804428044285e-06, + "loss": 1.7004, + "step": 291 + }, + { + "epoch": 0.3634100808960797, + "grad_norm": 3.2793378829956055, + "learning_rate": 4.916974169741698e-06, + "loss": 1.7028, + "step": 292 + }, + { + "epoch": 0.3646546359676416, + "grad_norm": 2.960880756378174, + "learning_rate": 4.914667896678967e-06, + "loss": 1.6759, + "step": 293 + }, + { + "epoch": 0.3658991910392035, + "grad_norm": 2.836421012878418, + "learning_rate": 4.912361623616237e-06, + "loss": 1.6475, + "step": 294 + }, + { + "epoch": 0.3671437461107654, + "grad_norm": 2.9495465755462646, + "learning_rate": 4.910055350553506e-06, + "loss": 1.7076, + "step": 295 + }, + { + "epoch": 0.3683883011823273, + "grad_norm": 2.954730272293091, + "learning_rate": 4.907749077490776e-06, + "loss": 1.7261, + "step": 296 + }, + { + "epoch": 0.36963285625388925, + "grad_norm": 2.8167543411254883, + "learning_rate": 4.905442804428044e-06, + "loss": 1.6939, + "step": 297 + }, + { + "epoch": 0.37087741132545116, + "grad_norm": 2.857316017150879, + "learning_rate": 4.903136531365314e-06, + "loss": 1.7362, + "step": 298 + }, + { + "epoch": 0.37212196639701306, + "grad_norm": 2.8612918853759766, + "learning_rate": 4.900830258302584e-06, + "loss": 1.6489, + "step": 299 + }, + { + "epoch": 0.37336652146857496, + "grad_norm": 2.9618914127349854, + "learning_rate": 4.898523985239853e-06, + "loss": 1.718, + "step": 300 + }, + { + "epoch": 0.37336652146857496, + "eval_loss": 1.827571988105774, + "eval_runtime": 46.4552, + "eval_samples_per_second": 21.526, + "eval_steps_per_second": 0.904, + "step": 300 + }, + { + "epoch": 0.3746110765401369, + "grad_norm": 2.8706953525543213, + "learning_rate": 4.896217712177122e-06, + "loss": 1.6925, + "step": 301 + }, + { + "epoch": 0.3758556316116988, + "grad_norm": 3.2565090656280518, + "learning_rate": 4.893911439114391e-06, + "loss": 1.7369, + "step": 302 + }, + { + "epoch": 0.37710018668326073, + "grad_norm": 2.928858995437622, + "learning_rate": 4.891605166051661e-06, + "loss": 1.6882, + "step": 303 + }, + { + "epoch": 0.37834474175482263, + "grad_norm": 3.042314052581787, + "learning_rate": 4.88929889298893e-06, + "loss": 1.6466, + "step": 304 + }, + { + "epoch": 0.3795892968263846, + "grad_norm": 2.7509143352508545, + "learning_rate": 4.8869926199262e-06, + "loss": 1.685, + "step": 305 + }, + { + "epoch": 0.3808338518979465, + "grad_norm": 2.9601588249206543, + "learning_rate": 4.884686346863469e-06, + "loss": 1.6772, + "step": 306 + }, + { + "epoch": 0.3820784069695084, + "grad_norm": 2.618608236312866, + "learning_rate": 4.8823800738007384e-06, + "loss": 1.7127, + "step": 307 + }, + { + "epoch": 0.3833229620410703, + "grad_norm": 3.047581195831299, + "learning_rate": 4.880073800738008e-06, + "loss": 1.7174, + "step": 308 + }, + { + "epoch": 0.3845675171126322, + "grad_norm": 2.87735652923584, + "learning_rate": 4.877767527675277e-06, + "loss": 1.6677, + "step": 309 + }, + { + "epoch": 0.38581207218419417, + "grad_norm": 2.7169721126556396, + "learning_rate": 4.875461254612546e-06, + "loss": 1.7486, + "step": 310 + }, + { + "epoch": 0.38581207218419417, + "eval_loss": 1.8221794366836548, + "eval_runtime": 43.1681, + "eval_samples_per_second": 23.165, + "eval_steps_per_second": 0.973, + "step": 310 + }, + { + "epoch": 0.38705662725575607, + "grad_norm": 2.7094991207122803, + "learning_rate": 4.873154981549816e-06, + "loss": 1.7348, + "step": 311 + }, + { + "epoch": 0.388301182327318, + "grad_norm": 2.8989078998565674, + "learning_rate": 4.8708487084870856e-06, + "loss": 1.7352, + "step": 312 + }, + { + "epoch": 0.3895457373988799, + "grad_norm": 2.9565601348876953, + "learning_rate": 4.868542435424355e-06, + "loss": 1.7063, + "step": 313 + }, + { + "epoch": 0.39079029247044184, + "grad_norm": 2.8354082107543945, + "learning_rate": 4.8662361623616235e-06, + "loss": 1.6808, + "step": 314 + }, + { + "epoch": 0.39203484754200374, + "grad_norm": 2.8287479877471924, + "learning_rate": 4.863929889298894e-06, + "loss": 1.6731, + "step": 315 + }, + { + "epoch": 0.39327940261356564, + "grad_norm": 2.6537821292877197, + "learning_rate": 4.861623616236163e-06, + "loss": 1.695, + "step": 316 + }, + { + "epoch": 0.39452395768512755, + "grad_norm": 2.7135190963745117, + "learning_rate": 4.859317343173432e-06, + "loss": 1.6897, + "step": 317 + }, + { + "epoch": 0.3957685127566895, + "grad_norm": 3.1019129753112793, + "learning_rate": 4.857011070110701e-06, + "loss": 1.7219, + "step": 318 + }, + { + "epoch": 0.3970130678282514, + "grad_norm": 2.945737838745117, + "learning_rate": 4.854704797047971e-06, + "loss": 1.7316, + "step": 319 + }, + { + "epoch": 0.3982576228998133, + "grad_norm": 2.7668251991271973, + "learning_rate": 4.85239852398524e-06, + "loss": 1.6461, + "step": 320 + }, + { + "epoch": 0.3982576228998133, + "eval_loss": 1.8210570812225342, + "eval_runtime": 46.2552, + "eval_samples_per_second": 21.619, + "eval_steps_per_second": 0.908, + "step": 320 + }, + { + "epoch": 0.3995021779713752, + "grad_norm": 2.738109588623047, + "learning_rate": 4.8500922509225095e-06, + "loss": 1.6907, + "step": 321 + }, + { + "epoch": 0.4007467330429372, + "grad_norm": 3.0195062160491943, + "learning_rate": 4.847785977859779e-06, + "loss": 1.7099, + "step": 322 + }, + { + "epoch": 0.4019912881144991, + "grad_norm": 2.8672404289245605, + "learning_rate": 4.845479704797048e-06, + "loss": 1.6509, + "step": 323 + }, + { + "epoch": 0.403235843186061, + "grad_norm": 2.9041008949279785, + "learning_rate": 4.843173431734318e-06, + "loss": 1.5818, + "step": 324 + }, + { + "epoch": 0.4044803982576229, + "grad_norm": 2.9105985164642334, + "learning_rate": 4.840867158671587e-06, + "loss": 1.7002, + "step": 325 + }, + { + "epoch": 0.4057249533291848, + "grad_norm": 2.749009847640991, + "learning_rate": 4.838560885608857e-06, + "loss": 1.7021, + "step": 326 + }, + { + "epoch": 0.40696950840074675, + "grad_norm": 2.6998822689056396, + "learning_rate": 4.836254612546126e-06, + "loss": 1.6853, + "step": 327 + }, + { + "epoch": 0.40821406347230865, + "grad_norm": 2.678201675415039, + "learning_rate": 4.8339483394833955e-06, + "loss": 1.6934, + "step": 328 + }, + { + "epoch": 0.40945861854387056, + "grad_norm": 2.774845600128174, + "learning_rate": 4.831642066420665e-06, + "loss": 1.657, + "step": 329 + }, + { + "epoch": 0.41070317361543246, + "grad_norm": 2.6363000869750977, + "learning_rate": 4.8293357933579335e-06, + "loss": 1.6555, + "step": 330 + }, + { + "epoch": 0.41070317361543246, + "eval_loss": 1.8126581907272339, + "eval_runtime": 52.7717, + "eval_samples_per_second": 18.95, + "eval_steps_per_second": 0.796, + "step": 330 + }, + { + "epoch": 0.4119477286869944, + "grad_norm": 3.011659622192383, + "learning_rate": 4.827029520295204e-06, + "loss": 1.7202, + "step": 331 + }, + { + "epoch": 0.4131922837585563, + "grad_norm": 2.8322594165802, + "learning_rate": 4.824723247232473e-06, + "loss": 1.6736, + "step": 332 + }, + { + "epoch": 0.4144368388301182, + "grad_norm": 2.9360146522521973, + "learning_rate": 4.822416974169742e-06, + "loss": 1.7213, + "step": 333 + }, + { + "epoch": 0.41568139390168013, + "grad_norm": 2.810206413269043, + "learning_rate": 4.820110701107011e-06, + "loss": 1.6867, + "step": 334 + }, + { + "epoch": 0.4169259489732421, + "grad_norm": 2.6864097118377686, + "learning_rate": 4.817804428044281e-06, + "loss": 1.6854, + "step": 335 + }, + { + "epoch": 0.418170504044804, + "grad_norm": 2.7396061420440674, + "learning_rate": 4.81549815498155e-06, + "loss": 1.68, + "step": 336 + }, + { + "epoch": 0.4194150591163659, + "grad_norm": 2.602771520614624, + "learning_rate": 4.8131918819188194e-06, + "loss": 1.6269, + "step": 337 + }, + { + "epoch": 0.4206596141879278, + "grad_norm": 2.8174757957458496, + "learning_rate": 4.810885608856089e-06, + "loss": 1.6981, + "step": 338 + }, + { + "epoch": 0.42190416925948976, + "grad_norm": 3.2398130893707275, + "learning_rate": 4.808579335793358e-06, + "loss": 1.7061, + "step": 339 + }, + { + "epoch": 0.42314872433105166, + "grad_norm": 2.7119736671447754, + "learning_rate": 4.806273062730628e-06, + "loss": 1.6875, + "step": 340 + }, + { + "epoch": 0.42314872433105166, + "eval_loss": 1.7973068952560425, + "eval_runtime": 53.5791, + "eval_samples_per_second": 18.664, + "eval_steps_per_second": 0.784, + "step": 340 + }, + { + "epoch": 0.42439327940261357, + "grad_norm": 2.868533134460449, + "learning_rate": 4.803966789667897e-06, + "loss": 1.6967, + "step": 341 + }, + { + "epoch": 0.42563783447417547, + "grad_norm": 2.528083562850952, + "learning_rate": 4.8016605166051665e-06, + "loss": 1.6195, + "step": 342 + }, + { + "epoch": 0.4268823895457374, + "grad_norm": 2.7885115146636963, + "learning_rate": 4.799354243542436e-06, + "loss": 1.6197, + "step": 343 + }, + { + "epoch": 0.42812694461729933, + "grad_norm": 2.768247604370117, + "learning_rate": 4.797047970479705e-06, + "loss": 1.6271, + "step": 344 + }, + { + "epoch": 0.42937149968886124, + "grad_norm": 2.7710275650024414, + "learning_rate": 4.794741697416975e-06, + "loss": 1.6768, + "step": 345 + }, + { + "epoch": 0.43061605476042314, + "grad_norm": 2.968337297439575, + "learning_rate": 4.792435424354243e-06, + "loss": 1.6433, + "step": 346 + }, + { + "epoch": 0.43186060983198504, + "grad_norm": 2.5739808082580566, + "learning_rate": 4.790129151291514e-06, + "loss": 1.6327, + "step": 347 + }, + { + "epoch": 0.433105164903547, + "grad_norm": 2.7676408290863037, + "learning_rate": 4.787822878228783e-06, + "loss": 1.6178, + "step": 348 + }, + { + "epoch": 0.4343497199751089, + "grad_norm": 2.959059476852417, + "learning_rate": 4.7855166051660525e-06, + "loss": 1.6809, + "step": 349 + }, + { + "epoch": 0.4355942750466708, + "grad_norm": 2.559967279434204, + "learning_rate": 4.783210332103321e-06, + "loss": 1.6704, + "step": 350 + }, + { + "epoch": 0.4355942750466708, + "eval_loss": 1.8013949394226074, + "eval_runtime": 50.4649, + "eval_samples_per_second": 19.816, + "eval_steps_per_second": 0.832, + "step": 350 + }, + { + "epoch": 0.4368388301182327, + "grad_norm": 2.817901611328125, + "learning_rate": 4.7809040590405905e-06, + "loss": 1.6296, + "step": 351 + }, + { + "epoch": 0.43808338518979467, + "grad_norm": 2.7116055488586426, + "learning_rate": 4.778597785977861e-06, + "loss": 1.6652, + "step": 352 + }, + { + "epoch": 0.4393279402613566, + "grad_norm": 2.544750452041626, + "learning_rate": 4.776291512915129e-06, + "loss": 1.5668, + "step": 353 + }, + { + "epoch": 0.4405724953329185, + "grad_norm": 2.720534086227417, + "learning_rate": 4.773985239852399e-06, + "loss": 1.6152, + "step": 354 + }, + { + "epoch": 0.4418170504044804, + "grad_norm": 2.7270126342773438, + "learning_rate": 4.771678966789668e-06, + "loss": 1.5624, + "step": 355 + }, + { + "epoch": 0.44306160547604234, + "grad_norm": 2.7986624240875244, + "learning_rate": 4.769372693726938e-06, + "loss": 1.6621, + "step": 356 + }, + { + "epoch": 0.44430616054760425, + "grad_norm": 2.943107843399048, + "learning_rate": 4.767066420664207e-06, + "loss": 1.6713, + "step": 357 + }, + { + "epoch": 0.44555071561916615, + "grad_norm": 2.739898681640625, + "learning_rate": 4.7647601476014765e-06, + "loss": 1.6363, + "step": 358 + }, + { + "epoch": 0.44679527069072805, + "grad_norm": 2.8729329109191895, + "learning_rate": 4.762453874538746e-06, + "loss": 1.6605, + "step": 359 + }, + { + "epoch": 0.44803982576228996, + "grad_norm": 2.909029483795166, + "learning_rate": 4.760147601476015e-06, + "loss": 1.6277, + "step": 360 + }, + { + "epoch": 0.44803982576228996, + "eval_loss": 1.8110179901123047, + "eval_runtime": 47.924, + "eval_samples_per_second": 20.866, + "eval_steps_per_second": 0.876, + "step": 360 + }, + { + "epoch": 0.4492843808338519, + "grad_norm": 3.3826682567596436, + "learning_rate": 4.757841328413285e-06, + "loss": 1.7155, + "step": 361 + }, + { + "epoch": 0.4505289359054138, + "grad_norm": 3.0586729049682617, + "learning_rate": 4.755535055350554e-06, + "loss": 1.5991, + "step": 362 + }, + { + "epoch": 0.4517734909769757, + "grad_norm": 2.645576238632202, + "learning_rate": 4.753228782287823e-06, + "loss": 1.6504, + "step": 363 + }, + { + "epoch": 0.4530180460485376, + "grad_norm": 2.705967664718628, + "learning_rate": 4.750922509225093e-06, + "loss": 1.6761, + "step": 364 + }, + { + "epoch": 0.4542626011200996, + "grad_norm": 2.7651922702789307, + "learning_rate": 4.748616236162362e-06, + "loss": 1.61, + "step": 365 + }, + { + "epoch": 0.4555071561916615, + "grad_norm": 2.5654966831207275, + "learning_rate": 4.746309963099631e-06, + "loss": 1.6335, + "step": 366 + }, + { + "epoch": 0.4567517112632234, + "grad_norm": 2.6606173515319824, + "learning_rate": 4.7440036900369e-06, + "loss": 1.6207, + "step": 367 + }, + { + "epoch": 0.4579962663347853, + "grad_norm": 2.724755048751831, + "learning_rate": 4.741697416974171e-06, + "loss": 1.6329, + "step": 368 + }, + { + "epoch": 0.45924082140634725, + "grad_norm": 2.7019615173339844, + "learning_rate": 4.739391143911439e-06, + "loss": 1.6763, + "step": 369 + }, + { + "epoch": 0.46048537647790916, + "grad_norm": 2.6594254970550537, + "learning_rate": 4.737084870848709e-06, + "loss": 1.6174, + "step": 370 + }, + { + "epoch": 0.46048537647790916, + "eval_loss": 1.7924141883850098, + "eval_runtime": 53.0349, + "eval_samples_per_second": 18.855, + "eval_steps_per_second": 0.792, + "step": 370 + }, + { + "epoch": 0.46172993154947106, + "grad_norm": 2.668421745300293, + "learning_rate": 4.734778597785978e-06, + "loss": 1.6421, + "step": 371 + }, + { + "epoch": 0.46297448662103297, + "grad_norm": 2.7148995399475098, + "learning_rate": 4.7324723247232475e-06, + "loss": 1.6635, + "step": 372 + }, + { + "epoch": 0.4642190416925949, + "grad_norm": 3.156646490097046, + "learning_rate": 4.730166051660517e-06, + "loss": 1.6679, + "step": 373 + }, + { + "epoch": 0.46546359676415683, + "grad_norm": 2.8170645236968994, + "learning_rate": 4.727859778597786e-06, + "loss": 1.6492, + "step": 374 + }, + { + "epoch": 0.46670815183571873, + "grad_norm": 2.9796457290649414, + "learning_rate": 4.725553505535056e-06, + "loss": 1.6025, + "step": 375 + }, + { + "epoch": 0.46795270690728064, + "grad_norm": 2.6493406295776367, + "learning_rate": 4.723247232472325e-06, + "loss": 1.6398, + "step": 376 + }, + { + "epoch": 0.46919726197884254, + "grad_norm": 3.2643542289733887, + "learning_rate": 4.720940959409595e-06, + "loss": 1.6413, + "step": 377 + }, + { + "epoch": 0.4704418170504045, + "grad_norm": 2.7613179683685303, + "learning_rate": 4.718634686346864e-06, + "loss": 1.6251, + "step": 378 + }, + { + "epoch": 0.4716863721219664, + "grad_norm": 3.1216518878936768, + "learning_rate": 4.716328413284133e-06, + "loss": 1.6268, + "step": 379 + }, + { + "epoch": 0.4729309271935283, + "grad_norm": 2.571647882461548, + "learning_rate": 4.714022140221403e-06, + "loss": 1.6245, + "step": 380 + }, + { + "epoch": 0.4729309271935283, + "eval_loss": 1.7875893115997314, + "eval_runtime": 49.3058, + "eval_samples_per_second": 20.282, + "eval_steps_per_second": 0.852, + "step": 380 + }, + { + "epoch": 0.4741754822650902, + "grad_norm": 3.443906545639038, + "learning_rate": 4.711715867158672e-06, + "loss": 1.6799, + "step": 381 + }, + { + "epoch": 0.47542003733665217, + "grad_norm": 2.7887957096099854, + "learning_rate": 4.709409594095941e-06, + "loss": 1.6589, + "step": 382 + }, + { + "epoch": 0.47666459240821407, + "grad_norm": 3.2833151817321777, + "learning_rate": 4.70710332103321e-06, + "loss": 1.6064, + "step": 383 + }, + { + "epoch": 0.477909147479776, + "grad_norm": 2.6975057125091553, + "learning_rate": 4.704797047970481e-06, + "loss": 1.6029, + "step": 384 + }, + { + "epoch": 0.4791537025513379, + "grad_norm": 2.745246171951294, + "learning_rate": 4.702490774907749e-06, + "loss": 1.7127, + "step": 385 + }, + { + "epoch": 0.48039825762289984, + "grad_norm": 3.0522775650024414, + "learning_rate": 4.700184501845019e-06, + "loss": 1.6508, + "step": 386 + }, + { + "epoch": 0.48164281269446174, + "grad_norm": 2.689218044281006, + "learning_rate": 4.697878228782288e-06, + "loss": 1.6971, + "step": 387 + }, + { + "epoch": 0.48288736776602365, + "grad_norm": 3.060361862182617, + "learning_rate": 4.6955719557195575e-06, + "loss": 1.6464, + "step": 388 + }, + { + "epoch": 0.48413192283758555, + "grad_norm": 2.872262477874756, + "learning_rate": 4.693265682656827e-06, + "loss": 1.6293, + "step": 389 + }, + { + "epoch": 0.48537647790914745, + "grad_norm": 2.980050563812256, + "learning_rate": 4.690959409594096e-06, + "loss": 1.6435, + "step": 390 + }, + { + "epoch": 0.48537647790914745, + "eval_loss": 1.7791496515274048, + "eval_runtime": 50.0299, + "eval_samples_per_second": 19.988, + "eval_steps_per_second": 0.839, + "step": 390 + }, + { + "epoch": 0.4866210329807094, + "grad_norm": 2.764827251434326, + "learning_rate": 4.688653136531366e-06, + "loss": 1.6279, + "step": 391 + }, + { + "epoch": 0.4878655880522713, + "grad_norm": 2.6912333965301514, + "learning_rate": 4.686346863468635e-06, + "loss": 1.7276, + "step": 392 + }, + { + "epoch": 0.4891101431238332, + "grad_norm": 2.8358330726623535, + "learning_rate": 4.6840405904059046e-06, + "loss": 1.6482, + "step": 393 + }, + { + "epoch": 0.4903546981953951, + "grad_norm": 2.6581575870513916, + "learning_rate": 4.681734317343174e-06, + "loss": 1.6853, + "step": 394 + }, + { + "epoch": 0.4915992532669571, + "grad_norm": 2.941425323486328, + "learning_rate": 4.6794280442804426e-06, + "loss": 1.6075, + "step": 395 + }, + { + "epoch": 0.492843808338519, + "grad_norm": 3.2334814071655273, + "learning_rate": 4.677121771217713e-06, + "loss": 1.6157, + "step": 396 + }, + { + "epoch": 0.4940883634100809, + "grad_norm": 2.984410524368286, + "learning_rate": 4.674815498154982e-06, + "loss": 1.6257, + "step": 397 + }, + { + "epoch": 0.4953329184816428, + "grad_norm": 3.0650789737701416, + "learning_rate": 4.672509225092252e-06, + "loss": 1.6573, + "step": 398 + }, + { + "epoch": 0.49657747355320475, + "grad_norm": 2.798664093017578, + "learning_rate": 4.67020295202952e-06, + "loss": 1.638, + "step": 399 + }, + { + "epoch": 0.49782202862476665, + "grad_norm": 2.8266706466674805, + "learning_rate": 4.66789667896679e-06, + "loss": 1.6484, + "step": 400 + }, + { + "epoch": 0.49782202862476665, + "eval_loss": 1.7714862823486328, + "eval_runtime": 48.0657, + "eval_samples_per_second": 20.805, + "eval_steps_per_second": 0.874, + "step": 400 + }, + { + "epoch": 0.49906658369632856, + "grad_norm": 2.9928388595581055, + "learning_rate": 4.66559040590406e-06, + "loss": 1.6721, + "step": 401 + }, + { + "epoch": 0.5003111387678905, + "grad_norm": 2.573514461517334, + "learning_rate": 4.6632841328413285e-06, + "loss": 1.679, + "step": 402 + }, + { + "epoch": 0.5015556938394524, + "grad_norm": 3.1736996173858643, + "learning_rate": 4.660977859778598e-06, + "loss": 1.7124, + "step": 403 + }, + { + "epoch": 0.5028002489110143, + "grad_norm": 2.858849287033081, + "learning_rate": 4.658671586715867e-06, + "loss": 1.6622, + "step": 404 + }, + { + "epoch": 0.5040448039825762, + "grad_norm": 2.825698137283325, + "learning_rate": 4.656365313653137e-06, + "loss": 1.6464, + "step": 405 + }, + { + "epoch": 0.5052893590541382, + "grad_norm": 3.783891439437866, + "learning_rate": 4.654059040590406e-06, + "loss": 1.6665, + "step": 406 + }, + { + "epoch": 0.5065339141257, + "grad_norm": 2.789813756942749, + "learning_rate": 4.651752767527676e-06, + "loss": 1.6064, + "step": 407 + }, + { + "epoch": 0.507778469197262, + "grad_norm": 3.3106348514556885, + "learning_rate": 4.649446494464945e-06, + "loss": 1.6542, + "step": 408 + }, + { + "epoch": 0.5090230242688238, + "grad_norm": 2.8805112838745117, + "learning_rate": 4.6471402214022145e-06, + "loss": 1.6058, + "step": 409 + }, + { + "epoch": 0.5102675793403858, + "grad_norm": 3.30898118019104, + "learning_rate": 4.644833948339484e-06, + "loss": 1.5933, + "step": 410 + }, + { + "epoch": 0.5102675793403858, + "eval_loss": 1.7787818908691406, + "eval_runtime": 44.0876, + "eval_samples_per_second": 22.682, + "eval_steps_per_second": 0.953, + "step": 410 + }, + { + "epoch": 0.5115121344119478, + "grad_norm": 3.3563625812530518, + "learning_rate": 4.642527675276753e-06, + "loss": 1.6751, + "step": 411 + }, + { + "epoch": 0.5127566894835096, + "grad_norm": 2.968120574951172, + "learning_rate": 4.640221402214023e-06, + "loss": 1.663, + "step": 412 + }, + { + "epoch": 0.5140012445550716, + "grad_norm": 3.310248374938965, + "learning_rate": 4.637915129151292e-06, + "loss": 1.658, + "step": 413 + }, + { + "epoch": 0.5152457996266335, + "grad_norm": 2.7011170387268066, + "learning_rate": 4.635608856088562e-06, + "loss": 1.6894, + "step": 414 + }, + { + "epoch": 0.5164903546981954, + "grad_norm": 2.804901361465454, + "learning_rate": 4.63330258302583e-06, + "loss": 1.602, + "step": 415 + }, + { + "epoch": 0.5177349097697573, + "grad_norm": 3.2412750720977783, + "learning_rate": 4.6309963099631e-06, + "loss": 1.6158, + "step": 416 + }, + { + "epoch": 0.5189794648413192, + "grad_norm": 2.699909210205078, + "learning_rate": 4.62869003690037e-06, + "loss": 1.5842, + "step": 417 + }, + { + "epoch": 0.5202240199128811, + "grad_norm": 3.065929412841797, + "learning_rate": 4.6263837638376384e-06, + "loss": 1.6434, + "step": 418 + }, + { + "epoch": 0.5214685749844431, + "grad_norm": 2.895305633544922, + "learning_rate": 4.624077490774908e-06, + "loss": 1.5969, + "step": 419 + }, + { + "epoch": 0.522713130056005, + "grad_norm": 3.1590664386749268, + "learning_rate": 4.621771217712177e-06, + "loss": 1.6211, + "step": 420 + }, + { + "epoch": 0.522713130056005, + "eval_loss": 1.7674189805984497, + "eval_runtime": 53.328, + "eval_samples_per_second": 18.752, + "eval_steps_per_second": 0.788, + "step": 420 + }, + { + "epoch": 0.5239576851275669, + "grad_norm": 2.7126312255859375, + "learning_rate": 4.619464944649447e-06, + "loss": 1.6246, + "step": 421 + }, + { + "epoch": 0.5252022401991289, + "grad_norm": 2.918508768081665, + "learning_rate": 4.617158671586716e-06, + "loss": 1.6937, + "step": 422 + }, + { + "epoch": 0.5264467952706907, + "grad_norm": 3.008610725402832, + "learning_rate": 4.6148523985239856e-06, + "loss": 1.6864, + "step": 423 + }, + { + "epoch": 0.5276913503422527, + "grad_norm": 2.6612043380737305, + "learning_rate": 4.612546125461255e-06, + "loss": 1.5785, + "step": 424 + }, + { + "epoch": 0.5289359054138145, + "grad_norm": 2.850679874420166, + "learning_rate": 4.610239852398524e-06, + "loss": 1.5899, + "step": 425 + }, + { + "epoch": 0.5301804604853765, + "grad_norm": 2.9205031394958496, + "learning_rate": 4.607933579335794e-06, + "loss": 1.5867, + "step": 426 + }, + { + "epoch": 0.5314250155569384, + "grad_norm": 2.929532527923584, + "learning_rate": 4.605627306273063e-06, + "loss": 1.6182, + "step": 427 + }, + { + "epoch": 0.5326695706285003, + "grad_norm": 2.874936103820801, + "learning_rate": 4.603321033210333e-06, + "loss": 1.5789, + "step": 428 + }, + { + "epoch": 0.5339141257000622, + "grad_norm": 2.8703525066375732, + "learning_rate": 4.601014760147602e-06, + "loss": 1.5859, + "step": 429 + }, + { + "epoch": 0.5351586807716241, + "grad_norm": 2.7076902389526367, + "learning_rate": 4.5987084870848715e-06, + "loss": 1.644, + "step": 430 + }, + { + "epoch": 0.5351586807716241, + "eval_loss": 1.7592095136642456, + "eval_runtime": 47.9438, + "eval_samples_per_second": 20.858, + "eval_steps_per_second": 0.876, + "step": 430 + }, + { + "epoch": 0.536403235843186, + "grad_norm": 2.839160680770874, + "learning_rate": 4.59640221402214e-06, + "loss": 1.6357, + "step": 431 + }, + { + "epoch": 0.537647790914748, + "grad_norm": 2.674025535583496, + "learning_rate": 4.5940959409594095e-06, + "loss": 1.676, + "step": 432 + }, + { + "epoch": 0.5388923459863099, + "grad_norm": 3.0619537830352783, + "learning_rate": 4.59178966789668e-06, + "loss": 1.5617, + "step": 433 + }, + { + "epoch": 0.5401369010578718, + "grad_norm": 3.058418035507202, + "learning_rate": 4.589483394833948e-06, + "loss": 1.5951, + "step": 434 + }, + { + "epoch": 0.5413814561294338, + "grad_norm": 3.0046396255493164, + "learning_rate": 4.587177121771218e-06, + "loss": 1.6473, + "step": 435 + }, + { + "epoch": 0.5426260112009956, + "grad_norm": 2.7736752033233643, + "learning_rate": 4.584870848708487e-06, + "loss": 1.6262, + "step": 436 + }, + { + "epoch": 0.5438705662725576, + "grad_norm": 2.7804994583129883, + "learning_rate": 4.5825645756457575e-06, + "loss": 1.6886, + "step": 437 + }, + { + "epoch": 0.5451151213441194, + "grad_norm": 3.0717954635620117, + "learning_rate": 4.580258302583026e-06, + "loss": 1.6287, + "step": 438 + }, + { + "epoch": 0.5463596764156814, + "grad_norm": 2.800243854522705, + "learning_rate": 4.5779520295202955e-06, + "loss": 1.6169, + "step": 439 + }, + { + "epoch": 0.5476042314872434, + "grad_norm": 2.9581644535064697, + "learning_rate": 4.575645756457565e-06, + "loss": 1.6617, + "step": 440 + }, + { + "epoch": 0.5476042314872434, + "eval_loss": 1.7642868757247925, + "eval_runtime": 51.2646, + "eval_samples_per_second": 19.507, + "eval_steps_per_second": 0.819, + "step": 440 + }, + { + "epoch": 0.5488487865588052, + "grad_norm": 3.0067460536956787, + "learning_rate": 4.573339483394834e-06, + "loss": 1.6482, + "step": 441 + }, + { + "epoch": 0.5500933416303672, + "grad_norm": 2.9075019359588623, + "learning_rate": 4.571033210332104e-06, + "loss": 1.6595, + "step": 442 + }, + { + "epoch": 0.551337896701929, + "grad_norm": 2.7788755893707275, + "learning_rate": 4.568726937269373e-06, + "loss": 1.5733, + "step": 443 + }, + { + "epoch": 0.552582451773491, + "grad_norm": 2.7775425910949707, + "learning_rate": 4.566420664206643e-06, + "loss": 1.6925, + "step": 444 + }, + { + "epoch": 0.5538270068450529, + "grad_norm": 2.6949567794799805, + "learning_rate": 4.564114391143912e-06, + "loss": 1.5738, + "step": 445 + }, + { + "epoch": 0.5550715619166148, + "grad_norm": 2.7093007564544678, + "learning_rate": 4.5618081180811814e-06, + "loss": 1.5848, + "step": 446 + }, + { + "epoch": 0.5563161169881767, + "grad_norm": 2.6920173168182373, + "learning_rate": 4.559501845018451e-06, + "loss": 1.6092, + "step": 447 + }, + { + "epoch": 0.5575606720597387, + "grad_norm": 2.733731985092163, + "learning_rate": 4.5571955719557194e-06, + "loss": 1.6056, + "step": 448 + }, + { + "epoch": 0.5588052271313005, + "grad_norm": 2.9156274795532227, + "learning_rate": 4.55488929889299e-06, + "loss": 1.6436, + "step": 449 + }, + { + "epoch": 0.5600497822028625, + "grad_norm": 2.8802616596221924, + "learning_rate": 4.552583025830259e-06, + "loss": 1.6309, + "step": 450 + }, + { + "epoch": 0.5600497822028625, + "eval_loss": 1.7486340999603271, + "eval_runtime": 47.2219, + "eval_samples_per_second": 21.177, + "eval_steps_per_second": 0.889, + "step": 450 + }, + { + "epoch": 0.5612943372744243, + "grad_norm": 2.729743242263794, + "learning_rate": 4.550276752767528e-06, + "loss": 1.7171, + "step": 451 + }, + { + "epoch": 0.5625388923459863, + "grad_norm": 2.800049066543579, + "learning_rate": 4.547970479704797e-06, + "loss": 1.6098, + "step": 452 + }, + { + "epoch": 0.5637834474175483, + "grad_norm": 2.9659311771392822, + "learning_rate": 4.5456642066420666e-06, + "loss": 1.6646, + "step": 453 + }, + { + "epoch": 0.5650280024891101, + "grad_norm": 2.5417754650115967, + "learning_rate": 4.543357933579336e-06, + "loss": 1.5847, + "step": 454 + }, + { + "epoch": 0.5662725575606721, + "grad_norm": 2.8471338748931885, + "learning_rate": 4.541051660516605e-06, + "loss": 1.6235, + "step": 455 + }, + { + "epoch": 0.5675171126322339, + "grad_norm": 2.7039637565612793, + "learning_rate": 4.538745387453875e-06, + "loss": 1.5613, + "step": 456 + }, + { + "epoch": 0.5687616677037959, + "grad_norm": 3.0383341312408447, + "learning_rate": 4.536439114391144e-06, + "loss": 1.6485, + "step": 457 + }, + { + "epoch": 0.5700062227753578, + "grad_norm": 2.528388261795044, + "learning_rate": 4.534132841328414e-06, + "loss": 1.6117, + "step": 458 + }, + { + "epoch": 0.5712507778469197, + "grad_norm": 2.7623119354248047, + "learning_rate": 4.531826568265683e-06, + "loss": 1.6017, + "step": 459 + }, + { + "epoch": 0.5724953329184816, + "grad_norm": 2.9213945865631104, + "learning_rate": 4.5295202952029525e-06, + "loss": 1.643, + "step": 460 + }, + { + "epoch": 0.5724953329184816, + "eval_loss": 1.747575044631958, + "eval_runtime": 45.2844, + "eval_samples_per_second": 22.083, + "eval_steps_per_second": 0.927, + "step": 460 + }, + { + "epoch": 0.5737398879900436, + "grad_norm": 2.664275884628296, + "learning_rate": 4.527214022140222e-06, + "loss": 1.6154, + "step": 461 + }, + { + "epoch": 0.5749844430616055, + "grad_norm": 2.5891788005828857, + "learning_rate": 4.524907749077491e-06, + "loss": 1.5975, + "step": 462 + }, + { + "epoch": 0.5762289981331674, + "grad_norm": 2.8126487731933594, + "learning_rate": 4.522601476014761e-06, + "loss": 1.6306, + "step": 463 + }, + { + "epoch": 0.5774735532047293, + "grad_norm": 2.799391031265259, + "learning_rate": 4.520295202952029e-06, + "loss": 1.6139, + "step": 464 + }, + { + "epoch": 0.5787181082762912, + "grad_norm": 2.7323718070983887, + "learning_rate": 4.5179889298893e-06, + "loss": 1.632, + "step": 465 + }, + { + "epoch": 0.5799626633478532, + "grad_norm": 2.601386785507202, + "learning_rate": 4.515682656826569e-06, + "loss": 1.6139, + "step": 466 + }, + { + "epoch": 0.581207218419415, + "grad_norm": 2.562162399291992, + "learning_rate": 4.513376383763838e-06, + "loss": 1.5168, + "step": 467 + }, + { + "epoch": 0.582451773490977, + "grad_norm": 2.6912002563476562, + "learning_rate": 4.511070110701107e-06, + "loss": 1.6249, + "step": 468 + }, + { + "epoch": 0.583696328562539, + "grad_norm": 2.6971495151519775, + "learning_rate": 4.5087638376383765e-06, + "loss": 1.616, + "step": 469 + }, + { + "epoch": 0.5849408836341008, + "grad_norm": 2.6898910999298096, + "learning_rate": 4.506457564575646e-06, + "loss": 1.614, + "step": 470 + }, + { + "epoch": 0.5849408836341008, + "eval_loss": 1.7460769414901733, + "eval_runtime": 45.3055, + "eval_samples_per_second": 22.072, + "eval_steps_per_second": 0.927, + "step": 470 + }, + { + "epoch": 0.5861854387056628, + "grad_norm": 2.698180675506592, + "learning_rate": 4.504151291512915e-06, + "loss": 1.5993, + "step": 471 + }, + { + "epoch": 0.5874299937772246, + "grad_norm": 2.7125210762023926, + "learning_rate": 4.501845018450185e-06, + "loss": 1.6113, + "step": 472 + }, + { + "epoch": 0.5886745488487866, + "grad_norm": 2.754445791244507, + "learning_rate": 4.499538745387454e-06, + "loss": 1.6719, + "step": 473 + }, + { + "epoch": 0.5899191039203485, + "grad_norm": 2.6979362964630127, + "learning_rate": 4.497232472324724e-06, + "loss": 1.608, + "step": 474 + }, + { + "epoch": 0.5911636589919104, + "grad_norm": 2.7694528102874756, + "learning_rate": 4.494926199261993e-06, + "loss": 1.5695, + "step": 475 + }, + { + "epoch": 0.5924082140634723, + "grad_norm": 2.653353214263916, + "learning_rate": 4.4926199261992624e-06, + "loss": 1.5523, + "step": 476 + }, + { + "epoch": 0.5936527691350342, + "grad_norm": 2.6632070541381836, + "learning_rate": 4.490313653136532e-06, + "loss": 1.5675, + "step": 477 + }, + { + "epoch": 0.5948973242065961, + "grad_norm": 3.038543939590454, + "learning_rate": 4.488007380073801e-06, + "loss": 1.5692, + "step": 478 + }, + { + "epoch": 0.5961418792781581, + "grad_norm": 2.8123953342437744, + "learning_rate": 4.485701107011071e-06, + "loss": 1.6223, + "step": 479 + }, + { + "epoch": 0.5973864343497199, + "grad_norm": 2.5845773220062256, + "learning_rate": 4.483394833948339e-06, + "loss": 1.5773, + "step": 480 + }, + { + "epoch": 0.5973864343497199, + "eval_loss": 1.7419319152832031, + "eval_runtime": 44.6842, + "eval_samples_per_second": 22.379, + "eval_steps_per_second": 0.94, + "step": 480 + }, + { + "epoch": 0.5986309894212819, + "grad_norm": 3.069936752319336, + "learning_rate": 4.4810885608856096e-06, + "loss": 1.6323, + "step": 481 + }, + { + "epoch": 0.5998755444928439, + "grad_norm": 2.9085781574249268, + "learning_rate": 4.478782287822879e-06, + "loss": 1.5845, + "step": 482 + }, + { + "epoch": 0.6011200995644057, + "grad_norm": 2.8568010330200195, + "learning_rate": 4.4764760147601476e-06, + "loss": 1.5898, + "step": 483 + }, + { + "epoch": 0.6023646546359677, + "grad_norm": 3.089081048965454, + "learning_rate": 4.474169741697417e-06, + "loss": 1.6622, + "step": 484 + }, + { + "epoch": 0.6036092097075295, + "grad_norm": 2.8515470027923584, + "learning_rate": 4.471863468634686e-06, + "loss": 1.5859, + "step": 485 + }, + { + "epoch": 0.6048537647790915, + "grad_norm": 2.647765874862671, + "learning_rate": 4.469557195571957e-06, + "loss": 1.5926, + "step": 486 + }, + { + "epoch": 0.6060983198506534, + "grad_norm": 2.61676025390625, + "learning_rate": 4.467250922509225e-06, + "loss": 1.5684, + "step": 487 + }, + { + "epoch": 0.6073428749222153, + "grad_norm": 2.7127039432525635, + "learning_rate": 4.464944649446495e-06, + "loss": 1.5702, + "step": 488 + }, + { + "epoch": 0.6085874299937772, + "grad_norm": 2.9932055473327637, + "learning_rate": 4.462638376383764e-06, + "loss": 1.5976, + "step": 489 + }, + { + "epoch": 0.6098319850653391, + "grad_norm": 2.6508774757385254, + "learning_rate": 4.4603321033210335e-06, + "loss": 1.586, + "step": 490 + }, + { + "epoch": 0.6098319850653391, + "eval_loss": 1.7357326745986938, + "eval_runtime": 45.393, + "eval_samples_per_second": 22.03, + "eval_steps_per_second": 0.925, + "step": 490 + }, + { + "epoch": 0.611076540136901, + "grad_norm": 2.7626044750213623, + "learning_rate": 4.458025830258303e-06, + "loss": 1.5773, + "step": 491 + }, + { + "epoch": 0.612321095208463, + "grad_norm": 2.595003604888916, + "learning_rate": 4.455719557195572e-06, + "loss": 1.601, + "step": 492 + }, + { + "epoch": 0.6135656502800249, + "grad_norm": 2.595767021179199, + "learning_rate": 4.453413284132842e-06, + "loss": 1.6287, + "step": 493 + }, + { + "epoch": 0.6148102053515868, + "grad_norm": 2.755845308303833, + "learning_rate": 4.451107011070111e-06, + "loss": 1.593, + "step": 494 + }, + { + "epoch": 0.6160547604231488, + "grad_norm": 2.7302653789520264, + "learning_rate": 4.448800738007381e-06, + "loss": 1.5768, + "step": 495 + }, + { + "epoch": 0.6172993154947106, + "grad_norm": 2.6493024826049805, + "learning_rate": 4.446494464944649e-06, + "loss": 1.6059, + "step": 496 + }, + { + "epoch": 0.6185438705662726, + "grad_norm": 2.8633735179901123, + "learning_rate": 4.4441881918819195e-06, + "loss": 1.598, + "step": 497 + }, + { + "epoch": 0.6197884256378344, + "grad_norm": 2.65639066696167, + "learning_rate": 4.441881918819189e-06, + "loss": 1.6036, + "step": 498 + }, + { + "epoch": 0.6210329807093964, + "grad_norm": 2.6867759227752686, + "learning_rate": 4.439575645756458e-06, + "loss": 1.5676, + "step": 499 + }, + { + "epoch": 0.6222775357809583, + "grad_norm": 2.7986443042755127, + "learning_rate": 4.437269372693727e-06, + "loss": 1.524, + "step": 500 + }, + { + "epoch": 0.6222775357809583, + "eval_loss": 1.7449125051498413, + "eval_runtime": 43.9329, + "eval_samples_per_second": 22.762, + "eval_steps_per_second": 0.956, + "step": 500 + }, + { + "epoch": 0.6235220908525202, + "grad_norm": 2.665905714035034, + "learning_rate": 4.434963099630996e-06, + "loss": 1.5635, + "step": 501 + }, + { + "epoch": 0.6247666459240822, + "grad_norm": 2.902435541152954, + "learning_rate": 4.432656826568267e-06, + "loss": 1.6377, + "step": 502 + }, + { + "epoch": 0.6260112009956441, + "grad_norm": 2.6872262954711914, + "learning_rate": 4.430350553505535e-06, + "loss": 1.5553, + "step": 503 + }, + { + "epoch": 0.627255756067206, + "grad_norm": 2.676621913909912, + "learning_rate": 4.428044280442805e-06, + "loss": 1.6101, + "step": 504 + }, + { + "epoch": 0.6285003111387679, + "grad_norm": 2.756347179412842, + "learning_rate": 4.425738007380074e-06, + "loss": 1.5884, + "step": 505 + }, + { + "epoch": 0.6297448662103298, + "grad_norm": 2.856882333755493, + "learning_rate": 4.4234317343173434e-06, + "loss": 1.5635, + "step": 506 + }, + { + "epoch": 0.6309894212818917, + "grad_norm": 2.7708330154418945, + "learning_rate": 4.421125461254613e-06, + "loss": 1.6028, + "step": 507 + }, + { + "epoch": 0.6322339763534537, + "grad_norm": 2.8167600631713867, + "learning_rate": 4.418819188191882e-06, + "loss": 1.6026, + "step": 508 + }, + { + "epoch": 0.6334785314250155, + "grad_norm": 2.54194974899292, + "learning_rate": 4.416512915129152e-06, + "loss": 1.6001, + "step": 509 + }, + { + "epoch": 0.6347230864965775, + "grad_norm": 2.683037519454956, + "learning_rate": 4.414206642066421e-06, + "loss": 1.6084, + "step": 510 + }, + { + "epoch": 0.6347230864965775, + "eval_loss": 1.7376186847686768, + "eval_runtime": 42.3026, + "eval_samples_per_second": 23.639, + "eval_steps_per_second": 0.993, + "step": 510 + }, + { + "epoch": 0.6359676415681393, + "grad_norm": 2.82627010345459, + "learning_rate": 4.4119003690036905e-06, + "loss": 1.6258, + "step": 511 + }, + { + "epoch": 0.6372121966397013, + "grad_norm": 2.6139848232269287, + "learning_rate": 4.40959409594096e-06, + "loss": 1.6029, + "step": 512 + }, + { + "epoch": 0.6384567517112633, + "grad_norm": 2.646712303161621, + "learning_rate": 4.407287822878229e-06, + "loss": 1.5602, + "step": 513 + }, + { + "epoch": 0.6397013067828251, + "grad_norm": 2.910935163497925, + "learning_rate": 4.404981549815499e-06, + "loss": 1.5757, + "step": 514 + }, + { + "epoch": 0.6409458618543871, + "grad_norm": 2.6001148223876953, + "learning_rate": 4.402675276752768e-06, + "loss": 1.5857, + "step": 515 + }, + { + "epoch": 0.642190416925949, + "grad_norm": 2.8525235652923584, + "learning_rate": 4.400369003690037e-06, + "loss": 1.6092, + "step": 516 + }, + { + "epoch": 0.6434349719975109, + "grad_norm": 2.5335488319396973, + "learning_rate": 4.398062730627306e-06, + "loss": 1.5849, + "step": 517 + }, + { + "epoch": 0.6446795270690728, + "grad_norm": 2.5787103176116943, + "learning_rate": 4.3957564575645765e-06, + "loss": 1.634, + "step": 518 + }, + { + "epoch": 0.6459240821406347, + "grad_norm": 2.6188197135925293, + "learning_rate": 4.393450184501845e-06, + "loss": 1.5477, + "step": 519 + }, + { + "epoch": 0.6471686372121966, + "grad_norm": 2.6548666954040527, + "learning_rate": 4.3911439114391145e-06, + "loss": 1.6133, + "step": 520 + }, + { + "epoch": 0.6471686372121966, + "eval_loss": 1.7393039464950562, + "eval_runtime": 45.9234, + "eval_samples_per_second": 21.775, + "eval_steps_per_second": 0.915, + "step": 520 + }, + { + "epoch": 0.6484131922837586, + "grad_norm": 2.907928228378296, + "learning_rate": 4.388837638376384e-06, + "loss": 1.587, + "step": 521 + }, + { + "epoch": 0.6496577473553204, + "grad_norm": 2.7899692058563232, + "learning_rate": 4.386531365313653e-06, + "loss": 1.6126, + "step": 522 + }, + { + "epoch": 0.6509023024268824, + "grad_norm": 2.680147886276245, + "learning_rate": 4.384225092250923e-06, + "loss": 1.565, + "step": 523 + }, + { + "epoch": 0.6521468574984443, + "grad_norm": 2.6590754985809326, + "learning_rate": 4.381918819188192e-06, + "loss": 1.6127, + "step": 524 + }, + { + "epoch": 0.6533914125700062, + "grad_norm": 2.6638906002044678, + "learning_rate": 4.379612546125462e-06, + "loss": 1.5615, + "step": 525 + }, + { + "epoch": 0.6546359676415682, + "grad_norm": 2.6518194675445557, + "learning_rate": 4.377306273062731e-06, + "loss": 1.6202, + "step": 526 + }, + { + "epoch": 0.65588052271313, + "grad_norm": 2.5323619842529297, + "learning_rate": 4.3750000000000005e-06, + "loss": 1.5305, + "step": 527 + }, + { + "epoch": 0.657125077784692, + "grad_norm": 2.6871442794799805, + "learning_rate": 4.37269372693727e-06, + "loss": 1.6043, + "step": 528 + }, + { + "epoch": 0.6583696328562539, + "grad_norm": 2.760746717453003, + "learning_rate": 4.370387453874539e-06, + "loss": 1.6059, + "step": 529 + }, + { + "epoch": 0.6596141879278158, + "grad_norm": 2.780482769012451, + "learning_rate": 4.368081180811809e-06, + "loss": 1.6082, + "step": 530 + }, + { + "epoch": 0.6596141879278158, + "eval_loss": 1.7219713926315308, + "eval_runtime": 51.8478, + "eval_samples_per_second": 19.287, + "eval_steps_per_second": 0.81, + "step": 530 + }, + { + "epoch": 0.6608587429993777, + "grad_norm": 2.605890989303589, + "learning_rate": 4.365774907749078e-06, + "loss": 1.5554, + "step": 531 + }, + { + "epoch": 0.6621032980709396, + "grad_norm": 2.731555461883545, + "learning_rate": 4.363468634686347e-06, + "loss": 1.606, + "step": 532 + }, + { + "epoch": 0.6633478531425016, + "grad_norm": 2.8356943130493164, + "learning_rate": 4.361162361623616e-06, + "loss": 1.5601, + "step": 533 + }, + { + "epoch": 0.6645924082140635, + "grad_norm": 2.7196593284606934, + "learning_rate": 4.3588560885608864e-06, + "loss": 1.5722, + "step": 534 + }, + { + "epoch": 0.6658369632856254, + "grad_norm": 2.621371269226074, + "learning_rate": 4.356549815498156e-06, + "loss": 1.6194, + "step": 535 + }, + { + "epoch": 0.6670815183571873, + "grad_norm": 2.653916120529175, + "learning_rate": 4.354243542435424e-06, + "loss": 1.5592, + "step": 536 + }, + { + "epoch": 0.6683260734287492, + "grad_norm": 2.88431453704834, + "learning_rate": 4.351937269372694e-06, + "loss": 1.6296, + "step": 537 + }, + { + "epoch": 0.6695706285003111, + "grad_norm": 2.667130470275879, + "learning_rate": 4.349630996309963e-06, + "loss": 1.5624, + "step": 538 + }, + { + "epoch": 0.6708151835718731, + "grad_norm": 2.6453566551208496, + "learning_rate": 4.347324723247233e-06, + "loss": 1.6025, + "step": 539 + }, + { + "epoch": 0.6720597386434349, + "grad_norm": 3.032271146774292, + "learning_rate": 4.345018450184502e-06, + "loss": 1.6771, + "step": 540 + }, + { + "epoch": 0.6720597386434349, + "eval_loss": 1.721895456314087, + "eval_runtime": 50.62, + "eval_samples_per_second": 19.755, + "eval_steps_per_second": 0.83, + "step": 540 + }, + { + "epoch": 0.6733042937149969, + "grad_norm": 2.8476362228393555, + "learning_rate": 4.3427121771217715e-06, + "loss": 1.5663, + "step": 541 + }, + { + "epoch": 0.6745488487865589, + "grad_norm": 2.615602970123291, + "learning_rate": 4.340405904059041e-06, + "loss": 1.5815, + "step": 542 + }, + { + "epoch": 0.6757934038581207, + "grad_norm": 3.221571683883667, + "learning_rate": 4.33809963099631e-06, + "loss": 1.5955, + "step": 543 + }, + { + "epoch": 0.6770379589296827, + "grad_norm": 2.602994203567505, + "learning_rate": 4.33579335793358e-06, + "loss": 1.5667, + "step": 544 + }, + { + "epoch": 0.6782825140012445, + "grad_norm": 2.9380433559417725, + "learning_rate": 4.333487084870848e-06, + "loss": 1.5651, + "step": 545 + }, + { + "epoch": 0.6795270690728065, + "grad_norm": 3.0689425468444824, + "learning_rate": 4.331180811808119e-06, + "loss": 1.6003, + "step": 546 + }, + { + "epoch": 0.6807716241443684, + "grad_norm": 2.770317316055298, + "learning_rate": 4.328874538745388e-06, + "loss": 1.5576, + "step": 547 + }, + { + "epoch": 0.6820161792159303, + "grad_norm": 2.8631815910339355, + "learning_rate": 4.3265682656826575e-06, + "loss": 1.5308, + "step": 548 + }, + { + "epoch": 0.6832607342874922, + "grad_norm": 2.785576820373535, + "learning_rate": 4.324261992619926e-06, + "loss": 1.6113, + "step": 549 + }, + { + "epoch": 0.6845052893590542, + "grad_norm": 2.723919630050659, + "learning_rate": 4.321955719557196e-06, + "loss": 1.5606, + "step": 550 + }, + { + "epoch": 0.6845052893590542, + "eval_loss": 1.721803069114685, + "eval_runtime": 51.6036, + "eval_samples_per_second": 19.378, + "eval_steps_per_second": 0.814, + "step": 550 + }, + { + "epoch": 0.685749844430616, + "grad_norm": 3.081198215484619, + "learning_rate": 4.319649446494466e-06, + "loss": 1.5643, + "step": 551 + }, + { + "epoch": 0.686994399502178, + "grad_norm": 2.757392168045044, + "learning_rate": 4.317343173431734e-06, + "loss": 1.5909, + "step": 552 + }, + { + "epoch": 0.6882389545737398, + "grad_norm": 3.038753032684326, + "learning_rate": 4.315036900369004e-06, + "loss": 1.6023, + "step": 553 + }, + { + "epoch": 0.6894835096453018, + "grad_norm": 2.8246452808380127, + "learning_rate": 4.312730627306273e-06, + "loss": 1.5548, + "step": 554 + }, + { + "epoch": 0.6907280647168638, + "grad_norm": 2.589320182800293, + "learning_rate": 4.310424354243543e-06, + "loss": 1.5424, + "step": 555 + }, + { + "epoch": 0.6919726197884256, + "grad_norm": 2.6413373947143555, + "learning_rate": 4.308118081180812e-06, + "loss": 1.6243, + "step": 556 + }, + { + "epoch": 0.6932171748599876, + "grad_norm": 2.764784336090088, + "learning_rate": 4.3058118081180815e-06, + "loss": 1.589, + "step": 557 + }, + { + "epoch": 0.6944617299315494, + "grad_norm": 2.806821823120117, + "learning_rate": 4.303505535055351e-06, + "loss": 1.621, + "step": 558 + }, + { + "epoch": 0.6957062850031114, + "grad_norm": 2.718017578125, + "learning_rate": 4.30119926199262e-06, + "loss": 1.5773, + "step": 559 + }, + { + "epoch": 0.6969508400746733, + "grad_norm": 2.7620160579681396, + "learning_rate": 4.29889298892989e-06, + "loss": 1.5868, + "step": 560 + }, + { + "epoch": 0.6969508400746733, + "eval_loss": 1.7152249813079834, + "eval_runtime": 52.0463, + "eval_samples_per_second": 19.214, + "eval_steps_per_second": 0.807, + "step": 560 + }, + { + "epoch": 0.6981953951462352, + "grad_norm": 2.66686749458313, + "learning_rate": 4.296586715867159e-06, + "loss": 1.534, + "step": 561 + }, + { + "epoch": 0.6994399502177971, + "grad_norm": 2.812577247619629, + "learning_rate": 4.2942804428044286e-06, + "loss": 1.5501, + "step": 562 + }, + { + "epoch": 0.7006845052893591, + "grad_norm": 2.578508138656616, + "learning_rate": 4.291974169741698e-06, + "loss": 1.6286, + "step": 563 + }, + { + "epoch": 0.701929060360921, + "grad_norm": 2.524249315261841, + "learning_rate": 4.289667896678967e-06, + "loss": 1.5719, + "step": 564 + }, + { + "epoch": 0.7031736154324829, + "grad_norm": 2.827235460281372, + "learning_rate": 4.287361623616236e-06, + "loss": 1.5333, + "step": 565 + }, + { + "epoch": 0.7044181705040448, + "grad_norm": 2.6359963417053223, + "learning_rate": 4.285055350553506e-06, + "loss": 1.5849, + "step": 566 + }, + { + "epoch": 0.7056627255756067, + "grad_norm": 2.930530071258545, + "learning_rate": 4.282749077490776e-06, + "loss": 1.5672, + "step": 567 + }, + { + "epoch": 0.7069072806471687, + "grad_norm": 2.750102996826172, + "learning_rate": 4.280442804428044e-06, + "loss": 1.5696, + "step": 568 + }, + { + "epoch": 0.7081518357187305, + "grad_norm": 2.869690418243408, + "learning_rate": 4.278136531365314e-06, + "loss": 1.5689, + "step": 569 + }, + { + "epoch": 0.7093963907902925, + "grad_norm": 2.954852819442749, + "learning_rate": 4.275830258302583e-06, + "loss": 1.5931, + "step": 570 + }, + { + "epoch": 0.7093963907902925, + "eval_loss": 1.7143300771713257, + "eval_runtime": 52.7293, + "eval_samples_per_second": 18.965, + "eval_steps_per_second": 0.797, + "step": 570 + }, + { + "epoch": 0.7106409458618543, + "grad_norm": 2.702223539352417, + "learning_rate": 4.273523985239853e-06, + "loss": 1.5635, + "step": 571 + }, + { + "epoch": 0.7118855009334163, + "grad_norm": 2.689995050430298, + "learning_rate": 4.271217712177122e-06, + "loss": 1.5545, + "step": 572 + }, + { + "epoch": 0.7131300560049783, + "grad_norm": 2.68979811668396, + "learning_rate": 4.268911439114391e-06, + "loss": 1.5404, + "step": 573 + }, + { + "epoch": 0.7143746110765401, + "grad_norm": 2.7477986812591553, + "learning_rate": 4.266605166051661e-06, + "loss": 1.5719, + "step": 574 + }, + { + "epoch": 0.7156191661481021, + "grad_norm": 2.975778341293335, + "learning_rate": 4.26429889298893e-06, + "loss": 1.5587, + "step": 575 + }, + { + "epoch": 0.716863721219664, + "grad_norm": 2.658170700073242, + "learning_rate": 4.2619926199262e-06, + "loss": 1.5746, + "step": 576 + }, + { + "epoch": 0.7181082762912259, + "grad_norm": 2.8825011253356934, + "learning_rate": 4.259686346863469e-06, + "loss": 1.5246, + "step": 577 + }, + { + "epoch": 0.7193528313627878, + "grad_norm": 2.845280170440674, + "learning_rate": 4.2573800738007385e-06, + "loss": 1.5703, + "step": 578 + }, + { + "epoch": 0.7205973864343497, + "grad_norm": 2.660616636276245, + "learning_rate": 4.255073800738008e-06, + "loss": 1.5373, + "step": 579 + }, + { + "epoch": 0.7218419415059116, + "grad_norm": 2.749447822570801, + "learning_rate": 4.252767527675277e-06, + "loss": 1.59, + "step": 580 + }, + { + "epoch": 0.7218419415059116, + "eval_loss": 1.7149444818496704, + "eval_runtime": 52.2099, + "eval_samples_per_second": 19.153, + "eval_steps_per_second": 0.804, + "step": 580 + }, + { + "epoch": 0.7230864965774736, + "grad_norm": 2.813328742980957, + "learning_rate": 4.250461254612546e-06, + "loss": 1.5694, + "step": 581 + }, + { + "epoch": 0.7243310516490354, + "grad_norm": 2.823866844177246, + "learning_rate": 4.248154981549816e-06, + "loss": 1.5143, + "step": 582 + }, + { + "epoch": 0.7255756067205974, + "grad_norm": 2.71337890625, + "learning_rate": 4.245848708487086e-06, + "loss": 1.5915, + "step": 583 + }, + { + "epoch": 0.7268201617921594, + "grad_norm": 2.718085765838623, + "learning_rate": 4.243542435424355e-06, + "loss": 1.604, + "step": 584 + }, + { + "epoch": 0.7280647168637212, + "grad_norm": 2.5741796493530273, + "learning_rate": 4.241236162361624e-06, + "loss": 1.5398, + "step": 585 + }, + { + "epoch": 0.7293092719352832, + "grad_norm": 2.727114200592041, + "learning_rate": 4.238929889298893e-06, + "loss": 1.5886, + "step": 586 + }, + { + "epoch": 0.730553827006845, + "grad_norm": 2.6703338623046875, + "learning_rate": 4.236623616236163e-06, + "loss": 1.5959, + "step": 587 + }, + { + "epoch": 0.731798382078407, + "grad_norm": 2.750326633453369, + "learning_rate": 4.234317343173432e-06, + "loss": 1.5916, + "step": 588 + }, + { + "epoch": 0.7330429371499689, + "grad_norm": 2.6680715084075928, + "learning_rate": 4.232011070110701e-06, + "loss": 1.562, + "step": 589 + }, + { + "epoch": 0.7342874922215308, + "grad_norm": 2.6603832244873047, + "learning_rate": 4.229704797047971e-06, + "loss": 1.6121, + "step": 590 + }, + { + "epoch": 0.7342874922215308, + "eval_loss": 1.7144734859466553, + "eval_runtime": 50.5317, + "eval_samples_per_second": 19.79, + "eval_steps_per_second": 0.831, + "step": 590 + }, + { + "epoch": 0.7355320472930927, + "grad_norm": 2.7868523597717285, + "learning_rate": 4.22739852398524e-06, + "loss": 1.5488, + "step": 591 + }, + { + "epoch": 0.7367766023646546, + "grad_norm": 2.755270004272461, + "learning_rate": 4.2250922509225096e-06, + "loss": 1.5801, + "step": 592 + }, + { + "epoch": 0.7380211574362165, + "grad_norm": 2.7614126205444336, + "learning_rate": 4.222785977859779e-06, + "loss": 1.5507, + "step": 593 + }, + { + "epoch": 0.7392657125077785, + "grad_norm": 2.8674862384796143, + "learning_rate": 4.220479704797048e-06, + "loss": 1.5807, + "step": 594 + }, + { + "epoch": 0.7405102675793404, + "grad_norm": 2.7366859912872314, + "learning_rate": 4.218173431734318e-06, + "loss": 1.6594, + "step": 595 + }, + { + "epoch": 0.7417548226509023, + "grad_norm": 2.6231045722961426, + "learning_rate": 4.215867158671587e-06, + "loss": 1.5609, + "step": 596 + }, + { + "epoch": 0.7429993777224643, + "grad_norm": 2.7694146633148193, + "learning_rate": 4.213560885608857e-06, + "loss": 1.5889, + "step": 597 + }, + { + "epoch": 0.7442439327940261, + "grad_norm": 2.6275386810302734, + "learning_rate": 4.211254612546125e-06, + "loss": 1.575, + "step": 598 + }, + { + "epoch": 0.7454884878655881, + "grad_norm": 2.9413866996765137, + "learning_rate": 4.2089483394833955e-06, + "loss": 1.589, + "step": 599 + }, + { + "epoch": 0.7467330429371499, + "grad_norm": 2.768606185913086, + "learning_rate": 4.206642066420665e-06, + "loss": 1.5738, + "step": 600 + }, + { + "epoch": 0.7467330429371499, + "eval_loss": 1.7148027420043945, + "eval_runtime": 54.7031, + "eval_samples_per_second": 18.28, + "eval_steps_per_second": 0.768, + "step": 600 + }, + { + "epoch": 0.7479775980087119, + "grad_norm": 2.7917706966400146, + "learning_rate": 4.2043357933579335e-06, + "loss": 1.499, + "step": 601 + }, + { + "epoch": 0.7492221530802738, + "grad_norm": 2.5982208251953125, + "learning_rate": 4.202029520295203e-06, + "loss": 1.5113, + "step": 602 + }, + { + "epoch": 0.7504667081518357, + "grad_norm": 2.692032814025879, + "learning_rate": 4.199723247232473e-06, + "loss": 1.4721, + "step": 603 + }, + { + "epoch": 0.7517112632233977, + "grad_norm": 2.975860834121704, + "learning_rate": 4.197416974169742e-06, + "loss": 1.5147, + "step": 604 + }, + { + "epoch": 0.7529558182949595, + "grad_norm": 2.758610248565674, + "learning_rate": 4.195110701107011e-06, + "loss": 1.5674, + "step": 605 + }, + { + "epoch": 0.7542003733665215, + "grad_norm": 2.7620184421539307, + "learning_rate": 4.192804428044281e-06, + "loss": 1.5669, + "step": 606 + }, + { + "epoch": 0.7554449284380834, + "grad_norm": 3.043940305709839, + "learning_rate": 4.19049815498155e-06, + "loss": 1.587, + "step": 607 + }, + { + "epoch": 0.7566894835096453, + "grad_norm": 2.7482151985168457, + "learning_rate": 4.1881918819188195e-06, + "loss": 1.5451, + "step": 608 + }, + { + "epoch": 0.7579340385812072, + "grad_norm": 2.7259294986724854, + "learning_rate": 4.185885608856089e-06, + "loss": 1.5295, + "step": 609 + }, + { + "epoch": 0.7591785936527692, + "grad_norm": 2.6340832710266113, + "learning_rate": 4.183579335793358e-06, + "loss": 1.5335, + "step": 610 + }, + { + "epoch": 0.7591785936527692, + "eval_loss": 1.7038393020629883, + "eval_runtime": 44.0792, + "eval_samples_per_second": 22.686, + "eval_steps_per_second": 0.953, + "step": 610 + }, + { + "epoch": 0.760423148724331, + "grad_norm": 2.6144909858703613, + "learning_rate": 4.181273062730628e-06, + "loss": 1.5907, + "step": 611 + }, + { + "epoch": 0.761667703795893, + "grad_norm": 2.7754175662994385, + "learning_rate": 4.178966789667897e-06, + "loss": 1.5773, + "step": 612 + }, + { + "epoch": 0.7629122588674548, + "grad_norm": 2.6313252449035645, + "learning_rate": 4.176660516605167e-06, + "loss": 1.5675, + "step": 613 + }, + { + "epoch": 0.7641568139390168, + "grad_norm": 2.549074649810791, + "learning_rate": 4.174354243542435e-06, + "loss": 1.595, + "step": 614 + }, + { + "epoch": 0.7654013690105788, + "grad_norm": 2.611804246902466, + "learning_rate": 4.1720479704797054e-06, + "loss": 1.5416, + "step": 615 + }, + { + "epoch": 0.7666459240821406, + "grad_norm": 2.6322927474975586, + "learning_rate": 4.169741697416975e-06, + "loss": 1.604, + "step": 616 + }, + { + "epoch": 0.7678904791537026, + "grad_norm": 2.5792219638824463, + "learning_rate": 4.1674354243542434e-06, + "loss": 1.5976, + "step": 617 + }, + { + "epoch": 0.7691350342252644, + "grad_norm": 2.845416307449341, + "learning_rate": 4.165129151291513e-06, + "loss": 1.5945, + "step": 618 + }, + { + "epoch": 0.7703795892968264, + "grad_norm": 2.8647871017456055, + "learning_rate": 4.162822878228783e-06, + "loss": 1.5637, + "step": 619 + }, + { + "epoch": 0.7716241443683883, + "grad_norm": 2.590719699859619, + "learning_rate": 4.1605166051660526e-06, + "loss": 1.556, + "step": 620 + }, + { + "epoch": 0.7716241443683883, + "eval_loss": 1.7115222215652466, + "eval_runtime": 50.3905, + "eval_samples_per_second": 19.845, + "eval_steps_per_second": 0.833, + "step": 620 + }, + { + "epoch": 0.7728686994399502, + "grad_norm": 2.626723527908325, + "learning_rate": 4.158210332103321e-06, + "loss": 1.5486, + "step": 621 + }, + { + "epoch": 0.7741132545115121, + "grad_norm": 2.713069438934326, + "learning_rate": 4.1559040590405906e-06, + "loss": 1.5474, + "step": 622 + }, + { + "epoch": 0.7753578095830741, + "grad_norm": 2.685563087463379, + "learning_rate": 4.15359778597786e-06, + "loss": 1.5529, + "step": 623 + }, + { + "epoch": 0.776602364654636, + "grad_norm": 2.690586805343628, + "learning_rate": 4.151291512915129e-06, + "loss": 1.5293, + "step": 624 + }, + { + "epoch": 0.7778469197261979, + "grad_norm": 2.865345001220703, + "learning_rate": 4.148985239852399e-06, + "loss": 1.5095, + "step": 625 + }, + { + "epoch": 0.7790914747977598, + "grad_norm": 2.92393159866333, + "learning_rate": 4.146678966789668e-06, + "loss": 1.5561, + "step": 626 + }, + { + "epoch": 0.7803360298693217, + "grad_norm": 2.554480791091919, + "learning_rate": 4.144372693726938e-06, + "loss": 1.4876, + "step": 627 + }, + { + "epoch": 0.7815805849408837, + "grad_norm": 2.6128833293914795, + "learning_rate": 4.142066420664207e-06, + "loss": 1.5378, + "step": 628 + }, + { + "epoch": 0.7828251400124455, + "grad_norm": 3.0110888481140137, + "learning_rate": 4.1397601476014765e-06, + "loss": 1.5859, + "step": 629 + }, + { + "epoch": 0.7840696950840075, + "grad_norm": 2.8152706623077393, + "learning_rate": 4.137453874538745e-06, + "loss": 1.5841, + "step": 630 + }, + { + "epoch": 0.7840696950840075, + "eval_loss": 1.702181339263916, + "eval_runtime": 53.6926, + "eval_samples_per_second": 18.625, + "eval_steps_per_second": 0.782, + "step": 630 + }, + { + "epoch": 0.7853142501555694, + "grad_norm": 2.8634233474731445, + "learning_rate": 4.135147601476015e-06, + "loss": 1.564, + "step": 631 + }, + { + "epoch": 0.7865588052271313, + "grad_norm": 2.8939666748046875, + "learning_rate": 4.132841328413285e-06, + "loss": 1.5859, + "step": 632 + }, + { + "epoch": 0.7878033602986932, + "grad_norm": 2.6560444831848145, + "learning_rate": 4.130535055350554e-06, + "loss": 1.5199, + "step": 633 + }, + { + "epoch": 0.7890479153702551, + "grad_norm": 2.964721202850342, + "learning_rate": 4.128228782287823e-06, + "loss": 1.5888, + "step": 634 + }, + { + "epoch": 0.790292470441817, + "grad_norm": 2.74668288230896, + "learning_rate": 4.125922509225092e-06, + "loss": 1.5847, + "step": 635 + }, + { + "epoch": 0.791537025513379, + "grad_norm": 2.723123550415039, + "learning_rate": 4.1236162361623625e-06, + "loss": 1.5166, + "step": 636 + }, + { + "epoch": 0.7927815805849409, + "grad_norm": 2.7265713214874268, + "learning_rate": 4.121309963099631e-06, + "loss": 1.5954, + "step": 637 + }, + { + "epoch": 0.7940261356565028, + "grad_norm": 2.979126214981079, + "learning_rate": 4.1190036900369005e-06, + "loss": 1.5583, + "step": 638 + }, + { + "epoch": 0.7952706907280647, + "grad_norm": 2.844376802444458, + "learning_rate": 4.11669741697417e-06, + "loss": 1.5564, + "step": 639 + }, + { + "epoch": 0.7965152457996266, + "grad_norm": 2.7583703994750977, + "learning_rate": 4.114391143911439e-06, + "loss": 1.5094, + "step": 640 + }, + { + "epoch": 0.7965152457996266, + "eval_loss": 1.6986565589904785, + "eval_runtime": 47.3549, + "eval_samples_per_second": 21.117, + "eval_steps_per_second": 0.887, + "step": 640 + }, + { + "epoch": 0.7977598008711886, + "grad_norm": 2.8942224979400635, + "learning_rate": 4.112084870848709e-06, + "loss": 1.5504, + "step": 641 + }, + { + "epoch": 0.7990043559427504, + "grad_norm": 2.6590495109558105, + "learning_rate": 4.109778597785978e-06, + "loss": 1.5227, + "step": 642 + }, + { + "epoch": 0.8002489110143124, + "grad_norm": 2.5988378524780273, + "learning_rate": 4.107472324723248e-06, + "loss": 1.5054, + "step": 643 + }, + { + "epoch": 0.8014934660858744, + "grad_norm": 2.787335157394409, + "learning_rate": 4.105166051660517e-06, + "loss": 1.5844, + "step": 644 + }, + { + "epoch": 0.8027380211574362, + "grad_norm": 2.8202896118164062, + "learning_rate": 4.1028597785977864e-06, + "loss": 1.5596, + "step": 645 + }, + { + "epoch": 0.8039825762289982, + "grad_norm": 2.65376615524292, + "learning_rate": 4.100553505535056e-06, + "loss": 1.5893, + "step": 646 + }, + { + "epoch": 0.80522713130056, + "grad_norm": 2.5933308601379395, + "learning_rate": 4.098247232472325e-06, + "loss": 1.5403, + "step": 647 + }, + { + "epoch": 0.806471686372122, + "grad_norm": 2.777070999145508, + "learning_rate": 4.095940959409595e-06, + "loss": 1.5231, + "step": 648 + }, + { + "epoch": 0.8077162414436839, + "grad_norm": 2.6427664756774902, + "learning_rate": 4.093634686346864e-06, + "loss": 1.5937, + "step": 649 + }, + { + "epoch": 0.8089607965152458, + "grad_norm": 2.651561737060547, + "learning_rate": 4.091328413284133e-06, + "loss": 1.4847, + "step": 650 + }, + { + "epoch": 0.8089607965152458, + "eval_loss": 1.7089996337890625, + "eval_runtime": 47.4346, + "eval_samples_per_second": 21.082, + "eval_steps_per_second": 0.885, + "step": 650 + }, + { + "epoch": 0.8102053515868077, + "grad_norm": 2.844067335128784, + "learning_rate": 4.089022140221402e-06, + "loss": 1.488, + "step": 651 + }, + { + "epoch": 0.8114499066583696, + "grad_norm": 2.603480577468872, + "learning_rate": 4.086715867158672e-06, + "loss": 1.5248, + "step": 652 + }, + { + "epoch": 0.8126944617299315, + "grad_norm": 2.789322853088379, + "learning_rate": 4.084409594095941e-06, + "loss": 1.5119, + "step": 653 + }, + { + "epoch": 0.8139390168014935, + "grad_norm": 2.7183773517608643, + "learning_rate": 4.08210332103321e-06, + "loss": 1.5123, + "step": 654 + }, + { + "epoch": 0.8151835718730553, + "grad_norm": 2.80613374710083, + "learning_rate": 4.07979704797048e-06, + "loss": 1.5526, + "step": 655 + }, + { + "epoch": 0.8164281269446173, + "grad_norm": 2.641035795211792, + "learning_rate": 4.077490774907749e-06, + "loss": 1.6127, + "step": 656 + }, + { + "epoch": 0.8176726820161793, + "grad_norm": 2.773684024810791, + "learning_rate": 4.075184501845019e-06, + "loss": 1.5404, + "step": 657 + }, + { + "epoch": 0.8189172370877411, + "grad_norm": 2.6951382160186768, + "learning_rate": 4.072878228782288e-06, + "loss": 1.4928, + "step": 658 + }, + { + "epoch": 0.8201617921593031, + "grad_norm": 2.931771755218506, + "learning_rate": 4.0705719557195575e-06, + "loss": 1.5949, + "step": 659 + }, + { + "epoch": 0.8214063472308649, + "grad_norm": 2.8786075115203857, + "learning_rate": 4.068265682656827e-06, + "loss": 1.5582, + "step": 660 + }, + { + "epoch": 0.8214063472308649, + "eval_loss": 1.698564052581787, + "eval_runtime": 46.6299, + "eval_samples_per_second": 21.445, + "eval_steps_per_second": 0.901, + "step": 660 + }, + { + "epoch": 0.8226509023024269, + "grad_norm": 2.7402451038360596, + "learning_rate": 4.065959409594096e-06, + "loss": 1.5562, + "step": 661 + }, + { + "epoch": 0.8238954573739888, + "grad_norm": 2.7968692779541016, + "learning_rate": 4.063653136531366e-06, + "loss": 1.556, + "step": 662 + }, + { + "epoch": 0.8251400124455507, + "grad_norm": 2.8233132362365723, + "learning_rate": 4.061346863468635e-06, + "loss": 1.5527, + "step": 663 + }, + { + "epoch": 0.8263845675171126, + "grad_norm": 2.713390827178955, + "learning_rate": 4.059040590405905e-06, + "loss": 1.6012, + "step": 664 + }, + { + "epoch": 0.8276291225886746, + "grad_norm": 2.7783989906311035, + "learning_rate": 4.056734317343174e-06, + "loss": 1.5247, + "step": 665 + }, + { + "epoch": 0.8288736776602365, + "grad_norm": 2.808554172515869, + "learning_rate": 4.054428044280443e-06, + "loss": 1.5412, + "step": 666 + }, + { + "epoch": 0.8301182327317984, + "grad_norm": 2.8476831912994385, + "learning_rate": 4.052121771217712e-06, + "loss": 1.5307, + "step": 667 + }, + { + "epoch": 0.8313627878033603, + "grad_norm": 2.8287601470947266, + "learning_rate": 4.049815498154982e-06, + "loss": 1.6021, + "step": 668 + }, + { + "epoch": 0.8326073428749222, + "grad_norm": 2.975510835647583, + "learning_rate": 4.047509225092252e-06, + "loss": 1.5474, + "step": 669 + }, + { + "epoch": 0.8338518979464842, + "grad_norm": 2.723236560821533, + "learning_rate": 4.04520295202952e-06, + "loss": 1.5242, + "step": 670 + }, + { + "epoch": 0.8338518979464842, + "eval_loss": 1.7005239725112915, + "eval_runtime": 49.4504, + "eval_samples_per_second": 20.222, + "eval_steps_per_second": 0.849, + "step": 670 + }, + { + "epoch": 0.835096453018046, + "grad_norm": 2.7521748542785645, + "learning_rate": 4.04289667896679e-06, + "loss": 1.5467, + "step": 671 + }, + { + "epoch": 0.836341008089608, + "grad_norm": 2.7770044803619385, + "learning_rate": 4.04059040590406e-06, + "loss": 1.5407, + "step": 672 + }, + { + "epoch": 0.8375855631611698, + "grad_norm": 2.744323492050171, + "learning_rate": 4.038284132841329e-06, + "loss": 1.5422, + "step": 673 + }, + { + "epoch": 0.8388301182327318, + "grad_norm": 2.6699817180633545, + "learning_rate": 4.035977859778598e-06, + "loss": 1.5508, + "step": 674 + }, + { + "epoch": 0.8400746733042938, + "grad_norm": 2.705273151397705, + "learning_rate": 4.0336715867158674e-06, + "loss": 1.5099, + "step": 675 + }, + { + "epoch": 0.8413192283758556, + "grad_norm": 2.9485747814178467, + "learning_rate": 4.031365313653137e-06, + "loss": 1.5529, + "step": 676 + }, + { + "epoch": 0.8425637834474176, + "grad_norm": 3.1750423908233643, + "learning_rate": 4.029059040590406e-06, + "loss": 1.5894, + "step": 677 + }, + { + "epoch": 0.8438083385189795, + "grad_norm": 2.76448655128479, + "learning_rate": 4.026752767527676e-06, + "loss": 1.5752, + "step": 678 + }, + { + "epoch": 0.8450528935905414, + "grad_norm": 2.676708698272705, + "learning_rate": 4.024446494464945e-06, + "loss": 1.5487, + "step": 679 + }, + { + "epoch": 0.8462974486621033, + "grad_norm": 2.6832051277160645, + "learning_rate": 4.0221402214022145e-06, + "loss": 1.5055, + "step": 680 + }, + { + "epoch": 0.8462974486621033, + "eval_loss": 1.6945017576217651, + "eval_runtime": 43.9369, + "eval_samples_per_second": 22.76, + "eval_steps_per_second": 0.956, + "step": 680 + }, + { + "epoch": 0.8475420037336652, + "grad_norm": 2.6812336444854736, + "learning_rate": 4.019833948339484e-06, + "loss": 1.5315, + "step": 681 + }, + { + "epoch": 0.8487865588052271, + "grad_norm": 2.9394326210021973, + "learning_rate": 4.017527675276753e-06, + "loss": 1.5636, + "step": 682 + }, + { + "epoch": 0.8500311138767891, + "grad_norm": 2.944952964782715, + "learning_rate": 4.015221402214022e-06, + "loss": 1.56, + "step": 683 + }, + { + "epoch": 0.8512756689483509, + "grad_norm": 2.7475314140319824, + "learning_rate": 4.012915129151292e-06, + "loss": 1.5442, + "step": 684 + }, + { + "epoch": 0.8525202240199129, + "grad_norm": 3.0572879314422607, + "learning_rate": 4.010608856088562e-06, + "loss": 1.5023, + "step": 685 + }, + { + "epoch": 0.8537647790914747, + "grad_norm": 2.75365948677063, + "learning_rate": 4.00830258302583e-06, + "loss": 1.5414, + "step": 686 + }, + { + "epoch": 0.8550093341630367, + "grad_norm": 2.5714449882507324, + "learning_rate": 4.0059963099631e-06, + "loss": 1.5027, + "step": 687 + }, + { + "epoch": 0.8562538892345987, + "grad_norm": 2.755167007446289, + "learning_rate": 4.003690036900369e-06, + "loss": 1.5788, + "step": 688 + }, + { + "epoch": 0.8574984443061605, + "grad_norm": 2.798967123031616, + "learning_rate": 4.0013837638376385e-06, + "loss": 1.5162, + "step": 689 + }, + { + "epoch": 0.8587429993777225, + "grad_norm": 2.803614854812622, + "learning_rate": 3.999077490774908e-06, + "loss": 1.5079, + "step": 690 + }, + { + "epoch": 0.8587429993777225, + "eval_loss": 1.6924962997436523, + "eval_runtime": 44.47, + "eval_samples_per_second": 22.487, + "eval_steps_per_second": 0.944, + "step": 690 + }, + { + "epoch": 0.8599875544492844, + "grad_norm": 2.6898066997528076, + "learning_rate": 3.996771217712177e-06, + "loss": 1.5478, + "step": 691 + }, + { + "epoch": 0.8612321095208463, + "grad_norm": 2.6376144886016846, + "learning_rate": 3.994464944649447e-06, + "loss": 1.5362, + "step": 692 + }, + { + "epoch": 0.8624766645924082, + "grad_norm": 2.7240405082702637, + "learning_rate": 3.992158671586716e-06, + "loss": 1.5502, + "step": 693 + }, + { + "epoch": 0.8637212196639701, + "grad_norm": 2.929445266723633, + "learning_rate": 3.989852398523986e-06, + "loss": 1.5799, + "step": 694 + }, + { + "epoch": 0.864965774735532, + "grad_norm": 2.593223810195923, + "learning_rate": 3.987546125461255e-06, + "loss": 1.5352, + "step": 695 + }, + { + "epoch": 0.866210329807094, + "grad_norm": 2.7710154056549072, + "learning_rate": 3.9852398523985245e-06, + "loss": 1.5048, + "step": 696 + }, + { + "epoch": 0.8674548848786559, + "grad_norm": 2.7370848655700684, + "learning_rate": 3.982933579335794e-06, + "loss": 1.5233, + "step": 697 + }, + { + "epoch": 0.8686994399502178, + "grad_norm": 2.8632168769836426, + "learning_rate": 3.980627306273063e-06, + "loss": 1.5435, + "step": 698 + }, + { + "epoch": 0.8699439950217797, + "grad_norm": 2.752298593521118, + "learning_rate": 3.978321033210332e-06, + "loss": 1.5728, + "step": 699 + }, + { + "epoch": 0.8711885500933416, + "grad_norm": 2.8345625400543213, + "learning_rate": 3.976014760147602e-06, + "loss": 1.5727, + "step": 700 + }, + { + "epoch": 0.8711885500933416, + "eval_loss": 1.6855305433273315, + "eval_runtime": 43.3683, + "eval_samples_per_second": 23.058, + "eval_steps_per_second": 0.968, + "step": 700 + }, + { + "epoch": 0.8724331051649036, + "grad_norm": 2.8135335445404053, + "learning_rate": 3.973708487084872e-06, + "loss": 1.5086, + "step": 701 + }, + { + "epoch": 0.8736776602364654, + "grad_norm": 2.7416603565216064, + "learning_rate": 3.97140221402214e-06, + "loss": 1.545, + "step": 702 + }, + { + "epoch": 0.8749222153080274, + "grad_norm": 2.6902289390563965, + "learning_rate": 3.96909594095941e-06, + "loss": 1.5509, + "step": 703 + }, + { + "epoch": 0.8761667703795893, + "grad_norm": 2.6537084579467773, + "learning_rate": 3.966789667896679e-06, + "loss": 1.562, + "step": 704 + }, + { + "epoch": 0.8774113254511512, + "grad_norm": 2.6979284286499023, + "learning_rate": 3.964483394833948e-06, + "loss": 1.4829, + "step": 705 + }, + { + "epoch": 0.8786558805227132, + "grad_norm": 2.5795764923095703, + "learning_rate": 3.962177121771218e-06, + "loss": 1.4677, + "step": 706 + }, + { + "epoch": 0.879900435594275, + "grad_norm": 3.0185086727142334, + "learning_rate": 3.959870848708487e-06, + "loss": 1.5608, + "step": 707 + }, + { + "epoch": 0.881144990665837, + "grad_norm": 2.6458113193511963, + "learning_rate": 3.957564575645757e-06, + "loss": 1.5342, + "step": 708 + }, + { + "epoch": 0.8823895457373989, + "grad_norm": 2.7985665798187256, + "learning_rate": 3.955258302583026e-06, + "loss": 1.527, + "step": 709 + }, + { + "epoch": 0.8836341008089608, + "grad_norm": 2.9069082736968994, + "learning_rate": 3.9529520295202955e-06, + "loss": 1.5156, + "step": 710 + }, + { + "epoch": 0.8836341008089608, + "eval_loss": 1.6898822784423828, + "eval_runtime": 42.8749, + "eval_samples_per_second": 23.324, + "eval_steps_per_second": 0.98, + "step": 710 + }, + { + "epoch": 0.8848786558805227, + "grad_norm": 2.760143995285034, + "learning_rate": 3.950645756457565e-06, + "loss": 1.5124, + "step": 711 + }, + { + "epoch": 0.8861232109520847, + "grad_norm": 2.8085529804229736, + "learning_rate": 3.948339483394834e-06, + "loss": 1.5862, + "step": 712 + }, + { + "epoch": 0.8873677660236465, + "grad_norm": 2.909905195236206, + "learning_rate": 3.946033210332104e-06, + "loss": 1.5557, + "step": 713 + }, + { + "epoch": 0.8886123210952085, + "grad_norm": 2.826899290084839, + "learning_rate": 3.943726937269373e-06, + "loss": 1.5277, + "step": 714 + }, + { + "epoch": 0.8898568761667703, + "grad_norm": 2.6269052028656006, + "learning_rate": 3.941420664206642e-06, + "loss": 1.4819, + "step": 715 + }, + { + "epoch": 0.8911014312383323, + "grad_norm": 2.6687698364257812, + "learning_rate": 3.939114391143912e-06, + "loss": 1.5156, + "step": 716 + }, + { + "epoch": 0.8923459863098943, + "grad_norm": 2.9605629444122314, + "learning_rate": 3.9368081180811815e-06, + "loss": 1.5358, + "step": 717 + }, + { + "epoch": 0.8935905413814561, + "grad_norm": 3.0367166996002197, + "learning_rate": 3.934501845018451e-06, + "loss": 1.5847, + "step": 718 + }, + { + "epoch": 0.8948350964530181, + "grad_norm": 2.528796434402466, + "learning_rate": 3.9321955719557195e-06, + "loss": 1.4851, + "step": 719 + }, + { + "epoch": 0.8960796515245799, + "grad_norm": 2.7116613388061523, + "learning_rate": 3.929889298892989e-06, + "loss": 1.4706, + "step": 720 + }, + { + "epoch": 0.8960796515245799, + "eval_loss": 1.6845883131027222, + "eval_runtime": 44.4734, + "eval_samples_per_second": 22.485, + "eval_steps_per_second": 0.944, + "step": 720 + }, + { + "epoch": 0.8973242065961419, + "grad_norm": 2.771864414215088, + "learning_rate": 3.927583025830259e-06, + "loss": 1.5444, + "step": 721 + }, + { + "epoch": 0.8985687616677038, + "grad_norm": 2.773746967315674, + "learning_rate": 3.925276752767528e-06, + "loss": 1.564, + "step": 722 + }, + { + "epoch": 0.8998133167392657, + "grad_norm": 2.7069854736328125, + "learning_rate": 3.922970479704797e-06, + "loss": 1.5601, + "step": 723 + }, + { + "epoch": 0.9010578718108276, + "grad_norm": 2.840261697769165, + "learning_rate": 3.920664206642067e-06, + "loss": 1.5733, + "step": 724 + }, + { + "epoch": 0.9023024268823896, + "grad_norm": 2.709897041320801, + "learning_rate": 3.918357933579336e-06, + "loss": 1.528, + "step": 725 + }, + { + "epoch": 0.9035469819539514, + "grad_norm": 2.662367343902588, + "learning_rate": 3.9160516605166055e-06, + "loss": 1.5532, + "step": 726 + }, + { + "epoch": 0.9047915370255134, + "grad_norm": 2.6775717735290527, + "learning_rate": 3.913745387453875e-06, + "loss": 1.5464, + "step": 727 + }, + { + "epoch": 0.9060360920970753, + "grad_norm": 2.617842674255371, + "learning_rate": 3.911439114391144e-06, + "loss": 1.4539, + "step": 728 + }, + { + "epoch": 0.9072806471686372, + "grad_norm": 2.668649435043335, + "learning_rate": 3.909132841328414e-06, + "loss": 1.5627, + "step": 729 + }, + { + "epoch": 0.9085252022401992, + "grad_norm": 2.5331950187683105, + "learning_rate": 3.906826568265683e-06, + "loss": 1.5165, + "step": 730 + }, + { + "epoch": 0.9085252022401992, + "eval_loss": 1.6860331296920776, + "eval_runtime": 46.094, + "eval_samples_per_second": 21.695, + "eval_steps_per_second": 0.911, + "step": 730 + }, + { + "epoch": 0.909769757311761, + "grad_norm": 2.602492332458496, + "learning_rate": 3.9045202952029526e-06, + "loss": 1.5145, + "step": 731 + }, + { + "epoch": 0.911014312383323, + "grad_norm": 2.632782459259033, + "learning_rate": 3.902214022140222e-06, + "loss": 1.4413, + "step": 732 + }, + { + "epoch": 0.9122588674548848, + "grad_norm": 2.6025912761688232, + "learning_rate": 3.899907749077491e-06, + "loss": 1.5263, + "step": 733 + }, + { + "epoch": 0.9135034225264468, + "grad_norm": 2.770116090774536, + "learning_rate": 3.897601476014761e-06, + "loss": 1.5514, + "step": 734 + }, + { + "epoch": 0.9147479775980087, + "grad_norm": 2.7822961807250977, + "learning_rate": 3.895295202952029e-06, + "loss": 1.5296, + "step": 735 + }, + { + "epoch": 0.9159925326695706, + "grad_norm": 2.6689720153808594, + "learning_rate": 3.892988929889299e-06, + "loss": 1.4552, + "step": 736 + }, + { + "epoch": 0.9172370877411326, + "grad_norm": 2.8021798133850098, + "learning_rate": 3.890682656826569e-06, + "loss": 1.5308, + "step": 737 + }, + { + "epoch": 0.9184816428126945, + "grad_norm": 2.575313091278076, + "learning_rate": 3.888376383763838e-06, + "loss": 1.4807, + "step": 738 + }, + { + "epoch": 0.9197261978842564, + "grad_norm": 2.5955779552459717, + "learning_rate": 3.886070110701107e-06, + "loss": 1.4222, + "step": 739 + }, + { + "epoch": 0.9209707529558183, + "grad_norm": 2.647939920425415, + "learning_rate": 3.8837638376383765e-06, + "loss": 1.5285, + "step": 740 + }, + { + "epoch": 0.9209707529558183, + "eval_loss": 1.6907480955123901, + "eval_runtime": 45.7678, + "eval_samples_per_second": 21.849, + "eval_steps_per_second": 0.918, + "step": 740 + }, + { + "epoch": 0.9222153080273802, + "grad_norm": 2.7664523124694824, + "learning_rate": 3.881457564575646e-06, + "loss": 1.5349, + "step": 741 + }, + { + "epoch": 0.9234598630989421, + "grad_norm": 2.5770998001098633, + "learning_rate": 3.879151291512915e-06, + "loss": 1.5071, + "step": 742 + }, + { + "epoch": 0.9247044181705041, + "grad_norm": 2.502567768096924, + "learning_rate": 3.876845018450185e-06, + "loss": 1.492, + "step": 743 + }, + { + "epoch": 0.9259489732420659, + "grad_norm": 2.6531119346618652, + "learning_rate": 3.874538745387454e-06, + "loss": 1.5045, + "step": 744 + }, + { + "epoch": 0.9271935283136279, + "grad_norm": 2.7343876361846924, + "learning_rate": 3.872232472324724e-06, + "loss": 1.5194, + "step": 745 + }, + { + "epoch": 0.9284380833851898, + "grad_norm": 2.842313528060913, + "learning_rate": 3.869926199261993e-06, + "loss": 1.5588, + "step": 746 + }, + { + "epoch": 0.9296826384567517, + "grad_norm": 2.718245267868042, + "learning_rate": 3.8676199261992625e-06, + "loss": 1.5252, + "step": 747 + }, + { + "epoch": 0.9309271935283137, + "grad_norm": 2.695392370223999, + "learning_rate": 3.865313653136532e-06, + "loss": 1.5596, + "step": 748 + }, + { + "epoch": 0.9321717485998755, + "grad_norm": 2.7859959602355957, + "learning_rate": 3.863007380073801e-06, + "loss": 1.5151, + "step": 749 + }, + { + "epoch": 0.9334163036714375, + "grad_norm": 2.5199174880981445, + "learning_rate": 3.860701107011071e-06, + "loss": 1.4948, + "step": 750 + }, + { + "epoch": 0.9334163036714375, + "eval_loss": 1.6868674755096436, + "eval_runtime": 46.4115, + "eval_samples_per_second": 21.546, + "eval_steps_per_second": 0.905, + "step": 750 + }, + { + "epoch": 0.9346608587429994, + "grad_norm": 2.8194046020507812, + "learning_rate": 3.858394833948339e-06, + "loss": 1.5144, + "step": 751 + }, + { + "epoch": 0.9359054138145613, + "grad_norm": 2.810380697250366, + "learning_rate": 3.856088560885609e-06, + "loss": 1.4998, + "step": 752 + }, + { + "epoch": 0.9371499688861232, + "grad_norm": 2.6906750202178955, + "learning_rate": 3.853782287822879e-06, + "loss": 1.4748, + "step": 753 + }, + { + "epoch": 0.9383945239576851, + "grad_norm": 2.9030277729034424, + "learning_rate": 3.851476014760148e-06, + "loss": 1.615, + "step": 754 + }, + { + "epoch": 0.939639079029247, + "grad_norm": 2.8690176010131836, + "learning_rate": 3.849169741697417e-06, + "loss": 1.5083, + "step": 755 + }, + { + "epoch": 0.940883634100809, + "grad_norm": 2.669646978378296, + "learning_rate": 3.8468634686346865e-06, + "loss": 1.5572, + "step": 756 + }, + { + "epoch": 0.9421281891723708, + "grad_norm": 2.6517696380615234, + "learning_rate": 3.844557195571956e-06, + "loss": 1.5142, + "step": 757 + }, + { + "epoch": 0.9433727442439328, + "grad_norm": 2.5524444580078125, + "learning_rate": 3.842250922509225e-06, + "loss": 1.4924, + "step": 758 + }, + { + "epoch": 0.9446172993154948, + "grad_norm": 2.6532633304595947, + "learning_rate": 3.839944649446495e-06, + "loss": 1.484, + "step": 759 + }, + { + "epoch": 0.9458618543870566, + "grad_norm": 2.7779057025909424, + "learning_rate": 3.837638376383764e-06, + "loss": 1.5106, + "step": 760 + }, + { + "epoch": 0.9458618543870566, + "eval_loss": 1.6845752000808716, + "eval_runtime": 50.3415, + "eval_samples_per_second": 19.864, + "eval_steps_per_second": 0.834, + "step": 760 + }, + { + "epoch": 0.9471064094586186, + "grad_norm": 2.6192541122436523, + "learning_rate": 3.8353321033210336e-06, + "loss": 1.5454, + "step": 761 + }, + { + "epoch": 0.9483509645301804, + "grad_norm": 2.561861991882324, + "learning_rate": 3.833025830258303e-06, + "loss": 1.5141, + "step": 762 + }, + { + "epoch": 0.9495955196017424, + "grad_norm": 2.661829948425293, + "learning_rate": 3.830719557195572e-06, + "loss": 1.4997, + "step": 763 + }, + { + "epoch": 0.9508400746733043, + "grad_norm": 2.725275754928589, + "learning_rate": 3.828413284132842e-06, + "loss": 1.5224, + "step": 764 + }, + { + "epoch": 0.9520846297448662, + "grad_norm": 2.5664193630218506, + "learning_rate": 3.826107011070111e-06, + "loss": 1.5073, + "step": 765 + }, + { + "epoch": 0.9533291848164281, + "grad_norm": 2.595189332962036, + "learning_rate": 3.823800738007381e-06, + "loss": 1.5379, + "step": 766 + }, + { + "epoch": 0.95457373988799, + "grad_norm": 2.738060235977173, + "learning_rate": 3.821494464944649e-06, + "loss": 1.4814, + "step": 767 + }, + { + "epoch": 0.955818294959552, + "grad_norm": 2.601071357727051, + "learning_rate": 3.819188191881919e-06, + "loss": 1.4812, + "step": 768 + }, + { + "epoch": 0.9570628500311139, + "grad_norm": 2.657212257385254, + "learning_rate": 3.816881918819189e-06, + "loss": 1.4748, + "step": 769 + }, + { + "epoch": 0.9583074051026758, + "grad_norm": 2.7673351764678955, + "learning_rate": 3.814575645756458e-06, + "loss": 1.5535, + "step": 770 + }, + { + "epoch": 0.9583074051026758, + "eval_loss": 1.677243947982788, + "eval_runtime": 51.5537, + "eval_samples_per_second": 19.397, + "eval_steps_per_second": 0.815, + "step": 770 + }, + { + "epoch": 0.9595519601742377, + "grad_norm": 2.760890007019043, + "learning_rate": 3.812269372693727e-06, + "loss": 1.5654, + "step": 771 + }, + { + "epoch": 0.9607965152457997, + "grad_norm": 2.6934309005737305, + "learning_rate": 3.809963099630997e-06, + "loss": 1.5149, + "step": 772 + }, + { + "epoch": 0.9620410703173615, + "grad_norm": 2.729950428009033, + "learning_rate": 3.8076568265682662e-06, + "loss": 1.5579, + "step": 773 + }, + { + "epoch": 0.9632856253889235, + "grad_norm": 2.732926607131958, + "learning_rate": 3.8053505535055352e-06, + "loss": 1.5179, + "step": 774 + }, + { + "epoch": 0.9645301804604853, + "grad_norm": 2.745391368865967, + "learning_rate": 3.8030442804428046e-06, + "loss": 1.5124, + "step": 775 + }, + { + "epoch": 0.9657747355320473, + "grad_norm": 2.8050146102905273, + "learning_rate": 3.800738007380074e-06, + "loss": 1.519, + "step": 776 + }, + { + "epoch": 0.9670192906036092, + "grad_norm": 2.697171449661255, + "learning_rate": 3.798431734317343e-06, + "loss": 1.5219, + "step": 777 + }, + { + "epoch": 0.9682638456751711, + "grad_norm": 2.6375980377197266, + "learning_rate": 3.796125461254613e-06, + "loss": 1.5345, + "step": 778 + }, + { + "epoch": 0.9695084007467331, + "grad_norm": 2.586636781692505, + "learning_rate": 3.7938191881918823e-06, + "loss": 1.5182, + "step": 779 + }, + { + "epoch": 0.9707529558182949, + "grad_norm": 2.626453399658203, + "learning_rate": 3.7915129151291518e-06, + "loss": 1.4817, + "step": 780 + }, + { + "epoch": 0.9707529558182949, + "eval_loss": 1.6734713315963745, + "eval_runtime": 48.1436, + "eval_samples_per_second": 20.771, + "eval_steps_per_second": 0.872, + "step": 780 + }, + { + "epoch": 0.9719975108898569, + "grad_norm": 2.579970121383667, + "learning_rate": 3.7892066420664208e-06, + "loss": 1.4981, + "step": 781 + }, + { + "epoch": 0.9732420659614188, + "grad_norm": 2.6834843158721924, + "learning_rate": 3.7869003690036906e-06, + "loss": 1.5044, + "step": 782 + }, + { + "epoch": 0.9744866210329807, + "grad_norm": 2.586982488632202, + "learning_rate": 3.78459409594096e-06, + "loss": 1.5079, + "step": 783 + }, + { + "epoch": 0.9757311761045426, + "grad_norm": 2.910027503967285, + "learning_rate": 3.782287822878229e-06, + "loss": 1.5695, + "step": 784 + }, + { + "epoch": 0.9769757311761046, + "grad_norm": 2.7088494300842285, + "learning_rate": 3.7799815498154984e-06, + "loss": 1.4986, + "step": 785 + }, + { + "epoch": 0.9782202862476664, + "grad_norm": 2.581325054168701, + "learning_rate": 3.777675276752768e-06, + "loss": 1.5137, + "step": 786 + }, + { + "epoch": 0.9794648413192284, + "grad_norm": 2.700709819793701, + "learning_rate": 3.775369003690037e-06, + "loss": 1.4718, + "step": 787 + }, + { + "epoch": 0.9807093963907902, + "grad_norm": 2.6712708473205566, + "learning_rate": 3.7730627306273067e-06, + "loss": 1.5208, + "step": 788 + }, + { + "epoch": 0.9819539514623522, + "grad_norm": 2.6799817085266113, + "learning_rate": 3.770756457564576e-06, + "loss": 1.5374, + "step": 789 + }, + { + "epoch": 0.9831985065339142, + "grad_norm": 2.618988037109375, + "learning_rate": 3.768450184501845e-06, + "loss": 1.4997, + "step": 790 + }, + { + "epoch": 0.9831985065339142, + "eval_loss": 1.6860820055007935, + "eval_runtime": 51.7654, + "eval_samples_per_second": 19.318, + "eval_steps_per_second": 0.811, + "step": 790 + }, + { + "epoch": 0.984443061605476, + "grad_norm": 2.6899526119232178, + "learning_rate": 3.7661439114391146e-06, + "loss": 1.5216, + "step": 791 + }, + { + "epoch": 0.985687616677038, + "grad_norm": 2.8003487586975098, + "learning_rate": 3.763837638376384e-06, + "loss": 1.557, + "step": 792 + }, + { + "epoch": 0.9869321717485999, + "grad_norm": 2.744536876678467, + "learning_rate": 3.761531365313654e-06, + "loss": 1.5355, + "step": 793 + }, + { + "epoch": 0.9881767268201618, + "grad_norm": 2.587250232696533, + "learning_rate": 3.759225092250923e-06, + "loss": 1.5259, + "step": 794 + }, + { + "epoch": 0.9894212818917237, + "grad_norm": 2.616291046142578, + "learning_rate": 3.7569188191881922e-06, + "loss": 1.5176, + "step": 795 + }, + { + "epoch": 0.9906658369632856, + "grad_norm": 2.6410577297210693, + "learning_rate": 3.7546125461254617e-06, + "loss": 1.511, + "step": 796 + }, + { + "epoch": 0.9919103920348475, + "grad_norm": 2.577373504638672, + "learning_rate": 3.7523062730627307e-06, + "loss": 1.4715, + "step": 797 + }, + { + "epoch": 0.9931549471064095, + "grad_norm": 2.67305326461792, + "learning_rate": 3.7500000000000005e-06, + "loss": 1.4828, + "step": 798 + }, + { + "epoch": 0.9943995021779714, + "grad_norm": 3.1631500720977783, + "learning_rate": 3.74769372693727e-06, + "loss": 1.5602, + "step": 799 + }, + { + "epoch": 0.9956440572495333, + "grad_norm": 2.9222350120544434, + "learning_rate": 3.745387453874539e-06, + "loss": 1.4814, + "step": 800 + }, + { + "epoch": 0.9956440572495333, + "eval_loss": 1.6810544729232788, + "eval_runtime": 51.1262, + "eval_samples_per_second": 19.559, + "eval_steps_per_second": 0.821, + "step": 800 + }, + { + "epoch": 0.9968886123210952, + "grad_norm": 2.884312629699707, + "learning_rate": 3.7430811808118084e-06, + "loss": 1.4553, + "step": 801 + }, + { + "epoch": 0.9981331673926571, + "grad_norm": 2.712655782699585, + "learning_rate": 3.7407749077490778e-06, + "loss": 1.5275, + "step": 802 + }, + { + "epoch": 0.9993777224642191, + "grad_norm": 2.8700571060180664, + "learning_rate": 3.7384686346863468e-06, + "loss": 1.4923, + "step": 803 + }, + { + "epoch": 1.000622277535781, + "grad_norm": 2.7779245376586914, + "learning_rate": 3.7361623616236166e-06, + "loss": 1.5224, + "step": 804 + }, + { + "epoch": 1.0018668326073428, + "grad_norm": 2.7679810523986816, + "learning_rate": 3.733856088560886e-06, + "loss": 1.4977, + "step": 805 + }, + { + "epoch": 1.0031113876789048, + "grad_norm": 2.797023296356201, + "learning_rate": 3.7315498154981555e-06, + "loss": 1.4396, + "step": 806 + }, + { + "epoch": 1.0043559427504667, + "grad_norm": 3.007962465286255, + "learning_rate": 3.7292435424354245e-06, + "loss": 1.5015, + "step": 807 + }, + { + "epoch": 1.0056004978220285, + "grad_norm": 3.127639055252075, + "learning_rate": 3.726937269372694e-06, + "loss": 1.5176, + "step": 808 + }, + { + "epoch": 1.0068450528935906, + "grad_norm": 2.9001357555389404, + "learning_rate": 3.7246309963099637e-06, + "loss": 1.4735, + "step": 809 + }, + { + "epoch": 1.0080896079651525, + "grad_norm": 2.8245413303375244, + "learning_rate": 3.7223247232472327e-06, + "loss": 1.486, + "step": 810 + }, + { + "epoch": 1.0080896079651525, + "eval_loss": 1.6835130453109741, + "eval_runtime": 44.5682, + "eval_samples_per_second": 22.438, + "eval_steps_per_second": 0.942, + "step": 810 + }, + { + "epoch": 1.0093341630367143, + "grad_norm": 2.780172824859619, + "learning_rate": 3.720018450184502e-06, + "loss": 1.4437, + "step": 811 + }, + { + "epoch": 1.0105787181082764, + "grad_norm": 2.6013996601104736, + "learning_rate": 3.7177121771217716e-06, + "loss": 1.4529, + "step": 812 + }, + { + "epoch": 1.0118232731798382, + "grad_norm": 2.831015110015869, + "learning_rate": 3.7154059040590406e-06, + "loss": 1.4825, + "step": 813 + }, + { + "epoch": 1.0130678282514, + "grad_norm": 2.8341829776763916, + "learning_rate": 3.71309963099631e-06, + "loss": 1.4736, + "step": 814 + }, + { + "epoch": 1.0143123833229621, + "grad_norm": 2.769540309906006, + "learning_rate": 3.71079335793358e-06, + "loss": 1.4724, + "step": 815 + }, + { + "epoch": 1.015556938394524, + "grad_norm": 3.0083069801330566, + "learning_rate": 3.708487084870849e-06, + "loss": 1.4985, + "step": 816 + }, + { + "epoch": 1.0168014934660858, + "grad_norm": 3.052168607711792, + "learning_rate": 3.7061808118081183e-06, + "loss": 1.5193, + "step": 817 + }, + { + "epoch": 1.0180460485376477, + "grad_norm": 2.924877405166626, + "learning_rate": 3.7038745387453877e-06, + "loss": 1.5219, + "step": 818 + }, + { + "epoch": 1.0192906036092098, + "grad_norm": 3.150123357772827, + "learning_rate": 3.7015682656826576e-06, + "loss": 1.5026, + "step": 819 + }, + { + "epoch": 1.0205351586807716, + "grad_norm": 2.881655216217041, + "learning_rate": 3.6992619926199266e-06, + "loss": 1.5011, + "step": 820 + }, + { + "epoch": 1.0205351586807716, + "eval_loss": 1.6745303869247437, + "eval_runtime": 50.6791, + "eval_samples_per_second": 19.732, + "eval_steps_per_second": 0.829, + "step": 820 + }, + { + "epoch": 1.0217797137523335, + "grad_norm": 2.824249505996704, + "learning_rate": 3.696955719557196e-06, + "loss": 1.4642, + "step": 821 + }, + { + "epoch": 1.0230242688238955, + "grad_norm": 2.859144926071167, + "learning_rate": 3.6946494464944654e-06, + "loss": 1.4681, + "step": 822 + }, + { + "epoch": 1.0242688238954574, + "grad_norm": 2.90312123298645, + "learning_rate": 3.6923431734317344e-06, + "loss": 1.4926, + "step": 823 + }, + { + "epoch": 1.0255133789670192, + "grad_norm": 2.8340659141540527, + "learning_rate": 3.690036900369004e-06, + "loss": 1.4834, + "step": 824 + }, + { + "epoch": 1.0267579340385813, + "grad_norm": 2.9743151664733887, + "learning_rate": 3.6877306273062737e-06, + "loss": 1.4574, + "step": 825 + }, + { + "epoch": 1.0280024891101431, + "grad_norm": 3.032179594039917, + "learning_rate": 3.6854243542435427e-06, + "loss": 1.5571, + "step": 826 + }, + { + "epoch": 1.029247044181705, + "grad_norm": 2.816826343536377, + "learning_rate": 3.683118081180812e-06, + "loss": 1.455, + "step": 827 + }, + { + "epoch": 1.030491599253267, + "grad_norm": 3.046696186065674, + "learning_rate": 3.6808118081180815e-06, + "loss": 1.4871, + "step": 828 + }, + { + "epoch": 1.031736154324829, + "grad_norm": 2.8517391681671143, + "learning_rate": 3.678505535055351e-06, + "loss": 1.5292, + "step": 829 + }, + { + "epoch": 1.0329807093963908, + "grad_norm": 2.752811908721924, + "learning_rate": 3.67619926199262e-06, + "loss": 1.5091, + "step": 830 + }, + { + "epoch": 1.0329807093963908, + "eval_loss": 1.6692287921905518, + "eval_runtime": 50.8347, + "eval_samples_per_second": 19.672, + "eval_steps_per_second": 0.826, + "step": 830 + }, + { + "epoch": 1.0342252644679526, + "grad_norm": 2.9220356941223145, + "learning_rate": 3.6738929889298898e-06, + "loss": 1.4247, + "step": 831 + }, + { + "epoch": 1.0354698195395147, + "grad_norm": 2.8903002738952637, + "learning_rate": 3.671586715867159e-06, + "loss": 1.5595, + "step": 832 + }, + { + "epoch": 1.0367143746110765, + "grad_norm": 2.6495542526245117, + "learning_rate": 3.669280442804428e-06, + "loss": 1.4076, + "step": 833 + }, + { + "epoch": 1.0379589296826384, + "grad_norm": 2.880809783935547, + "learning_rate": 3.6669741697416976e-06, + "loss": 1.5285, + "step": 834 + }, + { + "epoch": 1.0392034847542004, + "grad_norm": 2.919142007827759, + "learning_rate": 3.6646678966789675e-06, + "loss": 1.5165, + "step": 835 + }, + { + "epoch": 1.0404480398257623, + "grad_norm": 3.1180522441864014, + "learning_rate": 3.6623616236162365e-06, + "loss": 1.5224, + "step": 836 + }, + { + "epoch": 1.0416925948973241, + "grad_norm": 2.767709255218506, + "learning_rate": 3.660055350553506e-06, + "loss": 1.4859, + "step": 837 + }, + { + "epoch": 1.0429371499688862, + "grad_norm": 2.708828926086426, + "learning_rate": 3.6577490774907753e-06, + "loss": 1.4803, + "step": 838 + }, + { + "epoch": 1.044181705040448, + "grad_norm": 2.688359260559082, + "learning_rate": 3.6554428044280443e-06, + "loss": 1.4571, + "step": 839 + }, + { + "epoch": 1.04542626011201, + "grad_norm": 2.691812753677368, + "learning_rate": 3.6531365313653137e-06, + "loss": 1.5043, + "step": 840 + }, + { + "epoch": 1.04542626011201, + "eval_loss": 1.6769559383392334, + "eval_runtime": 49.6233, + "eval_samples_per_second": 20.152, + "eval_steps_per_second": 0.846, + "step": 840 + }, + { + "epoch": 1.046670815183572, + "grad_norm": 2.686372995376587, + "learning_rate": 3.6508302583025836e-06, + "loss": 1.5221, + "step": 841 + }, + { + "epoch": 1.0479153702551338, + "grad_norm": 2.742859125137329, + "learning_rate": 3.648523985239853e-06, + "loss": 1.4685, + "step": 842 + }, + { + "epoch": 1.0491599253266957, + "grad_norm": 2.874018907546997, + "learning_rate": 3.646217712177122e-06, + "loss": 1.4813, + "step": 843 + }, + { + "epoch": 1.0504044803982575, + "grad_norm": 2.8159990310668945, + "learning_rate": 3.6439114391143914e-06, + "loss": 1.4421, + "step": 844 + }, + { + "epoch": 1.0516490354698196, + "grad_norm": 2.714787006378174, + "learning_rate": 3.641605166051661e-06, + "loss": 1.4879, + "step": 845 + }, + { + "epoch": 1.0528935905413814, + "grad_norm": 2.885148525238037, + "learning_rate": 3.63929889298893e-06, + "loss": 1.4725, + "step": 846 + }, + { + "epoch": 1.0541381456129433, + "grad_norm": 2.7272610664367676, + "learning_rate": 3.6369926199261997e-06, + "loss": 1.4276, + "step": 847 + }, + { + "epoch": 1.0553827006845053, + "grad_norm": 2.9114978313446045, + "learning_rate": 3.634686346863469e-06, + "loss": 1.4194, + "step": 848 + }, + { + "epoch": 1.0566272557560672, + "grad_norm": 3.019313335418701, + "learning_rate": 3.632380073800738e-06, + "loss": 1.5266, + "step": 849 + }, + { + "epoch": 1.057871810827629, + "grad_norm": 2.714751958847046, + "learning_rate": 3.6300738007380075e-06, + "loss": 1.4593, + "step": 850 + }, + { + "epoch": 1.057871810827629, + "eval_loss": 1.6760298013687134, + "eval_runtime": 52.1681, + "eval_samples_per_second": 19.169, + "eval_steps_per_second": 0.805, + "step": 850 + }, + { + "epoch": 1.0591163658991911, + "grad_norm": 3.0507936477661133, + "learning_rate": 3.627767527675277e-06, + "loss": 1.5064, + "step": 851 + }, + { + "epoch": 1.060360920970753, + "grad_norm": 2.8116295337677, + "learning_rate": 3.625461254612546e-06, + "loss": 1.5218, + "step": 852 + }, + { + "epoch": 1.0616054760423148, + "grad_norm": 2.8801465034484863, + "learning_rate": 3.623154981549816e-06, + "loss": 1.4641, + "step": 853 + }, + { + "epoch": 1.0628500311138769, + "grad_norm": 3.0581862926483154, + "learning_rate": 3.6208487084870852e-06, + "loss": 1.483, + "step": 854 + }, + { + "epoch": 1.0640945861854387, + "grad_norm": 2.898780584335327, + "learning_rate": 3.6185424354243547e-06, + "loss": 1.4623, + "step": 855 + }, + { + "epoch": 1.0653391412570006, + "grad_norm": 2.971820116043091, + "learning_rate": 3.6162361623616237e-06, + "loss": 1.4455, + "step": 856 + }, + { + "epoch": 1.0665836963285624, + "grad_norm": 3.0021913051605225, + "learning_rate": 3.6139298892988935e-06, + "loss": 1.4878, + "step": 857 + }, + { + "epoch": 1.0678282514001245, + "grad_norm": 3.0697457790374756, + "learning_rate": 3.611623616236163e-06, + "loss": 1.5008, + "step": 858 + }, + { + "epoch": 1.0690728064716863, + "grad_norm": 3.0787923336029053, + "learning_rate": 3.609317343173432e-06, + "loss": 1.4612, + "step": 859 + }, + { + "epoch": 1.0703173615432482, + "grad_norm": 2.789339780807495, + "learning_rate": 3.6070110701107014e-06, + "loss": 1.4287, + "step": 860 + }, + { + "epoch": 1.0703173615432482, + "eval_loss": 1.6747506856918335, + "eval_runtime": 51.4912, + "eval_samples_per_second": 19.421, + "eval_steps_per_second": 0.816, + "step": 860 + }, + { + "epoch": 1.0715619166148103, + "grad_norm": 2.9622788429260254, + "learning_rate": 3.6047047970479708e-06, + "loss": 1.4905, + "step": 861 + }, + { + "epoch": 1.072806471686372, + "grad_norm": 3.0675268173217773, + "learning_rate": 3.6023985239852398e-06, + "loss": 1.461, + "step": 862 + }, + { + "epoch": 1.074051026757934, + "grad_norm": 2.820996046066284, + "learning_rate": 3.6000922509225096e-06, + "loss": 1.4948, + "step": 863 + }, + { + "epoch": 1.075295581829496, + "grad_norm": 2.833761215209961, + "learning_rate": 3.597785977859779e-06, + "loss": 1.4683, + "step": 864 + }, + { + "epoch": 1.0765401369010579, + "grad_norm": 2.917313814163208, + "learning_rate": 3.595479704797048e-06, + "loss": 1.5191, + "step": 865 + }, + { + "epoch": 1.0777846919726197, + "grad_norm": 2.876166820526123, + "learning_rate": 3.5931734317343175e-06, + "loss": 1.4854, + "step": 866 + }, + { + "epoch": 1.0790292470441818, + "grad_norm": 3.139049768447876, + "learning_rate": 3.590867158671587e-06, + "loss": 1.5134, + "step": 867 + }, + { + "epoch": 1.0802738021157436, + "grad_norm": 2.9956753253936768, + "learning_rate": 3.5885608856088567e-06, + "loss": 1.5003, + "step": 868 + }, + { + "epoch": 1.0815183571873055, + "grad_norm": 2.957181930541992, + "learning_rate": 3.5862546125461257e-06, + "loss": 1.4588, + "step": 869 + }, + { + "epoch": 1.0827629122588673, + "grad_norm": 2.8576438426971436, + "learning_rate": 3.583948339483395e-06, + "loss": 1.48, + "step": 870 + }, + { + "epoch": 1.0827629122588673, + "eval_loss": 1.6724848747253418, + "eval_runtime": 50.9669, + "eval_samples_per_second": 19.621, + "eval_steps_per_second": 0.824, + "step": 870 + }, + { + "epoch": 1.0840074673304294, + "grad_norm": 2.8183560371398926, + "learning_rate": 3.5816420664206646e-06, + "loss": 1.4818, + "step": 871 + }, + { + "epoch": 1.0852520224019913, + "grad_norm": 2.7531797885894775, + "learning_rate": 3.5793357933579336e-06, + "loss": 1.4712, + "step": 872 + }, + { + "epoch": 1.086496577473553, + "grad_norm": 2.923962354660034, + "learning_rate": 3.5770295202952034e-06, + "loss": 1.4864, + "step": 873 + }, + { + "epoch": 1.0877411325451152, + "grad_norm": 2.851724147796631, + "learning_rate": 3.574723247232473e-06, + "loss": 1.4771, + "step": 874 + }, + { + "epoch": 1.088985687616677, + "grad_norm": 3.024634838104248, + "learning_rate": 3.572416974169742e-06, + "loss": 1.4643, + "step": 875 + }, + { + "epoch": 1.0902302426882389, + "grad_norm": 2.7100844383239746, + "learning_rate": 3.5701107011070113e-06, + "loss": 1.4773, + "step": 876 + }, + { + "epoch": 1.091474797759801, + "grad_norm": 2.730502128601074, + "learning_rate": 3.5678044280442807e-06, + "loss": 1.5051, + "step": 877 + }, + { + "epoch": 1.0927193528313628, + "grad_norm": 2.747082233428955, + "learning_rate": 3.5654981549815497e-06, + "loss": 1.5207, + "step": 878 + }, + { + "epoch": 1.0939639079029246, + "grad_norm": 2.718358039855957, + "learning_rate": 3.5631918819188195e-06, + "loss": 1.5067, + "step": 879 + }, + { + "epoch": 1.0952084629744867, + "grad_norm": 2.7794573307037354, + "learning_rate": 3.560885608856089e-06, + "loss": 1.4616, + "step": 880 + }, + { + "epoch": 1.0952084629744867, + "eval_loss": 1.6676132678985596, + "eval_runtime": 51.6157, + "eval_samples_per_second": 19.374, + "eval_steps_per_second": 0.814, + "step": 880 + }, + { + "epoch": 1.0964530180460486, + "grad_norm": 2.727416753768921, + "learning_rate": 3.5585793357933584e-06, + "loss": 1.4043, + "step": 881 + }, + { + "epoch": 1.0976975731176104, + "grad_norm": 2.856900930404663, + "learning_rate": 3.5562730627306274e-06, + "loss": 1.5198, + "step": 882 + }, + { + "epoch": 1.0989421281891725, + "grad_norm": 2.872823715209961, + "learning_rate": 3.553966789667897e-06, + "loss": 1.4981, + "step": 883 + }, + { + "epoch": 1.1001866832607343, + "grad_norm": 2.7724361419677734, + "learning_rate": 3.5516605166051667e-06, + "loss": 1.4819, + "step": 884 + }, + { + "epoch": 1.1014312383322962, + "grad_norm": 2.884434461593628, + "learning_rate": 3.5493542435424357e-06, + "loss": 1.433, + "step": 885 + }, + { + "epoch": 1.102675793403858, + "grad_norm": 2.887873411178589, + "learning_rate": 3.547047970479705e-06, + "loss": 1.4677, + "step": 886 + }, + { + "epoch": 1.10392034847542, + "grad_norm": 2.946845054626465, + "learning_rate": 3.5447416974169745e-06, + "loss": 1.4983, + "step": 887 + }, + { + "epoch": 1.105164903546982, + "grad_norm": 2.752552032470703, + "learning_rate": 3.5424354243542435e-06, + "loss": 1.4766, + "step": 888 + }, + { + "epoch": 1.1064094586185438, + "grad_norm": 2.788634777069092, + "learning_rate": 3.5401291512915133e-06, + "loss": 1.4747, + "step": 889 + }, + { + "epoch": 1.1076540136901059, + "grad_norm": 2.782792568206787, + "learning_rate": 3.5378228782287828e-06, + "loss": 1.5503, + "step": 890 + }, + { + "epoch": 1.1076540136901059, + "eval_loss": 1.6698333024978638, + "eval_runtime": 49.1373, + "eval_samples_per_second": 20.351, + "eval_steps_per_second": 0.855, + "step": 890 + }, + { + "epoch": 1.1088985687616677, + "grad_norm": 2.8351166248321533, + "learning_rate": 3.535516605166052e-06, + "loss": 1.4589, + "step": 891 + }, + { + "epoch": 1.1101431238332296, + "grad_norm": 2.798872709274292, + "learning_rate": 3.533210332103321e-06, + "loss": 1.4496, + "step": 892 + }, + { + "epoch": 1.1113876789047916, + "grad_norm": 3.0863239765167236, + "learning_rate": 3.5309040590405906e-06, + "loss": 1.4698, + "step": 893 + }, + { + "epoch": 1.1126322339763535, + "grad_norm": 2.853862762451172, + "learning_rate": 3.5285977859778605e-06, + "loss": 1.4429, + "step": 894 + }, + { + "epoch": 1.1138767890479153, + "grad_norm": 2.8946170806884766, + "learning_rate": 3.5262915129151295e-06, + "loss": 1.4697, + "step": 895 + }, + { + "epoch": 1.1151213441194774, + "grad_norm": 3.043823003768921, + "learning_rate": 3.523985239852399e-06, + "loss": 1.4666, + "step": 896 + }, + { + "epoch": 1.1163658991910392, + "grad_norm": 2.7822980880737305, + "learning_rate": 3.5216789667896683e-06, + "loss": 1.4523, + "step": 897 + }, + { + "epoch": 1.117610454262601, + "grad_norm": 2.9454171657562256, + "learning_rate": 3.5193726937269373e-06, + "loss": 1.5009, + "step": 898 + }, + { + "epoch": 1.1188550093341632, + "grad_norm": 2.8533740043640137, + "learning_rate": 3.5170664206642067e-06, + "loss": 1.4265, + "step": 899 + }, + { + "epoch": 1.120099564405725, + "grad_norm": 2.894937038421631, + "learning_rate": 3.5147601476014766e-06, + "loss": 1.4895, + "step": 900 + }, + { + "epoch": 1.120099564405725, + "eval_loss": 1.6701407432556152, + "eval_runtime": 45.477, + "eval_samples_per_second": 21.989, + "eval_steps_per_second": 0.924, + "step": 900 + }, + { + "epoch": 1.1213441194772868, + "grad_norm": 2.933821201324463, + "learning_rate": 3.5124538745387456e-06, + "loss": 1.468, + "step": 901 + }, + { + "epoch": 1.1225886745488487, + "grad_norm": 2.834265947341919, + "learning_rate": 3.510147601476015e-06, + "loss": 1.4621, + "step": 902 + }, + { + "epoch": 1.1238332296204108, + "grad_norm": 2.9064722061157227, + "learning_rate": 3.5078413284132844e-06, + "loss": 1.504, + "step": 903 + }, + { + "epoch": 1.1250777846919726, + "grad_norm": 2.8753886222839355, + "learning_rate": 3.505535055350554e-06, + "loss": 1.4643, + "step": 904 + }, + { + "epoch": 1.1263223397635345, + "grad_norm": 2.894659996032715, + "learning_rate": 3.503228782287823e-06, + "loss": 1.4909, + "step": 905 + }, + { + "epoch": 1.1275668948350965, + "grad_norm": 3.085977792739868, + "learning_rate": 3.5009225092250927e-06, + "loss": 1.459, + "step": 906 + }, + { + "epoch": 1.1288114499066584, + "grad_norm": 2.9958064556121826, + "learning_rate": 3.498616236162362e-06, + "loss": 1.4152, + "step": 907 + }, + { + "epoch": 1.1300560049782202, + "grad_norm": 2.9136762619018555, + "learning_rate": 3.496309963099631e-06, + "loss": 1.473, + "step": 908 + }, + { + "epoch": 1.1313005600497823, + "grad_norm": 2.956437826156616, + "learning_rate": 3.4940036900369005e-06, + "loss": 1.5167, + "step": 909 + }, + { + "epoch": 1.1325451151213441, + "grad_norm": 2.7354865074157715, + "learning_rate": 3.4916974169741704e-06, + "loss": 1.4413, + "step": 910 + }, + { + "epoch": 1.1325451151213441, + "eval_loss": 1.6629912853240967, + "eval_runtime": 41.5478, + "eval_samples_per_second": 24.069, + "eval_steps_per_second": 1.011, + "step": 910 + }, + { + "epoch": 1.133789670192906, + "grad_norm": 2.7443008422851562, + "learning_rate": 3.4893911439114394e-06, + "loss": 1.4475, + "step": 911 + }, + { + "epoch": 1.135034225264468, + "grad_norm": 2.7811238765716553, + "learning_rate": 3.487084870848709e-06, + "loss": 1.4257, + "step": 912 + }, + { + "epoch": 1.13627878033603, + "grad_norm": 2.7916479110717773, + "learning_rate": 3.4847785977859782e-06, + "loss": 1.4048, + "step": 913 + }, + { + "epoch": 1.1375233354075918, + "grad_norm": 2.7835633754730225, + "learning_rate": 3.4824723247232472e-06, + "loss": 1.3996, + "step": 914 + }, + { + "epoch": 1.1387678904791536, + "grad_norm": 2.8188705444335938, + "learning_rate": 3.4801660516605166e-06, + "loss": 1.4541, + "step": 915 + }, + { + "epoch": 1.1400124455507157, + "grad_norm": 3.0134100914001465, + "learning_rate": 3.4778597785977865e-06, + "loss": 1.4353, + "step": 916 + }, + { + "epoch": 1.1412570006222775, + "grad_norm": 2.6965606212615967, + "learning_rate": 3.475553505535056e-06, + "loss": 1.4308, + "step": 917 + }, + { + "epoch": 1.1425015556938394, + "grad_norm": 2.8179919719696045, + "learning_rate": 3.473247232472325e-06, + "loss": 1.5204, + "step": 918 + }, + { + "epoch": 1.1437461107654014, + "grad_norm": 2.7213902473449707, + "learning_rate": 3.4709409594095943e-06, + "loss": 1.4344, + "step": 919 + }, + { + "epoch": 1.1449906658369633, + "grad_norm": 2.850193738937378, + "learning_rate": 3.4686346863468638e-06, + "loss": 1.473, + "step": 920 + }, + { + "epoch": 1.1449906658369633, + "eval_loss": 1.663345217704773, + "eval_runtime": 43.8397, + "eval_samples_per_second": 22.81, + "eval_steps_per_second": 0.958, + "step": 920 + }, + { + "epoch": 1.1462352209085251, + "grad_norm": 2.780198574066162, + "learning_rate": 3.4663284132841328e-06, + "loss": 1.4472, + "step": 921 + }, + { + "epoch": 1.1474797759800872, + "grad_norm": 2.9373831748962402, + "learning_rate": 3.4640221402214026e-06, + "loss": 1.426, + "step": 922 + }, + { + "epoch": 1.148724331051649, + "grad_norm": 2.6986873149871826, + "learning_rate": 3.461715867158672e-06, + "loss": 1.3953, + "step": 923 + }, + { + "epoch": 1.149968886123211, + "grad_norm": 2.927441120147705, + "learning_rate": 3.459409594095941e-06, + "loss": 1.4578, + "step": 924 + }, + { + "epoch": 1.151213441194773, + "grad_norm": 2.8469648361206055, + "learning_rate": 3.4571033210332105e-06, + "loss": 1.4313, + "step": 925 + }, + { + "epoch": 1.1524579962663348, + "grad_norm": 2.8114013671875, + "learning_rate": 3.4547970479704803e-06, + "loss": 1.4267, + "step": 926 + }, + { + "epoch": 1.1537025513378967, + "grad_norm": 2.977605104446411, + "learning_rate": 3.4524907749077493e-06, + "loss": 1.4747, + "step": 927 + }, + { + "epoch": 1.1549471064094585, + "grad_norm": 2.9327852725982666, + "learning_rate": 3.4501845018450187e-06, + "loss": 1.4219, + "step": 928 + }, + { + "epoch": 1.1561916614810206, + "grad_norm": 2.844038963317871, + "learning_rate": 3.447878228782288e-06, + "loss": 1.5146, + "step": 929 + }, + { + "epoch": 1.1574362165525824, + "grad_norm": 2.9402201175689697, + "learning_rate": 3.4455719557195576e-06, + "loss": 1.45, + "step": 930 + }, + { + "epoch": 1.1574362165525824, + "eval_loss": 1.6620122194290161, + "eval_runtime": 46.1748, + "eval_samples_per_second": 21.657, + "eval_steps_per_second": 0.91, + "step": 930 + }, + { + "epoch": 1.1586807716241443, + "grad_norm": 2.842651128768921, + "learning_rate": 3.4432656826568266e-06, + "loss": 1.4415, + "step": 931 + }, + { + "epoch": 1.1599253266957064, + "grad_norm": 2.8909685611724854, + "learning_rate": 3.4409594095940964e-06, + "loss": 1.4529, + "step": 932 + }, + { + "epoch": 1.1611698817672682, + "grad_norm": 2.8025808334350586, + "learning_rate": 3.438653136531366e-06, + "loss": 1.4552, + "step": 933 + }, + { + "epoch": 1.16241443683883, + "grad_norm": 2.6995794773101807, + "learning_rate": 3.436346863468635e-06, + "loss": 1.4435, + "step": 934 + }, + { + "epoch": 1.1636589919103921, + "grad_norm": 2.7307121753692627, + "learning_rate": 3.4340405904059043e-06, + "loss": 1.4447, + "step": 935 + }, + { + "epoch": 1.164903546981954, + "grad_norm": 2.9532501697540283, + "learning_rate": 3.4317343173431737e-06, + "loss": 1.4747, + "step": 936 + }, + { + "epoch": 1.1661481020535158, + "grad_norm": 3.0135788917541504, + "learning_rate": 3.4294280442804427e-06, + "loss": 1.525, + "step": 937 + }, + { + "epoch": 1.167392657125078, + "grad_norm": 2.796355962753296, + "learning_rate": 3.4271217712177125e-06, + "loss": 1.4539, + "step": 938 + }, + { + "epoch": 1.1686372121966397, + "grad_norm": 2.9422309398651123, + "learning_rate": 3.424815498154982e-06, + "loss": 1.4664, + "step": 939 + }, + { + "epoch": 1.1698817672682016, + "grad_norm": 2.7680039405822754, + "learning_rate": 3.4225092250922514e-06, + "loss": 1.4837, + "step": 940 + }, + { + "epoch": 1.1698817672682016, + "eval_loss": 1.660908818244934, + "eval_runtime": 45.6998, + "eval_samples_per_second": 21.882, + "eval_steps_per_second": 0.919, + "step": 940 + }, + { + "epoch": 1.1711263223397634, + "grad_norm": 2.8888845443725586, + "learning_rate": 3.4202029520295204e-06, + "loss": 1.5315, + "step": 941 + }, + { + "epoch": 1.1723708774113255, + "grad_norm": 2.7124130725860596, + "learning_rate": 3.41789667896679e-06, + "loss": 1.4188, + "step": 942 + }, + { + "epoch": 1.1736154324828874, + "grad_norm": 2.75203013420105, + "learning_rate": 3.4155904059040596e-06, + "loss": 1.4533, + "step": 943 + }, + { + "epoch": 1.1748599875544492, + "grad_norm": 2.865480422973633, + "learning_rate": 3.4132841328413286e-06, + "loss": 1.4532, + "step": 944 + }, + { + "epoch": 1.1761045426260113, + "grad_norm": 2.8618321418762207, + "learning_rate": 3.410977859778598e-06, + "loss": 1.4386, + "step": 945 + }, + { + "epoch": 1.1773490976975731, + "grad_norm": 2.7656145095825195, + "learning_rate": 3.4086715867158675e-06, + "loss": 1.3996, + "step": 946 + }, + { + "epoch": 1.178593652769135, + "grad_norm": 2.76213002204895, + "learning_rate": 3.4063653136531365e-06, + "loss": 1.4979, + "step": 947 + }, + { + "epoch": 1.179838207840697, + "grad_norm": 2.8581957817077637, + "learning_rate": 3.4040590405904063e-06, + "loss": 1.4673, + "step": 948 + }, + { + "epoch": 1.1810827629122589, + "grad_norm": 2.7825586795806885, + "learning_rate": 3.4017527675276758e-06, + "loss": 1.4453, + "step": 949 + }, + { + "epoch": 1.1823273179838207, + "grad_norm": 2.9226131439208984, + "learning_rate": 3.3994464944649448e-06, + "loss": 1.4879, + "step": 950 + }, + { + "epoch": 1.1823273179838207, + "eval_loss": 1.6621263027191162, + "eval_runtime": 45.1222, + "eval_samples_per_second": 22.162, + "eval_steps_per_second": 0.931, + "step": 950 + }, + { + "epoch": 1.1835718730553828, + "grad_norm": 2.9997975826263428, + "learning_rate": 3.397140221402214e-06, + "loss": 1.505, + "step": 951 + }, + { + "epoch": 1.1848164281269447, + "grad_norm": 2.7225453853607178, + "learning_rate": 3.3948339483394836e-06, + "loss": 1.4695, + "step": 952 + }, + { + "epoch": 1.1860609831985065, + "grad_norm": 2.916473150253296, + "learning_rate": 3.3925276752767534e-06, + "loss": 1.4901, + "step": 953 + }, + { + "epoch": 1.1873055382700684, + "grad_norm": 2.8371644020080566, + "learning_rate": 3.3902214022140224e-06, + "loss": 1.4944, + "step": 954 + }, + { + "epoch": 1.1885500933416304, + "grad_norm": 2.813662528991699, + "learning_rate": 3.387915129151292e-06, + "loss": 1.4603, + "step": 955 + }, + { + "epoch": 1.1897946484131923, + "grad_norm": 2.991931676864624, + "learning_rate": 3.3856088560885613e-06, + "loss": 1.5496, + "step": 956 + }, + { + "epoch": 1.1910392034847541, + "grad_norm": 2.6252501010894775, + "learning_rate": 3.3833025830258303e-06, + "loss": 1.4405, + "step": 957 + }, + { + "epoch": 1.1922837585563162, + "grad_norm": 2.747725009918213, + "learning_rate": 3.3809963099630997e-06, + "loss": 1.4689, + "step": 958 + }, + { + "epoch": 1.193528313627878, + "grad_norm": 2.887763261795044, + "learning_rate": 3.3786900369003696e-06, + "loss": 1.4501, + "step": 959 + }, + { + "epoch": 1.1947728686994399, + "grad_norm": 2.9954354763031006, + "learning_rate": 3.3763837638376386e-06, + "loss": 1.4978, + "step": 960 + }, + { + "epoch": 1.1947728686994399, + "eval_loss": 1.664442539215088, + "eval_runtime": 49.1266, + "eval_samples_per_second": 20.356, + "eval_steps_per_second": 0.855, + "step": 960 + }, + { + "epoch": 1.196017423771002, + "grad_norm": 3.0174152851104736, + "learning_rate": 3.374077490774908e-06, + "loss": 1.4353, + "step": 961 + }, + { + "epoch": 1.1972619788425638, + "grad_norm": 2.9082882404327393, + "learning_rate": 3.3717712177121774e-06, + "loss": 1.4878, + "step": 962 + }, + { + "epoch": 1.1985065339141256, + "grad_norm": 2.897993326187134, + "learning_rate": 3.3694649446494464e-06, + "loss": 1.4975, + "step": 963 + }, + { + "epoch": 1.1997510889856877, + "grad_norm": 2.739470958709717, + "learning_rate": 3.3671586715867163e-06, + "loss": 1.4202, + "step": 964 + }, + { + "epoch": 1.2009956440572496, + "grad_norm": 2.817786693572998, + "learning_rate": 3.3648523985239857e-06, + "loss": 1.5195, + "step": 965 + }, + { + "epoch": 1.2022401991288114, + "grad_norm": 2.8476386070251465, + "learning_rate": 3.362546125461255e-06, + "loss": 1.444, + "step": 966 + }, + { + "epoch": 1.2034847542003733, + "grad_norm": 2.804047107696533, + "learning_rate": 3.360239852398524e-06, + "loss": 1.4442, + "step": 967 + }, + { + "epoch": 1.2047293092719353, + "grad_norm": 2.9285430908203125, + "learning_rate": 3.3579335793357935e-06, + "loss": 1.4877, + "step": 968 + }, + { + "epoch": 1.2059738643434972, + "grad_norm": 2.8814878463745117, + "learning_rate": 3.3556273062730634e-06, + "loss": 1.5078, + "step": 969 + }, + { + "epoch": 1.207218419415059, + "grad_norm": 2.970728635787964, + "learning_rate": 3.3533210332103324e-06, + "loss": 1.4166, + "step": 970 + }, + { + "epoch": 1.207218419415059, + "eval_loss": 1.6635315418243408, + "eval_runtime": 47.2321, + "eval_samples_per_second": 21.172, + "eval_steps_per_second": 0.889, + "step": 970 + }, + { + "epoch": 1.208462974486621, + "grad_norm": 2.9836292266845703, + "learning_rate": 3.3510147601476018e-06, + "loss": 1.4777, + "step": 971 + }, + { + "epoch": 1.209707529558183, + "grad_norm": 2.8495216369628906, + "learning_rate": 3.348708487084871e-06, + "loss": 1.4829, + "step": 972 + }, + { + "epoch": 1.2109520846297448, + "grad_norm": 2.7709312438964844, + "learning_rate": 3.34640221402214e-06, + "loss": 1.4242, + "step": 973 + }, + { + "epoch": 1.2121966397013069, + "grad_norm": 2.999462366104126, + "learning_rate": 3.3440959409594096e-06, + "loss": 1.4148, + "step": 974 + }, + { + "epoch": 1.2134411947728687, + "grad_norm": 2.8719074726104736, + "learning_rate": 3.3417896678966795e-06, + "loss": 1.4196, + "step": 975 + }, + { + "epoch": 1.2146857498444306, + "grad_norm": 2.8469338417053223, + "learning_rate": 3.3394833948339485e-06, + "loss": 1.4941, + "step": 976 + }, + { + "epoch": 1.2159303049159926, + "grad_norm": 2.9780938625335693, + "learning_rate": 3.337177121771218e-06, + "loss": 1.513, + "step": 977 + }, + { + "epoch": 1.2171748599875545, + "grad_norm": 2.814760684967041, + "learning_rate": 3.3348708487084873e-06, + "loss": 1.4204, + "step": 978 + }, + { + "epoch": 1.2184194150591163, + "grad_norm": 3.016261339187622, + "learning_rate": 3.332564575645757e-06, + "loss": 1.4389, + "step": 979 + }, + { + "epoch": 1.2196639701306782, + "grad_norm": 2.9385855197906494, + "learning_rate": 3.330258302583026e-06, + "loss": 1.4885, + "step": 980 + }, + { + "epoch": 1.2196639701306782, + "eval_loss": 1.6688873767852783, + "eval_runtime": 46.2942, + "eval_samples_per_second": 21.601, + "eval_steps_per_second": 0.907, + "step": 980 + }, + { + "epoch": 1.2209085252022402, + "grad_norm": 3.1434757709503174, + "learning_rate": 3.3279520295202956e-06, + "loss": 1.5157, + "step": 981 + }, + { + "epoch": 1.222153080273802, + "grad_norm": 3.0104827880859375, + "learning_rate": 3.325645756457565e-06, + "loss": 1.49, + "step": 982 + }, + { + "epoch": 1.223397635345364, + "grad_norm": 3.1913645267486572, + "learning_rate": 3.323339483394834e-06, + "loss": 1.4937, + "step": 983 + }, + { + "epoch": 1.224642190416926, + "grad_norm": 3.19496750831604, + "learning_rate": 3.3210332103321034e-06, + "loss": 1.5197, + "step": 984 + }, + { + "epoch": 1.2258867454884879, + "grad_norm": 3.046232223510742, + "learning_rate": 3.3187269372693733e-06, + "loss": 1.4497, + "step": 985 + }, + { + "epoch": 1.2271313005600497, + "grad_norm": 2.850675344467163, + "learning_rate": 3.3164206642066423e-06, + "loss": 1.4595, + "step": 986 + }, + { + "epoch": 1.2283758556316118, + "grad_norm": 2.7744576930999756, + "learning_rate": 3.3141143911439117e-06, + "loss": 1.4726, + "step": 987 + }, + { + "epoch": 1.2296204107031736, + "grad_norm": 3.017153024673462, + "learning_rate": 3.311808118081181e-06, + "loss": 1.493, + "step": 988 + }, + { + "epoch": 1.2308649657747355, + "grad_norm": 2.8821120262145996, + "learning_rate": 3.3095018450184506e-06, + "loss": 1.4734, + "step": 989 + }, + { + "epoch": 1.2321095208462975, + "grad_norm": 2.8805840015411377, + "learning_rate": 3.3071955719557196e-06, + "loss": 1.4562, + "step": 990 + }, + { + "epoch": 1.2321095208462975, + "eval_loss": 1.669345736503601, + "eval_runtime": 47.0893, + "eval_samples_per_second": 21.236, + "eval_steps_per_second": 0.892, + "step": 990 + }, + { + "epoch": 1.2333540759178594, + "grad_norm": 2.7599008083343506, + "learning_rate": 3.3048892988929894e-06, + "loss": 1.4218, + "step": 991 + }, + { + "epoch": 1.2345986309894212, + "grad_norm": 2.9835240840911865, + "learning_rate": 3.302583025830259e-06, + "loss": 1.4596, + "step": 992 + }, + { + "epoch": 1.235843186060983, + "grad_norm": 2.843358039855957, + "learning_rate": 3.300276752767528e-06, + "loss": 1.4561, + "step": 993 + }, + { + "epoch": 1.2370877411325452, + "grad_norm": 2.7386093139648438, + "learning_rate": 3.2979704797047972e-06, + "loss": 1.4475, + "step": 994 + }, + { + "epoch": 1.238332296204107, + "grad_norm": 2.668161392211914, + "learning_rate": 3.2956642066420667e-06, + "loss": 1.4661, + "step": 995 + }, + { + "epoch": 1.2395768512756689, + "grad_norm": 2.947235107421875, + "learning_rate": 3.2933579335793357e-06, + "loss": 1.4358, + "step": 996 + }, + { + "epoch": 1.240821406347231, + "grad_norm": 2.8159878253936768, + "learning_rate": 3.2910516605166055e-06, + "loss": 1.4432, + "step": 997 + }, + { + "epoch": 1.2420659614187928, + "grad_norm": 2.896597385406494, + "learning_rate": 3.288745387453875e-06, + "loss": 1.5585, + "step": 998 + }, + { + "epoch": 1.2433105164903546, + "grad_norm": 2.80241322517395, + "learning_rate": 3.286439114391144e-06, + "loss": 1.5076, + "step": 999 + }, + { + "epoch": 1.2445550715619167, + "grad_norm": 3.008939027786255, + "learning_rate": 3.2841328413284134e-06, + "loss": 1.4234, + "step": 1000 + }, + { + "epoch": 1.2445550715619167, + "eval_loss": 1.6621588468551636, + "eval_runtime": 50.4926, + "eval_samples_per_second": 19.805, + "eval_steps_per_second": 0.832, + "step": 1000 + }, + { + "epoch": 1.2457996266334785, + "grad_norm": 2.9872381687164307, + "learning_rate": 3.281826568265683e-06, + "loss": 1.453, + "step": 1001 + }, + { + "epoch": 1.2470441817050404, + "grad_norm": 2.704676628112793, + "learning_rate": 3.2795202952029526e-06, + "loss": 1.4711, + "step": 1002 + }, + { + "epoch": 1.2482887367766025, + "grad_norm": 2.8100757598876953, + "learning_rate": 3.2772140221402216e-06, + "loss": 1.3884, + "step": 1003 + }, + { + "epoch": 1.2495332918481643, + "grad_norm": 2.902953624725342, + "learning_rate": 3.274907749077491e-06, + "loss": 1.4986, + "step": 1004 + }, + { + "epoch": 1.2507778469197262, + "grad_norm": 2.980163097381592, + "learning_rate": 3.2726014760147605e-06, + "loss": 1.5067, + "step": 1005 + }, + { + "epoch": 1.252022401991288, + "grad_norm": 2.8820345401763916, + "learning_rate": 3.2702952029520295e-06, + "loss": 1.4995, + "step": 1006 + }, + { + "epoch": 1.25326695706285, + "grad_norm": 2.8219635486602783, + "learning_rate": 3.2679889298892993e-06, + "loss": 1.4881, + "step": 1007 + }, + { + "epoch": 1.254511512134412, + "grad_norm": 2.8593199253082275, + "learning_rate": 3.2656826568265687e-06, + "loss": 1.4469, + "step": 1008 + }, + { + "epoch": 1.255756067205974, + "grad_norm": 2.888357639312744, + "learning_rate": 3.2633763837638377e-06, + "loss": 1.4869, + "step": 1009 + }, + { + "epoch": 1.2570006222775358, + "grad_norm": 2.9985013008117676, + "learning_rate": 3.261070110701107e-06, + "loss": 1.4724, + "step": 1010 + }, + { + "epoch": 1.2570006222775358, + "eval_loss": 1.6492141485214233, + "eval_runtime": 45.2197, + "eval_samples_per_second": 22.114, + "eval_steps_per_second": 0.929, + "step": 1010 + }, + { + "epoch": 1.2582451773490977, + "grad_norm": 2.860780715942383, + "learning_rate": 3.2587638376383766e-06, + "loss": 1.4618, + "step": 1011 + }, + { + "epoch": 1.2594897324206595, + "grad_norm": 2.766479730606079, + "learning_rate": 3.2564575645756456e-06, + "loss": 1.4642, + "step": 1012 + }, + { + "epoch": 1.2607342874922216, + "grad_norm": 2.711759328842163, + "learning_rate": 3.2541512915129154e-06, + "loss": 1.4283, + "step": 1013 + }, + { + "epoch": 1.2619788425637835, + "grad_norm": 2.8365769386291504, + "learning_rate": 3.251845018450185e-06, + "loss": 1.4248, + "step": 1014 + }, + { + "epoch": 1.2632233976353453, + "grad_norm": 2.957620143890381, + "learning_rate": 3.2495387453874543e-06, + "loss": 1.4662, + "step": 1015 + }, + { + "epoch": 1.2644679527069074, + "grad_norm": 2.8066208362579346, + "learning_rate": 3.2472324723247233e-06, + "loss": 1.4447, + "step": 1016 + }, + { + "epoch": 1.2657125077784692, + "grad_norm": 2.902040958404541, + "learning_rate": 3.244926199261993e-06, + "loss": 1.4743, + "step": 1017 + }, + { + "epoch": 1.266957062850031, + "grad_norm": 2.7636687755584717, + "learning_rate": 3.2426199261992625e-06, + "loss": 1.4465, + "step": 1018 + }, + { + "epoch": 1.268201617921593, + "grad_norm": 2.95261549949646, + "learning_rate": 3.2403136531365315e-06, + "loss": 1.4769, + "step": 1019 + }, + { + "epoch": 1.269446172993155, + "grad_norm": 2.911128520965576, + "learning_rate": 3.238007380073801e-06, + "loss": 1.3843, + "step": 1020 + }, + { + "epoch": 1.269446172993155, + "eval_loss": 1.6526964902877808, + "eval_runtime": 48.2126, + "eval_samples_per_second": 20.741, + "eval_steps_per_second": 0.871, + "step": 1020 + }, + { + "epoch": 1.2706907280647168, + "grad_norm": 2.769951820373535, + "learning_rate": 3.2357011070110704e-06, + "loss": 1.4847, + "step": 1021 + }, + { + "epoch": 1.271935283136279, + "grad_norm": 3.0451674461364746, + "learning_rate": 3.2333948339483394e-06, + "loss": 1.4453, + "step": 1022 + }, + { + "epoch": 1.2731798382078408, + "grad_norm": 2.9543521404266357, + "learning_rate": 3.2310885608856092e-06, + "loss": 1.4789, + "step": 1023 + }, + { + "epoch": 1.2744243932794026, + "grad_norm": 2.9174394607543945, + "learning_rate": 3.2287822878228787e-06, + "loss": 1.4807, + "step": 1024 + }, + { + "epoch": 1.2756689483509644, + "grad_norm": 2.888258934020996, + "learning_rate": 3.2264760147601477e-06, + "loss": 1.4367, + "step": 1025 + }, + { + "epoch": 1.2769135034225265, + "grad_norm": 2.8346047401428223, + "learning_rate": 3.224169741697417e-06, + "loss": 1.4313, + "step": 1026 + }, + { + "epoch": 1.2781580584940884, + "grad_norm": 2.7531332969665527, + "learning_rate": 3.2218634686346865e-06, + "loss": 1.4018, + "step": 1027 + }, + { + "epoch": 1.2794026135656502, + "grad_norm": 3.136946439743042, + "learning_rate": 3.2195571955719564e-06, + "loss": 1.4417, + "step": 1028 + }, + { + "epoch": 1.2806471686372123, + "grad_norm": 2.9618165493011475, + "learning_rate": 3.2172509225092254e-06, + "loss": 1.5117, + "step": 1029 + }, + { + "epoch": 1.2818917237087741, + "grad_norm": 3.0740747451782227, + "learning_rate": 3.2149446494464948e-06, + "loss": 1.4641, + "step": 1030 + }, + { + "epoch": 1.2818917237087741, + "eval_loss": 1.6572761535644531, + "eval_runtime": 51.6696, + "eval_samples_per_second": 19.354, + "eval_steps_per_second": 0.813, + "step": 1030 + }, + { + "epoch": 1.283136278780336, + "grad_norm": 2.9225106239318848, + "learning_rate": 3.212638376383764e-06, + "loss": 1.4421, + "step": 1031 + }, + { + "epoch": 1.2843808338518978, + "grad_norm": 2.9530649185180664, + "learning_rate": 3.210332103321033e-06, + "loss": 1.5138, + "step": 1032 + }, + { + "epoch": 1.28562538892346, + "grad_norm": 2.8513238430023193, + "learning_rate": 3.208025830258303e-06, + "loss": 1.411, + "step": 1033 + }, + { + "epoch": 1.2868699439950217, + "grad_norm": 2.7440171241760254, + "learning_rate": 3.2057195571955725e-06, + "loss": 1.4182, + "step": 1034 + }, + { + "epoch": 1.2881144990665838, + "grad_norm": 2.8126630783081055, + "learning_rate": 3.2034132841328415e-06, + "loss": 1.4333, + "step": 1035 + }, + { + "epoch": 1.2893590541381457, + "grad_norm": 2.8598744869232178, + "learning_rate": 3.201107011070111e-06, + "loss": 1.439, + "step": 1036 + }, + { + "epoch": 1.2906036092097075, + "grad_norm": 2.901622772216797, + "learning_rate": 3.1988007380073803e-06, + "loss": 1.4461, + "step": 1037 + }, + { + "epoch": 1.2918481642812694, + "grad_norm": 2.7670180797576904, + "learning_rate": 3.1964944649446493e-06, + "loss": 1.4772, + "step": 1038 + }, + { + "epoch": 1.2930927193528314, + "grad_norm": 2.7860350608825684, + "learning_rate": 3.194188191881919e-06, + "loss": 1.4414, + "step": 1039 + }, + { + "epoch": 1.2943372744243933, + "grad_norm": 2.822052478790283, + "learning_rate": 3.1918819188191886e-06, + "loss": 1.4192, + "step": 1040 + }, + { + "epoch": 1.2943372744243933, + "eval_loss": 1.663284420967102, + "eval_runtime": 47.2822, + "eval_samples_per_second": 21.15, + "eval_steps_per_second": 0.888, + "step": 1040 + }, + { + "epoch": 1.2955818294959551, + "grad_norm": 2.909109115600586, + "learning_rate": 3.189575645756458e-06, + "loss": 1.4795, + "step": 1041 + }, + { + "epoch": 1.2968263845675172, + "grad_norm": 2.933673143386841, + "learning_rate": 3.187269372693727e-06, + "loss": 1.5038, + "step": 1042 + }, + { + "epoch": 1.298070939639079, + "grad_norm": 3.032518148422241, + "learning_rate": 3.1849630996309964e-06, + "loss": 1.5133, + "step": 1043 + }, + { + "epoch": 1.299315494710641, + "grad_norm": 2.826040506362915, + "learning_rate": 3.1826568265682663e-06, + "loss": 1.4395, + "step": 1044 + }, + { + "epoch": 1.3005600497822027, + "grad_norm": 2.8152658939361572, + "learning_rate": 3.1803505535055353e-06, + "loss": 1.4787, + "step": 1045 + }, + { + "epoch": 1.3018046048537648, + "grad_norm": 2.9410431385040283, + "learning_rate": 3.1780442804428047e-06, + "loss": 1.4375, + "step": 1046 + }, + { + "epoch": 1.3030491599253267, + "grad_norm": 2.948636531829834, + "learning_rate": 3.175738007380074e-06, + "loss": 1.4847, + "step": 1047 + }, + { + "epoch": 1.3042937149968887, + "grad_norm": 2.6287615299224854, + "learning_rate": 3.173431734317343e-06, + "loss": 1.4186, + "step": 1048 + }, + { + "epoch": 1.3055382700684506, + "grad_norm": 2.836029052734375, + "learning_rate": 3.1711254612546125e-06, + "loss": 1.4251, + "step": 1049 + }, + { + "epoch": 1.3067828251400124, + "grad_norm": 2.982081651687622, + "learning_rate": 3.1688191881918824e-06, + "loss": 1.4951, + "step": 1050 + }, + { + "epoch": 1.3067828251400124, + "eval_loss": 1.6578067541122437, + "eval_runtime": 47.5847, + "eval_samples_per_second": 21.015, + "eval_steps_per_second": 0.883, + "step": 1050 + }, + { + "epoch": 1.3080273802115743, + "grad_norm": 2.9297118186950684, + "learning_rate": 3.166512915129152e-06, + "loss": 1.4699, + "step": 1051 + }, + { + "epoch": 1.3092719352831363, + "grad_norm": 2.906435251235962, + "learning_rate": 3.164206642066421e-06, + "loss": 1.4562, + "step": 1052 + }, + { + "epoch": 1.3105164903546982, + "grad_norm": 2.8655896186828613, + "learning_rate": 3.1619003690036902e-06, + "loss": 1.4351, + "step": 1053 + }, + { + "epoch": 1.31176104542626, + "grad_norm": 2.779244899749756, + "learning_rate": 3.15959409594096e-06, + "loss": 1.4479, + "step": 1054 + }, + { + "epoch": 1.3130056004978221, + "grad_norm": 2.9863739013671875, + "learning_rate": 3.157287822878229e-06, + "loss": 1.4964, + "step": 1055 + }, + { + "epoch": 1.314250155569384, + "grad_norm": 2.821871042251587, + "learning_rate": 3.1549815498154985e-06, + "loss": 1.4173, + "step": 1056 + }, + { + "epoch": 1.3154947106409458, + "grad_norm": 2.8573882579803467, + "learning_rate": 3.152675276752768e-06, + "loss": 1.4877, + "step": 1057 + }, + { + "epoch": 1.3167392657125077, + "grad_norm": 2.9824137687683105, + "learning_rate": 3.150369003690037e-06, + "loss": 1.5124, + "step": 1058 + }, + { + "epoch": 1.3179838207840697, + "grad_norm": 2.881098747253418, + "learning_rate": 3.1480627306273063e-06, + "loss": 1.4346, + "step": 1059 + }, + { + "epoch": 1.3192283758556316, + "grad_norm": 2.918782949447632, + "learning_rate": 3.145756457564576e-06, + "loss": 1.4797, + "step": 1060 + }, + { + "epoch": 1.3192283758556316, + "eval_loss": 1.6510179042816162, + "eval_runtime": 53.6356, + "eval_samples_per_second": 18.644, + "eval_steps_per_second": 0.783, + "step": 1060 + }, + { + "epoch": 1.3204729309271936, + "grad_norm": 3.045776128768921, + "learning_rate": 3.143450184501845e-06, + "loss": 1.5039, + "step": 1061 + }, + { + "epoch": 1.3217174859987555, + "grad_norm": 3.041501760482788, + "learning_rate": 3.1411439114391146e-06, + "loss": 1.4878, + "step": 1062 + }, + { + "epoch": 1.3229620410703173, + "grad_norm": 2.917837142944336, + "learning_rate": 3.138837638376384e-06, + "loss": 1.4202, + "step": 1063 + }, + { + "epoch": 1.3242065961418792, + "grad_norm": 2.708070755004883, + "learning_rate": 3.1365313653136535e-06, + "loss": 1.4071, + "step": 1064 + }, + { + "epoch": 1.3254511512134413, + "grad_norm": 2.9146156311035156, + "learning_rate": 3.1342250922509225e-06, + "loss": 1.4428, + "step": 1065 + }, + { + "epoch": 1.326695706285003, + "grad_norm": 3.220919132232666, + "learning_rate": 3.1319188191881923e-06, + "loss": 1.4733, + "step": 1066 + }, + { + "epoch": 1.327940261356565, + "grad_norm": 2.9127554893493652, + "learning_rate": 3.1296125461254617e-06, + "loss": 1.4554, + "step": 1067 + }, + { + "epoch": 1.329184816428127, + "grad_norm": 3.053191661834717, + "learning_rate": 3.1273062730627307e-06, + "loss": 1.4044, + "step": 1068 + }, + { + "epoch": 1.3304293714996889, + "grad_norm": 3.2937021255493164, + "learning_rate": 3.125e-06, + "loss": 1.4949, + "step": 1069 + }, + { + "epoch": 1.3316739265712507, + "grad_norm": 2.976717948913574, + "learning_rate": 3.12269372693727e-06, + "loss": 1.446, + "step": 1070 + }, + { + "epoch": 1.3316739265712507, + "eval_loss": 1.6503105163574219, + "eval_runtime": 50.7579, + "eval_samples_per_second": 19.701, + "eval_steps_per_second": 0.827, + "step": 1070 + }, + { + "epoch": 1.3329184816428126, + "grad_norm": 3.0860936641693115, + "learning_rate": 3.120387453874539e-06, + "loss": 1.4055, + "step": 1071 + }, + { + "epoch": 1.3341630367143746, + "grad_norm": 3.0401217937469482, + "learning_rate": 3.1180811808118084e-06, + "loss": 1.4688, + "step": 1072 + }, + { + "epoch": 1.3354075917859365, + "grad_norm": 2.953253984451294, + "learning_rate": 3.115774907749078e-06, + "loss": 1.5001, + "step": 1073 + }, + { + "epoch": 1.3366521468574986, + "grad_norm": 3.0665059089660645, + "learning_rate": 3.113468634686347e-06, + "loss": 1.4217, + "step": 1074 + }, + { + "epoch": 1.3378967019290604, + "grad_norm": 3.121600389480591, + "learning_rate": 3.1111623616236163e-06, + "loss": 1.4508, + "step": 1075 + }, + { + "epoch": 1.3391412570006223, + "grad_norm": 2.8057339191436768, + "learning_rate": 3.108856088560886e-06, + "loss": 1.4241, + "step": 1076 + }, + { + "epoch": 1.340385812072184, + "grad_norm": 3.0147411823272705, + "learning_rate": 3.1065498154981555e-06, + "loss": 1.479, + "step": 1077 + }, + { + "epoch": 1.3416303671437462, + "grad_norm": 3.030010223388672, + "learning_rate": 3.1042435424354245e-06, + "loss": 1.4861, + "step": 1078 + }, + { + "epoch": 1.342874922215308, + "grad_norm": 3.027998208999634, + "learning_rate": 3.101937269372694e-06, + "loss": 1.4916, + "step": 1079 + }, + { + "epoch": 1.3441194772868699, + "grad_norm": 2.9252536296844482, + "learning_rate": 3.0996309963099634e-06, + "loss": 1.418, + "step": 1080 + }, + { + "epoch": 1.3441194772868699, + "eval_loss": 1.6490333080291748, + "eval_runtime": 47.4026, + "eval_samples_per_second": 21.096, + "eval_steps_per_second": 0.886, + "step": 1080 + }, + { + "epoch": 1.345364032358432, + "grad_norm": 3.110847234725952, + "learning_rate": 3.0973247232472324e-06, + "loss": 1.4127, + "step": 1081 + }, + { + "epoch": 1.3466085874299938, + "grad_norm": 2.846491575241089, + "learning_rate": 3.0950184501845022e-06, + "loss": 1.444, + "step": 1082 + }, + { + "epoch": 1.3478531425015556, + "grad_norm": 2.950533151626587, + "learning_rate": 3.0927121771217716e-06, + "loss": 1.4456, + "step": 1083 + }, + { + "epoch": 1.3490976975731175, + "grad_norm": 2.875964403152466, + "learning_rate": 3.0904059040590406e-06, + "loss": 1.4485, + "step": 1084 + }, + { + "epoch": 1.3503422526446796, + "grad_norm": 2.9612326622009277, + "learning_rate": 3.08809963099631e-06, + "loss": 1.4446, + "step": 1085 + }, + { + "epoch": 1.3515868077162414, + "grad_norm": 2.925297975540161, + "learning_rate": 3.0857933579335795e-06, + "loss": 1.4847, + "step": 1086 + }, + { + "epoch": 1.3528313627878035, + "grad_norm": 2.991616725921631, + "learning_rate": 3.0834870848708485e-06, + "loss": 1.5151, + "step": 1087 + }, + { + "epoch": 1.3540759178593653, + "grad_norm": 2.96329927444458, + "learning_rate": 3.0811808118081183e-06, + "loss": 1.4574, + "step": 1088 + }, + { + "epoch": 1.3553204729309272, + "grad_norm": 2.948702573776245, + "learning_rate": 3.0788745387453878e-06, + "loss": 1.4585, + "step": 1089 + }, + { + "epoch": 1.356565028002489, + "grad_norm": 3.218463659286499, + "learning_rate": 3.076568265682657e-06, + "loss": 1.4439, + "step": 1090 + }, + { + "epoch": 1.356565028002489, + "eval_loss": 1.645838975906372, + "eval_runtime": 47.048, + "eval_samples_per_second": 21.255, + "eval_steps_per_second": 0.893, + "step": 1090 + }, + { + "epoch": 1.357809583074051, + "grad_norm": 2.9799578189849854, + "learning_rate": 3.074261992619926e-06, + "loss": 1.4883, + "step": 1091 + }, + { + "epoch": 1.359054138145613, + "grad_norm": 2.7660505771636963, + "learning_rate": 3.071955719557196e-06, + "loss": 1.4349, + "step": 1092 + }, + { + "epoch": 1.3602986932171748, + "grad_norm": 3.0201916694641113, + "learning_rate": 3.0696494464944655e-06, + "loss": 1.418, + "step": 1093 + }, + { + "epoch": 1.3615432482887369, + "grad_norm": 2.844207763671875, + "learning_rate": 3.0673431734317345e-06, + "loss": 1.4769, + "step": 1094 + }, + { + "epoch": 1.3627878033602987, + "grad_norm": 2.8306753635406494, + "learning_rate": 3.065036900369004e-06, + "loss": 1.4653, + "step": 1095 + }, + { + "epoch": 1.3640323584318605, + "grad_norm": 2.805023431777954, + "learning_rate": 3.0627306273062733e-06, + "loss": 1.4771, + "step": 1096 + }, + { + "epoch": 1.3652769135034224, + "grad_norm": 3.020883083343506, + "learning_rate": 3.0604243542435423e-06, + "loss": 1.4237, + "step": 1097 + }, + { + "epoch": 1.3665214685749845, + "grad_norm": 2.8395731449127197, + "learning_rate": 3.058118081180812e-06, + "loss": 1.4647, + "step": 1098 + }, + { + "epoch": 1.3677660236465463, + "grad_norm": 2.871156692504883, + "learning_rate": 3.0558118081180816e-06, + "loss": 1.4196, + "step": 1099 + }, + { + "epoch": 1.3690105787181084, + "grad_norm": 2.974905014038086, + "learning_rate": 3.053505535055351e-06, + "loss": 1.4779, + "step": 1100 + }, + { + "epoch": 1.3690105787181084, + "eval_loss": 1.651503562927246, + "eval_runtime": 54.2713, + "eval_samples_per_second": 18.426, + "eval_steps_per_second": 0.774, + "step": 1100 + }, + { + "epoch": 1.3702551337896702, + "grad_norm": 2.7591328620910645, + "learning_rate": 3.05119926199262e-06, + "loss": 1.3899, + "step": 1101 + }, + { + "epoch": 1.371499688861232, + "grad_norm": 2.8575668334960938, + "learning_rate": 3.0488929889298894e-06, + "loss": 1.4482, + "step": 1102 + }, + { + "epoch": 1.372744243932794, + "grad_norm": 2.9049339294433594, + "learning_rate": 3.0465867158671593e-06, + "loss": 1.4785, + "step": 1103 + }, + { + "epoch": 1.373988799004356, + "grad_norm": 2.816453695297241, + "learning_rate": 3.0442804428044283e-06, + "loss": 1.4601, + "step": 1104 + }, + { + "epoch": 1.3752333540759178, + "grad_norm": 2.8844189643859863, + "learning_rate": 3.0419741697416977e-06, + "loss": 1.496, + "step": 1105 + }, + { + "epoch": 1.3764779091474797, + "grad_norm": 3.014744281768799, + "learning_rate": 3.039667896678967e-06, + "loss": 1.4098, + "step": 1106 + }, + { + "epoch": 1.3777224642190418, + "grad_norm": 2.8243560791015625, + "learning_rate": 3.037361623616236e-06, + "loss": 1.4086, + "step": 1107 + }, + { + "epoch": 1.3789670192906036, + "grad_norm": 2.8333253860473633, + "learning_rate": 3.035055350553506e-06, + "loss": 1.3516, + "step": 1108 + }, + { + "epoch": 1.3802115743621655, + "grad_norm": 2.926455020904541, + "learning_rate": 3.0327490774907754e-06, + "loss": 1.3976, + "step": 1109 + }, + { + "epoch": 1.3814561294337273, + "grad_norm": 2.900937080383301, + "learning_rate": 3.0304428044280444e-06, + "loss": 1.4336, + "step": 1110 + }, + { + "epoch": 1.3814561294337273, + "eval_loss": 1.647048830986023, + "eval_runtime": 42.3955, + "eval_samples_per_second": 23.587, + "eval_steps_per_second": 0.991, + "step": 1110 + }, + { + "epoch": 1.3827006845052894, + "grad_norm": 3.0756969451904297, + "learning_rate": 3.028136531365314e-06, + "loss": 1.4674, + "step": 1111 + }, + { + "epoch": 1.3839452395768512, + "grad_norm": 2.7754578590393066, + "learning_rate": 3.0258302583025832e-06, + "loss": 1.4435, + "step": 1112 + }, + { + "epoch": 1.3851897946484133, + "grad_norm": 2.8011393547058105, + "learning_rate": 3.023523985239853e-06, + "loss": 1.4212, + "step": 1113 + }, + { + "epoch": 1.3864343497199751, + "grad_norm": 2.7063710689544678, + "learning_rate": 3.021217712177122e-06, + "loss": 1.3769, + "step": 1114 + }, + { + "epoch": 1.387678904791537, + "grad_norm": 2.830411434173584, + "learning_rate": 3.0189114391143915e-06, + "loss": 1.4832, + "step": 1115 + }, + { + "epoch": 1.3889234598630988, + "grad_norm": 2.9075911045074463, + "learning_rate": 3.016605166051661e-06, + "loss": 1.486, + "step": 1116 + }, + { + "epoch": 1.390168014934661, + "grad_norm": 2.847822666168213, + "learning_rate": 3.01429889298893e-06, + "loss": 1.424, + "step": 1117 + }, + { + "epoch": 1.3914125700062228, + "grad_norm": 2.8822033405303955, + "learning_rate": 3.0119926199261993e-06, + "loss": 1.4218, + "step": 1118 + }, + { + "epoch": 1.3926571250777848, + "grad_norm": 2.8944740295410156, + "learning_rate": 3.009686346863469e-06, + "loss": 1.4283, + "step": 1119 + }, + { + "epoch": 1.3939016801493467, + "grad_norm": 2.765190839767456, + "learning_rate": 3.007380073800738e-06, + "loss": 1.4622, + "step": 1120 + }, + { + "epoch": 1.3939016801493467, + "eval_loss": 1.6451724767684937, + "eval_runtime": 43.7449, + "eval_samples_per_second": 22.86, + "eval_steps_per_second": 0.96, + "step": 1120 + }, + { + "epoch": 1.3951462352209085, + "grad_norm": 2.877002716064453, + "learning_rate": 3.0050738007380076e-06, + "loss": 1.4145, + "step": 1121 + }, + { + "epoch": 1.3963907902924704, + "grad_norm": 2.870389699935913, + "learning_rate": 3.002767527675277e-06, + "loss": 1.4401, + "step": 1122 + }, + { + "epoch": 1.3976353453640322, + "grad_norm": 2.7788920402526855, + "learning_rate": 3.000461254612546e-06, + "loss": 1.4514, + "step": 1123 + }, + { + "epoch": 1.3988799004355943, + "grad_norm": 2.9269332885742188, + "learning_rate": 2.998154981549816e-06, + "loss": 1.3961, + "step": 1124 + }, + { + "epoch": 1.4001244555071561, + "grad_norm": 3.013291835784912, + "learning_rate": 2.9958487084870853e-06, + "loss": 1.4597, + "step": 1125 + }, + { + "epoch": 1.4013690105787182, + "grad_norm": 2.9063880443573, + "learning_rate": 2.9935424354243547e-06, + "loss": 1.4668, + "step": 1126 + }, + { + "epoch": 1.40261356565028, + "grad_norm": 2.8926970958709717, + "learning_rate": 2.9912361623616237e-06, + "loss": 1.4332, + "step": 1127 + }, + { + "epoch": 1.403858120721842, + "grad_norm": 2.8563647270202637, + "learning_rate": 2.988929889298893e-06, + "loss": 1.407, + "step": 1128 + }, + { + "epoch": 1.4051026757934038, + "grad_norm": 2.8725359439849854, + "learning_rate": 2.986623616236163e-06, + "loss": 1.4434, + "step": 1129 + }, + { + "epoch": 1.4063472308649658, + "grad_norm": 2.906898260116577, + "learning_rate": 2.984317343173432e-06, + "loss": 1.4745, + "step": 1130 + }, + { + "epoch": 1.4063472308649658, + "eval_loss": 1.647884726524353, + "eval_runtime": 44.9221, + "eval_samples_per_second": 22.261, + "eval_steps_per_second": 0.935, + "step": 1130 + }, + { + "epoch": 1.4075917859365277, + "grad_norm": 2.8297719955444336, + "learning_rate": 2.9820110701107014e-06, + "loss": 1.4542, + "step": 1131 + }, + { + "epoch": 1.4088363410080897, + "grad_norm": 2.841031551361084, + "learning_rate": 2.979704797047971e-06, + "loss": 1.4105, + "step": 1132 + }, + { + "epoch": 1.4100808960796516, + "grad_norm": 2.867791175842285, + "learning_rate": 2.97739852398524e-06, + "loss": 1.4409, + "step": 1133 + }, + { + "epoch": 1.4113254511512134, + "grad_norm": 2.8156039714813232, + "learning_rate": 2.9750922509225093e-06, + "loss": 1.4284, + "step": 1134 + }, + { + "epoch": 1.4125700062227753, + "grad_norm": 2.825985908508301, + "learning_rate": 2.972785977859779e-06, + "loss": 1.4594, + "step": 1135 + }, + { + "epoch": 1.4138145612943374, + "grad_norm": 3.0134711265563965, + "learning_rate": 2.970479704797048e-06, + "loss": 1.4743, + "step": 1136 + }, + { + "epoch": 1.4150591163658992, + "grad_norm": 2.9829142093658447, + "learning_rate": 2.9681734317343175e-06, + "loss": 1.4935, + "step": 1137 + }, + { + "epoch": 1.416303671437461, + "grad_norm": 3.067620277404785, + "learning_rate": 2.965867158671587e-06, + "loss": 1.4266, + "step": 1138 + }, + { + "epoch": 1.4175482265090231, + "grad_norm": 2.8969919681549072, + "learning_rate": 2.9635608856088564e-06, + "loss": 1.451, + "step": 1139 + }, + { + "epoch": 1.418792781580585, + "grad_norm": 2.8706412315368652, + "learning_rate": 2.9612546125461254e-06, + "loss": 1.4834, + "step": 1140 + }, + { + "epoch": 1.418792781580585, + "eval_loss": 1.643480658531189, + "eval_runtime": 44.743, + "eval_samples_per_second": 22.35, + "eval_steps_per_second": 0.939, + "step": 1140 + }, + { + "epoch": 1.4200373366521468, + "grad_norm": 2.8801848888397217, + "learning_rate": 2.958948339483395e-06, + "loss": 1.4258, + "step": 1141 + }, + { + "epoch": 1.4212818917237087, + "grad_norm": 2.774635076522827, + "learning_rate": 2.9566420664206646e-06, + "loss": 1.416, + "step": 1142 + }, + { + "epoch": 1.4225264467952707, + "grad_norm": 2.8460140228271484, + "learning_rate": 2.9543357933579336e-06, + "loss": 1.4213, + "step": 1143 + }, + { + "epoch": 1.4237710018668326, + "grad_norm": 2.907888889312744, + "learning_rate": 2.952029520295203e-06, + "loss": 1.4507, + "step": 1144 + }, + { + "epoch": 1.4250155569383947, + "grad_norm": 2.789041757583618, + "learning_rate": 2.949723247232473e-06, + "loss": 1.4257, + "step": 1145 + }, + { + "epoch": 1.4262601120099565, + "grad_norm": 2.78859806060791, + "learning_rate": 2.947416974169742e-06, + "loss": 1.4377, + "step": 1146 + }, + { + "epoch": 1.4275046670815184, + "grad_norm": 2.840303421020508, + "learning_rate": 2.9451107011070113e-06, + "loss": 1.4684, + "step": 1147 + }, + { + "epoch": 1.4287492221530802, + "grad_norm": 2.8800530433654785, + "learning_rate": 2.9428044280442807e-06, + "loss": 1.4912, + "step": 1148 + }, + { + "epoch": 1.4299937772246423, + "grad_norm": 2.841092824935913, + "learning_rate": 2.9404981549815497e-06, + "loss": 1.4234, + "step": 1149 + }, + { + "epoch": 1.4312383322962041, + "grad_norm": 2.8226537704467773, + "learning_rate": 2.938191881918819e-06, + "loss": 1.4578, + "step": 1150 + }, + { + "epoch": 1.4312383322962041, + "eval_loss": 1.643249273300171, + "eval_runtime": 43.5586, + "eval_samples_per_second": 22.958, + "eval_steps_per_second": 0.964, + "step": 1150 + }, + { + "epoch": 1.432482887367766, + "grad_norm": 2.869736909866333, + "learning_rate": 2.935885608856089e-06, + "loss": 1.4701, + "step": 1151 + }, + { + "epoch": 1.433727442439328, + "grad_norm": 2.689211130142212, + "learning_rate": 2.9335793357933584e-06, + "loss": 1.4483, + "step": 1152 + }, + { + "epoch": 1.4349719975108899, + "grad_norm": 2.9267077445983887, + "learning_rate": 2.9312730627306274e-06, + "loss": 1.4446, + "step": 1153 + }, + { + "epoch": 1.4362165525824517, + "grad_norm": 2.906862497329712, + "learning_rate": 2.928966789667897e-06, + "loss": 1.4524, + "step": 1154 + }, + { + "epoch": 1.4374611076540136, + "grad_norm": 2.8249800205230713, + "learning_rate": 2.9266605166051663e-06, + "loss": 1.4208, + "step": 1155 + }, + { + "epoch": 1.4387056627255757, + "grad_norm": 2.9836032390594482, + "learning_rate": 2.9243542435424353e-06, + "loss": 1.4581, + "step": 1156 + }, + { + "epoch": 1.4399502177971375, + "grad_norm": 2.9595284461975098, + "learning_rate": 2.922047970479705e-06, + "loss": 1.4141, + "step": 1157 + }, + { + "epoch": 1.4411947728686996, + "grad_norm": 3.0363423824310303, + "learning_rate": 2.9197416974169746e-06, + "loss": 1.4859, + "step": 1158 + }, + { + "epoch": 1.4424393279402614, + "grad_norm": 2.911201000213623, + "learning_rate": 2.9174354243542436e-06, + "loss": 1.4346, + "step": 1159 + }, + { + "epoch": 1.4436838830118233, + "grad_norm": 2.8617193698883057, + "learning_rate": 2.915129151291513e-06, + "loss": 1.433, + "step": 1160 + }, + { + "epoch": 1.4436838830118233, + "eval_loss": 1.6437232494354248, + "eval_runtime": 43.961, + "eval_samples_per_second": 22.747, + "eval_steps_per_second": 0.955, + "step": 1160 + }, + { + "epoch": 1.4449284380833851, + "grad_norm": 2.9584295749664307, + "learning_rate": 2.912822878228783e-06, + "loss": 1.4847, + "step": 1161 + }, + { + "epoch": 1.4461729931549472, + "grad_norm": 2.8209400177001953, + "learning_rate": 2.9105166051660522e-06, + "loss": 1.421, + "step": 1162 + }, + { + "epoch": 1.447417548226509, + "grad_norm": 2.847637414932251, + "learning_rate": 2.9082103321033212e-06, + "loss": 1.4326, + "step": 1163 + }, + { + "epoch": 1.4486621032980709, + "grad_norm": 2.836228132247925, + "learning_rate": 2.9059040590405907e-06, + "loss": 1.4459, + "step": 1164 + }, + { + "epoch": 1.449906658369633, + "grad_norm": 2.8327207565307617, + "learning_rate": 2.90359778597786e-06, + "loss": 1.4189, + "step": 1165 + }, + { + "epoch": 1.4511512134411948, + "grad_norm": 2.880643606185913, + "learning_rate": 2.901291512915129e-06, + "loss": 1.4214, + "step": 1166 + }, + { + "epoch": 1.4523957685127566, + "grad_norm": 2.9637348651885986, + "learning_rate": 2.898985239852399e-06, + "loss": 1.4471, + "step": 1167 + }, + { + "epoch": 1.4536403235843185, + "grad_norm": 3.006145477294922, + "learning_rate": 2.8966789667896684e-06, + "loss": 1.4228, + "step": 1168 + }, + { + "epoch": 1.4548848786558806, + "grad_norm": 2.8547627925872803, + "learning_rate": 2.8943726937269374e-06, + "loss": 1.4707, + "step": 1169 + }, + { + "epoch": 1.4561294337274424, + "grad_norm": 2.973092794418335, + "learning_rate": 2.8920664206642068e-06, + "loss": 1.4804, + "step": 1170 + }, + { + "epoch": 1.4561294337274424, + "eval_loss": 1.6340677738189697, + "eval_runtime": 44.6094, + "eval_samples_per_second": 22.417, + "eval_steps_per_second": 0.942, + "step": 1170 + }, + { + "epoch": 1.4573739887990045, + "grad_norm": 2.9433603286743164, + "learning_rate": 2.889760147601476e-06, + "loss": 1.4354, + "step": 1171 + }, + { + "epoch": 1.4586185438705663, + "grad_norm": 2.872236728668213, + "learning_rate": 2.887453874538745e-06, + "loss": 1.418, + "step": 1172 + }, + { + "epoch": 1.4598630989421282, + "grad_norm": 2.847557783126831, + "learning_rate": 2.885147601476015e-06, + "loss": 1.4306, + "step": 1173 + }, + { + "epoch": 1.46110765401369, + "grad_norm": 2.7554969787597656, + "learning_rate": 2.8828413284132845e-06, + "loss": 1.4295, + "step": 1174 + }, + { + "epoch": 1.462352209085252, + "grad_norm": 2.8754842281341553, + "learning_rate": 2.880535055350554e-06, + "loss": 1.4482, + "step": 1175 + }, + { + "epoch": 1.463596764156814, + "grad_norm": 3.0115292072296143, + "learning_rate": 2.878228782287823e-06, + "loss": 1.4428, + "step": 1176 + }, + { + "epoch": 1.4648413192283758, + "grad_norm": 2.8976168632507324, + "learning_rate": 2.8759225092250923e-06, + "loss": 1.4707, + "step": 1177 + }, + { + "epoch": 1.4660858742999379, + "grad_norm": 2.7981812953948975, + "learning_rate": 2.873616236162362e-06, + "loss": 1.4693, + "step": 1178 + }, + { + "epoch": 1.4673304293714997, + "grad_norm": 2.8562371730804443, + "learning_rate": 2.871309963099631e-06, + "loss": 1.4242, + "step": 1179 + }, + { + "epoch": 1.4685749844430616, + "grad_norm": 2.8705570697784424, + "learning_rate": 2.8690036900369006e-06, + "loss": 1.4725, + "step": 1180 + }, + { + "epoch": 1.4685749844430616, + "eval_loss": 1.6338286399841309, + "eval_runtime": 44.7171, + "eval_samples_per_second": 22.363, + "eval_steps_per_second": 0.939, + "step": 1180 + }, + { + "epoch": 1.4698195395146234, + "grad_norm": 2.8025710582733154, + "learning_rate": 2.86669741697417e-06, + "loss": 1.427, + "step": 1181 + }, + { + "epoch": 1.4710640945861855, + "grad_norm": 2.845151901245117, + "learning_rate": 2.864391143911439e-06, + "loss": 1.4402, + "step": 1182 + }, + { + "epoch": 1.4723086496577473, + "grad_norm": 2.943373680114746, + "learning_rate": 2.862084870848709e-06, + "loss": 1.5194, + "step": 1183 + }, + { + "epoch": 1.4735532047293094, + "grad_norm": 2.896955966949463, + "learning_rate": 2.8597785977859783e-06, + "loss": 1.4242, + "step": 1184 + }, + { + "epoch": 1.4747977598008712, + "grad_norm": 3.131305456161499, + "learning_rate": 2.8574723247232473e-06, + "loss": 1.5008, + "step": 1185 + }, + { + "epoch": 1.476042314872433, + "grad_norm": 2.8605735301971436, + "learning_rate": 2.8551660516605167e-06, + "loss": 1.3533, + "step": 1186 + }, + { + "epoch": 1.477286869943995, + "grad_norm": 2.9158811569213867, + "learning_rate": 2.852859778597786e-06, + "loss": 1.4404, + "step": 1187 + }, + { + "epoch": 1.478531425015557, + "grad_norm": 2.7795321941375732, + "learning_rate": 2.850553505535056e-06, + "loss": 1.3977, + "step": 1188 + }, + { + "epoch": 1.4797759800871189, + "grad_norm": 2.8194212913513184, + "learning_rate": 2.848247232472325e-06, + "loss": 1.4111, + "step": 1189 + }, + { + "epoch": 1.4810205351586807, + "grad_norm": 2.983750104904175, + "learning_rate": 2.8459409594095944e-06, + "loss": 1.4061, + "step": 1190 + }, + { + "epoch": 1.4810205351586807, + "eval_loss": 1.6375409364700317, + "eval_runtime": 45.4026, + "eval_samples_per_second": 22.025, + "eval_steps_per_second": 0.925, + "step": 1190 + }, + { + "epoch": 1.4822650902302428, + "grad_norm": 3.0246665477752686, + "learning_rate": 2.843634686346864e-06, + "loss": 1.4532, + "step": 1191 + }, + { + "epoch": 1.4835096453018046, + "grad_norm": 2.8856449127197266, + "learning_rate": 2.841328413284133e-06, + "loss": 1.4528, + "step": 1192 + }, + { + "epoch": 1.4847542003733665, + "grad_norm": 2.848987102508545, + "learning_rate": 2.8390221402214022e-06, + "loss": 1.4369, + "step": 1193 + }, + { + "epoch": 1.4859987554449283, + "grad_norm": 2.970419406890869, + "learning_rate": 2.836715867158672e-06, + "loss": 1.4605, + "step": 1194 + }, + { + "epoch": 1.4872433105164904, + "grad_norm": 3.156613826751709, + "learning_rate": 2.834409594095941e-06, + "loss": 1.4489, + "step": 1195 + }, + { + "epoch": 1.4884878655880522, + "grad_norm": 3.095349073410034, + "learning_rate": 2.8321033210332105e-06, + "loss": 1.4604, + "step": 1196 + }, + { + "epoch": 1.4897324206596143, + "grad_norm": 2.870742082595825, + "learning_rate": 2.82979704797048e-06, + "loss": 1.4284, + "step": 1197 + }, + { + "epoch": 1.4909769757311762, + "grad_norm": 2.9415056705474854, + "learning_rate": 2.827490774907749e-06, + "loss": 1.4715, + "step": 1198 + }, + { + "epoch": 1.492221530802738, + "grad_norm": 2.8297767639160156, + "learning_rate": 2.8251845018450188e-06, + "loss": 1.4508, + "step": 1199 + }, + { + "epoch": 1.4934660858742999, + "grad_norm": 2.872086524963379, + "learning_rate": 2.822878228782288e-06, + "loss": 1.5098, + "step": 1200 + }, + { + "epoch": 1.4934660858742999, + "eval_loss": 1.6357526779174805, + "eval_runtime": 51.422, + "eval_samples_per_second": 19.447, + "eval_steps_per_second": 0.817, + "step": 1200 + }, + { + "epoch": 1.494710640945862, + "grad_norm": 2.7152082920074463, + "learning_rate": 2.8205719557195576e-06, + "loss": 1.4329, + "step": 1201 + }, + { + "epoch": 1.4959551960174238, + "grad_norm": 2.866170883178711, + "learning_rate": 2.8182656826568266e-06, + "loss": 1.4455, + "step": 1202 + }, + { + "epoch": 1.4971997510889856, + "grad_norm": 2.9010684490203857, + "learning_rate": 2.815959409594096e-06, + "loss": 1.423, + "step": 1203 + }, + { + "epoch": 1.4984443061605477, + "grad_norm": 2.8588502407073975, + "learning_rate": 2.813653136531366e-06, + "loss": 1.4041, + "step": 1204 + }, + { + "epoch": 1.4996888612321095, + "grad_norm": 2.932544231414795, + "learning_rate": 2.811346863468635e-06, + "loss": 1.4559, + "step": 1205 + }, + { + "epoch": 1.5009334163036714, + "grad_norm": 2.7486014366149902, + "learning_rate": 2.8090405904059043e-06, + "loss": 1.4337, + "step": 1206 + }, + { + "epoch": 1.5021779713752332, + "grad_norm": 3.0866589546203613, + "learning_rate": 2.8067343173431737e-06, + "loss": 1.4186, + "step": 1207 + }, + { + "epoch": 1.5034225264467953, + "grad_norm": 2.9710276126861572, + "learning_rate": 2.8044280442804427e-06, + "loss": 1.419, + "step": 1208 + }, + { + "epoch": 1.5046670815183572, + "grad_norm": 2.8379950523376465, + "learning_rate": 2.802121771217712e-06, + "loss": 1.4922, + "step": 1209 + }, + { + "epoch": 1.5059116365899192, + "grad_norm": 2.939629554748535, + "learning_rate": 2.799815498154982e-06, + "loss": 1.4402, + "step": 1210 + }, + { + "epoch": 1.5059116365899192, + "eval_loss": 1.6419442892074585, + "eval_runtime": 43.1202, + "eval_samples_per_second": 23.191, + "eval_steps_per_second": 0.974, + "step": 1210 + }, + { + "epoch": 1.507156191661481, + "grad_norm": 3.041389226913452, + "learning_rate": 2.7975092250922514e-06, + "loss": 1.4873, + "step": 1211 + }, + { + "epoch": 1.508400746733043, + "grad_norm": 2.9776995182037354, + "learning_rate": 2.7952029520295204e-06, + "loss": 1.4422, + "step": 1212 + }, + { + "epoch": 1.5096453018046048, + "grad_norm": 2.8798792362213135, + "learning_rate": 2.79289667896679e-06, + "loss": 1.4535, + "step": 1213 + }, + { + "epoch": 1.5108898568761666, + "grad_norm": 2.9016385078430176, + "learning_rate": 2.7905904059040597e-06, + "loss": 1.4462, + "step": 1214 + }, + { + "epoch": 1.5121344119477287, + "grad_norm": 2.8987581729888916, + "learning_rate": 2.7882841328413287e-06, + "loss": 1.4477, + "step": 1215 + }, + { + "epoch": 1.5133789670192908, + "grad_norm": 2.960266351699829, + "learning_rate": 2.785977859778598e-06, + "loss": 1.4811, + "step": 1216 + }, + { + "epoch": 1.5146235220908526, + "grad_norm": 2.8965611457824707, + "learning_rate": 2.7836715867158675e-06, + "loss": 1.4366, + "step": 1217 + }, + { + "epoch": 1.5158680771624145, + "grad_norm": 3.0667364597320557, + "learning_rate": 2.7813653136531365e-06, + "loss": 1.3828, + "step": 1218 + }, + { + "epoch": 1.5171126322339763, + "grad_norm": 2.952362537384033, + "learning_rate": 2.779059040590406e-06, + "loss": 1.3903, + "step": 1219 + }, + { + "epoch": 1.5183571873055381, + "grad_norm": 2.9899795055389404, + "learning_rate": 2.776752767527676e-06, + "loss": 1.4312, + "step": 1220 + }, + { + "epoch": 1.5183571873055381, + "eval_loss": 1.640383005142212, + "eval_runtime": 45.4128, + "eval_samples_per_second": 22.02, + "eval_steps_per_second": 0.925, + "step": 1220 + }, + { + "epoch": 1.5196017423771002, + "grad_norm": 3.2202725410461426, + "learning_rate": 2.774446494464945e-06, + "loss": 1.4344, + "step": 1221 + }, + { + "epoch": 1.520846297448662, + "grad_norm": 3.2202725410461426, + "learning_rate": 2.774446494464945e-06, + "loss": 1.4386, + "step": 1222 + }, + { + "epoch": 1.5220908525202241, + "grad_norm": 2.883223295211792, + "learning_rate": 2.7721402214022142e-06, + "loss": 1.4401, + "step": 1223 + }, + { + "epoch": 1.523335407591786, + "grad_norm": 2.8346424102783203, + "learning_rate": 2.7698339483394837e-06, + "loss": 1.3926, + "step": 1224 + }, + { + "epoch": 1.5245799626633478, + "grad_norm": 2.8898870944976807, + "learning_rate": 2.767527675276753e-06, + "loss": 1.4128, + "step": 1225 + }, + { + "epoch": 1.5258245177349097, + "grad_norm": 2.828594923019409, + "learning_rate": 2.765221402214022e-06, + "loss": 1.4103, + "step": 1226 + }, + { + "epoch": 1.5270690728064715, + "grad_norm": 2.882558822631836, + "learning_rate": 2.762915129151292e-06, + "loss": 1.4186, + "step": 1227 + }, + { + "epoch": 1.5283136278780336, + "grad_norm": 2.955760955810547, + "learning_rate": 2.7606088560885613e-06, + "loss": 1.3705, + "step": 1228 + }, + { + "epoch": 1.5295581829495957, + "grad_norm": 2.862934112548828, + "learning_rate": 2.7583025830258303e-06, + "loss": 1.5012, + "step": 1229 + }, + { + "epoch": 1.5308027380211575, + "grad_norm": 2.926501989364624, + "learning_rate": 2.7559963099630998e-06, + "loss": 1.3875, + "step": 1230 + }, + { + "epoch": 1.5308027380211575, + "eval_loss": 1.6376254558563232, + "eval_runtime": 47.9535, + "eval_samples_per_second": 20.854, + "eval_steps_per_second": 0.876, + "step": 1230 + }, + { + "epoch": 1.5320472930927194, + "grad_norm": 2.7648823261260986, + "learning_rate": 2.753690036900369e-06, + "loss": 1.3781, + "step": 1231 + }, + { + "epoch": 1.5332918481642812, + "grad_norm": 2.927103281021118, + "learning_rate": 2.751383763837638e-06, + "loss": 1.4091, + "step": 1232 + }, + { + "epoch": 1.534536403235843, + "grad_norm": 2.964823007583618, + "learning_rate": 2.749077490774908e-06, + "loss": 1.5075, + "step": 1233 + }, + { + "epoch": 1.5357809583074051, + "grad_norm": 2.7944016456604004, + "learning_rate": 2.7467712177121775e-06, + "loss": 1.3888, + "step": 1234 + }, + { + "epoch": 1.537025513378967, + "grad_norm": 2.891592025756836, + "learning_rate": 2.7444649446494465e-06, + "loss": 1.4825, + "step": 1235 + }, + { + "epoch": 1.538270068450529, + "grad_norm": 2.8120031356811523, + "learning_rate": 2.742158671586716e-06, + "loss": 1.4246, + "step": 1236 + }, + { + "epoch": 1.539514623522091, + "grad_norm": 2.9384255409240723, + "learning_rate": 2.7398523985239857e-06, + "loss": 1.4657, + "step": 1237 + }, + { + "epoch": 1.5407591785936527, + "grad_norm": 2.73111891746521, + "learning_rate": 2.737546125461255e-06, + "loss": 1.4689, + "step": 1238 + }, + { + "epoch": 1.5420037336652146, + "grad_norm": 2.858963966369629, + "learning_rate": 2.735239852398524e-06, + "loss": 1.4626, + "step": 1239 + }, + { + "epoch": 1.5432482887367764, + "grad_norm": 2.9148507118225098, + "learning_rate": 2.7329335793357936e-06, + "loss": 1.4787, + "step": 1240 + }, + { + "epoch": 1.5432482887367764, + "eval_loss": 1.6436412334442139, + "eval_runtime": 47.3139, + "eval_samples_per_second": 21.135, + "eval_steps_per_second": 0.888, + "step": 1240 + }, + { + "epoch": 1.5444928438083385, + "grad_norm": 2.8731327056884766, + "learning_rate": 2.730627306273063e-06, + "loss": 1.4748, + "step": 1241 + }, + { + "epoch": 1.5457373988799006, + "grad_norm": 2.990265369415283, + "learning_rate": 2.728321033210332e-06, + "loss": 1.4375, + "step": 1242 + }, + { + "epoch": 1.5469819539514624, + "grad_norm": 2.874321460723877, + "learning_rate": 2.726014760147602e-06, + "loss": 1.4154, + "step": 1243 + }, + { + "epoch": 1.5482265090230243, + "grad_norm": 2.9316225051879883, + "learning_rate": 2.7237084870848713e-06, + "loss": 1.4285, + "step": 1244 + }, + { + "epoch": 1.5494710640945861, + "grad_norm": 2.783966302871704, + "learning_rate": 2.7214022140221403e-06, + "loss": 1.409, + "step": 1245 + }, + { + "epoch": 1.550715619166148, + "grad_norm": 2.9578707218170166, + "learning_rate": 2.7190959409594097e-06, + "loss": 1.4325, + "step": 1246 + }, + { + "epoch": 1.55196017423771, + "grad_norm": 2.869893789291382, + "learning_rate": 2.716789667896679e-06, + "loss": 1.3989, + "step": 1247 + }, + { + "epoch": 1.553204729309272, + "grad_norm": 2.962944269180298, + "learning_rate": 2.714483394833948e-06, + "loss": 1.4302, + "step": 1248 + }, + { + "epoch": 1.554449284380834, + "grad_norm": 2.974494695663452, + "learning_rate": 2.712177121771218e-06, + "loss": 1.46, + "step": 1249 + }, + { + "epoch": 1.5556938394523958, + "grad_norm": 2.9154603481292725, + "learning_rate": 2.7098708487084874e-06, + "loss": 1.4388, + "step": 1250 + }, + { + "epoch": 1.5556938394523958, + "eval_loss": 1.6357773542404175, + "eval_runtime": 47.6085, + "eval_samples_per_second": 21.005, + "eval_steps_per_second": 0.882, + "step": 1250 + }, + { + "epoch": 1.5569383945239577, + "grad_norm": 3.0333547592163086, + "learning_rate": 2.707564575645757e-06, + "loss": 1.4846, + "step": 1251 + }, + { + "epoch": 1.5581829495955195, + "grad_norm": 3.06097674369812, + "learning_rate": 2.705258302583026e-06, + "loss": 1.4355, + "step": 1252 + }, + { + "epoch": 1.5594275046670814, + "grad_norm": 2.872680425643921, + "learning_rate": 2.7029520295202956e-06, + "loss": 1.4219, + "step": 1253 + }, + { + "epoch": 1.5606720597386434, + "grad_norm": 2.9066731929779053, + "learning_rate": 2.700645756457565e-06, + "loss": 1.384, + "step": 1254 + }, + { + "epoch": 1.5619166148102055, + "grad_norm": 3.0570194721221924, + "learning_rate": 2.698339483394834e-06, + "loss": 1.4687, + "step": 1255 + }, + { + "epoch": 1.5631611698817673, + "grad_norm": 3.1955857276916504, + "learning_rate": 2.6960332103321035e-06, + "loss": 1.4736, + "step": 1256 + }, + { + "epoch": 1.5644057249533292, + "grad_norm": 2.9686198234558105, + "learning_rate": 2.693726937269373e-06, + "loss": 1.4176, + "step": 1257 + }, + { + "epoch": 1.565650280024891, + "grad_norm": 3.1161272525787354, + "learning_rate": 2.691420664206642e-06, + "loss": 1.443, + "step": 1258 + }, + { + "epoch": 1.5668948350964529, + "grad_norm": 2.9031193256378174, + "learning_rate": 2.6891143911439118e-06, + "loss": 1.3871, + "step": 1259 + }, + { + "epoch": 1.568139390168015, + "grad_norm": 2.9120476245880127, + "learning_rate": 2.686808118081181e-06, + "loss": 1.4406, + "step": 1260 + }, + { + "epoch": 1.568139390168015, + "eval_loss": 1.6291446685791016, + "eval_runtime": 49.7006, + "eval_samples_per_second": 20.12, + "eval_steps_per_second": 0.845, + "step": 1260 + }, + { + "epoch": 1.5693839452395768, + "grad_norm": 3.3181824684143066, + "learning_rate": 2.6845018450184506e-06, + "loss": 1.5278, + "step": 1261 + }, + { + "epoch": 1.5706285003111389, + "grad_norm": 2.9628148078918457, + "learning_rate": 2.6821955719557196e-06, + "loss": 1.4206, + "step": 1262 + }, + { + "epoch": 1.5718730553827007, + "grad_norm": 2.9828474521636963, + "learning_rate": 2.679889298892989e-06, + "loss": 1.5478, + "step": 1263 + }, + { + "epoch": 1.5731176104542626, + "grad_norm": 3.018711805343628, + "learning_rate": 2.677583025830259e-06, + "loss": 1.4442, + "step": 1264 + }, + { + "epoch": 1.5743621655258244, + "grad_norm": 2.8596792221069336, + "learning_rate": 2.675276752767528e-06, + "loss": 1.4325, + "step": 1265 + }, + { + "epoch": 1.5756067205973865, + "grad_norm": 2.9592769145965576, + "learning_rate": 2.6729704797047973e-06, + "loss": 1.4059, + "step": 1266 + }, + { + "epoch": 1.5768512756689483, + "grad_norm": 2.773589611053467, + "learning_rate": 2.6706642066420667e-06, + "loss": 1.3608, + "step": 1267 + }, + { + "epoch": 1.5780958307405104, + "grad_norm": 3.313255548477173, + "learning_rate": 2.6683579335793357e-06, + "loss": 1.4552, + "step": 1268 + }, + { + "epoch": 1.5793403858120723, + "grad_norm": 2.977649211883545, + "learning_rate": 2.6660516605166056e-06, + "loss": 1.4221, + "step": 1269 + }, + { + "epoch": 1.580584940883634, + "grad_norm": 3.0672051906585693, + "learning_rate": 2.663745387453875e-06, + "loss": 1.4568, + "step": 1270 + }, + { + "epoch": 1.580584940883634, + "eval_loss": 1.6317907571792603, + "eval_runtime": 47.2354, + "eval_samples_per_second": 21.171, + "eval_steps_per_second": 0.889, + "step": 1270 + }, + { + "epoch": 1.581829495955196, + "grad_norm": 3.029320240020752, + "learning_rate": 2.661439114391144e-06, + "loss": 1.4432, + "step": 1271 + }, + { + "epoch": 1.5830740510267578, + "grad_norm": 3.0183846950531006, + "learning_rate": 2.6591328413284134e-06, + "loss": 1.4332, + "step": 1272 + }, + { + "epoch": 1.5843186060983199, + "grad_norm": 2.856551170349121, + "learning_rate": 2.656826568265683e-06, + "loss": 1.4291, + "step": 1273 + }, + { + "epoch": 1.5855631611698817, + "grad_norm": 3.124971389770508, + "learning_rate": 2.6545202952029527e-06, + "loss": 1.4363, + "step": 1274 + }, + { + "epoch": 1.5868077162414438, + "grad_norm": 2.9470791816711426, + "learning_rate": 2.6522140221402217e-06, + "loss": 1.4502, + "step": 1275 + }, + { + "epoch": 1.5880522713130056, + "grad_norm": 2.9116358757019043, + "learning_rate": 2.649907749077491e-06, + "loss": 1.4314, + "step": 1276 + }, + { + "epoch": 1.5892968263845675, + "grad_norm": 3.2575323581695557, + "learning_rate": 2.6476014760147605e-06, + "loss": 1.4749, + "step": 1277 + }, + { + "epoch": 1.5905413814561293, + "grad_norm": 2.935518503189087, + "learning_rate": 2.6452952029520295e-06, + "loss": 1.4265, + "step": 1278 + }, + { + "epoch": 1.5917859365276914, + "grad_norm": 2.9750072956085205, + "learning_rate": 2.642988929889299e-06, + "loss": 1.4501, + "step": 1279 + }, + { + "epoch": 1.5930304915992533, + "grad_norm": 2.8086845874786377, + "learning_rate": 2.640682656826569e-06, + "loss": 1.4557, + "step": 1280 + }, + { + "epoch": 1.5930304915992533, + "eval_loss": 1.6302106380462646, + "eval_runtime": 52.3962, + "eval_samples_per_second": 19.085, + "eval_steps_per_second": 0.802, + "step": 1280 + }, + { + "epoch": 1.5942750466708153, + "grad_norm": 2.741849422454834, + "learning_rate": 2.638376383763838e-06, + "loss": 1.3918, + "step": 1281 + }, + { + "epoch": 1.5955196017423772, + "grad_norm": 2.9414806365966797, + "learning_rate": 2.6360701107011072e-06, + "loss": 1.3882, + "step": 1282 + }, + { + "epoch": 1.596764156813939, + "grad_norm": 2.901514768600464, + "learning_rate": 2.6337638376383766e-06, + "loss": 1.4331, + "step": 1283 + }, + { + "epoch": 1.5980087118855009, + "grad_norm": 2.8159122467041016, + "learning_rate": 2.6314575645756456e-06, + "loss": 1.4064, + "step": 1284 + }, + { + "epoch": 1.5992532669570627, + "grad_norm": 2.900561571121216, + "learning_rate": 2.629151291512915e-06, + "loss": 1.3731, + "step": 1285 + }, + { + "epoch": 1.6004978220286248, + "grad_norm": 3.019540548324585, + "learning_rate": 2.626845018450185e-06, + "loss": 1.4506, + "step": 1286 + }, + { + "epoch": 1.6017423771001866, + "grad_norm": 2.9263620376586914, + "learning_rate": 2.6245387453874543e-06, + "loss": 1.4754, + "step": 1287 + }, + { + "epoch": 1.6029869321717487, + "grad_norm": 2.8681881427764893, + "learning_rate": 2.6222324723247233e-06, + "loss": 1.4124, + "step": 1288 + }, + { + "epoch": 1.6042314872433105, + "grad_norm": 2.922945261001587, + "learning_rate": 2.6199261992619928e-06, + "loss": 1.5101, + "step": 1289 + }, + { + "epoch": 1.6054760423148724, + "grad_norm": 2.9548606872558594, + "learning_rate": 2.6176199261992626e-06, + "loss": 1.41, + "step": 1290 + }, + { + "epoch": 1.6054760423148724, + "eval_loss": 1.6357502937316895, + "eval_runtime": 46.9547, + "eval_samples_per_second": 21.297, + "eval_steps_per_second": 0.894, + "step": 1290 + }, + { + "epoch": 1.6067205973864342, + "grad_norm": 2.837184190750122, + "learning_rate": 2.6153136531365316e-06, + "loss": 1.4354, + "step": 1291 + }, + { + "epoch": 1.6079651524579963, + "grad_norm": 2.818990468978882, + "learning_rate": 2.613007380073801e-06, + "loss": 1.4474, + "step": 1292 + }, + { + "epoch": 1.6092097075295582, + "grad_norm": 2.7880146503448486, + "learning_rate": 2.6107011070110704e-06, + "loss": 1.4156, + "step": 1293 + }, + { + "epoch": 1.6104542626011202, + "grad_norm": 2.790971040725708, + "learning_rate": 2.6083948339483394e-06, + "loss": 1.4671, + "step": 1294 + }, + { + "epoch": 1.611698817672682, + "grad_norm": 2.943129777908325, + "learning_rate": 2.606088560885609e-06, + "loss": 1.4801, + "step": 1295 + }, + { + "epoch": 1.612943372744244, + "grad_norm": 2.8589110374450684, + "learning_rate": 2.6037822878228787e-06, + "loss": 1.4894, + "step": 1296 + }, + { + "epoch": 1.6141879278158058, + "grad_norm": 2.8929262161254883, + "learning_rate": 2.6014760147601477e-06, + "loss": 1.4022, + "step": 1297 + }, + { + "epoch": 1.6154324828873676, + "grad_norm": 2.970099925994873, + "learning_rate": 2.599169741697417e-06, + "loss": 1.5136, + "step": 1298 + }, + { + "epoch": 1.6166770379589297, + "grad_norm": 2.928466796875, + "learning_rate": 2.5968634686346866e-06, + "loss": 1.4743, + "step": 1299 + }, + { + "epoch": 1.6179215930304915, + "grad_norm": 2.941580057144165, + "learning_rate": 2.594557195571956e-06, + "loss": 1.4397, + "step": 1300 + }, + { + "epoch": 1.6179215930304915, + "eval_loss": 1.6348339319229126, + "eval_runtime": 50.433, + "eval_samples_per_second": 19.828, + "eval_steps_per_second": 0.833, + "step": 1300 + }, + { + "epoch": 1.6191661481020536, + "grad_norm": 2.903212308883667, + "learning_rate": 2.592250922509225e-06, + "loss": 1.4221, + "step": 1301 + }, + { + "epoch": 1.6204107031736155, + "grad_norm": 2.948651075363159, + "learning_rate": 2.589944649446495e-06, + "loss": 1.4363, + "step": 1302 + }, + { + "epoch": 1.6216552582451773, + "grad_norm": 2.8940505981445312, + "learning_rate": 2.5876383763837643e-06, + "loss": 1.398, + "step": 1303 + }, + { + "epoch": 1.6228998133167392, + "grad_norm": 2.950056552886963, + "learning_rate": 2.5853321033210333e-06, + "loss": 1.4425, + "step": 1304 + }, + { + "epoch": 1.6241443683883012, + "grad_norm": 2.9674088954925537, + "learning_rate": 2.5830258302583027e-06, + "loss": 1.4473, + "step": 1305 + }, + { + "epoch": 1.625388923459863, + "grad_norm": 2.7541685104370117, + "learning_rate": 2.5807195571955725e-06, + "loss": 1.4483, + "step": 1306 + }, + { + "epoch": 1.6266334785314251, + "grad_norm": 2.8897035121917725, + "learning_rate": 2.5784132841328415e-06, + "loss": 1.4572, + "step": 1307 + }, + { + "epoch": 1.627878033602987, + "grad_norm": 3.021559238433838, + "learning_rate": 2.576107011070111e-06, + "loss": 1.5164, + "step": 1308 + }, + { + "epoch": 1.6291225886745488, + "grad_norm": 2.884009838104248, + "learning_rate": 2.5738007380073804e-06, + "loss": 1.4196, + "step": 1309 + }, + { + "epoch": 1.6303671437461107, + "grad_norm": 2.749497890472412, + "learning_rate": 2.5714944649446494e-06, + "loss": 1.4324, + "step": 1310 + }, + { + "epoch": 1.6303671437461107, + "eval_loss": 1.6314265727996826, + "eval_runtime": 42.4647, + "eval_samples_per_second": 23.549, + "eval_steps_per_second": 0.989, + "step": 1310 + }, + { + "epoch": 1.6316116988176725, + "grad_norm": 2.8908045291900635, + "learning_rate": 2.5691881918819188e-06, + "loss": 1.4388, + "step": 1311 + }, + { + "epoch": 1.6328562538892346, + "grad_norm": 2.832808256149292, + "learning_rate": 2.5668819188191886e-06, + "loss": 1.4368, + "step": 1312 + }, + { + "epoch": 1.6341008089607967, + "grad_norm": 2.9047906398773193, + "learning_rate": 2.564575645756458e-06, + "loss": 1.4186, + "step": 1313 + }, + { + "epoch": 1.6353453640323585, + "grad_norm": 2.9271419048309326, + "learning_rate": 2.562269372693727e-06, + "loss": 1.3776, + "step": 1314 + }, + { + "epoch": 1.6365899191039204, + "grad_norm": 2.760314702987671, + "learning_rate": 2.5599630996309965e-06, + "loss": 1.386, + "step": 1315 + }, + { + "epoch": 1.6378344741754822, + "grad_norm": 2.9694864749908447, + "learning_rate": 2.557656826568266e-06, + "loss": 1.4542, + "step": 1316 + }, + { + "epoch": 1.639079029247044, + "grad_norm": 2.9950435161590576, + "learning_rate": 2.555350553505535e-06, + "loss": 1.4242, + "step": 1317 + }, + { + "epoch": 1.6403235843186061, + "grad_norm": 2.9291114807128906, + "learning_rate": 2.5530442804428047e-06, + "loss": 1.4289, + "step": 1318 + }, + { + "epoch": 1.641568139390168, + "grad_norm": 2.981961488723755, + "learning_rate": 2.550738007380074e-06, + "loss": 1.4283, + "step": 1319 + }, + { + "epoch": 1.64281269446173, + "grad_norm": 2.734123706817627, + "learning_rate": 2.548431734317343e-06, + "loss": 1.4144, + "step": 1320 + }, + { + "epoch": 1.64281269446173, + "eval_loss": 1.6249239444732666, + "eval_runtime": 48.8116, + "eval_samples_per_second": 20.487, + "eval_steps_per_second": 0.86, + "step": 1320 + }, + { + "epoch": 1.644057249533292, + "grad_norm": 2.9752919673919678, + "learning_rate": 2.5461254612546126e-06, + "loss": 1.3641, + "step": 1321 + }, + { + "epoch": 1.6453018046048538, + "grad_norm": 2.972463607788086, + "learning_rate": 2.543819188191882e-06, + "loss": 1.3797, + "step": 1322 + }, + { + "epoch": 1.6465463596764156, + "grad_norm": 2.905499219894409, + "learning_rate": 2.541512915129152e-06, + "loss": 1.4221, + "step": 1323 + }, + { + "epoch": 1.6477909147479775, + "grad_norm": 2.7921793460845947, + "learning_rate": 2.539206642066421e-06, + "loss": 1.4118, + "step": 1324 + }, + { + "epoch": 1.6490354698195395, + "grad_norm": 3.064558267593384, + "learning_rate": 2.5369003690036903e-06, + "loss": 1.4611, + "step": 1325 + }, + { + "epoch": 1.6502800248911016, + "grad_norm": 2.9690096378326416, + "learning_rate": 2.5345940959409597e-06, + "loss": 1.461, + "step": 1326 + }, + { + "epoch": 1.6515245799626634, + "grad_norm": 3.0379040241241455, + "learning_rate": 2.5322878228782287e-06, + "loss": 1.451, + "step": 1327 + }, + { + "epoch": 1.6527691350342253, + "grad_norm": 2.9426493644714355, + "learning_rate": 2.5299815498154986e-06, + "loss": 1.4671, + "step": 1328 + }, + { + "epoch": 1.6540136901057871, + "grad_norm": 2.915076732635498, + "learning_rate": 2.527675276752768e-06, + "loss": 1.4159, + "step": 1329 + }, + { + "epoch": 1.655258245177349, + "grad_norm": 2.8588879108428955, + "learning_rate": 2.525369003690037e-06, + "loss": 1.4978, + "step": 1330 + }, + { + "epoch": 1.655258245177349, + "eval_loss": 1.624009132385254, + "eval_runtime": 47.0461, + "eval_samples_per_second": 21.256, + "eval_steps_per_second": 0.893, + "step": 1330 + }, + { + "epoch": 1.656502800248911, + "grad_norm": 2.9942209720611572, + "learning_rate": 2.5230627306273064e-06, + "loss": 1.3872, + "step": 1331 + }, + { + "epoch": 1.657747355320473, + "grad_norm": 2.8424410820007324, + "learning_rate": 2.520756457564576e-06, + "loss": 1.4181, + "step": 1332 + }, + { + "epoch": 1.658991910392035, + "grad_norm": 2.9234843254089355, + "learning_rate": 2.518450184501845e-06, + "loss": 1.4516, + "step": 1333 + }, + { + "epoch": 1.6602364654635968, + "grad_norm": 2.8367815017700195, + "learning_rate": 2.5161439114391147e-06, + "loss": 1.4214, + "step": 1334 + }, + { + "epoch": 1.6614810205351587, + "grad_norm": 2.819149971008301, + "learning_rate": 2.513837638376384e-06, + "loss": 1.3883, + "step": 1335 + }, + { + "epoch": 1.6627255756067205, + "grad_norm": 3.024693250656128, + "learning_rate": 2.5115313653136535e-06, + "loss": 1.3902, + "step": 1336 + }, + { + "epoch": 1.6639701306782824, + "grad_norm": 2.903599262237549, + "learning_rate": 2.5092250922509225e-06, + "loss": 1.4238, + "step": 1337 + }, + { + "epoch": 1.6652146857498444, + "grad_norm": 2.8296589851379395, + "learning_rate": 2.506918819188192e-06, + "loss": 1.3993, + "step": 1338 + }, + { + "epoch": 1.6664592408214065, + "grad_norm": 2.873272657394409, + "learning_rate": 2.5046125461254618e-06, + "loss": 1.4186, + "step": 1339 + }, + { + "epoch": 1.6677037958929684, + "grad_norm": 2.9268789291381836, + "learning_rate": 2.5023062730627308e-06, + "loss": 1.4242, + "step": 1340 + }, + { + "epoch": 1.6677037958929684, + "eval_loss": 1.6260672807693481, + "eval_runtime": 47.5895, + "eval_samples_per_second": 21.013, + "eval_steps_per_second": 0.883, + "step": 1340 + }, + { + "epoch": 1.6689483509645302, + "grad_norm": 3.057659387588501, + "learning_rate": 2.5e-06, + "loss": 1.4077, + "step": 1341 + }, + { + "epoch": 1.670192906036092, + "grad_norm": 2.875095844268799, + "learning_rate": 2.4976937269372696e-06, + "loss": 1.394, + "step": 1342 + }, + { + "epoch": 1.671437461107654, + "grad_norm": 2.7579872608184814, + "learning_rate": 2.495387453874539e-06, + "loss": 1.4029, + "step": 1343 + }, + { + "epoch": 1.672682016179216, + "grad_norm": 2.8981001377105713, + "learning_rate": 2.4930811808118085e-06, + "loss": 1.4494, + "step": 1344 + }, + { + "epoch": 1.6739265712507778, + "grad_norm": 2.9696900844573975, + "learning_rate": 2.490774907749078e-06, + "loss": 1.4692, + "step": 1345 + }, + { + "epoch": 1.6751711263223399, + "grad_norm": 2.8094892501831055, + "learning_rate": 2.488468634686347e-06, + "loss": 1.441, + "step": 1346 + }, + { + "epoch": 1.6764156813939017, + "grad_norm": 2.8619563579559326, + "learning_rate": 2.4861623616236163e-06, + "loss": 1.4885, + "step": 1347 + }, + { + "epoch": 1.6776602364654636, + "grad_norm": 2.854151487350464, + "learning_rate": 2.4838560885608857e-06, + "loss": 1.4338, + "step": 1348 + }, + { + "epoch": 1.6789047915370254, + "grad_norm": 2.9707868099212646, + "learning_rate": 2.481549815498155e-06, + "loss": 1.4529, + "step": 1349 + }, + { + "epoch": 1.6801493466085873, + "grad_norm": 3.0289969444274902, + "learning_rate": 2.4792435424354246e-06, + "loss": 1.4855, + "step": 1350 + }, + { + "epoch": 1.6801493466085873, + "eval_loss": 1.6256393194198608, + "eval_runtime": 45.9524, + "eval_samples_per_second": 21.762, + "eval_steps_per_second": 0.914, + "step": 1350 + }, + { + "epoch": 1.6813939016801493, + "grad_norm": 2.8407845497131348, + "learning_rate": 2.476937269372694e-06, + "loss": 1.3876, + "step": 1351 + }, + { + "epoch": 1.6826384567517114, + "grad_norm": 2.9876599311828613, + "learning_rate": 2.4746309963099634e-06, + "loss": 1.3931, + "step": 1352 + }, + { + "epoch": 1.6838830118232733, + "grad_norm": 3.008683919906616, + "learning_rate": 2.472324723247233e-06, + "loss": 1.4482, + "step": 1353 + }, + { + "epoch": 1.6851275668948351, + "grad_norm": 2.8110079765319824, + "learning_rate": 2.470018450184502e-06, + "loss": 1.3851, + "step": 1354 + }, + { + "epoch": 1.686372121966397, + "grad_norm": 2.9402263164520264, + "learning_rate": 2.4677121771217713e-06, + "loss": 1.4009, + "step": 1355 + }, + { + "epoch": 1.6876166770379588, + "grad_norm": 2.9846489429473877, + "learning_rate": 2.4654059040590407e-06, + "loss": 1.3975, + "step": 1356 + }, + { + "epoch": 1.6888612321095209, + "grad_norm": 2.835801362991333, + "learning_rate": 2.46309963099631e-06, + "loss": 1.3421, + "step": 1357 + }, + { + "epoch": 1.6901057871810827, + "grad_norm": 2.9438889026641846, + "learning_rate": 2.4607933579335795e-06, + "loss": 1.4422, + "step": 1358 + }, + { + "epoch": 1.6913503422526448, + "grad_norm": 3.168829917907715, + "learning_rate": 2.458487084870849e-06, + "loss": 1.4596, + "step": 1359 + }, + { + "epoch": 1.6925948973242066, + "grad_norm": 2.8412039279937744, + "learning_rate": 2.4561808118081184e-06, + "loss": 1.4263, + "step": 1360 + }, + { + "epoch": 1.6925948973242066, + "eval_loss": 1.6265885829925537, + "eval_runtime": 50.3402, + "eval_samples_per_second": 19.865, + "eval_steps_per_second": 0.834, + "step": 1360 + }, + { + "epoch": 1.6938394523957685, + "grad_norm": 3.0540060997009277, + "learning_rate": 2.453874538745388e-06, + "loss": 1.4123, + "step": 1361 + }, + { + "epoch": 1.6950840074673303, + "grad_norm": 3.0544989109039307, + "learning_rate": 2.451568265682657e-06, + "loss": 1.412, + "step": 1362 + }, + { + "epoch": 1.6963285625388922, + "grad_norm": 3.114126205444336, + "learning_rate": 2.4492619926199267e-06, + "loss": 1.4952, + "step": 1363 + }, + { + "epoch": 1.6975731176104543, + "grad_norm": 2.9366016387939453, + "learning_rate": 2.4469557195571957e-06, + "loss": 1.5155, + "step": 1364 + }, + { + "epoch": 1.6988176726820163, + "grad_norm": 2.910677671432495, + "learning_rate": 2.444649446494465e-06, + "loss": 1.4393, + "step": 1365 + }, + { + "epoch": 1.7000622277535782, + "grad_norm": 2.8373727798461914, + "learning_rate": 2.4423431734317345e-06, + "loss": 1.4223, + "step": 1366 + }, + { + "epoch": 1.70130678282514, + "grad_norm": 2.865161895751953, + "learning_rate": 2.440036900369004e-06, + "loss": 1.4605, + "step": 1367 + }, + { + "epoch": 1.7025513378967019, + "grad_norm": 2.7929177284240723, + "learning_rate": 2.437730627306273e-06, + "loss": 1.4695, + "step": 1368 + }, + { + "epoch": 1.7037958929682637, + "grad_norm": 2.867953300476074, + "learning_rate": 2.4354243542435428e-06, + "loss": 1.3524, + "step": 1369 + }, + { + "epoch": 1.7050404480398258, + "grad_norm": 2.8980207443237305, + "learning_rate": 2.4331180811808118e-06, + "loss": 1.4568, + "step": 1370 + }, + { + "epoch": 1.7050404480398258, + "eval_loss": 1.6297556161880493, + "eval_runtime": 51.447, + "eval_samples_per_second": 19.437, + "eval_steps_per_second": 0.816, + "step": 1370 + }, + { + "epoch": 1.7062850031113876, + "grad_norm": 2.954972505569458, + "learning_rate": 2.4308118081180816e-06, + "loss": 1.4, + "step": 1371 + }, + { + "epoch": 1.7075295581829497, + "grad_norm": 2.973191738128662, + "learning_rate": 2.4285055350553506e-06, + "loss": 1.4062, + "step": 1372 + }, + { + "epoch": 1.7087741132545116, + "grad_norm": 2.951444387435913, + "learning_rate": 2.42619926199262e-06, + "loss": 1.4523, + "step": 1373 + }, + { + "epoch": 1.7100186683260734, + "grad_norm": 2.850445508956909, + "learning_rate": 2.4238929889298895e-06, + "loss": 1.4282, + "step": 1374 + }, + { + "epoch": 1.7112632233976353, + "grad_norm": 2.7643561363220215, + "learning_rate": 2.421586715867159e-06, + "loss": 1.3807, + "step": 1375 + }, + { + "epoch": 1.712507778469197, + "grad_norm": 2.8872134685516357, + "learning_rate": 2.4192804428044283e-06, + "loss": 1.4801, + "step": 1376 + }, + { + "epoch": 1.7137523335407592, + "grad_norm": 2.8748483657836914, + "learning_rate": 2.4169741697416977e-06, + "loss": 1.381, + "step": 1377 + }, + { + "epoch": 1.7149968886123212, + "grad_norm": 2.7833831310272217, + "learning_rate": 2.4146678966789667e-06, + "loss": 1.389, + "step": 1378 + }, + { + "epoch": 1.716241443683883, + "grad_norm": 2.994715690612793, + "learning_rate": 2.4123616236162366e-06, + "loss": 1.3917, + "step": 1379 + }, + { + "epoch": 1.717485998755445, + "grad_norm": 2.887026786804199, + "learning_rate": 2.4100553505535056e-06, + "loss": 1.3725, + "step": 1380 + }, + { + "epoch": 1.717485998755445, + "eval_loss": 1.6283434629440308, + "eval_runtime": 50.596, + "eval_samples_per_second": 19.764, + "eval_steps_per_second": 0.83, + "step": 1380 + }, + { + "epoch": 1.7187305538270068, + "grad_norm": 3.2046546936035156, + "learning_rate": 2.407749077490775e-06, + "loss": 1.4422, + "step": 1381 + }, + { + "epoch": 1.7199751088985686, + "grad_norm": 3.0105698108673096, + "learning_rate": 2.4054428044280444e-06, + "loss": 1.4494, + "step": 1382 + }, + { + "epoch": 1.7212196639701307, + "grad_norm": 2.9917800426483154, + "learning_rate": 2.403136531365314e-06, + "loss": 1.4407, + "step": 1383 + }, + { + "epoch": 1.7224642190416926, + "grad_norm": 2.844325065612793, + "learning_rate": 2.4008302583025833e-06, + "loss": 1.4067, + "step": 1384 + }, + { + "epoch": 1.7237087741132546, + "grad_norm": 2.919576406478882, + "learning_rate": 2.3985239852398527e-06, + "loss": 1.4352, + "step": 1385 + }, + { + "epoch": 1.7249533291848165, + "grad_norm": 2.8357744216918945, + "learning_rate": 2.3962177121771217e-06, + "loss": 1.3978, + "step": 1386 + }, + { + "epoch": 1.7261978842563783, + "grad_norm": 2.82425856590271, + "learning_rate": 2.3939114391143915e-06, + "loss": 1.4439, + "step": 1387 + }, + { + "epoch": 1.7274424393279402, + "grad_norm": 2.9155638217926025, + "learning_rate": 2.3916051660516605e-06, + "loss": 1.4326, + "step": 1388 + }, + { + "epoch": 1.728686994399502, + "grad_norm": 2.862804651260376, + "learning_rate": 2.3892988929889304e-06, + "loss": 1.3766, + "step": 1389 + }, + { + "epoch": 1.729931549471064, + "grad_norm": 3.012230634689331, + "learning_rate": 2.3869926199261994e-06, + "loss": 1.4569, + "step": 1390 + }, + { + "epoch": 1.729931549471064, + "eval_loss": 1.6268510818481445, + "eval_runtime": 52.5438, + "eval_samples_per_second": 19.032, + "eval_steps_per_second": 0.799, + "step": 1390 + }, + { + "epoch": 1.7311761045426262, + "grad_norm": 2.9922425746917725, + "learning_rate": 2.384686346863469e-06, + "loss": 1.4066, + "step": 1391 + }, + { + "epoch": 1.732420659614188, + "grad_norm": 2.960145950317383, + "learning_rate": 2.3823800738007382e-06, + "loss": 1.4514, + "step": 1392 + }, + { + "epoch": 1.7336652146857499, + "grad_norm": 2.834998846054077, + "learning_rate": 2.3800738007380077e-06, + "loss": 1.4714, + "step": 1393 + }, + { + "epoch": 1.7349097697573117, + "grad_norm": 2.9646077156066895, + "learning_rate": 2.377767527675277e-06, + "loss": 1.4368, + "step": 1394 + }, + { + "epoch": 1.7361543248288736, + "grad_norm": 2.8233699798583984, + "learning_rate": 2.3754612546125465e-06, + "loss": 1.4598, + "step": 1395 + }, + { + "epoch": 1.7373988799004356, + "grad_norm": 2.895472764968872, + "learning_rate": 2.3731549815498155e-06, + "loss": 1.4551, + "step": 1396 + }, + { + "epoch": 1.7386434349719975, + "grad_norm": 2.7917752265930176, + "learning_rate": 2.3708487084870853e-06, + "loss": 1.4759, + "step": 1397 + }, + { + "epoch": 1.7398879900435595, + "grad_norm": 2.820672035217285, + "learning_rate": 2.3685424354243543e-06, + "loss": 1.3711, + "step": 1398 + }, + { + "epoch": 1.7411325451151214, + "grad_norm": 2.804952621459961, + "learning_rate": 2.3662361623616238e-06, + "loss": 1.4227, + "step": 1399 + }, + { + "epoch": 1.7423771001866832, + "grad_norm": 2.981553316116333, + "learning_rate": 2.363929889298893e-06, + "loss": 1.4639, + "step": 1400 + }, + { + "epoch": 1.7423771001866832, + "eval_loss": 1.6229957342147827, + "eval_runtime": 49.1507, + "eval_samples_per_second": 20.346, + "eval_steps_per_second": 0.855, + "step": 1400 + }, + { + "epoch": 1.743621655258245, + "grad_norm": 2.8649392127990723, + "learning_rate": 2.3616236162361626e-06, + "loss": 1.4283, + "step": 1401 + }, + { + "epoch": 1.744866210329807, + "grad_norm": 2.9438977241516113, + "learning_rate": 2.359317343173432e-06, + "loss": 1.444, + "step": 1402 + }, + { + "epoch": 1.746110765401369, + "grad_norm": 2.9270176887512207, + "learning_rate": 2.3570110701107015e-06, + "loss": 1.4044, + "step": 1403 + }, + { + "epoch": 1.747355320472931, + "grad_norm": 2.791527271270752, + "learning_rate": 2.3547047970479705e-06, + "loss": 1.4195, + "step": 1404 + }, + { + "epoch": 1.748599875544493, + "grad_norm": 2.8519608974456787, + "learning_rate": 2.3523985239852403e-06, + "loss": 1.4031, + "step": 1405 + }, + { + "epoch": 1.7498444306160548, + "grad_norm": 2.90872859954834, + "learning_rate": 2.3500922509225093e-06, + "loss": 1.4616, + "step": 1406 + }, + { + "epoch": 1.7510889856876166, + "grad_norm": 2.9190874099731445, + "learning_rate": 2.3477859778597787e-06, + "loss": 1.3917, + "step": 1407 + }, + { + "epoch": 1.7523335407591785, + "grad_norm": 2.868323802947998, + "learning_rate": 2.345479704797048e-06, + "loss": 1.4053, + "step": 1408 + }, + { + "epoch": 1.7535780958307405, + "grad_norm": 2.8638579845428467, + "learning_rate": 2.3431734317343176e-06, + "loss": 1.4042, + "step": 1409 + }, + { + "epoch": 1.7548226509023024, + "grad_norm": 3.1759817600250244, + "learning_rate": 2.340867158671587e-06, + "loss": 1.4937, + "step": 1410 + }, + { + "epoch": 1.7548226509023024, + "eval_loss": 1.6252617835998535, + "eval_runtime": 44.3041, + "eval_samples_per_second": 22.571, + "eval_steps_per_second": 0.948, + "step": 1410 + }, + { + "epoch": 1.7560672059738645, + "grad_norm": 2.8265459537506104, + "learning_rate": 2.3385608856088564e-06, + "loss": 1.406, + "step": 1411 + }, + { + "epoch": 1.7573117610454263, + "grad_norm": 2.856229305267334, + "learning_rate": 2.336254612546126e-06, + "loss": 1.4395, + "step": 1412 + }, + { + "epoch": 1.7585563161169881, + "grad_norm": 2.9112775325775146, + "learning_rate": 2.333948339483395e-06, + "loss": 1.427, + "step": 1413 + }, + { + "epoch": 1.75980087118855, + "grad_norm": 3.098846435546875, + "learning_rate": 2.3316420664206643e-06, + "loss": 1.4222, + "step": 1414 + }, + { + "epoch": 1.7610454262601118, + "grad_norm": 2.758100986480713, + "learning_rate": 2.3293357933579337e-06, + "loss": 1.3844, + "step": 1415 + }, + { + "epoch": 1.762289981331674, + "grad_norm": 2.8422274589538574, + "learning_rate": 2.327029520295203e-06, + "loss": 1.4033, + "step": 1416 + }, + { + "epoch": 1.763534536403236, + "grad_norm": 2.9257912635803223, + "learning_rate": 2.3247232472324725e-06, + "loss": 1.3961, + "step": 1417 + }, + { + "epoch": 1.7647790914747978, + "grad_norm": 2.9431657791137695, + "learning_rate": 2.322416974169742e-06, + "loss": 1.4451, + "step": 1418 + }, + { + "epoch": 1.7660236465463597, + "grad_norm": 2.8598814010620117, + "learning_rate": 2.3201107011070114e-06, + "loss": 1.3816, + "step": 1419 + }, + { + "epoch": 1.7672682016179215, + "grad_norm": 2.9868006706237793, + "learning_rate": 2.317804428044281e-06, + "loss": 1.4037, + "step": 1420 + }, + { + "epoch": 1.7672682016179215, + "eval_loss": 1.6231273412704468, + "eval_runtime": 48.8067, + "eval_samples_per_second": 20.489, + "eval_steps_per_second": 0.861, + "step": 1420 + }, + { + "epoch": 1.7685127566894834, + "grad_norm": 2.888206720352173, + "learning_rate": 2.31549815498155e-06, + "loss": 1.4087, + "step": 1421 + }, + { + "epoch": 1.7697573117610454, + "grad_norm": 2.9240078926086426, + "learning_rate": 2.3131918819188192e-06, + "loss": 1.3909, + "step": 1422 + }, + { + "epoch": 1.7710018668326073, + "grad_norm": 3.0401012897491455, + "learning_rate": 2.3108856088560886e-06, + "loss": 1.423, + "step": 1423 + }, + { + "epoch": 1.7722464219041694, + "grad_norm": 2.9484200477600098, + "learning_rate": 2.308579335793358e-06, + "loss": 1.449, + "step": 1424 + }, + { + "epoch": 1.7734909769757312, + "grad_norm": 2.863267660140991, + "learning_rate": 2.3062730627306275e-06, + "loss": 1.4215, + "step": 1425 + }, + { + "epoch": 1.774735532047293, + "grad_norm": 2.9615612030029297, + "learning_rate": 2.303966789667897e-06, + "loss": 1.4391, + "step": 1426 + }, + { + "epoch": 1.775980087118855, + "grad_norm": 2.7958569526672363, + "learning_rate": 2.3016605166051663e-06, + "loss": 1.3662, + "step": 1427 + }, + { + "epoch": 1.777224642190417, + "grad_norm": 2.898787021636963, + "learning_rate": 2.2993542435424358e-06, + "loss": 1.362, + "step": 1428 + }, + { + "epoch": 1.7784691972619788, + "grad_norm": 2.883049726486206, + "learning_rate": 2.2970479704797048e-06, + "loss": 1.4339, + "step": 1429 + }, + { + "epoch": 1.779713752333541, + "grad_norm": 3.0204827785491943, + "learning_rate": 2.294741697416974e-06, + "loss": 1.5174, + "step": 1430 + }, + { + "epoch": 1.779713752333541, + "eval_loss": 1.6246228218078613, + "eval_runtime": 47.5611, + "eval_samples_per_second": 21.026, + "eval_steps_per_second": 0.883, + "step": 1430 + }, + { + "epoch": 1.7809583074051027, + "grad_norm": 2.9354310035705566, + "learning_rate": 2.2924354243542436e-06, + "loss": 1.4115, + "step": 1431 + }, + { + "epoch": 1.7822028624766646, + "grad_norm": 2.9498608112335205, + "learning_rate": 2.290129151291513e-06, + "loss": 1.4058, + "step": 1432 + }, + { + "epoch": 1.7834474175482264, + "grad_norm": 2.9706921577453613, + "learning_rate": 2.2878228782287825e-06, + "loss": 1.426, + "step": 1433 + }, + { + "epoch": 1.7846919726197883, + "grad_norm": 2.8945024013519287, + "learning_rate": 2.285516605166052e-06, + "loss": 1.4011, + "step": 1434 + }, + { + "epoch": 1.7859365276913504, + "grad_norm": 2.8987081050872803, + "learning_rate": 2.2832103321033213e-06, + "loss": 1.4163, + "step": 1435 + }, + { + "epoch": 1.7871810827629122, + "grad_norm": 2.9797592163085938, + "learning_rate": 2.2809040590405907e-06, + "loss": 1.4367, + "step": 1436 + }, + { + "epoch": 1.7884256378344743, + "grad_norm": 2.8872063159942627, + "learning_rate": 2.2785977859778597e-06, + "loss": 1.4226, + "step": 1437 + }, + { + "epoch": 1.7896701929060361, + "grad_norm": 2.942655563354492, + "learning_rate": 2.2762915129151296e-06, + "loss": 1.4132, + "step": 1438 + }, + { + "epoch": 1.790914747977598, + "grad_norm": 3.027392625808716, + "learning_rate": 2.2739852398523986e-06, + "loss": 1.4148, + "step": 1439 + }, + { + "epoch": 1.7921593030491598, + "grad_norm": 2.9475181102752686, + "learning_rate": 2.271678966789668e-06, + "loss": 1.4585, + "step": 1440 + }, + { + "epoch": 1.7921593030491598, + "eval_loss": 1.6234897375106812, + "eval_runtime": 53.3737, + "eval_samples_per_second": 18.736, + "eval_steps_per_second": 0.787, + "step": 1440 + }, + { + "epoch": 1.793403858120722, + "grad_norm": 2.846304416656494, + "learning_rate": 2.2693726937269374e-06, + "loss": 1.3667, + "step": 1441 + }, + { + "epoch": 1.7946484131922837, + "grad_norm": 2.8925700187683105, + "learning_rate": 2.267066420664207e-06, + "loss": 1.4318, + "step": 1442 + }, + { + "epoch": 1.7958929682638458, + "grad_norm": 2.884025812149048, + "learning_rate": 2.2647601476014763e-06, + "loss": 1.4395, + "step": 1443 + }, + { + "epoch": 1.7971375233354077, + "grad_norm": 2.9763753414154053, + "learning_rate": 2.2624538745387457e-06, + "loss": 1.3742, + "step": 1444 + }, + { + "epoch": 1.7983820784069695, + "grad_norm": 3.0834898948669434, + "learning_rate": 2.2601476014760147e-06, + "loss": 1.4651, + "step": 1445 + }, + { + "epoch": 1.7996266334785314, + "grad_norm": 3.0673978328704834, + "learning_rate": 2.2578413284132845e-06, + "loss": 1.475, + "step": 1446 + }, + { + "epoch": 1.8008711885500932, + "grad_norm": 2.931521415710449, + "learning_rate": 2.2555350553505535e-06, + "loss": 1.4506, + "step": 1447 + }, + { + "epoch": 1.8021157436216553, + "grad_norm": 2.953301429748535, + "learning_rate": 2.253228782287823e-06, + "loss": 1.4224, + "step": 1448 + }, + { + "epoch": 1.8033602986932171, + "grad_norm": 3.0210883617401123, + "learning_rate": 2.2509225092250924e-06, + "loss": 1.4387, + "step": 1449 + }, + { + "epoch": 1.8046048537647792, + "grad_norm": 2.975741386413574, + "learning_rate": 2.248616236162362e-06, + "loss": 1.4318, + "step": 1450 + }, + { + "epoch": 1.8046048537647792, + "eval_loss": 1.618930697441101, + "eval_runtime": 49.6692, + "eval_samples_per_second": 20.133, + "eval_steps_per_second": 0.846, + "step": 1450 + }, + { + "epoch": 1.805849408836341, + "grad_norm": 2.9083876609802246, + "learning_rate": 2.2463099630996312e-06, + "loss": 1.3837, + "step": 1451 + }, + { + "epoch": 1.807093963907903, + "grad_norm": 3.364248275756836, + "learning_rate": 2.2440036900369006e-06, + "loss": 1.4589, + "step": 1452 + }, + { + "epoch": 1.8083385189794647, + "grad_norm": 2.935478925704956, + "learning_rate": 2.2416974169741696e-06, + "loss": 1.4119, + "step": 1453 + }, + { + "epoch": 1.8095830740510268, + "grad_norm": 3.1742911338806152, + "learning_rate": 2.2393911439114395e-06, + "loss": 1.4203, + "step": 1454 + }, + { + "epoch": 1.8108276291225887, + "grad_norm": 2.96966290473938, + "learning_rate": 2.2370848708487085e-06, + "loss": 1.4054, + "step": 1455 + }, + { + "epoch": 1.8120721841941507, + "grad_norm": 3.0919055938720703, + "learning_rate": 2.2347785977859783e-06, + "loss": 1.4289, + "step": 1456 + }, + { + "epoch": 1.8133167392657126, + "grad_norm": 2.9061055183410645, + "learning_rate": 2.2324723247232473e-06, + "loss": 1.4958, + "step": 1457 + }, + { + "epoch": 1.8145612943372744, + "grad_norm": 2.968348503112793, + "learning_rate": 2.2301660516605168e-06, + "loss": 1.4033, + "step": 1458 + }, + { + "epoch": 1.8158058494088363, + "grad_norm": 3.0435378551483154, + "learning_rate": 2.227859778597786e-06, + "loss": 1.414, + "step": 1459 + }, + { + "epoch": 1.8170504044803981, + "grad_norm": 2.9067916870117188, + "learning_rate": 2.2255535055350556e-06, + "loss": 1.3954, + "step": 1460 + }, + { + "epoch": 1.8170504044803981, + "eval_loss": 1.6240814924240112, + "eval_runtime": 51.7164, + "eval_samples_per_second": 19.336, + "eval_steps_per_second": 0.812, + "step": 1460 + }, + { + "epoch": 1.8182949595519602, + "grad_norm": 3.156384229660034, + "learning_rate": 2.2232472324723246e-06, + "loss": 1.4658, + "step": 1461 + }, + { + "epoch": 1.819539514623522, + "grad_norm": 2.9885313510894775, + "learning_rate": 2.2209409594095944e-06, + "loss": 1.4123, + "step": 1462 + }, + { + "epoch": 1.820784069695084, + "grad_norm": 3.227780818939209, + "learning_rate": 2.2186346863468634e-06, + "loss": 1.3587, + "step": 1463 + }, + { + "epoch": 1.822028624766646, + "grad_norm": 3.0375192165374756, + "learning_rate": 2.2163284132841333e-06, + "loss": 1.4015, + "step": 1464 + }, + { + "epoch": 1.8232731798382078, + "grad_norm": 3.0200846195220947, + "learning_rate": 2.2140221402214023e-06, + "loss": 1.419, + "step": 1465 + }, + { + "epoch": 1.8245177349097697, + "grad_norm": 2.9456615447998047, + "learning_rate": 2.2117158671586717e-06, + "loss": 1.4075, + "step": 1466 + }, + { + "epoch": 1.8257622899813317, + "grad_norm": 3.3129165172576904, + "learning_rate": 2.209409594095941e-06, + "loss": 1.4241, + "step": 1467 + }, + { + "epoch": 1.8270068450528936, + "grad_norm": 3.013141393661499, + "learning_rate": 2.2071033210332106e-06, + "loss": 1.4066, + "step": 1468 + }, + { + "epoch": 1.8282514001244556, + "grad_norm": 2.8414366245269775, + "learning_rate": 2.20479704797048e-06, + "loss": 1.4009, + "step": 1469 + }, + { + "epoch": 1.8294959551960175, + "grad_norm": 2.8920438289642334, + "learning_rate": 2.2024907749077494e-06, + "loss": 1.3916, + "step": 1470 + }, + { + "epoch": 1.8294959551960175, + "eval_loss": 1.6220412254333496, + "eval_runtime": 52.6199, + "eval_samples_per_second": 19.004, + "eval_steps_per_second": 0.798, + "step": 1470 + }, + { + "epoch": 1.8307405102675793, + "grad_norm": 3.032353639602661, + "learning_rate": 2.2001845018450184e-06, + "loss": 1.402, + "step": 1471 + }, + { + "epoch": 1.8319850653391412, + "grad_norm": 2.9512081146240234, + "learning_rate": 2.1978782287822883e-06, + "loss": 1.429, + "step": 1472 + }, + { + "epoch": 1.833229620410703, + "grad_norm": 2.846081018447876, + "learning_rate": 2.1955719557195573e-06, + "loss": 1.4203, + "step": 1473 + }, + { + "epoch": 1.834474175482265, + "grad_norm": 3.093378782272339, + "learning_rate": 2.1932656826568267e-06, + "loss": 1.4307, + "step": 1474 + }, + { + "epoch": 1.8357187305538272, + "grad_norm": 2.808326005935669, + "learning_rate": 2.190959409594096e-06, + "loss": 1.3753, + "step": 1475 + }, + { + "epoch": 1.836963285625389, + "grad_norm": 3.0328383445739746, + "learning_rate": 2.1886531365313655e-06, + "loss": 1.4534, + "step": 1476 + }, + { + "epoch": 1.8382078406969509, + "grad_norm": 3.045897960662842, + "learning_rate": 2.186346863468635e-06, + "loss": 1.3259, + "step": 1477 + }, + { + "epoch": 1.8394523957685127, + "grad_norm": 3.171846628189087, + "learning_rate": 2.1840405904059044e-06, + "loss": 1.397, + "step": 1478 + }, + { + "epoch": 1.8406969508400746, + "grad_norm": 2.893061637878418, + "learning_rate": 2.1817343173431734e-06, + "loss": 1.4201, + "step": 1479 + }, + { + "epoch": 1.8419415059116366, + "grad_norm": 3.0151467323303223, + "learning_rate": 2.1794280442804432e-06, + "loss": 1.4266, + "step": 1480 + }, + { + "epoch": 1.8419415059116366, + "eval_loss": 1.6166906356811523, + "eval_runtime": 51.8266, + "eval_samples_per_second": 19.295, + "eval_steps_per_second": 0.81, + "step": 1480 + }, + { + "epoch": 1.8431860609831985, + "grad_norm": 2.942431688308716, + "learning_rate": 2.177121771217712e-06, + "loss": 1.3744, + "step": 1481 + }, + { + "epoch": 1.8444306160547606, + "grad_norm": 2.8482232093811035, + "learning_rate": 2.1748154981549816e-06, + "loss": 1.3914, + "step": 1482 + }, + { + "epoch": 1.8456751711263224, + "grad_norm": 2.9315359592437744, + "learning_rate": 2.172509225092251e-06, + "loss": 1.419, + "step": 1483 + }, + { + "epoch": 1.8469197261978842, + "grad_norm": 3.191800594329834, + "learning_rate": 2.1702029520295205e-06, + "loss": 1.3524, + "step": 1484 + }, + { + "epoch": 1.848164281269446, + "grad_norm": 2.9308040142059326, + "learning_rate": 2.16789667896679e-06, + "loss": 1.3742, + "step": 1485 + }, + { + "epoch": 1.849408836341008, + "grad_norm": 2.7678964138031006, + "learning_rate": 2.1655904059040593e-06, + "loss": 1.3786, + "step": 1486 + }, + { + "epoch": 1.85065339141257, + "grad_norm": 3.0300636291503906, + "learning_rate": 2.1632841328413287e-06, + "loss": 1.4469, + "step": 1487 + }, + { + "epoch": 1.851897946484132, + "grad_norm": 3.142781972885132, + "learning_rate": 2.160977859778598e-06, + "loss": 1.4565, + "step": 1488 + }, + { + "epoch": 1.853142501555694, + "grad_norm": 2.954916000366211, + "learning_rate": 2.158671586715867e-06, + "loss": 1.4356, + "step": 1489 + }, + { + "epoch": 1.8543870566272558, + "grad_norm": 2.864875316619873, + "learning_rate": 2.1563653136531366e-06, + "loss": 1.4521, + "step": 1490 + }, + { + "epoch": 1.8543870566272558, + "eval_loss": 1.6207302808761597, + "eval_runtime": 49.2215, + "eval_samples_per_second": 20.316, + "eval_steps_per_second": 0.853, + "step": 1490 + }, + { + "epoch": 1.8556316116988176, + "grad_norm": 2.8908467292785645, + "learning_rate": 2.154059040590406e-06, + "loss": 1.4304, + "step": 1491 + }, + { + "epoch": 1.8568761667703795, + "grad_norm": 2.9439549446105957, + "learning_rate": 2.1517527675276754e-06, + "loss": 1.398, + "step": 1492 + }, + { + "epoch": 1.8581207218419415, + "grad_norm": 2.967728853225708, + "learning_rate": 2.149446494464945e-06, + "loss": 1.5038, + "step": 1493 + }, + { + "epoch": 1.8593652769135034, + "grad_norm": 2.938123941421509, + "learning_rate": 2.1471402214022143e-06, + "loss": 1.4395, + "step": 1494 + }, + { + "epoch": 1.8606098319850655, + "grad_norm": 2.93609881401062, + "learning_rate": 2.1448339483394837e-06, + "loss": 1.3939, + "step": 1495 + }, + { + "epoch": 1.8618543870566273, + "grad_norm": 2.9093775749206543, + "learning_rate": 2.142527675276753e-06, + "loss": 1.3942, + "step": 1496 + }, + { + "epoch": 1.8630989421281892, + "grad_norm": 2.773960590362549, + "learning_rate": 2.140221402214022e-06, + "loss": 1.3886, + "step": 1497 + }, + { + "epoch": 1.864343497199751, + "grad_norm": 2.9858205318450928, + "learning_rate": 2.1379151291512916e-06, + "loss": 1.3814, + "step": 1498 + }, + { + "epoch": 1.8655880522713129, + "grad_norm": 3.1233417987823486, + "learning_rate": 2.135608856088561e-06, + "loss": 1.4313, + "step": 1499 + }, + { + "epoch": 1.866832607342875, + "grad_norm": 2.9569480419158936, + "learning_rate": 2.1333025830258304e-06, + "loss": 1.4302, + "step": 1500 + }, + { + "epoch": 1.866832607342875, + "eval_loss": 1.6130231618881226, + "eval_runtime": 46.2834, + "eval_samples_per_second": 21.606, + "eval_steps_per_second": 0.907, + "step": 1500 + } + ], + "logging_steps": 1, + "max_steps": 2409, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 100, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 4.67596411404288e+17, + "train_batch_size": 3, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/training_args.bin b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..6cdf93817e8f6d23b60d79888b226cc66ed2e88b --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a50e4918f3c49370326e14a7ed2f73b48b05e521f4b3399b53438fac6abee926 +size 6011 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/zero_to_fp32.py b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..24cc342e78d1a006c782b3a4cd68d9ce786d8fd8 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/zero_to_fp32.py @@ -0,0 +1,604 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: python zero_to_fp32.py . pytorch_model.bin + +import argparse +import torch +import glob +import math +import os +import re +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + + total_files = len(files) + state_dicts = [] + for f in files: + state_dict = torch.load(f, map_location=device) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + if zero_stage <= 2: + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + elif zero_stage == 3: + # if there is more than one param group, there will be multiple flattened tensors - one + # flattened tensor per group - for simplicity merge them into a single tensor + # + # XXX: could make the script more memory efficient for when there are multiple groups - it + # will require matching the sub-lists of param_shapes for each param group flattened tensor + + fp32_flat_groups = [ + torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts)) + ] + + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = fp32_flat_groups[0].numel() * world_size + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + for name, shape in param_shapes.items(): + + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # XXX: memory usage doubles here + state_dict[name] = torch.cat( + tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)), + 0).narrow(0, 0, unpartitioned_numel).view(shape) + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + + Returns: + - pytorch ``state_dict`` + + Note: this approach may not work if your application doesn't have sufficient free CPU memory and + you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters) + print(f"Saving fp32 state dict to {output_file}") + torch.save(state_dict, output_file) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info(f"Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info(f"Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument( + "output_file", + type=str, + help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_file, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fb0f9a53a507ffcdb60410deb71d60ef801bf350 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/config.json @@ -0,0 +1,36 @@ +{ + "_name_or_path": "meta-llama/Llama-3.2-3B", + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "eos_token_id": 128001, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 3072, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 24, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "torch_dtype": "float16", + "transformers_version": "4.45.1", + "use_cache": true, + "vocab_size": 128256 +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/generation_config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9c389b841a9a29ddd904e02d1eb0e08dcce82ad9 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": 128001, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.45.1" +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/latest b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/latest new file mode 100644 index 0000000000000000000000000000000000000000..10b09fac99bc80ff931649e8b3378aab683b28be --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/latest @@ -0,0 +1 @@ +global_step1600 \ No newline at end of file diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/model.safetensors.index.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..ed64de846d720b9a7859dc20575fea8e8ca51940 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/model.safetensors.index.json @@ -0,0 +1,262 @@ +{ + "metadata": { + "total_size": 7213504512 + }, + "weight_map": { + "lm_head.weight": "model-00002-of-00002.safetensors", + "model.embed_tokens.weight": "model-00001-of-00002.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.norm.weight": "model-00002-of-00002.safetensors" + } +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/rng_state_0.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..5509f0e1ece25901639e38b63337f0f28e33b98d --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0123f1b841cbb90ad2cc1a10c38027bc63872dc02a831b18d254a6186f0ffff0 +size 16567 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/rng_state_1.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..bdaddc4981c690246dfa665b10da40b38a679bc3 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86e82dbca3ee98b4aff4d87464271202c99a718b90530a7719851b8b6a191547 +size 16567 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/rng_state_2.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..fd1ca9202bb0004e7be63108d043e05db918daf3 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fdc0a29e8caac820aa227d61191352d70074e7bc59c8656d34721d88d568ccb2 +size 16567 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/scheduler.pt b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..9206ddeeffec149ed3d4e16f994082d5aebc21d0 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff2f11dbfd060ce016b21218b873a40ef74c7e31eecd585532ac412097f6fdc2 +size 627 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/special_tokens_map.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..6949c5975ee0e961ef61cf31010dce04df0a03f8 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "[PAD]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/tokenizer.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f28ecaeab53ae07feed29ccf8624d2b0a8344df9 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19fb2e1e3cdd6f7433d89fd6d62c82042599dd4984f342efe7fec6e159e6a8f6 +size 17210734 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/tokenizer_config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8991b49e9c2a43fc527dab9e09ad8171f0cc5943 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/tokenizer_config.json @@ -0,0 +1,2086 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "[PAD]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128257": { + "content": "🅂", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "128258": { + "content": "🄿", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "[PAD]", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/trainer_state.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..df3c62cc7540daa15fa545a2543789b53f1be33a --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/trainer_state.json @@ -0,0 +1,12513 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.9912881144990666, + "eval_steps": 10, + "global_step": 1600, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0012445550715619166, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8368, + "step": 1 + }, + { + "epoch": 0.002489110143123833, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.86, + "step": 2 + }, + { + "epoch": 0.00373366521468575, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8265, + "step": 3 + }, + { + "epoch": 0.004978220286247666, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8271, + "step": 4 + }, + { + "epoch": 0.006222775357809583, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8768, + "step": 5 + }, + { + "epoch": 0.0074673304293715, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8402, + "step": 6 + }, + { + "epoch": 0.008711885500933417, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.785, + "step": 7 + }, + { + "epoch": 0.009956440572495333, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8466, + "step": 8 + }, + { + "epoch": 0.01120099564405725, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.7673, + "step": 9 + }, + { + "epoch": 0.012445550715619166, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8074, + "step": 10 + }, + { + "epoch": 0.012445550715619166, + "eval_loss": 2.8535053730010986, + "eval_runtime": 42.446, + "eval_samples_per_second": 23.559, + "eval_steps_per_second": 0.989, + "step": 10 + }, + { + "epoch": 0.013690105787181083, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8592, + "step": 11 + }, + { + "epoch": 0.014934660858743, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.804, + "step": 12 + }, + { + "epoch": 0.016179215930304917, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8076, + "step": 13 + }, + { + "epoch": 0.017423771001866834, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8219, + "step": 14 + }, + { + "epoch": 0.018668326073428748, + "grad_norm": 5.105710506439209, + "learning_rate": 2.0746887966804982e-08, + "loss": 2.7995, + "step": 15 + }, + { + "epoch": 0.019912881144990666, + "grad_norm": 5.156588077545166, + "learning_rate": 4.1493775933609963e-08, + "loss": 2.8577, + "step": 16 + }, + { + "epoch": 0.021157436216552583, + "grad_norm": 5.709373950958252, + "learning_rate": 6.224066390041494e-08, + "loss": 2.8433, + "step": 17 + }, + { + "epoch": 0.0224019912881145, + "grad_norm": 5.64658784866333, + "learning_rate": 8.298755186721993e-08, + "loss": 2.8428, + "step": 18 + }, + { + "epoch": 0.023646546359676415, + "grad_norm": 5.541937351226807, + "learning_rate": 1.037344398340249e-07, + "loss": 2.7785, + "step": 19 + }, + { + "epoch": 0.024891101431238332, + "grad_norm": 5.233374118804932, + "learning_rate": 1.2448132780082988e-07, + "loss": 2.8599, + "step": 20 + }, + { + "epoch": 0.024891101431238332, + "eval_loss": 2.8521182537078857, + "eval_runtime": 43.5664, + "eval_samples_per_second": 22.953, + "eval_steps_per_second": 0.964, + "step": 20 + }, + { + "epoch": 0.02613565650280025, + "grad_norm": 5.407598972320557, + "learning_rate": 1.4522821576763488e-07, + "loss": 2.8081, + "step": 21 + }, + { + "epoch": 0.027380211574362167, + "grad_norm": 5.0483317375183105, + "learning_rate": 1.6597510373443985e-07, + "loss": 2.8489, + "step": 22 + }, + { + "epoch": 0.02862476664592408, + "grad_norm": 4.985588550567627, + "learning_rate": 1.8672199170124483e-07, + "loss": 2.8375, + "step": 23 + }, + { + "epoch": 0.029869321717486, + "grad_norm": 5.201700687408447, + "learning_rate": 2.074688796680498e-07, + "loss": 2.8355, + "step": 24 + }, + { + "epoch": 0.031113876789047916, + "grad_norm": 5.009235382080078, + "learning_rate": 2.2821576763485478e-07, + "loss": 2.7774, + "step": 25 + }, + { + "epoch": 0.03235843186060983, + "grad_norm": 4.509210109710693, + "learning_rate": 2.4896265560165975e-07, + "loss": 2.7992, + "step": 26 + }, + { + "epoch": 0.03360298693217175, + "grad_norm": 4.529262065887451, + "learning_rate": 2.6970954356846476e-07, + "loss": 2.836, + "step": 27 + }, + { + "epoch": 0.03484754200373367, + "grad_norm": 4.526787281036377, + "learning_rate": 2.9045643153526976e-07, + "loss": 2.8128, + "step": 28 + }, + { + "epoch": 0.03609209707529558, + "grad_norm": 4.016609191894531, + "learning_rate": 3.112033195020747e-07, + "loss": 2.7947, + "step": 29 + }, + { + "epoch": 0.037336652146857496, + "grad_norm": 3.4784562587738037, + "learning_rate": 3.319502074688797e-07, + "loss": 2.7854, + "step": 30 + }, + { + "epoch": 0.037336652146857496, + "eval_loss": 2.8020100593566895, + "eval_runtime": 44.8464, + "eval_samples_per_second": 22.298, + "eval_steps_per_second": 0.937, + "step": 30 + }, + { + "epoch": 0.038581207218419414, + "grad_norm": 3.4887945652008057, + "learning_rate": 3.5269709543568466e-07, + "loss": 2.7908, + "step": 31 + }, + { + "epoch": 0.03982576228998133, + "grad_norm": 3.4533374309539795, + "learning_rate": 3.7344398340248966e-07, + "loss": 2.7358, + "step": 32 + }, + { + "epoch": 0.04107031736154325, + "grad_norm": 3.270954132080078, + "learning_rate": 3.941908713692946e-07, + "loss": 2.8013, + "step": 33 + }, + { + "epoch": 0.042314872433105166, + "grad_norm": 3.61322283744812, + "learning_rate": 4.149377593360996e-07, + "loss": 2.7521, + "step": 34 + }, + { + "epoch": 0.043559427504667084, + "grad_norm": 3.158719062805176, + "learning_rate": 4.3568464730290456e-07, + "loss": 2.7573, + "step": 35 + }, + { + "epoch": 0.044803982576229, + "grad_norm": 2.888746500015259, + "learning_rate": 4.5643153526970956e-07, + "loss": 2.7795, + "step": 36 + }, + { + "epoch": 0.04604853764779091, + "grad_norm": 3.022629499435425, + "learning_rate": 4.771784232365145e-07, + "loss": 2.7891, + "step": 37 + }, + { + "epoch": 0.04729309271935283, + "grad_norm": 3.038097381591797, + "learning_rate": 4.979253112033195e-07, + "loss": 2.7376, + "step": 38 + }, + { + "epoch": 0.04853764779091475, + "grad_norm": 2.9992239475250244, + "learning_rate": 5.186721991701245e-07, + "loss": 2.7279, + "step": 39 + }, + { + "epoch": 0.049782202862476664, + "grad_norm": 2.908842086791992, + "learning_rate": 5.394190871369295e-07, + "loss": 2.6582, + "step": 40 + }, + { + "epoch": 0.049782202862476664, + "eval_loss": 2.725968837738037, + "eval_runtime": 44.9498, + "eval_samples_per_second": 22.247, + "eval_steps_per_second": 0.934, + "step": 40 + }, + { + "epoch": 0.05102675793403858, + "grad_norm": 2.8025174140930176, + "learning_rate": 5.601659751037345e-07, + "loss": 2.6809, + "step": 41 + }, + { + "epoch": 0.0522713130056005, + "grad_norm": 2.7555699348449707, + "learning_rate": 5.809128630705395e-07, + "loss": 2.6954, + "step": 42 + }, + { + "epoch": 0.053515868077162417, + "grad_norm": 2.4732933044433594, + "learning_rate": 6.016597510373444e-07, + "loss": 2.7307, + "step": 43 + }, + { + "epoch": 0.054760423148724334, + "grad_norm": 2.451366424560547, + "learning_rate": 6.224066390041494e-07, + "loss": 2.7064, + "step": 44 + }, + { + "epoch": 0.056004978220286245, + "grad_norm": 2.5768678188323975, + "learning_rate": 6.431535269709543e-07, + "loss": 2.6466, + "step": 45 + }, + { + "epoch": 0.05724953329184816, + "grad_norm": 2.579332113265991, + "learning_rate": 6.639004149377594e-07, + "loss": 2.6127, + "step": 46 + }, + { + "epoch": 0.05849408836341008, + "grad_norm": 2.231207847595215, + "learning_rate": 6.846473029045644e-07, + "loss": 2.6429, + "step": 47 + }, + { + "epoch": 0.059738643434972, + "grad_norm": 2.134375810623169, + "learning_rate": 7.053941908713693e-07, + "loss": 2.6423, + "step": 48 + }, + { + "epoch": 0.060983198506533914, + "grad_norm": 2.0485494136810303, + "learning_rate": 7.261410788381744e-07, + "loss": 2.581, + "step": 49 + }, + { + "epoch": 0.06222775357809583, + "grad_norm": 2.02909517288208, + "learning_rate": 7.468879668049793e-07, + "loss": 2.6357, + "step": 50 + }, + { + "epoch": 0.06222775357809583, + "eval_loss": 2.632822036743164, + "eval_runtime": 47.0104, + "eval_samples_per_second": 21.272, + "eval_steps_per_second": 0.893, + "step": 50 + }, + { + "epoch": 0.06347230864965775, + "grad_norm": 1.892500877380371, + "learning_rate": 7.676348547717843e-07, + "loss": 2.5945, + "step": 51 + }, + { + "epoch": 0.06471686372121967, + "grad_norm": 2.107921600341797, + "learning_rate": 7.883817427385892e-07, + "loss": 2.5867, + "step": 52 + }, + { + "epoch": 0.06596141879278158, + "grad_norm": 1.8287527561187744, + "learning_rate": 8.091286307053943e-07, + "loss": 2.5862, + "step": 53 + }, + { + "epoch": 0.0672059738643435, + "grad_norm": 1.7342065572738647, + "learning_rate": 8.298755186721992e-07, + "loss": 2.5735, + "step": 54 + }, + { + "epoch": 0.06845052893590542, + "grad_norm": 1.7905038595199585, + "learning_rate": 8.506224066390042e-07, + "loss": 2.572, + "step": 55 + }, + { + "epoch": 0.06969508400746734, + "grad_norm": 1.9715144634246826, + "learning_rate": 8.713692946058091e-07, + "loss": 2.5797, + "step": 56 + }, + { + "epoch": 0.07093963907902924, + "grad_norm": 1.788966417312622, + "learning_rate": 8.921161825726142e-07, + "loss": 2.5567, + "step": 57 + }, + { + "epoch": 0.07218419415059116, + "grad_norm": 1.8978915214538574, + "learning_rate": 9.128630705394191e-07, + "loss": 2.5776, + "step": 58 + }, + { + "epoch": 0.07342874922215308, + "grad_norm": 1.6401689052581787, + "learning_rate": 9.336099585062241e-07, + "loss": 2.5662, + "step": 59 + }, + { + "epoch": 0.07467330429371499, + "grad_norm": 1.8200798034667969, + "learning_rate": 9.54356846473029e-07, + "loss": 2.5001, + "step": 60 + }, + { + "epoch": 0.07467330429371499, + "eval_loss": 2.569110870361328, + "eval_runtime": 47.3292, + "eval_samples_per_second": 21.129, + "eval_steps_per_second": 0.887, + "step": 60 + }, + { + "epoch": 0.07591785936527691, + "grad_norm": 1.5468674898147583, + "learning_rate": 9.751037344398341e-07, + "loss": 2.5308, + "step": 61 + }, + { + "epoch": 0.07716241443683883, + "grad_norm": 1.723684310913086, + "learning_rate": 9.95850622406639e-07, + "loss": 2.4907, + "step": 62 + }, + { + "epoch": 0.07840696950840075, + "grad_norm": 1.6346293687820435, + "learning_rate": 1.0165975103734441e-06, + "loss": 2.514, + "step": 63 + }, + { + "epoch": 0.07965152457996266, + "grad_norm": 1.8902088403701782, + "learning_rate": 1.037344398340249e-06, + "loss": 2.4859, + "step": 64 + }, + { + "epoch": 0.08089607965152458, + "grad_norm": 2.092611312866211, + "learning_rate": 1.058091286307054e-06, + "loss": 2.4888, + "step": 65 + }, + { + "epoch": 0.0821406347230865, + "grad_norm": 1.758482575416565, + "learning_rate": 1.078838174273859e-06, + "loss": 2.4537, + "step": 66 + }, + { + "epoch": 0.08338518979464841, + "grad_norm": 1.8941714763641357, + "learning_rate": 1.099585062240664e-06, + "loss": 2.5416, + "step": 67 + }, + { + "epoch": 0.08462974486621033, + "grad_norm": 1.7030234336853027, + "learning_rate": 1.120331950207469e-06, + "loss": 2.4791, + "step": 68 + }, + { + "epoch": 0.08587429993777225, + "grad_norm": 1.7602269649505615, + "learning_rate": 1.141078838174274e-06, + "loss": 2.4316, + "step": 69 + }, + { + "epoch": 0.08711885500933417, + "grad_norm": 1.7635431289672852, + "learning_rate": 1.161825726141079e-06, + "loss": 2.4674, + "step": 70 + }, + { + "epoch": 0.08711885500933417, + "eval_loss": 2.4922080039978027, + "eval_runtime": 45.3028, + "eval_samples_per_second": 22.074, + "eval_steps_per_second": 0.927, + "step": 70 + }, + { + "epoch": 0.08836341008089608, + "grad_norm": 2.0913803577423096, + "learning_rate": 1.182572614107884e-06, + "loss": 2.4341, + "step": 71 + }, + { + "epoch": 0.089607965152458, + "grad_norm": 1.8292183876037598, + "learning_rate": 1.2033195020746888e-06, + "loss": 2.4291, + "step": 72 + }, + { + "epoch": 0.09085252022401992, + "grad_norm": 2.206770181655884, + "learning_rate": 1.224066390041494e-06, + "loss": 2.3764, + "step": 73 + }, + { + "epoch": 0.09209707529558182, + "grad_norm": 1.8263559341430664, + "learning_rate": 1.2448132780082988e-06, + "loss": 2.4082, + "step": 74 + }, + { + "epoch": 0.09334163036714374, + "grad_norm": 1.9405455589294434, + "learning_rate": 1.2655601659751037e-06, + "loss": 2.4673, + "step": 75 + }, + { + "epoch": 0.09458618543870566, + "grad_norm": 2.138108015060425, + "learning_rate": 1.2863070539419086e-06, + "loss": 2.3879, + "step": 76 + }, + { + "epoch": 0.09583074051026758, + "grad_norm": 1.9024745225906372, + "learning_rate": 1.307053941908714e-06, + "loss": 2.4344, + "step": 77 + }, + { + "epoch": 0.0970752955818295, + "grad_norm": 1.9136689901351929, + "learning_rate": 1.3278008298755188e-06, + "loss": 2.474, + "step": 78 + }, + { + "epoch": 0.09831985065339141, + "grad_norm": 2.4752795696258545, + "learning_rate": 1.3485477178423237e-06, + "loss": 2.3412, + "step": 79 + }, + { + "epoch": 0.09956440572495333, + "grad_norm": 1.8720006942749023, + "learning_rate": 1.3692946058091288e-06, + "loss": 2.3438, + "step": 80 + }, + { + "epoch": 0.09956440572495333, + "eval_loss": 2.3980512619018555, + "eval_runtime": 48.2788, + "eval_samples_per_second": 20.713, + "eval_steps_per_second": 0.87, + "step": 80 + }, + { + "epoch": 0.10080896079651525, + "grad_norm": 2.671691656112671, + "learning_rate": 1.3900414937759337e-06, + "loss": 2.3336, + "step": 81 + }, + { + "epoch": 0.10205351586807716, + "grad_norm": 2.2953391075134277, + "learning_rate": 1.4107883817427386e-06, + "loss": 2.377, + "step": 82 + }, + { + "epoch": 0.10329807093963908, + "grad_norm": 3.009018898010254, + "learning_rate": 1.4315352697095435e-06, + "loss": 2.2977, + "step": 83 + }, + { + "epoch": 0.104542626011201, + "grad_norm": 2.664454936981201, + "learning_rate": 1.4522821576763488e-06, + "loss": 2.3271, + "step": 84 + }, + { + "epoch": 0.10578718108276292, + "grad_norm": 3.017303705215454, + "learning_rate": 1.4730290456431537e-06, + "loss": 2.3251, + "step": 85 + }, + { + "epoch": 0.10703173615432483, + "grad_norm": 2.634716510772705, + "learning_rate": 1.4937759336099586e-06, + "loss": 2.332, + "step": 86 + }, + { + "epoch": 0.10827629122588675, + "grad_norm": 3.059644937515259, + "learning_rate": 1.5145228215767635e-06, + "loss": 2.3478, + "step": 87 + }, + { + "epoch": 0.10952084629744867, + "grad_norm": 2.6962637901306152, + "learning_rate": 1.5352697095435686e-06, + "loss": 2.2792, + "step": 88 + }, + { + "epoch": 0.11076540136901059, + "grad_norm": 3.419729709625244, + "learning_rate": 1.5560165975103735e-06, + "loss": 2.2571, + "step": 89 + }, + { + "epoch": 0.11200995644057249, + "grad_norm": 2.740781545639038, + "learning_rate": 1.5767634854771784e-06, + "loss": 2.2875, + "step": 90 + }, + { + "epoch": 0.11200995644057249, + "eval_loss": 2.30843186378479, + "eval_runtime": 49.9264, + "eval_samples_per_second": 20.029, + "eval_steps_per_second": 0.841, + "step": 90 + }, + { + "epoch": 0.1132545115121344, + "grad_norm": 2.5608789920806885, + "learning_rate": 1.5975103734439833e-06, + "loss": 2.2592, + "step": 91 + }, + { + "epoch": 0.11449906658369632, + "grad_norm": 2.687999963760376, + "learning_rate": 1.6182572614107886e-06, + "loss": 2.2546, + "step": 92 + }, + { + "epoch": 0.11574362165525824, + "grad_norm": 2.695909023284912, + "learning_rate": 1.6390041493775935e-06, + "loss": 2.2525, + "step": 93 + }, + { + "epoch": 0.11698817672682016, + "grad_norm": 2.818357467651367, + "learning_rate": 1.6597510373443984e-06, + "loss": 2.216, + "step": 94 + }, + { + "epoch": 0.11823273179838208, + "grad_norm": 2.884119987487793, + "learning_rate": 1.6804979253112035e-06, + "loss": 2.2321, + "step": 95 + }, + { + "epoch": 0.119477286869944, + "grad_norm": 2.52104115486145, + "learning_rate": 1.7012448132780084e-06, + "loss": 2.199, + "step": 96 + }, + { + "epoch": 0.12072184194150591, + "grad_norm": 2.420313596725464, + "learning_rate": 1.7219917012448133e-06, + "loss": 2.1862, + "step": 97 + }, + { + "epoch": 0.12196639701306783, + "grad_norm": 2.8047542572021484, + "learning_rate": 1.7427385892116182e-06, + "loss": 2.1793, + "step": 98 + }, + { + "epoch": 0.12321095208462975, + "grad_norm": 2.836482286453247, + "learning_rate": 1.7634854771784235e-06, + "loss": 2.2271, + "step": 99 + }, + { + "epoch": 0.12445550715619166, + "grad_norm": 2.5282301902770996, + "learning_rate": 1.7842323651452284e-06, + "loss": 2.1768, + "step": 100 + }, + { + "epoch": 0.12445550715619166, + "eval_loss": 2.2312686443328857, + "eval_runtime": 49.2382, + "eval_samples_per_second": 20.309, + "eval_steps_per_second": 0.853, + "step": 100 + }, + { + "epoch": 0.12570006222775357, + "grad_norm": 3.3407280445098877, + "learning_rate": 1.8049792531120333e-06, + "loss": 2.1666, + "step": 101 + }, + { + "epoch": 0.1269446172993155, + "grad_norm": 2.4754133224487305, + "learning_rate": 1.8257261410788382e-06, + "loss": 2.1768, + "step": 102 + }, + { + "epoch": 0.1281891723708774, + "grad_norm": 3.430889129638672, + "learning_rate": 1.8464730290456433e-06, + "loss": 2.1953, + "step": 103 + }, + { + "epoch": 0.12943372744243933, + "grad_norm": 2.835294246673584, + "learning_rate": 1.8672199170124482e-06, + "loss": 2.146, + "step": 104 + }, + { + "epoch": 0.13067828251400124, + "grad_norm": 3.1532323360443115, + "learning_rate": 1.8879668049792531e-06, + "loss": 2.1729, + "step": 105 + }, + { + "epoch": 0.13192283758556317, + "grad_norm": 3.2278342247009277, + "learning_rate": 1.908713692946058e-06, + "loss": 2.1336, + "step": 106 + }, + { + "epoch": 0.13316739265712507, + "grad_norm": 2.7892515659332275, + "learning_rate": 1.929460580912863e-06, + "loss": 2.141, + "step": 107 + }, + { + "epoch": 0.134411947728687, + "grad_norm": 3.0179977416992188, + "learning_rate": 1.9502074688796682e-06, + "loss": 2.1255, + "step": 108 + }, + { + "epoch": 0.1356565028002489, + "grad_norm": 2.977935552597046, + "learning_rate": 1.970954356846473e-06, + "loss": 2.0986, + "step": 109 + }, + { + "epoch": 0.13690105787181084, + "grad_norm": 3.3042492866516113, + "learning_rate": 1.991701244813278e-06, + "loss": 2.0992, + "step": 110 + }, + { + "epoch": 0.13690105787181084, + "eval_loss": 2.171299695968628, + "eval_runtime": 42.9511, + "eval_samples_per_second": 23.282, + "eval_steps_per_second": 0.978, + "step": 110 + }, + { + "epoch": 0.13814561294337274, + "grad_norm": 2.9750890731811523, + "learning_rate": 2.012448132780083e-06, + "loss": 2.1116, + "step": 111 + }, + { + "epoch": 0.13939016801493467, + "grad_norm": 2.8199286460876465, + "learning_rate": 2.0331950207468883e-06, + "loss": 2.0822, + "step": 112 + }, + { + "epoch": 0.14063472308649658, + "grad_norm": 2.930532217025757, + "learning_rate": 2.053941908713693e-06, + "loss": 2.161, + "step": 113 + }, + { + "epoch": 0.14187927815805848, + "grad_norm": 3.3400321006774902, + "learning_rate": 2.074688796680498e-06, + "loss": 2.0953, + "step": 114 + }, + { + "epoch": 0.1431238332296204, + "grad_norm": 2.716564655303955, + "learning_rate": 2.095435684647303e-06, + "loss": 2.0757, + "step": 115 + }, + { + "epoch": 0.14436838830118232, + "grad_norm": 2.7586236000061035, + "learning_rate": 2.116182572614108e-06, + "loss": 2.0979, + "step": 116 + }, + { + "epoch": 0.14561294337274425, + "grad_norm": 2.622126340866089, + "learning_rate": 2.136929460580913e-06, + "loss": 2.0748, + "step": 117 + }, + { + "epoch": 0.14685749844430615, + "grad_norm": 2.646477460861206, + "learning_rate": 2.157676348547718e-06, + "loss": 2.0605, + "step": 118 + }, + { + "epoch": 0.14810205351586808, + "grad_norm": 2.8882410526275635, + "learning_rate": 2.178423236514523e-06, + "loss": 2.0501, + "step": 119 + }, + { + "epoch": 0.14934660858742999, + "grad_norm": 2.8302247524261475, + "learning_rate": 2.199170124481328e-06, + "loss": 2.0592, + "step": 120 + }, + { + "epoch": 0.14934660858742999, + "eval_loss": 2.1291966438293457, + "eval_runtime": 47.5399, + "eval_samples_per_second": 21.035, + "eval_steps_per_second": 0.883, + "step": 120 + }, + { + "epoch": 0.15059116365899192, + "grad_norm": 3.26979660987854, + "learning_rate": 2.219917012448133e-06, + "loss": 2.0422, + "step": 121 + }, + { + "epoch": 0.15183571873055382, + "grad_norm": 2.7483913898468018, + "learning_rate": 2.240663900414938e-06, + "loss": 2.0872, + "step": 122 + }, + { + "epoch": 0.15308027380211575, + "grad_norm": 3.1252024173736572, + "learning_rate": 2.2614107883817427e-06, + "loss": 2.0684, + "step": 123 + }, + { + "epoch": 0.15432482887367766, + "grad_norm": 2.9175283908843994, + "learning_rate": 2.282157676348548e-06, + "loss": 2.0522, + "step": 124 + }, + { + "epoch": 0.1555693839452396, + "grad_norm": 2.7950754165649414, + "learning_rate": 2.302904564315353e-06, + "loss": 2.065, + "step": 125 + }, + { + "epoch": 0.1568139390168015, + "grad_norm": 2.84049654006958, + "learning_rate": 2.323651452282158e-06, + "loss": 2.0781, + "step": 126 + }, + { + "epoch": 0.15805849408836342, + "grad_norm": 2.7396647930145264, + "learning_rate": 2.3443983402489627e-06, + "loss": 2.1758, + "step": 127 + }, + { + "epoch": 0.15930304915992533, + "grad_norm": 3.0765926837921143, + "learning_rate": 2.365145228215768e-06, + "loss": 2.0172, + "step": 128 + }, + { + "epoch": 0.16054760423148726, + "grad_norm": 3.217189073562622, + "learning_rate": 2.385892116182573e-06, + "loss": 2.0286, + "step": 129 + }, + { + "epoch": 0.16179215930304916, + "grad_norm": 3.141545295715332, + "learning_rate": 2.4066390041493776e-06, + "loss": 2.0583, + "step": 130 + }, + { + "epoch": 0.16179215930304916, + "eval_loss": 2.0946149826049805, + "eval_runtime": 54.92, + "eval_samples_per_second": 18.208, + "eval_steps_per_second": 0.765, + "step": 130 + }, + { + "epoch": 0.16303671437461106, + "grad_norm": 2.850052833557129, + "learning_rate": 2.4273858921161828e-06, + "loss": 2.0746, + "step": 131 + }, + { + "epoch": 0.164281269446173, + "grad_norm": 3.28913950920105, + "learning_rate": 2.448132780082988e-06, + "loss": 2.0539, + "step": 132 + }, + { + "epoch": 0.1655258245177349, + "grad_norm": 2.8819124698638916, + "learning_rate": 2.468879668049793e-06, + "loss": 1.9843, + "step": 133 + }, + { + "epoch": 0.16677037958929683, + "grad_norm": 3.6254632472991943, + "learning_rate": 2.4896265560165977e-06, + "loss": 2.0233, + "step": 134 + }, + { + "epoch": 0.16801493466085873, + "grad_norm": 2.7385146617889404, + "learning_rate": 2.5103734439834028e-06, + "loss": 1.9667, + "step": 135 + }, + { + "epoch": 0.16925948973242066, + "grad_norm": 2.9722647666931152, + "learning_rate": 2.5311203319502074e-06, + "loss": 1.9527, + "step": 136 + }, + { + "epoch": 0.17050404480398257, + "grad_norm": 3.139526605606079, + "learning_rate": 2.5518672199170125e-06, + "loss": 1.9967, + "step": 137 + }, + { + "epoch": 0.1717485998755445, + "grad_norm": 3.399920701980591, + "learning_rate": 2.5726141078838172e-06, + "loss": 1.9798, + "step": 138 + }, + { + "epoch": 0.1729931549471064, + "grad_norm": 2.8017327785491943, + "learning_rate": 2.5933609958506228e-06, + "loss": 1.983, + "step": 139 + }, + { + "epoch": 0.17423771001866833, + "grad_norm": 3.9967198371887207, + "learning_rate": 2.614107883817428e-06, + "loss": 1.977, + "step": 140 + }, + { + "epoch": 0.17423771001866833, + "eval_loss": 2.059037685394287, + "eval_runtime": 52.6232, + "eval_samples_per_second": 19.003, + "eval_steps_per_second": 0.798, + "step": 140 + }, + { + "epoch": 0.17548226509023024, + "grad_norm": 2.893092393875122, + "learning_rate": 2.6348547717842326e-06, + "loss": 1.9888, + "step": 141 + }, + { + "epoch": 0.17672682016179217, + "grad_norm": 3.634352207183838, + "learning_rate": 2.6556016597510377e-06, + "loss": 2.0726, + "step": 142 + }, + { + "epoch": 0.17797137523335407, + "grad_norm": 3.3651444911956787, + "learning_rate": 2.6763485477178423e-06, + "loss": 1.973, + "step": 143 + }, + { + "epoch": 0.179215930304916, + "grad_norm": 3.968986988067627, + "learning_rate": 2.6970954356846475e-06, + "loss": 1.9948, + "step": 144 + }, + { + "epoch": 0.1804604853764779, + "grad_norm": 3.20105242729187, + "learning_rate": 2.717842323651452e-06, + "loss": 1.9238, + "step": 145 + }, + { + "epoch": 0.18170504044803984, + "grad_norm": 3.648339033126831, + "learning_rate": 2.7385892116182577e-06, + "loss": 1.9923, + "step": 146 + }, + { + "epoch": 0.18294959551960174, + "grad_norm": 3.0856316089630127, + "learning_rate": 2.7593360995850628e-06, + "loss": 1.9749, + "step": 147 + }, + { + "epoch": 0.18419415059116365, + "grad_norm": 3.2681071758270264, + "learning_rate": 2.7800829875518675e-06, + "loss": 1.9304, + "step": 148 + }, + { + "epoch": 0.18543870566272558, + "grad_norm": 2.634958267211914, + "learning_rate": 2.8008298755186726e-06, + "loss": 1.9237, + "step": 149 + }, + { + "epoch": 0.18668326073428748, + "grad_norm": 2.769491672515869, + "learning_rate": 2.8215767634854773e-06, + "loss": 1.8963, + "step": 150 + }, + { + "epoch": 0.18668326073428748, + "eval_loss": 2.0394654273986816, + "eval_runtime": 55.2136, + "eval_samples_per_second": 18.111, + "eval_steps_per_second": 0.761, + "step": 150 + }, + { + "epoch": 0.1879278158058494, + "grad_norm": 3.4345781803131104, + "learning_rate": 2.8423236514522824e-06, + "loss": 1.9836, + "step": 151 + }, + { + "epoch": 0.18917237087741132, + "grad_norm": 3.115727424621582, + "learning_rate": 2.863070539419087e-06, + "loss": 1.9352, + "step": 152 + }, + { + "epoch": 0.19041692594897325, + "grad_norm": 3.050652503967285, + "learning_rate": 2.883817427385892e-06, + "loss": 1.9031, + "step": 153 + }, + { + "epoch": 0.19166148102053515, + "grad_norm": 2.9404428005218506, + "learning_rate": 2.9045643153526977e-06, + "loss": 1.9196, + "step": 154 + }, + { + "epoch": 0.19290603609209708, + "grad_norm": 3.080810785293579, + "learning_rate": 2.9253112033195024e-06, + "loss": 1.9405, + "step": 155 + }, + { + "epoch": 0.194150591163659, + "grad_norm": 3.058558702468872, + "learning_rate": 2.9460580912863075e-06, + "loss": 1.9052, + "step": 156 + }, + { + "epoch": 0.19539514623522092, + "grad_norm": 3.307955026626587, + "learning_rate": 2.966804979253112e-06, + "loss": 1.9245, + "step": 157 + }, + { + "epoch": 0.19663970130678282, + "grad_norm": 2.845506191253662, + "learning_rate": 2.9875518672199173e-06, + "loss": 1.9461, + "step": 158 + }, + { + "epoch": 0.19788425637834475, + "grad_norm": 2.758654832839966, + "learning_rate": 3.008298755186722e-06, + "loss": 1.8885, + "step": 159 + }, + { + "epoch": 0.19912881144990666, + "grad_norm": 3.161252737045288, + "learning_rate": 3.029045643153527e-06, + "loss": 1.8535, + "step": 160 + }, + { + "epoch": 0.19912881144990666, + "eval_loss": 2.018101453781128, + "eval_runtime": 51.0744, + "eval_samples_per_second": 19.579, + "eval_steps_per_second": 0.822, + "step": 160 + }, + { + "epoch": 0.2003733665214686, + "grad_norm": 3.196847677230835, + "learning_rate": 3.0497925311203326e-06, + "loss": 1.8853, + "step": 161 + }, + { + "epoch": 0.2016179215930305, + "grad_norm": 2.8364663124084473, + "learning_rate": 3.0705394190871373e-06, + "loss": 1.9463, + "step": 162 + }, + { + "epoch": 0.2028624766645924, + "grad_norm": 3.0874054431915283, + "learning_rate": 3.0912863070539424e-06, + "loss": 1.9254, + "step": 163 + }, + { + "epoch": 0.20410703173615433, + "grad_norm": 2.7914493083953857, + "learning_rate": 3.112033195020747e-06, + "loss": 1.9213, + "step": 164 + }, + { + "epoch": 0.20535158680771623, + "grad_norm": 3.3871428966522217, + "learning_rate": 3.132780082987552e-06, + "loss": 1.8993, + "step": 165 + }, + { + "epoch": 0.20659614187927816, + "grad_norm": 3.096653461456299, + "learning_rate": 3.153526970954357e-06, + "loss": 1.8827, + "step": 166 + }, + { + "epoch": 0.20784069695084006, + "grad_norm": 2.8276076316833496, + "learning_rate": 3.174273858921162e-06, + "loss": 1.912, + "step": 167 + }, + { + "epoch": 0.209085252022402, + "grad_norm": 3.3058435916900635, + "learning_rate": 3.1950207468879666e-06, + "loss": 1.8381, + "step": 168 + }, + { + "epoch": 0.2103298070939639, + "grad_norm": 3.5017333030700684, + "learning_rate": 3.215767634854772e-06, + "loss": 1.9544, + "step": 169 + }, + { + "epoch": 0.21157436216552583, + "grad_norm": 3.457296133041382, + "learning_rate": 3.2365145228215773e-06, + "loss": 1.922, + "step": 170 + }, + { + "epoch": 0.21157436216552583, + "eval_loss": 1.988455891609192, + "eval_runtime": 56.2099, + "eval_samples_per_second": 17.79, + "eval_steps_per_second": 0.747, + "step": 170 + }, + { + "epoch": 0.21281891723708773, + "grad_norm": 3.1964566707611084, + "learning_rate": 3.257261410788382e-06, + "loss": 1.8925, + "step": 171 + }, + { + "epoch": 0.21406347230864967, + "grad_norm": 3.234652042388916, + "learning_rate": 3.278008298755187e-06, + "loss": 1.884, + "step": 172 + }, + { + "epoch": 0.21530802738021157, + "grad_norm": 3.14414119720459, + "learning_rate": 3.2987551867219918e-06, + "loss": 1.8905, + "step": 173 + }, + { + "epoch": 0.2165525824517735, + "grad_norm": 3.606379508972168, + "learning_rate": 3.319502074688797e-06, + "loss": 1.946, + "step": 174 + }, + { + "epoch": 0.2177971375233354, + "grad_norm": 2.986646890640259, + "learning_rate": 3.3402489626556016e-06, + "loss": 1.8691, + "step": 175 + }, + { + "epoch": 0.21904169259489734, + "grad_norm": 3.7298269271850586, + "learning_rate": 3.360995850622407e-06, + "loss": 1.8717, + "step": 176 + }, + { + "epoch": 0.22028624766645924, + "grad_norm": 3.422295093536377, + "learning_rate": 3.381742738589212e-06, + "loss": 1.8568, + "step": 177 + }, + { + "epoch": 0.22153080273802117, + "grad_norm": 3.2293782234191895, + "learning_rate": 3.402489626556017e-06, + "loss": 1.8471, + "step": 178 + }, + { + "epoch": 0.22277535780958307, + "grad_norm": 3.2293782234191895, + "learning_rate": 3.402489626556017e-06, + "loss": 1.9303, + "step": 179 + }, + { + "epoch": 0.22401991288114498, + "grad_norm": 3.506223440170288, + "learning_rate": 3.423236514522822e-06, + "loss": 1.8237, + "step": 180 + }, + { + "epoch": 0.22401991288114498, + "eval_loss": 1.9734643697738647, + "eval_runtime": 52.2815, + "eval_samples_per_second": 19.127, + "eval_steps_per_second": 0.803, + "step": 180 + }, + { + "epoch": 0.2252644679527069, + "grad_norm": 3.0156619548797607, + "learning_rate": 3.4439834024896267e-06, + "loss": 1.908, + "step": 181 + }, + { + "epoch": 0.2265090230242688, + "grad_norm": 3.55517315864563, + "learning_rate": 3.4647302904564318e-06, + "loss": 1.9104, + "step": 182 + }, + { + "epoch": 0.22775357809583074, + "grad_norm": 3.144984006881714, + "learning_rate": 3.4854771784232365e-06, + "loss": 1.8198, + "step": 183 + }, + { + "epoch": 0.22899813316739265, + "grad_norm": 4.302074432373047, + "learning_rate": 3.5062240663900416e-06, + "loss": 1.8237, + "step": 184 + }, + { + "epoch": 0.23024268823895458, + "grad_norm": 3.0522446632385254, + "learning_rate": 3.526970954356847e-06, + "loss": 1.8409, + "step": 185 + }, + { + "epoch": 0.23148724331051648, + "grad_norm": 3.8607394695281982, + "learning_rate": 3.5477178423236518e-06, + "loss": 1.8807, + "step": 186 + }, + { + "epoch": 0.23273179838207841, + "grad_norm": 2.9236302375793457, + "learning_rate": 3.568464730290457e-06, + "loss": 1.7873, + "step": 187 + }, + { + "epoch": 0.23397635345364032, + "grad_norm": 4.013780117034912, + "learning_rate": 3.5892116182572616e-06, + "loss": 1.7909, + "step": 188 + }, + { + "epoch": 0.23522090852520225, + "grad_norm": 3.0933122634887695, + "learning_rate": 3.6099585062240667e-06, + "loss": 1.8469, + "step": 189 + }, + { + "epoch": 0.23646546359676415, + "grad_norm": 3.487816095352173, + "learning_rate": 3.6307053941908714e-06, + "loss": 1.8469, + "step": 190 + }, + { + "epoch": 0.23646546359676415, + "eval_loss": 1.9604240655899048, + "eval_runtime": 53.4604, + "eval_samples_per_second": 18.705, + "eval_steps_per_second": 0.786, + "step": 190 + }, + { + "epoch": 0.23771001866832608, + "grad_norm": 2.829219341278076, + "learning_rate": 3.6514522821576765e-06, + "loss": 1.8639, + "step": 191 + }, + { + "epoch": 0.238954573739888, + "grad_norm": 3.595534324645996, + "learning_rate": 3.672199170124482e-06, + "loss": 1.8988, + "step": 192 + }, + { + "epoch": 0.24019912881144992, + "grad_norm": 2.9022483825683594, + "learning_rate": 3.6929460580912867e-06, + "loss": 1.8972, + "step": 193 + }, + { + "epoch": 0.24144368388301182, + "grad_norm": 3.0667994022369385, + "learning_rate": 3.713692946058092e-06, + "loss": 1.8499, + "step": 194 + }, + { + "epoch": 0.24268823895457373, + "grad_norm": 3.0498485565185547, + "learning_rate": 3.7344398340248965e-06, + "loss": 1.8341, + "step": 195 + }, + { + "epoch": 0.24393279402613566, + "grad_norm": 4.058084964752197, + "learning_rate": 3.7551867219917016e-06, + "loss": 1.8641, + "step": 196 + }, + { + "epoch": 0.24517734909769756, + "grad_norm": 2.9995028972625732, + "learning_rate": 3.7759336099585063e-06, + "loss": 1.7885, + "step": 197 + }, + { + "epoch": 0.2464219041692595, + "grad_norm": 3.254704475402832, + "learning_rate": 3.7966804979253114e-06, + "loss": 1.7544, + "step": 198 + }, + { + "epoch": 0.2476664592408214, + "grad_norm": 2.9126744270324707, + "learning_rate": 3.817427385892116e-06, + "loss": 1.8461, + "step": 199 + }, + { + "epoch": 0.24891101431238333, + "grad_norm": 3.475247621536255, + "learning_rate": 3.838174273858922e-06, + "loss": 1.8095, + "step": 200 + }, + { + "epoch": 0.24891101431238333, + "eval_loss": 1.9373760223388672, + "eval_runtime": 51.8733, + "eval_samples_per_second": 19.278, + "eval_steps_per_second": 0.81, + "step": 200 + }, + { + "epoch": 0.25015556938394523, + "grad_norm": 3.2663731575012207, + "learning_rate": 3.858921161825726e-06, + "loss": 1.8076, + "step": 201 + }, + { + "epoch": 0.25140012445550713, + "grad_norm": 3.1151726245880127, + "learning_rate": 3.879668049792531e-06, + "loss": 1.8143, + "step": 202 + }, + { + "epoch": 0.2526446795270691, + "grad_norm": 3.102038860321045, + "learning_rate": 3.9004149377593365e-06, + "loss": 1.8279, + "step": 203 + }, + { + "epoch": 0.253889234598631, + "grad_norm": 3.370642900466919, + "learning_rate": 3.921161825726142e-06, + "loss": 1.8522, + "step": 204 + }, + { + "epoch": 0.2551337896701929, + "grad_norm": 3.116128921508789, + "learning_rate": 3.941908713692946e-06, + "loss": 1.8104, + "step": 205 + }, + { + "epoch": 0.2563783447417548, + "grad_norm": 3.12028431892395, + "learning_rate": 3.962655601659751e-06, + "loss": 1.7723, + "step": 206 + }, + { + "epoch": 0.25762289981331676, + "grad_norm": 3.1251418590545654, + "learning_rate": 3.983402489626556e-06, + "loss": 1.863, + "step": 207 + }, + { + "epoch": 0.25886745488487867, + "grad_norm": 3.073702812194824, + "learning_rate": 4.004149377593361e-06, + "loss": 1.805, + "step": 208 + }, + { + "epoch": 0.26011200995644057, + "grad_norm": 2.9772469997406006, + "learning_rate": 4.024896265560166e-06, + "loss": 1.8251, + "step": 209 + }, + { + "epoch": 0.2613565650280025, + "grad_norm": 2.9970037937164307, + "learning_rate": 4.045643153526971e-06, + "loss": 1.8386, + "step": 210 + }, + { + "epoch": 0.2613565650280025, + "eval_loss": 1.9167065620422363, + "eval_runtime": 42.3328, + "eval_samples_per_second": 23.622, + "eval_steps_per_second": 0.992, + "step": 210 + }, + { + "epoch": 0.26260112009956443, + "grad_norm": 3.1391561031341553, + "learning_rate": 4.0663900414937765e-06, + "loss": 1.8109, + "step": 211 + }, + { + "epoch": 0.26384567517112634, + "grad_norm": 3.161538600921631, + "learning_rate": 4.087136929460581e-06, + "loss": 1.8441, + "step": 212 + }, + { + "epoch": 0.26509023024268824, + "grad_norm": 3.12412166595459, + "learning_rate": 4.107883817427386e-06, + "loss": 1.7736, + "step": 213 + }, + { + "epoch": 0.26633478531425014, + "grad_norm": 3.3241145610809326, + "learning_rate": 4.128630705394191e-06, + "loss": 1.8067, + "step": 214 + }, + { + "epoch": 0.26757934038581205, + "grad_norm": 3.503307819366455, + "learning_rate": 4.149377593360996e-06, + "loss": 1.7858, + "step": 215 + }, + { + "epoch": 0.268823895457374, + "grad_norm": 3.1009578704833984, + "learning_rate": 4.170124481327801e-06, + "loss": 1.7542, + "step": 216 + }, + { + "epoch": 0.2700684505289359, + "grad_norm": 3.2236011028289795, + "learning_rate": 4.190871369294606e-06, + "loss": 1.8006, + "step": 217 + }, + { + "epoch": 0.2713130056004978, + "grad_norm": 3.048935651779175, + "learning_rate": 4.211618257261411e-06, + "loss": 1.7706, + "step": 218 + }, + { + "epoch": 0.2725575606720597, + "grad_norm": 3.3416147232055664, + "learning_rate": 4.232365145228216e-06, + "loss": 1.8108, + "step": 219 + }, + { + "epoch": 0.2738021157436217, + "grad_norm": 3.2010738849639893, + "learning_rate": 4.253112033195021e-06, + "loss": 1.8561, + "step": 220 + }, + { + "epoch": 0.2738021157436217, + "eval_loss": 1.9111930131912231, + "eval_runtime": 50.4077, + "eval_samples_per_second": 19.838, + "eval_steps_per_second": 0.833, + "step": 220 + }, + { + "epoch": 0.2750466708151836, + "grad_norm": 3.4033374786376953, + "learning_rate": 4.273858921161826e-06, + "loss": 1.8218, + "step": 221 + }, + { + "epoch": 0.2762912258867455, + "grad_norm": 3.2086002826690674, + "learning_rate": 4.294605809128631e-06, + "loss": 1.8213, + "step": 222 + }, + { + "epoch": 0.2775357809583074, + "grad_norm": 4.090956211090088, + "learning_rate": 4.315352697095436e-06, + "loss": 1.8455, + "step": 223 + }, + { + "epoch": 0.27878033602986935, + "grad_norm": 3.1142985820770264, + "learning_rate": 4.336099585062241e-06, + "loss": 1.7737, + "step": 224 + }, + { + "epoch": 0.28002489110143125, + "grad_norm": 3.369669198989868, + "learning_rate": 4.356846473029046e-06, + "loss": 1.824, + "step": 225 + }, + { + "epoch": 0.28126944617299315, + "grad_norm": 3.165672779083252, + "learning_rate": 4.3775933609958506e-06, + "loss": 1.7768, + "step": 226 + }, + { + "epoch": 0.28251400124455506, + "grad_norm": 3.3343470096588135, + "learning_rate": 4.398340248962656e-06, + "loss": 1.7278, + "step": 227 + }, + { + "epoch": 0.28375855631611696, + "grad_norm": 3.2574994564056396, + "learning_rate": 4.419087136929461e-06, + "loss": 1.8439, + "step": 228 + }, + { + "epoch": 0.2850031113876789, + "grad_norm": 3.043928623199463, + "learning_rate": 4.439834024896266e-06, + "loss": 1.818, + "step": 229 + }, + { + "epoch": 0.2862476664592408, + "grad_norm": 2.9703125953674316, + "learning_rate": 4.460580912863071e-06, + "loss": 1.773, + "step": 230 + }, + { + "epoch": 0.2862476664592408, + "eval_loss": 1.898109793663025, + "eval_runtime": 45.6023, + "eval_samples_per_second": 21.929, + "eval_steps_per_second": 0.921, + "step": 230 + }, + { + "epoch": 0.2874922215308027, + "grad_norm": 2.9997220039367676, + "learning_rate": 4.481327800829876e-06, + "loss": 1.7727, + "step": 231 + }, + { + "epoch": 0.28873677660236463, + "grad_norm": 3.1358482837677, + "learning_rate": 4.502074688796681e-06, + "loss": 1.7764, + "step": 232 + }, + { + "epoch": 0.2899813316739266, + "grad_norm": 3.395747661590576, + "learning_rate": 4.5228215767634855e-06, + "loss": 1.7826, + "step": 233 + }, + { + "epoch": 0.2912258867454885, + "grad_norm": 3.7494754791259766, + "learning_rate": 4.543568464730291e-06, + "loss": 1.7181, + "step": 234 + }, + { + "epoch": 0.2924704418170504, + "grad_norm": 3.42293381690979, + "learning_rate": 4.564315352697096e-06, + "loss": 1.7773, + "step": 235 + }, + { + "epoch": 0.2937149968886123, + "grad_norm": 3.2524514198303223, + "learning_rate": 4.585062240663901e-06, + "loss": 1.7294, + "step": 236 + }, + { + "epoch": 0.29495955196017426, + "grad_norm": 3.657869577407837, + "learning_rate": 4.605809128630706e-06, + "loss": 1.762, + "step": 237 + }, + { + "epoch": 0.29620410703173616, + "grad_norm": 3.127372980117798, + "learning_rate": 4.626556016597511e-06, + "loss": 1.7623, + "step": 238 + }, + { + "epoch": 0.29744866210329807, + "grad_norm": 4.166962146759033, + "learning_rate": 4.647302904564316e-06, + "loss": 1.6995, + "step": 239 + }, + { + "epoch": 0.29869321717485997, + "grad_norm": 3.094264030456543, + "learning_rate": 4.66804979253112e-06, + "loss": 1.7469, + "step": 240 + }, + { + "epoch": 0.29869321717485997, + "eval_loss": 1.8908178806304932, + "eval_runtime": 44.6332, + "eval_samples_per_second": 22.405, + "eval_steps_per_second": 0.941, + "step": 240 + }, + { + "epoch": 0.29993777224642193, + "grad_norm": 3.8644745349884033, + "learning_rate": 4.6887966804979255e-06, + "loss": 1.7644, + "step": 241 + }, + { + "epoch": 0.30118232731798383, + "grad_norm": 3.1488852500915527, + "learning_rate": 4.709543568464731e-06, + "loss": 1.772, + "step": 242 + }, + { + "epoch": 0.30242688238954574, + "grad_norm": 3.3179638385772705, + "learning_rate": 4.730290456431536e-06, + "loss": 1.7588, + "step": 243 + }, + { + "epoch": 0.30367143746110764, + "grad_norm": 3.328355550765991, + "learning_rate": 4.751037344398341e-06, + "loss": 1.7417, + "step": 244 + }, + { + "epoch": 0.30491599253266954, + "grad_norm": 3.2690482139587402, + "learning_rate": 4.771784232365146e-06, + "loss": 1.7562, + "step": 245 + }, + { + "epoch": 0.3061605476042315, + "grad_norm": 3.2759454250335693, + "learning_rate": 4.792531120331951e-06, + "loss": 1.7536, + "step": 246 + }, + { + "epoch": 0.3074051026757934, + "grad_norm": 2.937964916229248, + "learning_rate": 4.813278008298755e-06, + "loss": 1.7518, + "step": 247 + }, + { + "epoch": 0.3086496577473553, + "grad_norm": 3.0995302200317383, + "learning_rate": 4.83402489626556e-06, + "loss": 1.7593, + "step": 248 + }, + { + "epoch": 0.3098942128189172, + "grad_norm": 3.230459213256836, + "learning_rate": 4.8547717842323655e-06, + "loss": 1.6962, + "step": 249 + }, + { + "epoch": 0.3111387678904792, + "grad_norm": 3.242576837539673, + "learning_rate": 4.875518672199171e-06, + "loss": 1.7307, + "step": 250 + }, + { + "epoch": 0.3111387678904792, + "eval_loss": 1.8805371522903442, + "eval_runtime": 45.9886, + "eval_samples_per_second": 21.745, + "eval_steps_per_second": 0.913, + "step": 250 + }, + { + "epoch": 0.3123833229620411, + "grad_norm": 3.097045421600342, + "learning_rate": 4.896265560165976e-06, + "loss": 1.7438, + "step": 251 + }, + { + "epoch": 0.313627878033603, + "grad_norm": 3.2428948879241943, + "learning_rate": 4.91701244813278e-06, + "loss": 1.7179, + "step": 252 + }, + { + "epoch": 0.3148724331051649, + "grad_norm": 3.196274518966675, + "learning_rate": 4.937759336099586e-06, + "loss": 1.7855, + "step": 253 + }, + { + "epoch": 0.31611698817672684, + "grad_norm": 2.978203535079956, + "learning_rate": 4.95850622406639e-06, + "loss": 1.7144, + "step": 254 + }, + { + "epoch": 0.31736154324828875, + "grad_norm": 3.2641701698303223, + "learning_rate": 4.979253112033195e-06, + "loss": 1.7427, + "step": 255 + }, + { + "epoch": 0.31860609831985065, + "grad_norm": 2.7441232204437256, + "learning_rate": 5e-06, + "loss": 1.7619, + "step": 256 + }, + { + "epoch": 0.31985065339141255, + "grad_norm": 3.0723495483398438, + "learning_rate": 4.99769372693727e-06, + "loss": 1.8017, + "step": 257 + }, + { + "epoch": 0.3210952084629745, + "grad_norm": 3.104752540588379, + "learning_rate": 4.995387453874539e-06, + "loss": 1.7761, + "step": 258 + }, + { + "epoch": 0.3223397635345364, + "grad_norm": 3.138627052307129, + "learning_rate": 4.993081180811809e-06, + "loss": 1.7879, + "step": 259 + }, + { + "epoch": 0.3235843186060983, + "grad_norm": 2.8567333221435547, + "learning_rate": 4.990774907749078e-06, + "loss": 1.7561, + "step": 260 + }, + { + "epoch": 0.3235843186060983, + "eval_loss": 1.8606494665145874, + "eval_runtime": 46.453, + "eval_samples_per_second": 21.527, + "eval_steps_per_second": 0.904, + "step": 260 + }, + { + "epoch": 0.3248288736776602, + "grad_norm": 3.2696540355682373, + "learning_rate": 4.988468634686347e-06, + "loss": 1.7201, + "step": 261 + }, + { + "epoch": 0.3260734287492221, + "grad_norm": 2.6692731380462646, + "learning_rate": 4.986162361623617e-06, + "loss": 1.7264, + "step": 262 + }, + { + "epoch": 0.3273179838207841, + "grad_norm": 3.047549247741699, + "learning_rate": 4.983856088560886e-06, + "loss": 1.7362, + "step": 263 + }, + { + "epoch": 0.328562538892346, + "grad_norm": 3.0667457580566406, + "learning_rate": 4.981549815498156e-06, + "loss": 1.7321, + "step": 264 + }, + { + "epoch": 0.3298070939639079, + "grad_norm": 2.560047149658203, + "learning_rate": 4.979243542435424e-06, + "loss": 1.7508, + "step": 265 + }, + { + "epoch": 0.3310516490354698, + "grad_norm": 3.26595401763916, + "learning_rate": 4.976937269372694e-06, + "loss": 1.7248, + "step": 266 + }, + { + "epoch": 0.33229620410703176, + "grad_norm": 2.929210662841797, + "learning_rate": 4.974630996309964e-06, + "loss": 1.697, + "step": 267 + }, + { + "epoch": 0.33354075917859366, + "grad_norm": 3.1526286602020264, + "learning_rate": 4.972324723247233e-06, + "loss": 1.7385, + "step": 268 + }, + { + "epoch": 0.33478531425015556, + "grad_norm": 3.001619577407837, + "learning_rate": 4.970018450184502e-06, + "loss": 1.7072, + "step": 269 + }, + { + "epoch": 0.33602986932171747, + "grad_norm": 3.2464189529418945, + "learning_rate": 4.9677121771217715e-06, + "loss": 1.7361, + "step": 270 + }, + { + "epoch": 0.33602986932171747, + "eval_loss": 1.860226035118103, + "eval_runtime": 46.6881, + "eval_samples_per_second": 21.419, + "eval_steps_per_second": 0.9, + "step": 270 + }, + { + "epoch": 0.3372744243932794, + "grad_norm": 3.2104530334472656, + "learning_rate": 4.965405904059041e-06, + "loss": 1.7171, + "step": 271 + }, + { + "epoch": 0.33851897946484133, + "grad_norm": 3.146847724914551, + "learning_rate": 4.96309963099631e-06, + "loss": 1.7311, + "step": 272 + }, + { + "epoch": 0.33976353453640323, + "grad_norm": 3.1431286334991455, + "learning_rate": 4.96079335793358e-06, + "loss": 1.7345, + "step": 273 + }, + { + "epoch": 0.34100808960796514, + "grad_norm": 2.847163677215576, + "learning_rate": 4.958487084870849e-06, + "loss": 1.7262, + "step": 274 + }, + { + "epoch": 0.3422526446795271, + "grad_norm": 3.1987810134887695, + "learning_rate": 4.956180811808119e-06, + "loss": 1.7945, + "step": 275 + }, + { + "epoch": 0.343497199751089, + "grad_norm": 2.7475385665893555, + "learning_rate": 4.953874538745388e-06, + "loss": 1.76, + "step": 276 + }, + { + "epoch": 0.3447417548226509, + "grad_norm": 3.0427663326263428, + "learning_rate": 4.9515682656826574e-06, + "loss": 1.7324, + "step": 277 + }, + { + "epoch": 0.3459863098942128, + "grad_norm": 3.605212688446045, + "learning_rate": 4.949261992619927e-06, + "loss": 1.7712, + "step": 278 + }, + { + "epoch": 0.3472308649657747, + "grad_norm": 3.0564935207366943, + "learning_rate": 4.946955719557196e-06, + "loss": 1.6792, + "step": 279 + }, + { + "epoch": 0.34847542003733667, + "grad_norm": 3.1069741249084473, + "learning_rate": 4.944649446494466e-06, + "loss": 1.7145, + "step": 280 + }, + { + "epoch": 0.34847542003733667, + "eval_loss": 1.8537051677703857, + "eval_runtime": 47.4813, + "eval_samples_per_second": 21.061, + "eval_steps_per_second": 0.885, + "step": 280 + }, + { + "epoch": 0.3497199751088986, + "grad_norm": 2.8021512031555176, + "learning_rate": 4.942343173431734e-06, + "loss": 1.7145, + "step": 281 + }, + { + "epoch": 0.3509645301804605, + "grad_norm": 3.5549023151397705, + "learning_rate": 4.940036900369004e-06, + "loss": 1.7642, + "step": 282 + }, + { + "epoch": 0.3522090852520224, + "grad_norm": 2.7648985385894775, + "learning_rate": 4.937730627306274e-06, + "loss": 1.7255, + "step": 283 + }, + { + "epoch": 0.35345364032358434, + "grad_norm": 3.0815863609313965, + "learning_rate": 4.9354243542435426e-06, + "loss": 1.7055, + "step": 284 + }, + { + "epoch": 0.35469819539514624, + "grad_norm": 2.9009227752685547, + "learning_rate": 4.933118081180812e-06, + "loss": 1.7019, + "step": 285 + }, + { + "epoch": 0.35594275046670815, + "grad_norm": 2.618429660797119, + "learning_rate": 4.930811808118081e-06, + "loss": 1.6318, + "step": 286 + }, + { + "epoch": 0.35718730553827005, + "grad_norm": 3.368230104446411, + "learning_rate": 4.928505535055351e-06, + "loss": 1.7304, + "step": 287 + }, + { + "epoch": 0.358431860609832, + "grad_norm": 3.256889820098877, + "learning_rate": 4.92619926199262e-06, + "loss": 1.7019, + "step": 288 + }, + { + "epoch": 0.3596764156813939, + "grad_norm": 2.9366304874420166, + "learning_rate": 4.92389298892989e-06, + "loss": 1.734, + "step": 289 + }, + { + "epoch": 0.3609209707529558, + "grad_norm": 3.0393142700195312, + "learning_rate": 4.921586715867159e-06, + "loss": 1.7504, + "step": 290 + }, + { + "epoch": 0.3609209707529558, + "eval_loss": 1.8359886407852173, + "eval_runtime": 49.7622, + "eval_samples_per_second": 20.096, + "eval_steps_per_second": 0.844, + "step": 290 + }, + { + "epoch": 0.3621655258245177, + "grad_norm": 2.7288589477539062, + "learning_rate": 4.9192804428044285e-06, + "loss": 1.7004, + "step": 291 + }, + { + "epoch": 0.3634100808960797, + "grad_norm": 3.2793378829956055, + "learning_rate": 4.916974169741698e-06, + "loss": 1.7028, + "step": 292 + }, + { + "epoch": 0.3646546359676416, + "grad_norm": 2.960880756378174, + "learning_rate": 4.914667896678967e-06, + "loss": 1.6759, + "step": 293 + }, + { + "epoch": 0.3658991910392035, + "grad_norm": 2.836421012878418, + "learning_rate": 4.912361623616237e-06, + "loss": 1.6475, + "step": 294 + }, + { + "epoch": 0.3671437461107654, + "grad_norm": 2.9495465755462646, + "learning_rate": 4.910055350553506e-06, + "loss": 1.7076, + "step": 295 + }, + { + "epoch": 0.3683883011823273, + "grad_norm": 2.954730272293091, + "learning_rate": 4.907749077490776e-06, + "loss": 1.7261, + "step": 296 + }, + { + "epoch": 0.36963285625388925, + "grad_norm": 2.8167543411254883, + "learning_rate": 4.905442804428044e-06, + "loss": 1.6939, + "step": 297 + }, + { + "epoch": 0.37087741132545116, + "grad_norm": 2.857316017150879, + "learning_rate": 4.903136531365314e-06, + "loss": 1.7362, + "step": 298 + }, + { + "epoch": 0.37212196639701306, + "grad_norm": 2.8612918853759766, + "learning_rate": 4.900830258302584e-06, + "loss": 1.6489, + "step": 299 + }, + { + "epoch": 0.37336652146857496, + "grad_norm": 2.9618914127349854, + "learning_rate": 4.898523985239853e-06, + "loss": 1.718, + "step": 300 + }, + { + "epoch": 0.37336652146857496, + "eval_loss": 1.827571988105774, + "eval_runtime": 46.4552, + "eval_samples_per_second": 21.526, + "eval_steps_per_second": 0.904, + "step": 300 + }, + { + "epoch": 0.3746110765401369, + "grad_norm": 2.8706953525543213, + "learning_rate": 4.896217712177122e-06, + "loss": 1.6925, + "step": 301 + }, + { + "epoch": 0.3758556316116988, + "grad_norm": 3.2565090656280518, + "learning_rate": 4.893911439114391e-06, + "loss": 1.7369, + "step": 302 + }, + { + "epoch": 0.37710018668326073, + "grad_norm": 2.928858995437622, + "learning_rate": 4.891605166051661e-06, + "loss": 1.6882, + "step": 303 + }, + { + "epoch": 0.37834474175482263, + "grad_norm": 3.042314052581787, + "learning_rate": 4.88929889298893e-06, + "loss": 1.6466, + "step": 304 + }, + { + "epoch": 0.3795892968263846, + "grad_norm": 2.7509143352508545, + "learning_rate": 4.8869926199262e-06, + "loss": 1.685, + "step": 305 + }, + { + "epoch": 0.3808338518979465, + "grad_norm": 2.9601588249206543, + "learning_rate": 4.884686346863469e-06, + "loss": 1.6772, + "step": 306 + }, + { + "epoch": 0.3820784069695084, + "grad_norm": 2.618608236312866, + "learning_rate": 4.8823800738007384e-06, + "loss": 1.7127, + "step": 307 + }, + { + "epoch": 0.3833229620410703, + "grad_norm": 3.047581195831299, + "learning_rate": 4.880073800738008e-06, + "loss": 1.7174, + "step": 308 + }, + { + "epoch": 0.3845675171126322, + "grad_norm": 2.87735652923584, + "learning_rate": 4.877767527675277e-06, + "loss": 1.6677, + "step": 309 + }, + { + "epoch": 0.38581207218419417, + "grad_norm": 2.7169721126556396, + "learning_rate": 4.875461254612546e-06, + "loss": 1.7486, + "step": 310 + }, + { + "epoch": 0.38581207218419417, + "eval_loss": 1.8221794366836548, + "eval_runtime": 43.1681, + "eval_samples_per_second": 23.165, + "eval_steps_per_second": 0.973, + "step": 310 + }, + { + "epoch": 0.38705662725575607, + "grad_norm": 2.7094991207122803, + "learning_rate": 4.873154981549816e-06, + "loss": 1.7348, + "step": 311 + }, + { + "epoch": 0.388301182327318, + "grad_norm": 2.8989078998565674, + "learning_rate": 4.8708487084870856e-06, + "loss": 1.7352, + "step": 312 + }, + { + "epoch": 0.3895457373988799, + "grad_norm": 2.9565601348876953, + "learning_rate": 4.868542435424355e-06, + "loss": 1.7063, + "step": 313 + }, + { + "epoch": 0.39079029247044184, + "grad_norm": 2.8354082107543945, + "learning_rate": 4.8662361623616235e-06, + "loss": 1.6808, + "step": 314 + }, + { + "epoch": 0.39203484754200374, + "grad_norm": 2.8287479877471924, + "learning_rate": 4.863929889298894e-06, + "loss": 1.6731, + "step": 315 + }, + { + "epoch": 0.39327940261356564, + "grad_norm": 2.6537821292877197, + "learning_rate": 4.861623616236163e-06, + "loss": 1.695, + "step": 316 + }, + { + "epoch": 0.39452395768512755, + "grad_norm": 2.7135190963745117, + "learning_rate": 4.859317343173432e-06, + "loss": 1.6897, + "step": 317 + }, + { + "epoch": 0.3957685127566895, + "grad_norm": 3.1019129753112793, + "learning_rate": 4.857011070110701e-06, + "loss": 1.7219, + "step": 318 + }, + { + "epoch": 0.3970130678282514, + "grad_norm": 2.945737838745117, + "learning_rate": 4.854704797047971e-06, + "loss": 1.7316, + "step": 319 + }, + { + "epoch": 0.3982576228998133, + "grad_norm": 2.7668251991271973, + "learning_rate": 4.85239852398524e-06, + "loss": 1.6461, + "step": 320 + }, + { + "epoch": 0.3982576228998133, + "eval_loss": 1.8210570812225342, + "eval_runtime": 46.2552, + "eval_samples_per_second": 21.619, + "eval_steps_per_second": 0.908, + "step": 320 + }, + { + "epoch": 0.3995021779713752, + "grad_norm": 2.738109588623047, + "learning_rate": 4.8500922509225095e-06, + "loss": 1.6907, + "step": 321 + }, + { + "epoch": 0.4007467330429372, + "grad_norm": 3.0195062160491943, + "learning_rate": 4.847785977859779e-06, + "loss": 1.7099, + "step": 322 + }, + { + "epoch": 0.4019912881144991, + "grad_norm": 2.8672404289245605, + "learning_rate": 4.845479704797048e-06, + "loss": 1.6509, + "step": 323 + }, + { + "epoch": 0.403235843186061, + "grad_norm": 2.9041008949279785, + "learning_rate": 4.843173431734318e-06, + "loss": 1.5818, + "step": 324 + }, + { + "epoch": 0.4044803982576229, + "grad_norm": 2.9105985164642334, + "learning_rate": 4.840867158671587e-06, + "loss": 1.7002, + "step": 325 + }, + { + "epoch": 0.4057249533291848, + "grad_norm": 2.749009847640991, + "learning_rate": 4.838560885608857e-06, + "loss": 1.7021, + "step": 326 + }, + { + "epoch": 0.40696950840074675, + "grad_norm": 2.6998822689056396, + "learning_rate": 4.836254612546126e-06, + "loss": 1.6853, + "step": 327 + }, + { + "epoch": 0.40821406347230865, + "grad_norm": 2.678201675415039, + "learning_rate": 4.8339483394833955e-06, + "loss": 1.6934, + "step": 328 + }, + { + "epoch": 0.40945861854387056, + "grad_norm": 2.774845600128174, + "learning_rate": 4.831642066420665e-06, + "loss": 1.657, + "step": 329 + }, + { + "epoch": 0.41070317361543246, + "grad_norm": 2.6363000869750977, + "learning_rate": 4.8293357933579335e-06, + "loss": 1.6555, + "step": 330 + }, + { + "epoch": 0.41070317361543246, + "eval_loss": 1.8126581907272339, + "eval_runtime": 52.7717, + "eval_samples_per_second": 18.95, + "eval_steps_per_second": 0.796, + "step": 330 + }, + { + "epoch": 0.4119477286869944, + "grad_norm": 3.011659622192383, + "learning_rate": 4.827029520295204e-06, + "loss": 1.7202, + "step": 331 + }, + { + "epoch": 0.4131922837585563, + "grad_norm": 2.8322594165802, + "learning_rate": 4.824723247232473e-06, + "loss": 1.6736, + "step": 332 + }, + { + "epoch": 0.4144368388301182, + "grad_norm": 2.9360146522521973, + "learning_rate": 4.822416974169742e-06, + "loss": 1.7213, + "step": 333 + }, + { + "epoch": 0.41568139390168013, + "grad_norm": 2.810206413269043, + "learning_rate": 4.820110701107011e-06, + "loss": 1.6867, + "step": 334 + }, + { + "epoch": 0.4169259489732421, + "grad_norm": 2.6864097118377686, + "learning_rate": 4.817804428044281e-06, + "loss": 1.6854, + "step": 335 + }, + { + "epoch": 0.418170504044804, + "grad_norm": 2.7396061420440674, + "learning_rate": 4.81549815498155e-06, + "loss": 1.68, + "step": 336 + }, + { + "epoch": 0.4194150591163659, + "grad_norm": 2.602771520614624, + "learning_rate": 4.8131918819188194e-06, + "loss": 1.6269, + "step": 337 + }, + { + "epoch": 0.4206596141879278, + "grad_norm": 2.8174757957458496, + "learning_rate": 4.810885608856089e-06, + "loss": 1.6981, + "step": 338 + }, + { + "epoch": 0.42190416925948976, + "grad_norm": 3.2398130893707275, + "learning_rate": 4.808579335793358e-06, + "loss": 1.7061, + "step": 339 + }, + { + "epoch": 0.42314872433105166, + "grad_norm": 2.7119736671447754, + "learning_rate": 4.806273062730628e-06, + "loss": 1.6875, + "step": 340 + }, + { + "epoch": 0.42314872433105166, + "eval_loss": 1.7973068952560425, + "eval_runtime": 53.5791, + "eval_samples_per_second": 18.664, + "eval_steps_per_second": 0.784, + "step": 340 + }, + { + "epoch": 0.42439327940261357, + "grad_norm": 2.868533134460449, + "learning_rate": 4.803966789667897e-06, + "loss": 1.6967, + "step": 341 + }, + { + "epoch": 0.42563783447417547, + "grad_norm": 2.528083562850952, + "learning_rate": 4.8016605166051665e-06, + "loss": 1.6195, + "step": 342 + }, + { + "epoch": 0.4268823895457374, + "grad_norm": 2.7885115146636963, + "learning_rate": 4.799354243542436e-06, + "loss": 1.6197, + "step": 343 + }, + { + "epoch": 0.42812694461729933, + "grad_norm": 2.768247604370117, + "learning_rate": 4.797047970479705e-06, + "loss": 1.6271, + "step": 344 + }, + { + "epoch": 0.42937149968886124, + "grad_norm": 2.7710275650024414, + "learning_rate": 4.794741697416975e-06, + "loss": 1.6768, + "step": 345 + }, + { + "epoch": 0.43061605476042314, + "grad_norm": 2.968337297439575, + "learning_rate": 4.792435424354243e-06, + "loss": 1.6433, + "step": 346 + }, + { + "epoch": 0.43186060983198504, + "grad_norm": 2.5739808082580566, + "learning_rate": 4.790129151291514e-06, + "loss": 1.6327, + "step": 347 + }, + { + "epoch": 0.433105164903547, + "grad_norm": 2.7676408290863037, + "learning_rate": 4.787822878228783e-06, + "loss": 1.6178, + "step": 348 + }, + { + "epoch": 0.4343497199751089, + "grad_norm": 2.959059476852417, + "learning_rate": 4.7855166051660525e-06, + "loss": 1.6809, + "step": 349 + }, + { + "epoch": 0.4355942750466708, + "grad_norm": 2.559967279434204, + "learning_rate": 4.783210332103321e-06, + "loss": 1.6704, + "step": 350 + }, + { + "epoch": 0.4355942750466708, + "eval_loss": 1.8013949394226074, + "eval_runtime": 50.4649, + "eval_samples_per_second": 19.816, + "eval_steps_per_second": 0.832, + "step": 350 + }, + { + "epoch": 0.4368388301182327, + "grad_norm": 2.817901611328125, + "learning_rate": 4.7809040590405905e-06, + "loss": 1.6296, + "step": 351 + }, + { + "epoch": 0.43808338518979467, + "grad_norm": 2.7116055488586426, + "learning_rate": 4.778597785977861e-06, + "loss": 1.6652, + "step": 352 + }, + { + "epoch": 0.4393279402613566, + "grad_norm": 2.544750452041626, + "learning_rate": 4.776291512915129e-06, + "loss": 1.5668, + "step": 353 + }, + { + "epoch": 0.4405724953329185, + "grad_norm": 2.720534086227417, + "learning_rate": 4.773985239852399e-06, + "loss": 1.6152, + "step": 354 + }, + { + "epoch": 0.4418170504044804, + "grad_norm": 2.7270126342773438, + "learning_rate": 4.771678966789668e-06, + "loss": 1.5624, + "step": 355 + }, + { + "epoch": 0.44306160547604234, + "grad_norm": 2.7986624240875244, + "learning_rate": 4.769372693726938e-06, + "loss": 1.6621, + "step": 356 + }, + { + "epoch": 0.44430616054760425, + "grad_norm": 2.943107843399048, + "learning_rate": 4.767066420664207e-06, + "loss": 1.6713, + "step": 357 + }, + { + "epoch": 0.44555071561916615, + "grad_norm": 2.739898681640625, + "learning_rate": 4.7647601476014765e-06, + "loss": 1.6363, + "step": 358 + }, + { + "epoch": 0.44679527069072805, + "grad_norm": 2.8729329109191895, + "learning_rate": 4.762453874538746e-06, + "loss": 1.6605, + "step": 359 + }, + { + "epoch": 0.44803982576228996, + "grad_norm": 2.909029483795166, + "learning_rate": 4.760147601476015e-06, + "loss": 1.6277, + "step": 360 + }, + { + "epoch": 0.44803982576228996, + "eval_loss": 1.8110179901123047, + "eval_runtime": 47.924, + "eval_samples_per_second": 20.866, + "eval_steps_per_second": 0.876, + "step": 360 + }, + { + "epoch": 0.4492843808338519, + "grad_norm": 3.3826682567596436, + "learning_rate": 4.757841328413285e-06, + "loss": 1.7155, + "step": 361 + }, + { + "epoch": 0.4505289359054138, + "grad_norm": 3.0586729049682617, + "learning_rate": 4.755535055350554e-06, + "loss": 1.5991, + "step": 362 + }, + { + "epoch": 0.4517734909769757, + "grad_norm": 2.645576238632202, + "learning_rate": 4.753228782287823e-06, + "loss": 1.6504, + "step": 363 + }, + { + "epoch": 0.4530180460485376, + "grad_norm": 2.705967664718628, + "learning_rate": 4.750922509225093e-06, + "loss": 1.6761, + "step": 364 + }, + { + "epoch": 0.4542626011200996, + "grad_norm": 2.7651922702789307, + "learning_rate": 4.748616236162362e-06, + "loss": 1.61, + "step": 365 + }, + { + "epoch": 0.4555071561916615, + "grad_norm": 2.5654966831207275, + "learning_rate": 4.746309963099631e-06, + "loss": 1.6335, + "step": 366 + }, + { + "epoch": 0.4567517112632234, + "grad_norm": 2.6606173515319824, + "learning_rate": 4.7440036900369e-06, + "loss": 1.6207, + "step": 367 + }, + { + "epoch": 0.4579962663347853, + "grad_norm": 2.724755048751831, + "learning_rate": 4.741697416974171e-06, + "loss": 1.6329, + "step": 368 + }, + { + "epoch": 0.45924082140634725, + "grad_norm": 2.7019615173339844, + "learning_rate": 4.739391143911439e-06, + "loss": 1.6763, + "step": 369 + }, + { + "epoch": 0.46048537647790916, + "grad_norm": 2.6594254970550537, + "learning_rate": 4.737084870848709e-06, + "loss": 1.6174, + "step": 370 + }, + { + "epoch": 0.46048537647790916, + "eval_loss": 1.7924141883850098, + "eval_runtime": 53.0349, + "eval_samples_per_second": 18.855, + "eval_steps_per_second": 0.792, + "step": 370 + }, + { + "epoch": 0.46172993154947106, + "grad_norm": 2.668421745300293, + "learning_rate": 4.734778597785978e-06, + "loss": 1.6421, + "step": 371 + }, + { + "epoch": 0.46297448662103297, + "grad_norm": 2.7148995399475098, + "learning_rate": 4.7324723247232475e-06, + "loss": 1.6635, + "step": 372 + }, + { + "epoch": 0.4642190416925949, + "grad_norm": 3.156646490097046, + "learning_rate": 4.730166051660517e-06, + "loss": 1.6679, + "step": 373 + }, + { + "epoch": 0.46546359676415683, + "grad_norm": 2.8170645236968994, + "learning_rate": 4.727859778597786e-06, + "loss": 1.6492, + "step": 374 + }, + { + "epoch": 0.46670815183571873, + "grad_norm": 2.9796457290649414, + "learning_rate": 4.725553505535056e-06, + "loss": 1.6025, + "step": 375 + }, + { + "epoch": 0.46795270690728064, + "grad_norm": 2.6493406295776367, + "learning_rate": 4.723247232472325e-06, + "loss": 1.6398, + "step": 376 + }, + { + "epoch": 0.46919726197884254, + "grad_norm": 3.2643542289733887, + "learning_rate": 4.720940959409595e-06, + "loss": 1.6413, + "step": 377 + }, + { + "epoch": 0.4704418170504045, + "grad_norm": 2.7613179683685303, + "learning_rate": 4.718634686346864e-06, + "loss": 1.6251, + "step": 378 + }, + { + "epoch": 0.4716863721219664, + "grad_norm": 3.1216518878936768, + "learning_rate": 4.716328413284133e-06, + "loss": 1.6268, + "step": 379 + }, + { + "epoch": 0.4729309271935283, + "grad_norm": 2.571647882461548, + "learning_rate": 4.714022140221403e-06, + "loss": 1.6245, + "step": 380 + }, + { + "epoch": 0.4729309271935283, + "eval_loss": 1.7875893115997314, + "eval_runtime": 49.3058, + "eval_samples_per_second": 20.282, + "eval_steps_per_second": 0.852, + "step": 380 + }, + { + "epoch": 0.4741754822650902, + "grad_norm": 3.443906545639038, + "learning_rate": 4.711715867158672e-06, + "loss": 1.6799, + "step": 381 + }, + { + "epoch": 0.47542003733665217, + "grad_norm": 2.7887957096099854, + "learning_rate": 4.709409594095941e-06, + "loss": 1.6589, + "step": 382 + }, + { + "epoch": 0.47666459240821407, + "grad_norm": 3.2833151817321777, + "learning_rate": 4.70710332103321e-06, + "loss": 1.6064, + "step": 383 + }, + { + "epoch": 0.477909147479776, + "grad_norm": 2.6975057125091553, + "learning_rate": 4.704797047970481e-06, + "loss": 1.6029, + "step": 384 + }, + { + "epoch": 0.4791537025513379, + "grad_norm": 2.745246171951294, + "learning_rate": 4.702490774907749e-06, + "loss": 1.7127, + "step": 385 + }, + { + "epoch": 0.48039825762289984, + "grad_norm": 3.0522775650024414, + "learning_rate": 4.700184501845019e-06, + "loss": 1.6508, + "step": 386 + }, + { + "epoch": 0.48164281269446174, + "grad_norm": 2.689218044281006, + "learning_rate": 4.697878228782288e-06, + "loss": 1.6971, + "step": 387 + }, + { + "epoch": 0.48288736776602365, + "grad_norm": 3.060361862182617, + "learning_rate": 4.6955719557195575e-06, + "loss": 1.6464, + "step": 388 + }, + { + "epoch": 0.48413192283758555, + "grad_norm": 2.872262477874756, + "learning_rate": 4.693265682656827e-06, + "loss": 1.6293, + "step": 389 + }, + { + "epoch": 0.48537647790914745, + "grad_norm": 2.980050563812256, + "learning_rate": 4.690959409594096e-06, + "loss": 1.6435, + "step": 390 + }, + { + "epoch": 0.48537647790914745, + "eval_loss": 1.7791496515274048, + "eval_runtime": 50.0299, + "eval_samples_per_second": 19.988, + "eval_steps_per_second": 0.839, + "step": 390 + }, + { + "epoch": 0.4866210329807094, + "grad_norm": 2.764827251434326, + "learning_rate": 4.688653136531366e-06, + "loss": 1.6279, + "step": 391 + }, + { + "epoch": 0.4878655880522713, + "grad_norm": 2.6912333965301514, + "learning_rate": 4.686346863468635e-06, + "loss": 1.7276, + "step": 392 + }, + { + "epoch": 0.4891101431238332, + "grad_norm": 2.8358330726623535, + "learning_rate": 4.6840405904059046e-06, + "loss": 1.6482, + "step": 393 + }, + { + "epoch": 0.4903546981953951, + "grad_norm": 2.6581575870513916, + "learning_rate": 4.681734317343174e-06, + "loss": 1.6853, + "step": 394 + }, + { + "epoch": 0.4915992532669571, + "grad_norm": 2.941425323486328, + "learning_rate": 4.6794280442804426e-06, + "loss": 1.6075, + "step": 395 + }, + { + "epoch": 0.492843808338519, + "grad_norm": 3.2334814071655273, + "learning_rate": 4.677121771217713e-06, + "loss": 1.6157, + "step": 396 + }, + { + "epoch": 0.4940883634100809, + "grad_norm": 2.984410524368286, + "learning_rate": 4.674815498154982e-06, + "loss": 1.6257, + "step": 397 + }, + { + "epoch": 0.4953329184816428, + "grad_norm": 3.0650789737701416, + "learning_rate": 4.672509225092252e-06, + "loss": 1.6573, + "step": 398 + }, + { + "epoch": 0.49657747355320475, + "grad_norm": 2.798664093017578, + "learning_rate": 4.67020295202952e-06, + "loss": 1.638, + "step": 399 + }, + { + "epoch": 0.49782202862476665, + "grad_norm": 2.8266706466674805, + "learning_rate": 4.66789667896679e-06, + "loss": 1.6484, + "step": 400 + }, + { + "epoch": 0.49782202862476665, + "eval_loss": 1.7714862823486328, + "eval_runtime": 48.0657, + "eval_samples_per_second": 20.805, + "eval_steps_per_second": 0.874, + "step": 400 + }, + { + "epoch": 0.49906658369632856, + "grad_norm": 2.9928388595581055, + "learning_rate": 4.66559040590406e-06, + "loss": 1.6721, + "step": 401 + }, + { + "epoch": 0.5003111387678905, + "grad_norm": 2.573514461517334, + "learning_rate": 4.6632841328413285e-06, + "loss": 1.679, + "step": 402 + }, + { + "epoch": 0.5015556938394524, + "grad_norm": 3.1736996173858643, + "learning_rate": 4.660977859778598e-06, + "loss": 1.7124, + "step": 403 + }, + { + "epoch": 0.5028002489110143, + "grad_norm": 2.858849287033081, + "learning_rate": 4.658671586715867e-06, + "loss": 1.6622, + "step": 404 + }, + { + "epoch": 0.5040448039825762, + "grad_norm": 2.825698137283325, + "learning_rate": 4.656365313653137e-06, + "loss": 1.6464, + "step": 405 + }, + { + "epoch": 0.5052893590541382, + "grad_norm": 3.783891439437866, + "learning_rate": 4.654059040590406e-06, + "loss": 1.6665, + "step": 406 + }, + { + "epoch": 0.5065339141257, + "grad_norm": 2.789813756942749, + "learning_rate": 4.651752767527676e-06, + "loss": 1.6064, + "step": 407 + }, + { + "epoch": 0.507778469197262, + "grad_norm": 3.3106348514556885, + "learning_rate": 4.649446494464945e-06, + "loss": 1.6542, + "step": 408 + }, + { + "epoch": 0.5090230242688238, + "grad_norm": 2.8805112838745117, + "learning_rate": 4.6471402214022145e-06, + "loss": 1.6058, + "step": 409 + }, + { + "epoch": 0.5102675793403858, + "grad_norm": 3.30898118019104, + "learning_rate": 4.644833948339484e-06, + "loss": 1.5933, + "step": 410 + }, + { + "epoch": 0.5102675793403858, + "eval_loss": 1.7787818908691406, + "eval_runtime": 44.0876, + "eval_samples_per_second": 22.682, + "eval_steps_per_second": 0.953, + "step": 410 + }, + { + "epoch": 0.5115121344119478, + "grad_norm": 3.3563625812530518, + "learning_rate": 4.642527675276753e-06, + "loss": 1.6751, + "step": 411 + }, + { + "epoch": 0.5127566894835096, + "grad_norm": 2.968120574951172, + "learning_rate": 4.640221402214023e-06, + "loss": 1.663, + "step": 412 + }, + { + "epoch": 0.5140012445550716, + "grad_norm": 3.310248374938965, + "learning_rate": 4.637915129151292e-06, + "loss": 1.658, + "step": 413 + }, + { + "epoch": 0.5152457996266335, + "grad_norm": 2.7011170387268066, + "learning_rate": 4.635608856088562e-06, + "loss": 1.6894, + "step": 414 + }, + { + "epoch": 0.5164903546981954, + "grad_norm": 2.804901361465454, + "learning_rate": 4.63330258302583e-06, + "loss": 1.602, + "step": 415 + }, + { + "epoch": 0.5177349097697573, + "grad_norm": 3.2412750720977783, + "learning_rate": 4.6309963099631e-06, + "loss": 1.6158, + "step": 416 + }, + { + "epoch": 0.5189794648413192, + "grad_norm": 2.699909210205078, + "learning_rate": 4.62869003690037e-06, + "loss": 1.5842, + "step": 417 + }, + { + "epoch": 0.5202240199128811, + "grad_norm": 3.065929412841797, + "learning_rate": 4.6263837638376384e-06, + "loss": 1.6434, + "step": 418 + }, + { + "epoch": 0.5214685749844431, + "grad_norm": 2.895305633544922, + "learning_rate": 4.624077490774908e-06, + "loss": 1.5969, + "step": 419 + }, + { + "epoch": 0.522713130056005, + "grad_norm": 3.1590664386749268, + "learning_rate": 4.621771217712177e-06, + "loss": 1.6211, + "step": 420 + }, + { + "epoch": 0.522713130056005, + "eval_loss": 1.7674189805984497, + "eval_runtime": 53.328, + "eval_samples_per_second": 18.752, + "eval_steps_per_second": 0.788, + "step": 420 + }, + { + "epoch": 0.5239576851275669, + "grad_norm": 2.7126312255859375, + "learning_rate": 4.619464944649447e-06, + "loss": 1.6246, + "step": 421 + }, + { + "epoch": 0.5252022401991289, + "grad_norm": 2.918508768081665, + "learning_rate": 4.617158671586716e-06, + "loss": 1.6937, + "step": 422 + }, + { + "epoch": 0.5264467952706907, + "grad_norm": 3.008610725402832, + "learning_rate": 4.6148523985239856e-06, + "loss": 1.6864, + "step": 423 + }, + { + "epoch": 0.5276913503422527, + "grad_norm": 2.6612043380737305, + "learning_rate": 4.612546125461255e-06, + "loss": 1.5785, + "step": 424 + }, + { + "epoch": 0.5289359054138145, + "grad_norm": 2.850679874420166, + "learning_rate": 4.610239852398524e-06, + "loss": 1.5899, + "step": 425 + }, + { + "epoch": 0.5301804604853765, + "grad_norm": 2.9205031394958496, + "learning_rate": 4.607933579335794e-06, + "loss": 1.5867, + "step": 426 + }, + { + "epoch": 0.5314250155569384, + "grad_norm": 2.929532527923584, + "learning_rate": 4.605627306273063e-06, + "loss": 1.6182, + "step": 427 + }, + { + "epoch": 0.5326695706285003, + "grad_norm": 2.874936103820801, + "learning_rate": 4.603321033210333e-06, + "loss": 1.5789, + "step": 428 + }, + { + "epoch": 0.5339141257000622, + "grad_norm": 2.8703525066375732, + "learning_rate": 4.601014760147602e-06, + "loss": 1.5859, + "step": 429 + }, + { + "epoch": 0.5351586807716241, + "grad_norm": 2.7076902389526367, + "learning_rate": 4.5987084870848715e-06, + "loss": 1.644, + "step": 430 + }, + { + "epoch": 0.5351586807716241, + "eval_loss": 1.7592095136642456, + "eval_runtime": 47.9438, + "eval_samples_per_second": 20.858, + "eval_steps_per_second": 0.876, + "step": 430 + }, + { + "epoch": 0.536403235843186, + "grad_norm": 2.839160680770874, + "learning_rate": 4.59640221402214e-06, + "loss": 1.6357, + "step": 431 + }, + { + "epoch": 0.537647790914748, + "grad_norm": 2.674025535583496, + "learning_rate": 4.5940959409594095e-06, + "loss": 1.676, + "step": 432 + }, + { + "epoch": 0.5388923459863099, + "grad_norm": 3.0619537830352783, + "learning_rate": 4.59178966789668e-06, + "loss": 1.5617, + "step": 433 + }, + { + "epoch": 0.5401369010578718, + "grad_norm": 3.058418035507202, + "learning_rate": 4.589483394833948e-06, + "loss": 1.5951, + "step": 434 + }, + { + "epoch": 0.5413814561294338, + "grad_norm": 3.0046396255493164, + "learning_rate": 4.587177121771218e-06, + "loss": 1.6473, + "step": 435 + }, + { + "epoch": 0.5426260112009956, + "grad_norm": 2.7736752033233643, + "learning_rate": 4.584870848708487e-06, + "loss": 1.6262, + "step": 436 + }, + { + "epoch": 0.5438705662725576, + "grad_norm": 2.7804994583129883, + "learning_rate": 4.5825645756457575e-06, + "loss": 1.6886, + "step": 437 + }, + { + "epoch": 0.5451151213441194, + "grad_norm": 3.0717954635620117, + "learning_rate": 4.580258302583026e-06, + "loss": 1.6287, + "step": 438 + }, + { + "epoch": 0.5463596764156814, + "grad_norm": 2.800243854522705, + "learning_rate": 4.5779520295202955e-06, + "loss": 1.6169, + "step": 439 + }, + { + "epoch": 0.5476042314872434, + "grad_norm": 2.9581644535064697, + "learning_rate": 4.575645756457565e-06, + "loss": 1.6617, + "step": 440 + }, + { + "epoch": 0.5476042314872434, + "eval_loss": 1.7642868757247925, + "eval_runtime": 51.2646, + "eval_samples_per_second": 19.507, + "eval_steps_per_second": 0.819, + "step": 440 + }, + { + "epoch": 0.5488487865588052, + "grad_norm": 3.0067460536956787, + "learning_rate": 4.573339483394834e-06, + "loss": 1.6482, + "step": 441 + }, + { + "epoch": 0.5500933416303672, + "grad_norm": 2.9075019359588623, + "learning_rate": 4.571033210332104e-06, + "loss": 1.6595, + "step": 442 + }, + { + "epoch": 0.551337896701929, + "grad_norm": 2.7788755893707275, + "learning_rate": 4.568726937269373e-06, + "loss": 1.5733, + "step": 443 + }, + { + "epoch": 0.552582451773491, + "grad_norm": 2.7775425910949707, + "learning_rate": 4.566420664206643e-06, + "loss": 1.6925, + "step": 444 + }, + { + "epoch": 0.5538270068450529, + "grad_norm": 2.6949567794799805, + "learning_rate": 4.564114391143912e-06, + "loss": 1.5738, + "step": 445 + }, + { + "epoch": 0.5550715619166148, + "grad_norm": 2.7093007564544678, + "learning_rate": 4.5618081180811814e-06, + "loss": 1.5848, + "step": 446 + }, + { + "epoch": 0.5563161169881767, + "grad_norm": 2.6920173168182373, + "learning_rate": 4.559501845018451e-06, + "loss": 1.6092, + "step": 447 + }, + { + "epoch": 0.5575606720597387, + "grad_norm": 2.733731985092163, + "learning_rate": 4.5571955719557194e-06, + "loss": 1.6056, + "step": 448 + }, + { + "epoch": 0.5588052271313005, + "grad_norm": 2.9156274795532227, + "learning_rate": 4.55488929889299e-06, + "loss": 1.6436, + "step": 449 + }, + { + "epoch": 0.5600497822028625, + "grad_norm": 2.8802616596221924, + "learning_rate": 4.552583025830259e-06, + "loss": 1.6309, + "step": 450 + }, + { + "epoch": 0.5600497822028625, + "eval_loss": 1.7486340999603271, + "eval_runtime": 47.2219, + "eval_samples_per_second": 21.177, + "eval_steps_per_second": 0.889, + "step": 450 + }, + { + "epoch": 0.5612943372744243, + "grad_norm": 2.729743242263794, + "learning_rate": 4.550276752767528e-06, + "loss": 1.7171, + "step": 451 + }, + { + "epoch": 0.5625388923459863, + "grad_norm": 2.800049066543579, + "learning_rate": 4.547970479704797e-06, + "loss": 1.6098, + "step": 452 + }, + { + "epoch": 0.5637834474175483, + "grad_norm": 2.9659311771392822, + "learning_rate": 4.5456642066420666e-06, + "loss": 1.6646, + "step": 453 + }, + { + "epoch": 0.5650280024891101, + "grad_norm": 2.5417754650115967, + "learning_rate": 4.543357933579336e-06, + "loss": 1.5847, + "step": 454 + }, + { + "epoch": 0.5662725575606721, + "grad_norm": 2.8471338748931885, + "learning_rate": 4.541051660516605e-06, + "loss": 1.6235, + "step": 455 + }, + { + "epoch": 0.5675171126322339, + "grad_norm": 2.7039637565612793, + "learning_rate": 4.538745387453875e-06, + "loss": 1.5613, + "step": 456 + }, + { + "epoch": 0.5687616677037959, + "grad_norm": 3.0383341312408447, + "learning_rate": 4.536439114391144e-06, + "loss": 1.6485, + "step": 457 + }, + { + "epoch": 0.5700062227753578, + "grad_norm": 2.528388261795044, + "learning_rate": 4.534132841328414e-06, + "loss": 1.6117, + "step": 458 + }, + { + "epoch": 0.5712507778469197, + "grad_norm": 2.7623119354248047, + "learning_rate": 4.531826568265683e-06, + "loss": 1.6017, + "step": 459 + }, + { + "epoch": 0.5724953329184816, + "grad_norm": 2.9213945865631104, + "learning_rate": 4.5295202952029525e-06, + "loss": 1.643, + "step": 460 + }, + { + "epoch": 0.5724953329184816, + "eval_loss": 1.747575044631958, + "eval_runtime": 45.2844, + "eval_samples_per_second": 22.083, + "eval_steps_per_second": 0.927, + "step": 460 + }, + { + "epoch": 0.5737398879900436, + "grad_norm": 2.664275884628296, + "learning_rate": 4.527214022140222e-06, + "loss": 1.6154, + "step": 461 + }, + { + "epoch": 0.5749844430616055, + "grad_norm": 2.5891788005828857, + "learning_rate": 4.524907749077491e-06, + "loss": 1.5975, + "step": 462 + }, + { + "epoch": 0.5762289981331674, + "grad_norm": 2.8126487731933594, + "learning_rate": 4.522601476014761e-06, + "loss": 1.6306, + "step": 463 + }, + { + "epoch": 0.5774735532047293, + "grad_norm": 2.799391031265259, + "learning_rate": 4.520295202952029e-06, + "loss": 1.6139, + "step": 464 + }, + { + "epoch": 0.5787181082762912, + "grad_norm": 2.7323718070983887, + "learning_rate": 4.5179889298893e-06, + "loss": 1.632, + "step": 465 + }, + { + "epoch": 0.5799626633478532, + "grad_norm": 2.601386785507202, + "learning_rate": 4.515682656826569e-06, + "loss": 1.6139, + "step": 466 + }, + { + "epoch": 0.581207218419415, + "grad_norm": 2.562162399291992, + "learning_rate": 4.513376383763838e-06, + "loss": 1.5168, + "step": 467 + }, + { + "epoch": 0.582451773490977, + "grad_norm": 2.6912002563476562, + "learning_rate": 4.511070110701107e-06, + "loss": 1.6249, + "step": 468 + }, + { + "epoch": 0.583696328562539, + "grad_norm": 2.6971495151519775, + "learning_rate": 4.5087638376383765e-06, + "loss": 1.616, + "step": 469 + }, + { + "epoch": 0.5849408836341008, + "grad_norm": 2.6898910999298096, + "learning_rate": 4.506457564575646e-06, + "loss": 1.614, + "step": 470 + }, + { + "epoch": 0.5849408836341008, + "eval_loss": 1.7460769414901733, + "eval_runtime": 45.3055, + "eval_samples_per_second": 22.072, + "eval_steps_per_second": 0.927, + "step": 470 + }, + { + "epoch": 0.5861854387056628, + "grad_norm": 2.698180675506592, + "learning_rate": 4.504151291512915e-06, + "loss": 1.5993, + "step": 471 + }, + { + "epoch": 0.5874299937772246, + "grad_norm": 2.7125210762023926, + "learning_rate": 4.501845018450185e-06, + "loss": 1.6113, + "step": 472 + }, + { + "epoch": 0.5886745488487866, + "grad_norm": 2.754445791244507, + "learning_rate": 4.499538745387454e-06, + "loss": 1.6719, + "step": 473 + }, + { + "epoch": 0.5899191039203485, + "grad_norm": 2.6979362964630127, + "learning_rate": 4.497232472324724e-06, + "loss": 1.608, + "step": 474 + }, + { + "epoch": 0.5911636589919104, + "grad_norm": 2.7694528102874756, + "learning_rate": 4.494926199261993e-06, + "loss": 1.5695, + "step": 475 + }, + { + "epoch": 0.5924082140634723, + "grad_norm": 2.653353214263916, + "learning_rate": 4.4926199261992624e-06, + "loss": 1.5523, + "step": 476 + }, + { + "epoch": 0.5936527691350342, + "grad_norm": 2.6632070541381836, + "learning_rate": 4.490313653136532e-06, + "loss": 1.5675, + "step": 477 + }, + { + "epoch": 0.5948973242065961, + "grad_norm": 3.038543939590454, + "learning_rate": 4.488007380073801e-06, + "loss": 1.5692, + "step": 478 + }, + { + "epoch": 0.5961418792781581, + "grad_norm": 2.8123953342437744, + "learning_rate": 4.485701107011071e-06, + "loss": 1.6223, + "step": 479 + }, + { + "epoch": 0.5973864343497199, + "grad_norm": 2.5845773220062256, + "learning_rate": 4.483394833948339e-06, + "loss": 1.5773, + "step": 480 + }, + { + "epoch": 0.5973864343497199, + "eval_loss": 1.7419319152832031, + "eval_runtime": 44.6842, + "eval_samples_per_second": 22.379, + "eval_steps_per_second": 0.94, + "step": 480 + }, + { + "epoch": 0.5986309894212819, + "grad_norm": 3.069936752319336, + "learning_rate": 4.4810885608856096e-06, + "loss": 1.6323, + "step": 481 + }, + { + "epoch": 0.5998755444928439, + "grad_norm": 2.9085781574249268, + "learning_rate": 4.478782287822879e-06, + "loss": 1.5845, + "step": 482 + }, + { + "epoch": 0.6011200995644057, + "grad_norm": 2.8568010330200195, + "learning_rate": 4.4764760147601476e-06, + "loss": 1.5898, + "step": 483 + }, + { + "epoch": 0.6023646546359677, + "grad_norm": 3.089081048965454, + "learning_rate": 4.474169741697417e-06, + "loss": 1.6622, + "step": 484 + }, + { + "epoch": 0.6036092097075295, + "grad_norm": 2.8515470027923584, + "learning_rate": 4.471863468634686e-06, + "loss": 1.5859, + "step": 485 + }, + { + "epoch": 0.6048537647790915, + "grad_norm": 2.647765874862671, + "learning_rate": 4.469557195571957e-06, + "loss": 1.5926, + "step": 486 + }, + { + "epoch": 0.6060983198506534, + "grad_norm": 2.61676025390625, + "learning_rate": 4.467250922509225e-06, + "loss": 1.5684, + "step": 487 + }, + { + "epoch": 0.6073428749222153, + "grad_norm": 2.7127039432525635, + "learning_rate": 4.464944649446495e-06, + "loss": 1.5702, + "step": 488 + }, + { + "epoch": 0.6085874299937772, + "grad_norm": 2.9932055473327637, + "learning_rate": 4.462638376383764e-06, + "loss": 1.5976, + "step": 489 + }, + { + "epoch": 0.6098319850653391, + "grad_norm": 2.6508774757385254, + "learning_rate": 4.4603321033210335e-06, + "loss": 1.586, + "step": 490 + }, + { + "epoch": 0.6098319850653391, + "eval_loss": 1.7357326745986938, + "eval_runtime": 45.393, + "eval_samples_per_second": 22.03, + "eval_steps_per_second": 0.925, + "step": 490 + }, + { + "epoch": 0.611076540136901, + "grad_norm": 2.7626044750213623, + "learning_rate": 4.458025830258303e-06, + "loss": 1.5773, + "step": 491 + }, + { + "epoch": 0.612321095208463, + "grad_norm": 2.595003604888916, + "learning_rate": 4.455719557195572e-06, + "loss": 1.601, + "step": 492 + }, + { + "epoch": 0.6135656502800249, + "grad_norm": 2.595767021179199, + "learning_rate": 4.453413284132842e-06, + "loss": 1.6287, + "step": 493 + }, + { + "epoch": 0.6148102053515868, + "grad_norm": 2.755845308303833, + "learning_rate": 4.451107011070111e-06, + "loss": 1.593, + "step": 494 + }, + { + "epoch": 0.6160547604231488, + "grad_norm": 2.7302653789520264, + "learning_rate": 4.448800738007381e-06, + "loss": 1.5768, + "step": 495 + }, + { + "epoch": 0.6172993154947106, + "grad_norm": 2.6493024826049805, + "learning_rate": 4.446494464944649e-06, + "loss": 1.6059, + "step": 496 + }, + { + "epoch": 0.6185438705662726, + "grad_norm": 2.8633735179901123, + "learning_rate": 4.4441881918819195e-06, + "loss": 1.598, + "step": 497 + }, + { + "epoch": 0.6197884256378344, + "grad_norm": 2.65639066696167, + "learning_rate": 4.441881918819189e-06, + "loss": 1.6036, + "step": 498 + }, + { + "epoch": 0.6210329807093964, + "grad_norm": 2.6867759227752686, + "learning_rate": 4.439575645756458e-06, + "loss": 1.5676, + "step": 499 + }, + { + "epoch": 0.6222775357809583, + "grad_norm": 2.7986443042755127, + "learning_rate": 4.437269372693727e-06, + "loss": 1.524, + "step": 500 + }, + { + "epoch": 0.6222775357809583, + "eval_loss": 1.7449125051498413, + "eval_runtime": 43.9329, + "eval_samples_per_second": 22.762, + "eval_steps_per_second": 0.956, + "step": 500 + }, + { + "epoch": 0.6235220908525202, + "grad_norm": 2.665905714035034, + "learning_rate": 4.434963099630996e-06, + "loss": 1.5635, + "step": 501 + }, + { + "epoch": 0.6247666459240822, + "grad_norm": 2.902435541152954, + "learning_rate": 4.432656826568267e-06, + "loss": 1.6377, + "step": 502 + }, + { + "epoch": 0.6260112009956441, + "grad_norm": 2.6872262954711914, + "learning_rate": 4.430350553505535e-06, + "loss": 1.5553, + "step": 503 + }, + { + "epoch": 0.627255756067206, + "grad_norm": 2.676621913909912, + "learning_rate": 4.428044280442805e-06, + "loss": 1.6101, + "step": 504 + }, + { + "epoch": 0.6285003111387679, + "grad_norm": 2.756347179412842, + "learning_rate": 4.425738007380074e-06, + "loss": 1.5884, + "step": 505 + }, + { + "epoch": 0.6297448662103298, + "grad_norm": 2.856882333755493, + "learning_rate": 4.4234317343173434e-06, + "loss": 1.5635, + "step": 506 + }, + { + "epoch": 0.6309894212818917, + "grad_norm": 2.7708330154418945, + "learning_rate": 4.421125461254613e-06, + "loss": 1.6028, + "step": 507 + }, + { + "epoch": 0.6322339763534537, + "grad_norm": 2.8167600631713867, + "learning_rate": 4.418819188191882e-06, + "loss": 1.6026, + "step": 508 + }, + { + "epoch": 0.6334785314250155, + "grad_norm": 2.54194974899292, + "learning_rate": 4.416512915129152e-06, + "loss": 1.6001, + "step": 509 + }, + { + "epoch": 0.6347230864965775, + "grad_norm": 2.683037519454956, + "learning_rate": 4.414206642066421e-06, + "loss": 1.6084, + "step": 510 + }, + { + "epoch": 0.6347230864965775, + "eval_loss": 1.7376186847686768, + "eval_runtime": 42.3026, + "eval_samples_per_second": 23.639, + "eval_steps_per_second": 0.993, + "step": 510 + }, + { + "epoch": 0.6359676415681393, + "grad_norm": 2.82627010345459, + "learning_rate": 4.4119003690036905e-06, + "loss": 1.6258, + "step": 511 + }, + { + "epoch": 0.6372121966397013, + "grad_norm": 2.6139848232269287, + "learning_rate": 4.40959409594096e-06, + "loss": 1.6029, + "step": 512 + }, + { + "epoch": 0.6384567517112633, + "grad_norm": 2.646712303161621, + "learning_rate": 4.407287822878229e-06, + "loss": 1.5602, + "step": 513 + }, + { + "epoch": 0.6397013067828251, + "grad_norm": 2.910935163497925, + "learning_rate": 4.404981549815499e-06, + "loss": 1.5757, + "step": 514 + }, + { + "epoch": 0.6409458618543871, + "grad_norm": 2.6001148223876953, + "learning_rate": 4.402675276752768e-06, + "loss": 1.5857, + "step": 515 + }, + { + "epoch": 0.642190416925949, + "grad_norm": 2.8525235652923584, + "learning_rate": 4.400369003690037e-06, + "loss": 1.6092, + "step": 516 + }, + { + "epoch": 0.6434349719975109, + "grad_norm": 2.5335488319396973, + "learning_rate": 4.398062730627306e-06, + "loss": 1.5849, + "step": 517 + }, + { + "epoch": 0.6446795270690728, + "grad_norm": 2.5787103176116943, + "learning_rate": 4.3957564575645765e-06, + "loss": 1.634, + "step": 518 + }, + { + "epoch": 0.6459240821406347, + "grad_norm": 2.6188197135925293, + "learning_rate": 4.393450184501845e-06, + "loss": 1.5477, + "step": 519 + }, + { + "epoch": 0.6471686372121966, + "grad_norm": 2.6548666954040527, + "learning_rate": 4.3911439114391145e-06, + "loss": 1.6133, + "step": 520 + }, + { + "epoch": 0.6471686372121966, + "eval_loss": 1.7393039464950562, + "eval_runtime": 45.9234, + "eval_samples_per_second": 21.775, + "eval_steps_per_second": 0.915, + "step": 520 + }, + { + "epoch": 0.6484131922837586, + "grad_norm": 2.907928228378296, + "learning_rate": 4.388837638376384e-06, + "loss": 1.587, + "step": 521 + }, + { + "epoch": 0.6496577473553204, + "grad_norm": 2.7899692058563232, + "learning_rate": 4.386531365313653e-06, + "loss": 1.6126, + "step": 522 + }, + { + "epoch": 0.6509023024268824, + "grad_norm": 2.680147886276245, + "learning_rate": 4.384225092250923e-06, + "loss": 1.565, + "step": 523 + }, + { + "epoch": 0.6521468574984443, + "grad_norm": 2.6590754985809326, + "learning_rate": 4.381918819188192e-06, + "loss": 1.6127, + "step": 524 + }, + { + "epoch": 0.6533914125700062, + "grad_norm": 2.6638906002044678, + "learning_rate": 4.379612546125462e-06, + "loss": 1.5615, + "step": 525 + }, + { + "epoch": 0.6546359676415682, + "grad_norm": 2.6518194675445557, + "learning_rate": 4.377306273062731e-06, + "loss": 1.6202, + "step": 526 + }, + { + "epoch": 0.65588052271313, + "grad_norm": 2.5323619842529297, + "learning_rate": 4.3750000000000005e-06, + "loss": 1.5305, + "step": 527 + }, + { + "epoch": 0.657125077784692, + "grad_norm": 2.6871442794799805, + "learning_rate": 4.37269372693727e-06, + "loss": 1.6043, + "step": 528 + }, + { + "epoch": 0.6583696328562539, + "grad_norm": 2.760746717453003, + "learning_rate": 4.370387453874539e-06, + "loss": 1.6059, + "step": 529 + }, + { + "epoch": 0.6596141879278158, + "grad_norm": 2.780482769012451, + "learning_rate": 4.368081180811809e-06, + "loss": 1.6082, + "step": 530 + }, + { + "epoch": 0.6596141879278158, + "eval_loss": 1.7219713926315308, + "eval_runtime": 51.8478, + "eval_samples_per_second": 19.287, + "eval_steps_per_second": 0.81, + "step": 530 + }, + { + "epoch": 0.6608587429993777, + "grad_norm": 2.605890989303589, + "learning_rate": 4.365774907749078e-06, + "loss": 1.5554, + "step": 531 + }, + { + "epoch": 0.6621032980709396, + "grad_norm": 2.731555461883545, + "learning_rate": 4.363468634686347e-06, + "loss": 1.606, + "step": 532 + }, + { + "epoch": 0.6633478531425016, + "grad_norm": 2.8356943130493164, + "learning_rate": 4.361162361623616e-06, + "loss": 1.5601, + "step": 533 + }, + { + "epoch": 0.6645924082140635, + "grad_norm": 2.7196593284606934, + "learning_rate": 4.3588560885608864e-06, + "loss": 1.5722, + "step": 534 + }, + { + "epoch": 0.6658369632856254, + "grad_norm": 2.621371269226074, + "learning_rate": 4.356549815498156e-06, + "loss": 1.6194, + "step": 535 + }, + { + "epoch": 0.6670815183571873, + "grad_norm": 2.653916120529175, + "learning_rate": 4.354243542435424e-06, + "loss": 1.5592, + "step": 536 + }, + { + "epoch": 0.6683260734287492, + "grad_norm": 2.88431453704834, + "learning_rate": 4.351937269372694e-06, + "loss": 1.6296, + "step": 537 + }, + { + "epoch": 0.6695706285003111, + "grad_norm": 2.667130470275879, + "learning_rate": 4.349630996309963e-06, + "loss": 1.5624, + "step": 538 + }, + { + "epoch": 0.6708151835718731, + "grad_norm": 2.6453566551208496, + "learning_rate": 4.347324723247233e-06, + "loss": 1.6025, + "step": 539 + }, + { + "epoch": 0.6720597386434349, + "grad_norm": 3.032271146774292, + "learning_rate": 4.345018450184502e-06, + "loss": 1.6771, + "step": 540 + }, + { + "epoch": 0.6720597386434349, + "eval_loss": 1.721895456314087, + "eval_runtime": 50.62, + "eval_samples_per_second": 19.755, + "eval_steps_per_second": 0.83, + "step": 540 + }, + { + "epoch": 0.6733042937149969, + "grad_norm": 2.8476362228393555, + "learning_rate": 4.3427121771217715e-06, + "loss": 1.5663, + "step": 541 + }, + { + "epoch": 0.6745488487865589, + "grad_norm": 2.615602970123291, + "learning_rate": 4.340405904059041e-06, + "loss": 1.5815, + "step": 542 + }, + { + "epoch": 0.6757934038581207, + "grad_norm": 3.221571683883667, + "learning_rate": 4.33809963099631e-06, + "loss": 1.5955, + "step": 543 + }, + { + "epoch": 0.6770379589296827, + "grad_norm": 2.602994203567505, + "learning_rate": 4.33579335793358e-06, + "loss": 1.5667, + "step": 544 + }, + { + "epoch": 0.6782825140012445, + "grad_norm": 2.9380433559417725, + "learning_rate": 4.333487084870848e-06, + "loss": 1.5651, + "step": 545 + }, + { + "epoch": 0.6795270690728065, + "grad_norm": 3.0689425468444824, + "learning_rate": 4.331180811808119e-06, + "loss": 1.6003, + "step": 546 + }, + { + "epoch": 0.6807716241443684, + "grad_norm": 2.770317316055298, + "learning_rate": 4.328874538745388e-06, + "loss": 1.5576, + "step": 547 + }, + { + "epoch": 0.6820161792159303, + "grad_norm": 2.8631815910339355, + "learning_rate": 4.3265682656826575e-06, + "loss": 1.5308, + "step": 548 + }, + { + "epoch": 0.6832607342874922, + "grad_norm": 2.785576820373535, + "learning_rate": 4.324261992619926e-06, + "loss": 1.6113, + "step": 549 + }, + { + "epoch": 0.6845052893590542, + "grad_norm": 2.723919630050659, + "learning_rate": 4.321955719557196e-06, + "loss": 1.5606, + "step": 550 + }, + { + "epoch": 0.6845052893590542, + "eval_loss": 1.721803069114685, + "eval_runtime": 51.6036, + "eval_samples_per_second": 19.378, + "eval_steps_per_second": 0.814, + "step": 550 + }, + { + "epoch": 0.685749844430616, + "grad_norm": 3.081198215484619, + "learning_rate": 4.319649446494466e-06, + "loss": 1.5643, + "step": 551 + }, + { + "epoch": 0.686994399502178, + "grad_norm": 2.757392168045044, + "learning_rate": 4.317343173431734e-06, + "loss": 1.5909, + "step": 552 + }, + { + "epoch": 0.6882389545737398, + "grad_norm": 3.038753032684326, + "learning_rate": 4.315036900369004e-06, + "loss": 1.6023, + "step": 553 + }, + { + "epoch": 0.6894835096453018, + "grad_norm": 2.8246452808380127, + "learning_rate": 4.312730627306273e-06, + "loss": 1.5548, + "step": 554 + }, + { + "epoch": 0.6907280647168638, + "grad_norm": 2.589320182800293, + "learning_rate": 4.310424354243543e-06, + "loss": 1.5424, + "step": 555 + }, + { + "epoch": 0.6919726197884256, + "grad_norm": 2.6413373947143555, + "learning_rate": 4.308118081180812e-06, + "loss": 1.6243, + "step": 556 + }, + { + "epoch": 0.6932171748599876, + "grad_norm": 2.764784336090088, + "learning_rate": 4.3058118081180815e-06, + "loss": 1.589, + "step": 557 + }, + { + "epoch": 0.6944617299315494, + "grad_norm": 2.806821823120117, + "learning_rate": 4.303505535055351e-06, + "loss": 1.621, + "step": 558 + }, + { + "epoch": 0.6957062850031114, + "grad_norm": 2.718017578125, + "learning_rate": 4.30119926199262e-06, + "loss": 1.5773, + "step": 559 + }, + { + "epoch": 0.6969508400746733, + "grad_norm": 2.7620160579681396, + "learning_rate": 4.29889298892989e-06, + "loss": 1.5868, + "step": 560 + }, + { + "epoch": 0.6969508400746733, + "eval_loss": 1.7152249813079834, + "eval_runtime": 52.0463, + "eval_samples_per_second": 19.214, + "eval_steps_per_second": 0.807, + "step": 560 + }, + { + "epoch": 0.6981953951462352, + "grad_norm": 2.66686749458313, + "learning_rate": 4.296586715867159e-06, + "loss": 1.534, + "step": 561 + }, + { + "epoch": 0.6994399502177971, + "grad_norm": 2.812577247619629, + "learning_rate": 4.2942804428044286e-06, + "loss": 1.5501, + "step": 562 + }, + { + "epoch": 0.7006845052893591, + "grad_norm": 2.578508138656616, + "learning_rate": 4.291974169741698e-06, + "loss": 1.6286, + "step": 563 + }, + { + "epoch": 0.701929060360921, + "grad_norm": 2.524249315261841, + "learning_rate": 4.289667896678967e-06, + "loss": 1.5719, + "step": 564 + }, + { + "epoch": 0.7031736154324829, + "grad_norm": 2.827235460281372, + "learning_rate": 4.287361623616236e-06, + "loss": 1.5333, + "step": 565 + }, + { + "epoch": 0.7044181705040448, + "grad_norm": 2.6359963417053223, + "learning_rate": 4.285055350553506e-06, + "loss": 1.5849, + "step": 566 + }, + { + "epoch": 0.7056627255756067, + "grad_norm": 2.930530071258545, + "learning_rate": 4.282749077490776e-06, + "loss": 1.5672, + "step": 567 + }, + { + "epoch": 0.7069072806471687, + "grad_norm": 2.750102996826172, + "learning_rate": 4.280442804428044e-06, + "loss": 1.5696, + "step": 568 + }, + { + "epoch": 0.7081518357187305, + "grad_norm": 2.869690418243408, + "learning_rate": 4.278136531365314e-06, + "loss": 1.5689, + "step": 569 + }, + { + "epoch": 0.7093963907902925, + "grad_norm": 2.954852819442749, + "learning_rate": 4.275830258302583e-06, + "loss": 1.5931, + "step": 570 + }, + { + "epoch": 0.7093963907902925, + "eval_loss": 1.7143300771713257, + "eval_runtime": 52.7293, + "eval_samples_per_second": 18.965, + "eval_steps_per_second": 0.797, + "step": 570 + }, + { + "epoch": 0.7106409458618543, + "grad_norm": 2.702223539352417, + "learning_rate": 4.273523985239853e-06, + "loss": 1.5635, + "step": 571 + }, + { + "epoch": 0.7118855009334163, + "grad_norm": 2.689995050430298, + "learning_rate": 4.271217712177122e-06, + "loss": 1.5545, + "step": 572 + }, + { + "epoch": 0.7131300560049783, + "grad_norm": 2.68979811668396, + "learning_rate": 4.268911439114391e-06, + "loss": 1.5404, + "step": 573 + }, + { + "epoch": 0.7143746110765401, + "grad_norm": 2.7477986812591553, + "learning_rate": 4.266605166051661e-06, + "loss": 1.5719, + "step": 574 + }, + { + "epoch": 0.7156191661481021, + "grad_norm": 2.975778341293335, + "learning_rate": 4.26429889298893e-06, + "loss": 1.5587, + "step": 575 + }, + { + "epoch": 0.716863721219664, + "grad_norm": 2.658170700073242, + "learning_rate": 4.2619926199262e-06, + "loss": 1.5746, + "step": 576 + }, + { + "epoch": 0.7181082762912259, + "grad_norm": 2.8825011253356934, + "learning_rate": 4.259686346863469e-06, + "loss": 1.5246, + "step": 577 + }, + { + "epoch": 0.7193528313627878, + "grad_norm": 2.845280170440674, + "learning_rate": 4.2573800738007385e-06, + "loss": 1.5703, + "step": 578 + }, + { + "epoch": 0.7205973864343497, + "grad_norm": 2.660616636276245, + "learning_rate": 4.255073800738008e-06, + "loss": 1.5373, + "step": 579 + }, + { + "epoch": 0.7218419415059116, + "grad_norm": 2.749447822570801, + "learning_rate": 4.252767527675277e-06, + "loss": 1.59, + "step": 580 + }, + { + "epoch": 0.7218419415059116, + "eval_loss": 1.7149444818496704, + "eval_runtime": 52.2099, + "eval_samples_per_second": 19.153, + "eval_steps_per_second": 0.804, + "step": 580 + }, + { + "epoch": 0.7230864965774736, + "grad_norm": 2.813328742980957, + "learning_rate": 4.250461254612546e-06, + "loss": 1.5694, + "step": 581 + }, + { + "epoch": 0.7243310516490354, + "grad_norm": 2.823866844177246, + "learning_rate": 4.248154981549816e-06, + "loss": 1.5143, + "step": 582 + }, + { + "epoch": 0.7255756067205974, + "grad_norm": 2.71337890625, + "learning_rate": 4.245848708487086e-06, + "loss": 1.5915, + "step": 583 + }, + { + "epoch": 0.7268201617921594, + "grad_norm": 2.718085765838623, + "learning_rate": 4.243542435424355e-06, + "loss": 1.604, + "step": 584 + }, + { + "epoch": 0.7280647168637212, + "grad_norm": 2.5741796493530273, + "learning_rate": 4.241236162361624e-06, + "loss": 1.5398, + "step": 585 + }, + { + "epoch": 0.7293092719352832, + "grad_norm": 2.727114200592041, + "learning_rate": 4.238929889298893e-06, + "loss": 1.5886, + "step": 586 + }, + { + "epoch": 0.730553827006845, + "grad_norm": 2.6703338623046875, + "learning_rate": 4.236623616236163e-06, + "loss": 1.5959, + "step": 587 + }, + { + "epoch": 0.731798382078407, + "grad_norm": 2.750326633453369, + "learning_rate": 4.234317343173432e-06, + "loss": 1.5916, + "step": 588 + }, + { + "epoch": 0.7330429371499689, + "grad_norm": 2.6680715084075928, + "learning_rate": 4.232011070110701e-06, + "loss": 1.562, + "step": 589 + }, + { + "epoch": 0.7342874922215308, + "grad_norm": 2.6603832244873047, + "learning_rate": 4.229704797047971e-06, + "loss": 1.6121, + "step": 590 + }, + { + "epoch": 0.7342874922215308, + "eval_loss": 1.7144734859466553, + "eval_runtime": 50.5317, + "eval_samples_per_second": 19.79, + "eval_steps_per_second": 0.831, + "step": 590 + }, + { + "epoch": 0.7355320472930927, + "grad_norm": 2.7868523597717285, + "learning_rate": 4.22739852398524e-06, + "loss": 1.5488, + "step": 591 + }, + { + "epoch": 0.7367766023646546, + "grad_norm": 2.755270004272461, + "learning_rate": 4.2250922509225096e-06, + "loss": 1.5801, + "step": 592 + }, + { + "epoch": 0.7380211574362165, + "grad_norm": 2.7614126205444336, + "learning_rate": 4.222785977859779e-06, + "loss": 1.5507, + "step": 593 + }, + { + "epoch": 0.7392657125077785, + "grad_norm": 2.8674862384796143, + "learning_rate": 4.220479704797048e-06, + "loss": 1.5807, + "step": 594 + }, + { + "epoch": 0.7405102675793404, + "grad_norm": 2.7366859912872314, + "learning_rate": 4.218173431734318e-06, + "loss": 1.6594, + "step": 595 + }, + { + "epoch": 0.7417548226509023, + "grad_norm": 2.6231045722961426, + "learning_rate": 4.215867158671587e-06, + "loss": 1.5609, + "step": 596 + }, + { + "epoch": 0.7429993777224643, + "grad_norm": 2.7694146633148193, + "learning_rate": 4.213560885608857e-06, + "loss": 1.5889, + "step": 597 + }, + { + "epoch": 0.7442439327940261, + "grad_norm": 2.6275386810302734, + "learning_rate": 4.211254612546125e-06, + "loss": 1.575, + "step": 598 + }, + { + "epoch": 0.7454884878655881, + "grad_norm": 2.9413866996765137, + "learning_rate": 4.2089483394833955e-06, + "loss": 1.589, + "step": 599 + }, + { + "epoch": 0.7467330429371499, + "grad_norm": 2.768606185913086, + "learning_rate": 4.206642066420665e-06, + "loss": 1.5738, + "step": 600 + }, + { + "epoch": 0.7467330429371499, + "eval_loss": 1.7148027420043945, + "eval_runtime": 54.7031, + "eval_samples_per_second": 18.28, + "eval_steps_per_second": 0.768, + "step": 600 + }, + { + "epoch": 0.7479775980087119, + "grad_norm": 2.7917706966400146, + "learning_rate": 4.2043357933579335e-06, + "loss": 1.499, + "step": 601 + }, + { + "epoch": 0.7492221530802738, + "grad_norm": 2.5982208251953125, + "learning_rate": 4.202029520295203e-06, + "loss": 1.5113, + "step": 602 + }, + { + "epoch": 0.7504667081518357, + "grad_norm": 2.692032814025879, + "learning_rate": 4.199723247232473e-06, + "loss": 1.4721, + "step": 603 + }, + { + "epoch": 0.7517112632233977, + "grad_norm": 2.975860834121704, + "learning_rate": 4.197416974169742e-06, + "loss": 1.5147, + "step": 604 + }, + { + "epoch": 0.7529558182949595, + "grad_norm": 2.758610248565674, + "learning_rate": 4.195110701107011e-06, + "loss": 1.5674, + "step": 605 + }, + { + "epoch": 0.7542003733665215, + "grad_norm": 2.7620184421539307, + "learning_rate": 4.192804428044281e-06, + "loss": 1.5669, + "step": 606 + }, + { + "epoch": 0.7554449284380834, + "grad_norm": 3.043940305709839, + "learning_rate": 4.19049815498155e-06, + "loss": 1.587, + "step": 607 + }, + { + "epoch": 0.7566894835096453, + "grad_norm": 2.7482151985168457, + "learning_rate": 4.1881918819188195e-06, + "loss": 1.5451, + "step": 608 + }, + { + "epoch": 0.7579340385812072, + "grad_norm": 2.7259294986724854, + "learning_rate": 4.185885608856089e-06, + "loss": 1.5295, + "step": 609 + }, + { + "epoch": 0.7591785936527692, + "grad_norm": 2.6340832710266113, + "learning_rate": 4.183579335793358e-06, + "loss": 1.5335, + "step": 610 + }, + { + "epoch": 0.7591785936527692, + "eval_loss": 1.7038393020629883, + "eval_runtime": 44.0792, + "eval_samples_per_second": 22.686, + "eval_steps_per_second": 0.953, + "step": 610 + }, + { + "epoch": 0.760423148724331, + "grad_norm": 2.6144909858703613, + "learning_rate": 4.181273062730628e-06, + "loss": 1.5907, + "step": 611 + }, + { + "epoch": 0.761667703795893, + "grad_norm": 2.7754175662994385, + "learning_rate": 4.178966789667897e-06, + "loss": 1.5773, + "step": 612 + }, + { + "epoch": 0.7629122588674548, + "grad_norm": 2.6313252449035645, + "learning_rate": 4.176660516605167e-06, + "loss": 1.5675, + "step": 613 + }, + { + "epoch": 0.7641568139390168, + "grad_norm": 2.549074649810791, + "learning_rate": 4.174354243542435e-06, + "loss": 1.595, + "step": 614 + }, + { + "epoch": 0.7654013690105788, + "grad_norm": 2.611804246902466, + "learning_rate": 4.1720479704797054e-06, + "loss": 1.5416, + "step": 615 + }, + { + "epoch": 0.7666459240821406, + "grad_norm": 2.6322927474975586, + "learning_rate": 4.169741697416975e-06, + "loss": 1.604, + "step": 616 + }, + { + "epoch": 0.7678904791537026, + "grad_norm": 2.5792219638824463, + "learning_rate": 4.1674354243542434e-06, + "loss": 1.5976, + "step": 617 + }, + { + "epoch": 0.7691350342252644, + "grad_norm": 2.845416307449341, + "learning_rate": 4.165129151291513e-06, + "loss": 1.5945, + "step": 618 + }, + { + "epoch": 0.7703795892968264, + "grad_norm": 2.8647871017456055, + "learning_rate": 4.162822878228783e-06, + "loss": 1.5637, + "step": 619 + }, + { + "epoch": 0.7716241443683883, + "grad_norm": 2.590719699859619, + "learning_rate": 4.1605166051660526e-06, + "loss": 1.556, + "step": 620 + }, + { + "epoch": 0.7716241443683883, + "eval_loss": 1.7115222215652466, + "eval_runtime": 50.3905, + "eval_samples_per_second": 19.845, + "eval_steps_per_second": 0.833, + "step": 620 + }, + { + "epoch": 0.7728686994399502, + "grad_norm": 2.626723527908325, + "learning_rate": 4.158210332103321e-06, + "loss": 1.5486, + "step": 621 + }, + { + "epoch": 0.7741132545115121, + "grad_norm": 2.713069438934326, + "learning_rate": 4.1559040590405906e-06, + "loss": 1.5474, + "step": 622 + }, + { + "epoch": 0.7753578095830741, + "grad_norm": 2.685563087463379, + "learning_rate": 4.15359778597786e-06, + "loss": 1.5529, + "step": 623 + }, + { + "epoch": 0.776602364654636, + "grad_norm": 2.690586805343628, + "learning_rate": 4.151291512915129e-06, + "loss": 1.5293, + "step": 624 + }, + { + "epoch": 0.7778469197261979, + "grad_norm": 2.865345001220703, + "learning_rate": 4.148985239852399e-06, + "loss": 1.5095, + "step": 625 + }, + { + "epoch": 0.7790914747977598, + "grad_norm": 2.92393159866333, + "learning_rate": 4.146678966789668e-06, + "loss": 1.5561, + "step": 626 + }, + { + "epoch": 0.7803360298693217, + "grad_norm": 2.554480791091919, + "learning_rate": 4.144372693726938e-06, + "loss": 1.4876, + "step": 627 + }, + { + "epoch": 0.7815805849408837, + "grad_norm": 2.6128833293914795, + "learning_rate": 4.142066420664207e-06, + "loss": 1.5378, + "step": 628 + }, + { + "epoch": 0.7828251400124455, + "grad_norm": 3.0110888481140137, + "learning_rate": 4.1397601476014765e-06, + "loss": 1.5859, + "step": 629 + }, + { + "epoch": 0.7840696950840075, + "grad_norm": 2.8152706623077393, + "learning_rate": 4.137453874538745e-06, + "loss": 1.5841, + "step": 630 + }, + { + "epoch": 0.7840696950840075, + "eval_loss": 1.702181339263916, + "eval_runtime": 53.6926, + "eval_samples_per_second": 18.625, + "eval_steps_per_second": 0.782, + "step": 630 + }, + { + "epoch": 0.7853142501555694, + "grad_norm": 2.8634233474731445, + "learning_rate": 4.135147601476015e-06, + "loss": 1.564, + "step": 631 + }, + { + "epoch": 0.7865588052271313, + "grad_norm": 2.8939666748046875, + "learning_rate": 4.132841328413285e-06, + "loss": 1.5859, + "step": 632 + }, + { + "epoch": 0.7878033602986932, + "grad_norm": 2.6560444831848145, + "learning_rate": 4.130535055350554e-06, + "loss": 1.5199, + "step": 633 + }, + { + "epoch": 0.7890479153702551, + "grad_norm": 2.964721202850342, + "learning_rate": 4.128228782287823e-06, + "loss": 1.5888, + "step": 634 + }, + { + "epoch": 0.790292470441817, + "grad_norm": 2.74668288230896, + "learning_rate": 4.125922509225092e-06, + "loss": 1.5847, + "step": 635 + }, + { + "epoch": 0.791537025513379, + "grad_norm": 2.723123550415039, + "learning_rate": 4.1236162361623625e-06, + "loss": 1.5166, + "step": 636 + }, + { + "epoch": 0.7927815805849409, + "grad_norm": 2.7265713214874268, + "learning_rate": 4.121309963099631e-06, + "loss": 1.5954, + "step": 637 + }, + { + "epoch": 0.7940261356565028, + "grad_norm": 2.979126214981079, + "learning_rate": 4.1190036900369005e-06, + "loss": 1.5583, + "step": 638 + }, + { + "epoch": 0.7952706907280647, + "grad_norm": 2.844376802444458, + "learning_rate": 4.11669741697417e-06, + "loss": 1.5564, + "step": 639 + }, + { + "epoch": 0.7965152457996266, + "grad_norm": 2.7583703994750977, + "learning_rate": 4.114391143911439e-06, + "loss": 1.5094, + "step": 640 + }, + { + "epoch": 0.7965152457996266, + "eval_loss": 1.6986565589904785, + "eval_runtime": 47.3549, + "eval_samples_per_second": 21.117, + "eval_steps_per_second": 0.887, + "step": 640 + }, + { + "epoch": 0.7977598008711886, + "grad_norm": 2.8942224979400635, + "learning_rate": 4.112084870848709e-06, + "loss": 1.5504, + "step": 641 + }, + { + "epoch": 0.7990043559427504, + "grad_norm": 2.6590495109558105, + "learning_rate": 4.109778597785978e-06, + "loss": 1.5227, + "step": 642 + }, + { + "epoch": 0.8002489110143124, + "grad_norm": 2.5988378524780273, + "learning_rate": 4.107472324723248e-06, + "loss": 1.5054, + "step": 643 + }, + { + "epoch": 0.8014934660858744, + "grad_norm": 2.787335157394409, + "learning_rate": 4.105166051660517e-06, + "loss": 1.5844, + "step": 644 + }, + { + "epoch": 0.8027380211574362, + "grad_norm": 2.8202896118164062, + "learning_rate": 4.1028597785977864e-06, + "loss": 1.5596, + "step": 645 + }, + { + "epoch": 0.8039825762289982, + "grad_norm": 2.65376615524292, + "learning_rate": 4.100553505535056e-06, + "loss": 1.5893, + "step": 646 + }, + { + "epoch": 0.80522713130056, + "grad_norm": 2.5933308601379395, + "learning_rate": 4.098247232472325e-06, + "loss": 1.5403, + "step": 647 + }, + { + "epoch": 0.806471686372122, + "grad_norm": 2.777070999145508, + "learning_rate": 4.095940959409595e-06, + "loss": 1.5231, + "step": 648 + }, + { + "epoch": 0.8077162414436839, + "grad_norm": 2.6427664756774902, + "learning_rate": 4.093634686346864e-06, + "loss": 1.5937, + "step": 649 + }, + { + "epoch": 0.8089607965152458, + "grad_norm": 2.651561737060547, + "learning_rate": 4.091328413284133e-06, + "loss": 1.4847, + "step": 650 + }, + { + "epoch": 0.8089607965152458, + "eval_loss": 1.7089996337890625, + "eval_runtime": 47.4346, + "eval_samples_per_second": 21.082, + "eval_steps_per_second": 0.885, + "step": 650 + }, + { + "epoch": 0.8102053515868077, + "grad_norm": 2.844067335128784, + "learning_rate": 4.089022140221402e-06, + "loss": 1.488, + "step": 651 + }, + { + "epoch": 0.8114499066583696, + "grad_norm": 2.603480577468872, + "learning_rate": 4.086715867158672e-06, + "loss": 1.5248, + "step": 652 + }, + { + "epoch": 0.8126944617299315, + "grad_norm": 2.789322853088379, + "learning_rate": 4.084409594095941e-06, + "loss": 1.5119, + "step": 653 + }, + { + "epoch": 0.8139390168014935, + "grad_norm": 2.7183773517608643, + "learning_rate": 4.08210332103321e-06, + "loss": 1.5123, + "step": 654 + }, + { + "epoch": 0.8151835718730553, + "grad_norm": 2.80613374710083, + "learning_rate": 4.07979704797048e-06, + "loss": 1.5526, + "step": 655 + }, + { + "epoch": 0.8164281269446173, + "grad_norm": 2.641035795211792, + "learning_rate": 4.077490774907749e-06, + "loss": 1.6127, + "step": 656 + }, + { + "epoch": 0.8176726820161793, + "grad_norm": 2.773684024810791, + "learning_rate": 4.075184501845019e-06, + "loss": 1.5404, + "step": 657 + }, + { + "epoch": 0.8189172370877411, + "grad_norm": 2.6951382160186768, + "learning_rate": 4.072878228782288e-06, + "loss": 1.4928, + "step": 658 + }, + { + "epoch": 0.8201617921593031, + "grad_norm": 2.931771755218506, + "learning_rate": 4.0705719557195575e-06, + "loss": 1.5949, + "step": 659 + }, + { + "epoch": 0.8214063472308649, + "grad_norm": 2.8786075115203857, + "learning_rate": 4.068265682656827e-06, + "loss": 1.5582, + "step": 660 + }, + { + "epoch": 0.8214063472308649, + "eval_loss": 1.698564052581787, + "eval_runtime": 46.6299, + "eval_samples_per_second": 21.445, + "eval_steps_per_second": 0.901, + "step": 660 + }, + { + "epoch": 0.8226509023024269, + "grad_norm": 2.7402451038360596, + "learning_rate": 4.065959409594096e-06, + "loss": 1.5562, + "step": 661 + }, + { + "epoch": 0.8238954573739888, + "grad_norm": 2.7968692779541016, + "learning_rate": 4.063653136531366e-06, + "loss": 1.556, + "step": 662 + }, + { + "epoch": 0.8251400124455507, + "grad_norm": 2.8233132362365723, + "learning_rate": 4.061346863468635e-06, + "loss": 1.5527, + "step": 663 + }, + { + "epoch": 0.8263845675171126, + "grad_norm": 2.713390827178955, + "learning_rate": 4.059040590405905e-06, + "loss": 1.6012, + "step": 664 + }, + { + "epoch": 0.8276291225886746, + "grad_norm": 2.7783989906311035, + "learning_rate": 4.056734317343174e-06, + "loss": 1.5247, + "step": 665 + }, + { + "epoch": 0.8288736776602365, + "grad_norm": 2.808554172515869, + "learning_rate": 4.054428044280443e-06, + "loss": 1.5412, + "step": 666 + }, + { + "epoch": 0.8301182327317984, + "grad_norm": 2.8476831912994385, + "learning_rate": 4.052121771217712e-06, + "loss": 1.5307, + "step": 667 + }, + { + "epoch": 0.8313627878033603, + "grad_norm": 2.8287601470947266, + "learning_rate": 4.049815498154982e-06, + "loss": 1.6021, + "step": 668 + }, + { + "epoch": 0.8326073428749222, + "grad_norm": 2.975510835647583, + "learning_rate": 4.047509225092252e-06, + "loss": 1.5474, + "step": 669 + }, + { + "epoch": 0.8338518979464842, + "grad_norm": 2.723236560821533, + "learning_rate": 4.04520295202952e-06, + "loss": 1.5242, + "step": 670 + }, + { + "epoch": 0.8338518979464842, + "eval_loss": 1.7005239725112915, + "eval_runtime": 49.4504, + "eval_samples_per_second": 20.222, + "eval_steps_per_second": 0.849, + "step": 670 + }, + { + "epoch": 0.835096453018046, + "grad_norm": 2.7521748542785645, + "learning_rate": 4.04289667896679e-06, + "loss": 1.5467, + "step": 671 + }, + { + "epoch": 0.836341008089608, + "grad_norm": 2.7770044803619385, + "learning_rate": 4.04059040590406e-06, + "loss": 1.5407, + "step": 672 + }, + { + "epoch": 0.8375855631611698, + "grad_norm": 2.744323492050171, + "learning_rate": 4.038284132841329e-06, + "loss": 1.5422, + "step": 673 + }, + { + "epoch": 0.8388301182327318, + "grad_norm": 2.6699817180633545, + "learning_rate": 4.035977859778598e-06, + "loss": 1.5508, + "step": 674 + }, + { + "epoch": 0.8400746733042938, + "grad_norm": 2.705273151397705, + "learning_rate": 4.0336715867158674e-06, + "loss": 1.5099, + "step": 675 + }, + { + "epoch": 0.8413192283758556, + "grad_norm": 2.9485747814178467, + "learning_rate": 4.031365313653137e-06, + "loss": 1.5529, + "step": 676 + }, + { + "epoch": 0.8425637834474176, + "grad_norm": 3.1750423908233643, + "learning_rate": 4.029059040590406e-06, + "loss": 1.5894, + "step": 677 + }, + { + "epoch": 0.8438083385189795, + "grad_norm": 2.76448655128479, + "learning_rate": 4.026752767527676e-06, + "loss": 1.5752, + "step": 678 + }, + { + "epoch": 0.8450528935905414, + "grad_norm": 2.676708698272705, + "learning_rate": 4.024446494464945e-06, + "loss": 1.5487, + "step": 679 + }, + { + "epoch": 0.8462974486621033, + "grad_norm": 2.6832051277160645, + "learning_rate": 4.0221402214022145e-06, + "loss": 1.5055, + "step": 680 + }, + { + "epoch": 0.8462974486621033, + "eval_loss": 1.6945017576217651, + "eval_runtime": 43.9369, + "eval_samples_per_second": 22.76, + "eval_steps_per_second": 0.956, + "step": 680 + }, + { + "epoch": 0.8475420037336652, + "grad_norm": 2.6812336444854736, + "learning_rate": 4.019833948339484e-06, + "loss": 1.5315, + "step": 681 + }, + { + "epoch": 0.8487865588052271, + "grad_norm": 2.9394326210021973, + "learning_rate": 4.017527675276753e-06, + "loss": 1.5636, + "step": 682 + }, + { + "epoch": 0.8500311138767891, + "grad_norm": 2.944952964782715, + "learning_rate": 4.015221402214022e-06, + "loss": 1.56, + "step": 683 + }, + { + "epoch": 0.8512756689483509, + "grad_norm": 2.7475314140319824, + "learning_rate": 4.012915129151292e-06, + "loss": 1.5442, + "step": 684 + }, + { + "epoch": 0.8525202240199129, + "grad_norm": 3.0572879314422607, + "learning_rate": 4.010608856088562e-06, + "loss": 1.5023, + "step": 685 + }, + { + "epoch": 0.8537647790914747, + "grad_norm": 2.75365948677063, + "learning_rate": 4.00830258302583e-06, + "loss": 1.5414, + "step": 686 + }, + { + "epoch": 0.8550093341630367, + "grad_norm": 2.5714449882507324, + "learning_rate": 4.0059963099631e-06, + "loss": 1.5027, + "step": 687 + }, + { + "epoch": 0.8562538892345987, + "grad_norm": 2.755167007446289, + "learning_rate": 4.003690036900369e-06, + "loss": 1.5788, + "step": 688 + }, + { + "epoch": 0.8574984443061605, + "grad_norm": 2.798967123031616, + "learning_rate": 4.0013837638376385e-06, + "loss": 1.5162, + "step": 689 + }, + { + "epoch": 0.8587429993777225, + "grad_norm": 2.803614854812622, + "learning_rate": 3.999077490774908e-06, + "loss": 1.5079, + "step": 690 + }, + { + "epoch": 0.8587429993777225, + "eval_loss": 1.6924962997436523, + "eval_runtime": 44.47, + "eval_samples_per_second": 22.487, + "eval_steps_per_second": 0.944, + "step": 690 + }, + { + "epoch": 0.8599875544492844, + "grad_norm": 2.6898066997528076, + "learning_rate": 3.996771217712177e-06, + "loss": 1.5478, + "step": 691 + }, + { + "epoch": 0.8612321095208463, + "grad_norm": 2.6376144886016846, + "learning_rate": 3.994464944649447e-06, + "loss": 1.5362, + "step": 692 + }, + { + "epoch": 0.8624766645924082, + "grad_norm": 2.7240405082702637, + "learning_rate": 3.992158671586716e-06, + "loss": 1.5502, + "step": 693 + }, + { + "epoch": 0.8637212196639701, + "grad_norm": 2.929445266723633, + "learning_rate": 3.989852398523986e-06, + "loss": 1.5799, + "step": 694 + }, + { + "epoch": 0.864965774735532, + "grad_norm": 2.593223810195923, + "learning_rate": 3.987546125461255e-06, + "loss": 1.5352, + "step": 695 + }, + { + "epoch": 0.866210329807094, + "grad_norm": 2.7710154056549072, + "learning_rate": 3.9852398523985245e-06, + "loss": 1.5048, + "step": 696 + }, + { + "epoch": 0.8674548848786559, + "grad_norm": 2.7370848655700684, + "learning_rate": 3.982933579335794e-06, + "loss": 1.5233, + "step": 697 + }, + { + "epoch": 0.8686994399502178, + "grad_norm": 2.8632168769836426, + "learning_rate": 3.980627306273063e-06, + "loss": 1.5435, + "step": 698 + }, + { + "epoch": 0.8699439950217797, + "grad_norm": 2.752298593521118, + "learning_rate": 3.978321033210332e-06, + "loss": 1.5728, + "step": 699 + }, + { + "epoch": 0.8711885500933416, + "grad_norm": 2.8345625400543213, + "learning_rate": 3.976014760147602e-06, + "loss": 1.5727, + "step": 700 + }, + { + "epoch": 0.8711885500933416, + "eval_loss": 1.6855305433273315, + "eval_runtime": 43.3683, + "eval_samples_per_second": 23.058, + "eval_steps_per_second": 0.968, + "step": 700 + }, + { + "epoch": 0.8724331051649036, + "grad_norm": 2.8135335445404053, + "learning_rate": 3.973708487084872e-06, + "loss": 1.5086, + "step": 701 + }, + { + "epoch": 0.8736776602364654, + "grad_norm": 2.7416603565216064, + "learning_rate": 3.97140221402214e-06, + "loss": 1.545, + "step": 702 + }, + { + "epoch": 0.8749222153080274, + "grad_norm": 2.6902289390563965, + "learning_rate": 3.96909594095941e-06, + "loss": 1.5509, + "step": 703 + }, + { + "epoch": 0.8761667703795893, + "grad_norm": 2.6537084579467773, + "learning_rate": 3.966789667896679e-06, + "loss": 1.562, + "step": 704 + }, + { + "epoch": 0.8774113254511512, + "grad_norm": 2.6979284286499023, + "learning_rate": 3.964483394833948e-06, + "loss": 1.4829, + "step": 705 + }, + { + "epoch": 0.8786558805227132, + "grad_norm": 2.5795764923095703, + "learning_rate": 3.962177121771218e-06, + "loss": 1.4677, + "step": 706 + }, + { + "epoch": 0.879900435594275, + "grad_norm": 3.0185086727142334, + "learning_rate": 3.959870848708487e-06, + "loss": 1.5608, + "step": 707 + }, + { + "epoch": 0.881144990665837, + "grad_norm": 2.6458113193511963, + "learning_rate": 3.957564575645757e-06, + "loss": 1.5342, + "step": 708 + }, + { + "epoch": 0.8823895457373989, + "grad_norm": 2.7985665798187256, + "learning_rate": 3.955258302583026e-06, + "loss": 1.527, + "step": 709 + }, + { + "epoch": 0.8836341008089608, + "grad_norm": 2.9069082736968994, + "learning_rate": 3.9529520295202955e-06, + "loss": 1.5156, + "step": 710 + }, + { + "epoch": 0.8836341008089608, + "eval_loss": 1.6898822784423828, + "eval_runtime": 42.8749, + "eval_samples_per_second": 23.324, + "eval_steps_per_second": 0.98, + "step": 710 + }, + { + "epoch": 0.8848786558805227, + "grad_norm": 2.760143995285034, + "learning_rate": 3.950645756457565e-06, + "loss": 1.5124, + "step": 711 + }, + { + "epoch": 0.8861232109520847, + "grad_norm": 2.8085529804229736, + "learning_rate": 3.948339483394834e-06, + "loss": 1.5862, + "step": 712 + }, + { + "epoch": 0.8873677660236465, + "grad_norm": 2.909905195236206, + "learning_rate": 3.946033210332104e-06, + "loss": 1.5557, + "step": 713 + }, + { + "epoch": 0.8886123210952085, + "grad_norm": 2.826899290084839, + "learning_rate": 3.943726937269373e-06, + "loss": 1.5277, + "step": 714 + }, + { + "epoch": 0.8898568761667703, + "grad_norm": 2.6269052028656006, + "learning_rate": 3.941420664206642e-06, + "loss": 1.4819, + "step": 715 + }, + { + "epoch": 0.8911014312383323, + "grad_norm": 2.6687698364257812, + "learning_rate": 3.939114391143912e-06, + "loss": 1.5156, + "step": 716 + }, + { + "epoch": 0.8923459863098943, + "grad_norm": 2.9605629444122314, + "learning_rate": 3.9368081180811815e-06, + "loss": 1.5358, + "step": 717 + }, + { + "epoch": 0.8935905413814561, + "grad_norm": 3.0367166996002197, + "learning_rate": 3.934501845018451e-06, + "loss": 1.5847, + "step": 718 + }, + { + "epoch": 0.8948350964530181, + "grad_norm": 2.528796434402466, + "learning_rate": 3.9321955719557195e-06, + "loss": 1.4851, + "step": 719 + }, + { + "epoch": 0.8960796515245799, + "grad_norm": 2.7116613388061523, + "learning_rate": 3.929889298892989e-06, + "loss": 1.4706, + "step": 720 + }, + { + "epoch": 0.8960796515245799, + "eval_loss": 1.6845883131027222, + "eval_runtime": 44.4734, + "eval_samples_per_second": 22.485, + "eval_steps_per_second": 0.944, + "step": 720 + }, + { + "epoch": 0.8973242065961419, + "grad_norm": 2.771864414215088, + "learning_rate": 3.927583025830259e-06, + "loss": 1.5444, + "step": 721 + }, + { + "epoch": 0.8985687616677038, + "grad_norm": 2.773746967315674, + "learning_rate": 3.925276752767528e-06, + "loss": 1.564, + "step": 722 + }, + { + "epoch": 0.8998133167392657, + "grad_norm": 2.7069854736328125, + "learning_rate": 3.922970479704797e-06, + "loss": 1.5601, + "step": 723 + }, + { + "epoch": 0.9010578718108276, + "grad_norm": 2.840261697769165, + "learning_rate": 3.920664206642067e-06, + "loss": 1.5733, + "step": 724 + }, + { + "epoch": 0.9023024268823896, + "grad_norm": 2.709897041320801, + "learning_rate": 3.918357933579336e-06, + "loss": 1.528, + "step": 725 + }, + { + "epoch": 0.9035469819539514, + "grad_norm": 2.662367343902588, + "learning_rate": 3.9160516605166055e-06, + "loss": 1.5532, + "step": 726 + }, + { + "epoch": 0.9047915370255134, + "grad_norm": 2.6775717735290527, + "learning_rate": 3.913745387453875e-06, + "loss": 1.5464, + "step": 727 + }, + { + "epoch": 0.9060360920970753, + "grad_norm": 2.617842674255371, + "learning_rate": 3.911439114391144e-06, + "loss": 1.4539, + "step": 728 + }, + { + "epoch": 0.9072806471686372, + "grad_norm": 2.668649435043335, + "learning_rate": 3.909132841328414e-06, + "loss": 1.5627, + "step": 729 + }, + { + "epoch": 0.9085252022401992, + "grad_norm": 2.5331950187683105, + "learning_rate": 3.906826568265683e-06, + "loss": 1.5165, + "step": 730 + }, + { + "epoch": 0.9085252022401992, + "eval_loss": 1.6860331296920776, + "eval_runtime": 46.094, + "eval_samples_per_second": 21.695, + "eval_steps_per_second": 0.911, + "step": 730 + }, + { + "epoch": 0.909769757311761, + "grad_norm": 2.602492332458496, + "learning_rate": 3.9045202952029526e-06, + "loss": 1.5145, + "step": 731 + }, + { + "epoch": 0.911014312383323, + "grad_norm": 2.632782459259033, + "learning_rate": 3.902214022140222e-06, + "loss": 1.4413, + "step": 732 + }, + { + "epoch": 0.9122588674548848, + "grad_norm": 2.6025912761688232, + "learning_rate": 3.899907749077491e-06, + "loss": 1.5263, + "step": 733 + }, + { + "epoch": 0.9135034225264468, + "grad_norm": 2.770116090774536, + "learning_rate": 3.897601476014761e-06, + "loss": 1.5514, + "step": 734 + }, + { + "epoch": 0.9147479775980087, + "grad_norm": 2.7822961807250977, + "learning_rate": 3.895295202952029e-06, + "loss": 1.5296, + "step": 735 + }, + { + "epoch": 0.9159925326695706, + "grad_norm": 2.6689720153808594, + "learning_rate": 3.892988929889299e-06, + "loss": 1.4552, + "step": 736 + }, + { + "epoch": 0.9172370877411326, + "grad_norm": 2.8021798133850098, + "learning_rate": 3.890682656826569e-06, + "loss": 1.5308, + "step": 737 + }, + { + "epoch": 0.9184816428126945, + "grad_norm": 2.575313091278076, + "learning_rate": 3.888376383763838e-06, + "loss": 1.4807, + "step": 738 + }, + { + "epoch": 0.9197261978842564, + "grad_norm": 2.5955779552459717, + "learning_rate": 3.886070110701107e-06, + "loss": 1.4222, + "step": 739 + }, + { + "epoch": 0.9209707529558183, + "grad_norm": 2.647939920425415, + "learning_rate": 3.8837638376383765e-06, + "loss": 1.5285, + "step": 740 + }, + { + "epoch": 0.9209707529558183, + "eval_loss": 1.6907480955123901, + "eval_runtime": 45.7678, + "eval_samples_per_second": 21.849, + "eval_steps_per_second": 0.918, + "step": 740 + }, + { + "epoch": 0.9222153080273802, + "grad_norm": 2.7664523124694824, + "learning_rate": 3.881457564575646e-06, + "loss": 1.5349, + "step": 741 + }, + { + "epoch": 0.9234598630989421, + "grad_norm": 2.5770998001098633, + "learning_rate": 3.879151291512915e-06, + "loss": 1.5071, + "step": 742 + }, + { + "epoch": 0.9247044181705041, + "grad_norm": 2.502567768096924, + "learning_rate": 3.876845018450185e-06, + "loss": 1.492, + "step": 743 + }, + { + "epoch": 0.9259489732420659, + "grad_norm": 2.6531119346618652, + "learning_rate": 3.874538745387454e-06, + "loss": 1.5045, + "step": 744 + }, + { + "epoch": 0.9271935283136279, + "grad_norm": 2.7343876361846924, + "learning_rate": 3.872232472324724e-06, + "loss": 1.5194, + "step": 745 + }, + { + "epoch": 0.9284380833851898, + "grad_norm": 2.842313528060913, + "learning_rate": 3.869926199261993e-06, + "loss": 1.5588, + "step": 746 + }, + { + "epoch": 0.9296826384567517, + "grad_norm": 2.718245267868042, + "learning_rate": 3.8676199261992625e-06, + "loss": 1.5252, + "step": 747 + }, + { + "epoch": 0.9309271935283137, + "grad_norm": 2.695392370223999, + "learning_rate": 3.865313653136532e-06, + "loss": 1.5596, + "step": 748 + }, + { + "epoch": 0.9321717485998755, + "grad_norm": 2.7859959602355957, + "learning_rate": 3.863007380073801e-06, + "loss": 1.5151, + "step": 749 + }, + { + "epoch": 0.9334163036714375, + "grad_norm": 2.5199174880981445, + "learning_rate": 3.860701107011071e-06, + "loss": 1.4948, + "step": 750 + }, + { + "epoch": 0.9334163036714375, + "eval_loss": 1.6868674755096436, + "eval_runtime": 46.4115, + "eval_samples_per_second": 21.546, + "eval_steps_per_second": 0.905, + "step": 750 + }, + { + "epoch": 0.9346608587429994, + "grad_norm": 2.8194046020507812, + "learning_rate": 3.858394833948339e-06, + "loss": 1.5144, + "step": 751 + }, + { + "epoch": 0.9359054138145613, + "grad_norm": 2.810380697250366, + "learning_rate": 3.856088560885609e-06, + "loss": 1.4998, + "step": 752 + }, + { + "epoch": 0.9371499688861232, + "grad_norm": 2.6906750202178955, + "learning_rate": 3.853782287822879e-06, + "loss": 1.4748, + "step": 753 + }, + { + "epoch": 0.9383945239576851, + "grad_norm": 2.9030277729034424, + "learning_rate": 3.851476014760148e-06, + "loss": 1.615, + "step": 754 + }, + { + "epoch": 0.939639079029247, + "grad_norm": 2.8690176010131836, + "learning_rate": 3.849169741697417e-06, + "loss": 1.5083, + "step": 755 + }, + { + "epoch": 0.940883634100809, + "grad_norm": 2.669646978378296, + "learning_rate": 3.8468634686346865e-06, + "loss": 1.5572, + "step": 756 + }, + { + "epoch": 0.9421281891723708, + "grad_norm": 2.6517696380615234, + "learning_rate": 3.844557195571956e-06, + "loss": 1.5142, + "step": 757 + }, + { + "epoch": 0.9433727442439328, + "grad_norm": 2.5524444580078125, + "learning_rate": 3.842250922509225e-06, + "loss": 1.4924, + "step": 758 + }, + { + "epoch": 0.9446172993154948, + "grad_norm": 2.6532633304595947, + "learning_rate": 3.839944649446495e-06, + "loss": 1.484, + "step": 759 + }, + { + "epoch": 0.9458618543870566, + "grad_norm": 2.7779057025909424, + "learning_rate": 3.837638376383764e-06, + "loss": 1.5106, + "step": 760 + }, + { + "epoch": 0.9458618543870566, + "eval_loss": 1.6845752000808716, + "eval_runtime": 50.3415, + "eval_samples_per_second": 19.864, + "eval_steps_per_second": 0.834, + "step": 760 + }, + { + "epoch": 0.9471064094586186, + "grad_norm": 2.6192541122436523, + "learning_rate": 3.8353321033210336e-06, + "loss": 1.5454, + "step": 761 + }, + { + "epoch": 0.9483509645301804, + "grad_norm": 2.561861991882324, + "learning_rate": 3.833025830258303e-06, + "loss": 1.5141, + "step": 762 + }, + { + "epoch": 0.9495955196017424, + "grad_norm": 2.661829948425293, + "learning_rate": 3.830719557195572e-06, + "loss": 1.4997, + "step": 763 + }, + { + "epoch": 0.9508400746733043, + "grad_norm": 2.725275754928589, + "learning_rate": 3.828413284132842e-06, + "loss": 1.5224, + "step": 764 + }, + { + "epoch": 0.9520846297448662, + "grad_norm": 2.5664193630218506, + "learning_rate": 3.826107011070111e-06, + "loss": 1.5073, + "step": 765 + }, + { + "epoch": 0.9533291848164281, + "grad_norm": 2.595189332962036, + "learning_rate": 3.823800738007381e-06, + "loss": 1.5379, + "step": 766 + }, + { + "epoch": 0.95457373988799, + "grad_norm": 2.738060235977173, + "learning_rate": 3.821494464944649e-06, + "loss": 1.4814, + "step": 767 + }, + { + "epoch": 0.955818294959552, + "grad_norm": 2.601071357727051, + "learning_rate": 3.819188191881919e-06, + "loss": 1.4812, + "step": 768 + }, + { + "epoch": 0.9570628500311139, + "grad_norm": 2.657212257385254, + "learning_rate": 3.816881918819189e-06, + "loss": 1.4748, + "step": 769 + }, + { + "epoch": 0.9583074051026758, + "grad_norm": 2.7673351764678955, + "learning_rate": 3.814575645756458e-06, + "loss": 1.5535, + "step": 770 + }, + { + "epoch": 0.9583074051026758, + "eval_loss": 1.677243947982788, + "eval_runtime": 51.5537, + "eval_samples_per_second": 19.397, + "eval_steps_per_second": 0.815, + "step": 770 + }, + { + "epoch": 0.9595519601742377, + "grad_norm": 2.760890007019043, + "learning_rate": 3.812269372693727e-06, + "loss": 1.5654, + "step": 771 + }, + { + "epoch": 0.9607965152457997, + "grad_norm": 2.6934309005737305, + "learning_rate": 3.809963099630997e-06, + "loss": 1.5149, + "step": 772 + }, + { + "epoch": 0.9620410703173615, + "grad_norm": 2.729950428009033, + "learning_rate": 3.8076568265682662e-06, + "loss": 1.5579, + "step": 773 + }, + { + "epoch": 0.9632856253889235, + "grad_norm": 2.732926607131958, + "learning_rate": 3.8053505535055352e-06, + "loss": 1.5179, + "step": 774 + }, + { + "epoch": 0.9645301804604853, + "grad_norm": 2.745391368865967, + "learning_rate": 3.8030442804428046e-06, + "loss": 1.5124, + "step": 775 + }, + { + "epoch": 0.9657747355320473, + "grad_norm": 2.8050146102905273, + "learning_rate": 3.800738007380074e-06, + "loss": 1.519, + "step": 776 + }, + { + "epoch": 0.9670192906036092, + "grad_norm": 2.697171449661255, + "learning_rate": 3.798431734317343e-06, + "loss": 1.5219, + "step": 777 + }, + { + "epoch": 0.9682638456751711, + "grad_norm": 2.6375980377197266, + "learning_rate": 3.796125461254613e-06, + "loss": 1.5345, + "step": 778 + }, + { + "epoch": 0.9695084007467331, + "grad_norm": 2.586636781692505, + "learning_rate": 3.7938191881918823e-06, + "loss": 1.5182, + "step": 779 + }, + { + "epoch": 0.9707529558182949, + "grad_norm": 2.626453399658203, + "learning_rate": 3.7915129151291518e-06, + "loss": 1.4817, + "step": 780 + }, + { + "epoch": 0.9707529558182949, + "eval_loss": 1.6734713315963745, + "eval_runtime": 48.1436, + "eval_samples_per_second": 20.771, + "eval_steps_per_second": 0.872, + "step": 780 + }, + { + "epoch": 0.9719975108898569, + "grad_norm": 2.579970121383667, + "learning_rate": 3.7892066420664208e-06, + "loss": 1.4981, + "step": 781 + }, + { + "epoch": 0.9732420659614188, + "grad_norm": 2.6834843158721924, + "learning_rate": 3.7869003690036906e-06, + "loss": 1.5044, + "step": 782 + }, + { + "epoch": 0.9744866210329807, + "grad_norm": 2.586982488632202, + "learning_rate": 3.78459409594096e-06, + "loss": 1.5079, + "step": 783 + }, + { + "epoch": 0.9757311761045426, + "grad_norm": 2.910027503967285, + "learning_rate": 3.782287822878229e-06, + "loss": 1.5695, + "step": 784 + }, + { + "epoch": 0.9769757311761046, + "grad_norm": 2.7088494300842285, + "learning_rate": 3.7799815498154984e-06, + "loss": 1.4986, + "step": 785 + }, + { + "epoch": 0.9782202862476664, + "grad_norm": 2.581325054168701, + "learning_rate": 3.777675276752768e-06, + "loss": 1.5137, + "step": 786 + }, + { + "epoch": 0.9794648413192284, + "grad_norm": 2.700709819793701, + "learning_rate": 3.775369003690037e-06, + "loss": 1.4718, + "step": 787 + }, + { + "epoch": 0.9807093963907902, + "grad_norm": 2.6712708473205566, + "learning_rate": 3.7730627306273067e-06, + "loss": 1.5208, + "step": 788 + }, + { + "epoch": 0.9819539514623522, + "grad_norm": 2.6799817085266113, + "learning_rate": 3.770756457564576e-06, + "loss": 1.5374, + "step": 789 + }, + { + "epoch": 0.9831985065339142, + "grad_norm": 2.618988037109375, + "learning_rate": 3.768450184501845e-06, + "loss": 1.4997, + "step": 790 + }, + { + "epoch": 0.9831985065339142, + "eval_loss": 1.6860820055007935, + "eval_runtime": 51.7654, + "eval_samples_per_second": 19.318, + "eval_steps_per_second": 0.811, + "step": 790 + }, + { + "epoch": 0.984443061605476, + "grad_norm": 2.6899526119232178, + "learning_rate": 3.7661439114391146e-06, + "loss": 1.5216, + "step": 791 + }, + { + "epoch": 0.985687616677038, + "grad_norm": 2.8003487586975098, + "learning_rate": 3.763837638376384e-06, + "loss": 1.557, + "step": 792 + }, + { + "epoch": 0.9869321717485999, + "grad_norm": 2.744536876678467, + "learning_rate": 3.761531365313654e-06, + "loss": 1.5355, + "step": 793 + }, + { + "epoch": 0.9881767268201618, + "grad_norm": 2.587250232696533, + "learning_rate": 3.759225092250923e-06, + "loss": 1.5259, + "step": 794 + }, + { + "epoch": 0.9894212818917237, + "grad_norm": 2.616291046142578, + "learning_rate": 3.7569188191881922e-06, + "loss": 1.5176, + "step": 795 + }, + { + "epoch": 0.9906658369632856, + "grad_norm": 2.6410577297210693, + "learning_rate": 3.7546125461254617e-06, + "loss": 1.511, + "step": 796 + }, + { + "epoch": 0.9919103920348475, + "grad_norm": 2.577373504638672, + "learning_rate": 3.7523062730627307e-06, + "loss": 1.4715, + "step": 797 + }, + { + "epoch": 0.9931549471064095, + "grad_norm": 2.67305326461792, + "learning_rate": 3.7500000000000005e-06, + "loss": 1.4828, + "step": 798 + }, + { + "epoch": 0.9943995021779714, + "grad_norm": 3.1631500720977783, + "learning_rate": 3.74769372693727e-06, + "loss": 1.5602, + "step": 799 + }, + { + "epoch": 0.9956440572495333, + "grad_norm": 2.9222350120544434, + "learning_rate": 3.745387453874539e-06, + "loss": 1.4814, + "step": 800 + }, + { + "epoch": 0.9956440572495333, + "eval_loss": 1.6810544729232788, + "eval_runtime": 51.1262, + "eval_samples_per_second": 19.559, + "eval_steps_per_second": 0.821, + "step": 800 + }, + { + "epoch": 0.9968886123210952, + "grad_norm": 2.884312629699707, + "learning_rate": 3.7430811808118084e-06, + "loss": 1.4553, + "step": 801 + }, + { + "epoch": 0.9981331673926571, + "grad_norm": 2.712655782699585, + "learning_rate": 3.7407749077490778e-06, + "loss": 1.5275, + "step": 802 + }, + { + "epoch": 0.9993777224642191, + "grad_norm": 2.8700571060180664, + "learning_rate": 3.7384686346863468e-06, + "loss": 1.4923, + "step": 803 + }, + { + "epoch": 1.000622277535781, + "grad_norm": 2.7779245376586914, + "learning_rate": 3.7361623616236166e-06, + "loss": 1.5224, + "step": 804 + }, + { + "epoch": 1.0018668326073428, + "grad_norm": 2.7679810523986816, + "learning_rate": 3.733856088560886e-06, + "loss": 1.4977, + "step": 805 + }, + { + "epoch": 1.0031113876789048, + "grad_norm": 2.797023296356201, + "learning_rate": 3.7315498154981555e-06, + "loss": 1.4396, + "step": 806 + }, + { + "epoch": 1.0043559427504667, + "grad_norm": 3.007962465286255, + "learning_rate": 3.7292435424354245e-06, + "loss": 1.5015, + "step": 807 + }, + { + "epoch": 1.0056004978220285, + "grad_norm": 3.127639055252075, + "learning_rate": 3.726937269372694e-06, + "loss": 1.5176, + "step": 808 + }, + { + "epoch": 1.0068450528935906, + "grad_norm": 2.9001357555389404, + "learning_rate": 3.7246309963099637e-06, + "loss": 1.4735, + "step": 809 + }, + { + "epoch": 1.0080896079651525, + "grad_norm": 2.8245413303375244, + "learning_rate": 3.7223247232472327e-06, + "loss": 1.486, + "step": 810 + }, + { + "epoch": 1.0080896079651525, + "eval_loss": 1.6835130453109741, + "eval_runtime": 44.5682, + "eval_samples_per_second": 22.438, + "eval_steps_per_second": 0.942, + "step": 810 + }, + { + "epoch": 1.0093341630367143, + "grad_norm": 2.780172824859619, + "learning_rate": 3.720018450184502e-06, + "loss": 1.4437, + "step": 811 + }, + { + "epoch": 1.0105787181082764, + "grad_norm": 2.6013996601104736, + "learning_rate": 3.7177121771217716e-06, + "loss": 1.4529, + "step": 812 + }, + { + "epoch": 1.0118232731798382, + "grad_norm": 2.831015110015869, + "learning_rate": 3.7154059040590406e-06, + "loss": 1.4825, + "step": 813 + }, + { + "epoch": 1.0130678282514, + "grad_norm": 2.8341829776763916, + "learning_rate": 3.71309963099631e-06, + "loss": 1.4736, + "step": 814 + }, + { + "epoch": 1.0143123833229621, + "grad_norm": 2.769540309906006, + "learning_rate": 3.71079335793358e-06, + "loss": 1.4724, + "step": 815 + }, + { + "epoch": 1.015556938394524, + "grad_norm": 3.0083069801330566, + "learning_rate": 3.708487084870849e-06, + "loss": 1.4985, + "step": 816 + }, + { + "epoch": 1.0168014934660858, + "grad_norm": 3.052168607711792, + "learning_rate": 3.7061808118081183e-06, + "loss": 1.5193, + "step": 817 + }, + { + "epoch": 1.0180460485376477, + "grad_norm": 2.924877405166626, + "learning_rate": 3.7038745387453877e-06, + "loss": 1.5219, + "step": 818 + }, + { + "epoch": 1.0192906036092098, + "grad_norm": 3.150123357772827, + "learning_rate": 3.7015682656826576e-06, + "loss": 1.5026, + "step": 819 + }, + { + "epoch": 1.0205351586807716, + "grad_norm": 2.881655216217041, + "learning_rate": 3.6992619926199266e-06, + "loss": 1.5011, + "step": 820 + }, + { + "epoch": 1.0205351586807716, + "eval_loss": 1.6745303869247437, + "eval_runtime": 50.6791, + "eval_samples_per_second": 19.732, + "eval_steps_per_second": 0.829, + "step": 820 + }, + { + "epoch": 1.0217797137523335, + "grad_norm": 2.824249505996704, + "learning_rate": 3.696955719557196e-06, + "loss": 1.4642, + "step": 821 + }, + { + "epoch": 1.0230242688238955, + "grad_norm": 2.859144926071167, + "learning_rate": 3.6946494464944654e-06, + "loss": 1.4681, + "step": 822 + }, + { + "epoch": 1.0242688238954574, + "grad_norm": 2.90312123298645, + "learning_rate": 3.6923431734317344e-06, + "loss": 1.4926, + "step": 823 + }, + { + "epoch": 1.0255133789670192, + "grad_norm": 2.8340659141540527, + "learning_rate": 3.690036900369004e-06, + "loss": 1.4834, + "step": 824 + }, + { + "epoch": 1.0267579340385813, + "grad_norm": 2.9743151664733887, + "learning_rate": 3.6877306273062737e-06, + "loss": 1.4574, + "step": 825 + }, + { + "epoch": 1.0280024891101431, + "grad_norm": 3.032179594039917, + "learning_rate": 3.6854243542435427e-06, + "loss": 1.5571, + "step": 826 + }, + { + "epoch": 1.029247044181705, + "grad_norm": 2.816826343536377, + "learning_rate": 3.683118081180812e-06, + "loss": 1.455, + "step": 827 + }, + { + "epoch": 1.030491599253267, + "grad_norm": 3.046696186065674, + "learning_rate": 3.6808118081180815e-06, + "loss": 1.4871, + "step": 828 + }, + { + "epoch": 1.031736154324829, + "grad_norm": 2.8517391681671143, + "learning_rate": 3.678505535055351e-06, + "loss": 1.5292, + "step": 829 + }, + { + "epoch": 1.0329807093963908, + "grad_norm": 2.752811908721924, + "learning_rate": 3.67619926199262e-06, + "loss": 1.5091, + "step": 830 + }, + { + "epoch": 1.0329807093963908, + "eval_loss": 1.6692287921905518, + "eval_runtime": 50.8347, + "eval_samples_per_second": 19.672, + "eval_steps_per_second": 0.826, + "step": 830 + }, + { + "epoch": 1.0342252644679526, + "grad_norm": 2.9220356941223145, + "learning_rate": 3.6738929889298898e-06, + "loss": 1.4247, + "step": 831 + }, + { + "epoch": 1.0354698195395147, + "grad_norm": 2.8903002738952637, + "learning_rate": 3.671586715867159e-06, + "loss": 1.5595, + "step": 832 + }, + { + "epoch": 1.0367143746110765, + "grad_norm": 2.6495542526245117, + "learning_rate": 3.669280442804428e-06, + "loss": 1.4076, + "step": 833 + }, + { + "epoch": 1.0379589296826384, + "grad_norm": 2.880809783935547, + "learning_rate": 3.6669741697416976e-06, + "loss": 1.5285, + "step": 834 + }, + { + "epoch": 1.0392034847542004, + "grad_norm": 2.919142007827759, + "learning_rate": 3.6646678966789675e-06, + "loss": 1.5165, + "step": 835 + }, + { + "epoch": 1.0404480398257623, + "grad_norm": 3.1180522441864014, + "learning_rate": 3.6623616236162365e-06, + "loss": 1.5224, + "step": 836 + }, + { + "epoch": 1.0416925948973241, + "grad_norm": 2.767709255218506, + "learning_rate": 3.660055350553506e-06, + "loss": 1.4859, + "step": 837 + }, + { + "epoch": 1.0429371499688862, + "grad_norm": 2.708828926086426, + "learning_rate": 3.6577490774907753e-06, + "loss": 1.4803, + "step": 838 + }, + { + "epoch": 1.044181705040448, + "grad_norm": 2.688359260559082, + "learning_rate": 3.6554428044280443e-06, + "loss": 1.4571, + "step": 839 + }, + { + "epoch": 1.04542626011201, + "grad_norm": 2.691812753677368, + "learning_rate": 3.6531365313653137e-06, + "loss": 1.5043, + "step": 840 + }, + { + "epoch": 1.04542626011201, + "eval_loss": 1.6769559383392334, + "eval_runtime": 49.6233, + "eval_samples_per_second": 20.152, + "eval_steps_per_second": 0.846, + "step": 840 + }, + { + "epoch": 1.046670815183572, + "grad_norm": 2.686372995376587, + "learning_rate": 3.6508302583025836e-06, + "loss": 1.5221, + "step": 841 + }, + { + "epoch": 1.0479153702551338, + "grad_norm": 2.742859125137329, + "learning_rate": 3.648523985239853e-06, + "loss": 1.4685, + "step": 842 + }, + { + "epoch": 1.0491599253266957, + "grad_norm": 2.874018907546997, + "learning_rate": 3.646217712177122e-06, + "loss": 1.4813, + "step": 843 + }, + { + "epoch": 1.0504044803982575, + "grad_norm": 2.8159990310668945, + "learning_rate": 3.6439114391143914e-06, + "loss": 1.4421, + "step": 844 + }, + { + "epoch": 1.0516490354698196, + "grad_norm": 2.714787006378174, + "learning_rate": 3.641605166051661e-06, + "loss": 1.4879, + "step": 845 + }, + { + "epoch": 1.0528935905413814, + "grad_norm": 2.885148525238037, + "learning_rate": 3.63929889298893e-06, + "loss": 1.4725, + "step": 846 + }, + { + "epoch": 1.0541381456129433, + "grad_norm": 2.7272610664367676, + "learning_rate": 3.6369926199261997e-06, + "loss": 1.4276, + "step": 847 + }, + { + "epoch": 1.0553827006845053, + "grad_norm": 2.9114978313446045, + "learning_rate": 3.634686346863469e-06, + "loss": 1.4194, + "step": 848 + }, + { + "epoch": 1.0566272557560672, + "grad_norm": 3.019313335418701, + "learning_rate": 3.632380073800738e-06, + "loss": 1.5266, + "step": 849 + }, + { + "epoch": 1.057871810827629, + "grad_norm": 2.714751958847046, + "learning_rate": 3.6300738007380075e-06, + "loss": 1.4593, + "step": 850 + }, + { + "epoch": 1.057871810827629, + "eval_loss": 1.6760298013687134, + "eval_runtime": 52.1681, + "eval_samples_per_second": 19.169, + "eval_steps_per_second": 0.805, + "step": 850 + }, + { + "epoch": 1.0591163658991911, + "grad_norm": 3.0507936477661133, + "learning_rate": 3.627767527675277e-06, + "loss": 1.5064, + "step": 851 + }, + { + "epoch": 1.060360920970753, + "grad_norm": 2.8116295337677, + "learning_rate": 3.625461254612546e-06, + "loss": 1.5218, + "step": 852 + }, + { + "epoch": 1.0616054760423148, + "grad_norm": 2.8801465034484863, + "learning_rate": 3.623154981549816e-06, + "loss": 1.4641, + "step": 853 + }, + { + "epoch": 1.0628500311138769, + "grad_norm": 3.0581862926483154, + "learning_rate": 3.6208487084870852e-06, + "loss": 1.483, + "step": 854 + }, + { + "epoch": 1.0640945861854387, + "grad_norm": 2.898780584335327, + "learning_rate": 3.6185424354243547e-06, + "loss": 1.4623, + "step": 855 + }, + { + "epoch": 1.0653391412570006, + "grad_norm": 2.971820116043091, + "learning_rate": 3.6162361623616237e-06, + "loss": 1.4455, + "step": 856 + }, + { + "epoch": 1.0665836963285624, + "grad_norm": 3.0021913051605225, + "learning_rate": 3.6139298892988935e-06, + "loss": 1.4878, + "step": 857 + }, + { + "epoch": 1.0678282514001245, + "grad_norm": 3.0697457790374756, + "learning_rate": 3.611623616236163e-06, + "loss": 1.5008, + "step": 858 + }, + { + "epoch": 1.0690728064716863, + "grad_norm": 3.0787923336029053, + "learning_rate": 3.609317343173432e-06, + "loss": 1.4612, + "step": 859 + }, + { + "epoch": 1.0703173615432482, + "grad_norm": 2.789339780807495, + "learning_rate": 3.6070110701107014e-06, + "loss": 1.4287, + "step": 860 + }, + { + "epoch": 1.0703173615432482, + "eval_loss": 1.6747506856918335, + "eval_runtime": 51.4912, + "eval_samples_per_second": 19.421, + "eval_steps_per_second": 0.816, + "step": 860 + }, + { + "epoch": 1.0715619166148103, + "grad_norm": 2.9622788429260254, + "learning_rate": 3.6047047970479708e-06, + "loss": 1.4905, + "step": 861 + }, + { + "epoch": 1.072806471686372, + "grad_norm": 3.0675268173217773, + "learning_rate": 3.6023985239852398e-06, + "loss": 1.461, + "step": 862 + }, + { + "epoch": 1.074051026757934, + "grad_norm": 2.820996046066284, + "learning_rate": 3.6000922509225096e-06, + "loss": 1.4948, + "step": 863 + }, + { + "epoch": 1.075295581829496, + "grad_norm": 2.833761215209961, + "learning_rate": 3.597785977859779e-06, + "loss": 1.4683, + "step": 864 + }, + { + "epoch": 1.0765401369010579, + "grad_norm": 2.917313814163208, + "learning_rate": 3.595479704797048e-06, + "loss": 1.5191, + "step": 865 + }, + { + "epoch": 1.0777846919726197, + "grad_norm": 2.876166820526123, + "learning_rate": 3.5931734317343175e-06, + "loss": 1.4854, + "step": 866 + }, + { + "epoch": 1.0790292470441818, + "grad_norm": 3.139049768447876, + "learning_rate": 3.590867158671587e-06, + "loss": 1.5134, + "step": 867 + }, + { + "epoch": 1.0802738021157436, + "grad_norm": 2.9956753253936768, + "learning_rate": 3.5885608856088567e-06, + "loss": 1.5003, + "step": 868 + }, + { + "epoch": 1.0815183571873055, + "grad_norm": 2.957181930541992, + "learning_rate": 3.5862546125461257e-06, + "loss": 1.4588, + "step": 869 + }, + { + "epoch": 1.0827629122588673, + "grad_norm": 2.8576438426971436, + "learning_rate": 3.583948339483395e-06, + "loss": 1.48, + "step": 870 + }, + { + "epoch": 1.0827629122588673, + "eval_loss": 1.6724848747253418, + "eval_runtime": 50.9669, + "eval_samples_per_second": 19.621, + "eval_steps_per_second": 0.824, + "step": 870 + }, + { + "epoch": 1.0840074673304294, + "grad_norm": 2.8183560371398926, + "learning_rate": 3.5816420664206646e-06, + "loss": 1.4818, + "step": 871 + }, + { + "epoch": 1.0852520224019913, + "grad_norm": 2.7531797885894775, + "learning_rate": 3.5793357933579336e-06, + "loss": 1.4712, + "step": 872 + }, + { + "epoch": 1.086496577473553, + "grad_norm": 2.923962354660034, + "learning_rate": 3.5770295202952034e-06, + "loss": 1.4864, + "step": 873 + }, + { + "epoch": 1.0877411325451152, + "grad_norm": 2.851724147796631, + "learning_rate": 3.574723247232473e-06, + "loss": 1.4771, + "step": 874 + }, + { + "epoch": 1.088985687616677, + "grad_norm": 3.024634838104248, + "learning_rate": 3.572416974169742e-06, + "loss": 1.4643, + "step": 875 + }, + { + "epoch": 1.0902302426882389, + "grad_norm": 2.7100844383239746, + "learning_rate": 3.5701107011070113e-06, + "loss": 1.4773, + "step": 876 + }, + { + "epoch": 1.091474797759801, + "grad_norm": 2.730502128601074, + "learning_rate": 3.5678044280442807e-06, + "loss": 1.5051, + "step": 877 + }, + { + "epoch": 1.0927193528313628, + "grad_norm": 2.747082233428955, + "learning_rate": 3.5654981549815497e-06, + "loss": 1.5207, + "step": 878 + }, + { + "epoch": 1.0939639079029246, + "grad_norm": 2.718358039855957, + "learning_rate": 3.5631918819188195e-06, + "loss": 1.5067, + "step": 879 + }, + { + "epoch": 1.0952084629744867, + "grad_norm": 2.7794573307037354, + "learning_rate": 3.560885608856089e-06, + "loss": 1.4616, + "step": 880 + }, + { + "epoch": 1.0952084629744867, + "eval_loss": 1.6676132678985596, + "eval_runtime": 51.6157, + "eval_samples_per_second": 19.374, + "eval_steps_per_second": 0.814, + "step": 880 + }, + { + "epoch": 1.0964530180460486, + "grad_norm": 2.727416753768921, + "learning_rate": 3.5585793357933584e-06, + "loss": 1.4043, + "step": 881 + }, + { + "epoch": 1.0976975731176104, + "grad_norm": 2.856900930404663, + "learning_rate": 3.5562730627306274e-06, + "loss": 1.5198, + "step": 882 + }, + { + "epoch": 1.0989421281891725, + "grad_norm": 2.872823715209961, + "learning_rate": 3.553966789667897e-06, + "loss": 1.4981, + "step": 883 + }, + { + "epoch": 1.1001866832607343, + "grad_norm": 2.7724361419677734, + "learning_rate": 3.5516605166051667e-06, + "loss": 1.4819, + "step": 884 + }, + { + "epoch": 1.1014312383322962, + "grad_norm": 2.884434461593628, + "learning_rate": 3.5493542435424357e-06, + "loss": 1.433, + "step": 885 + }, + { + "epoch": 1.102675793403858, + "grad_norm": 2.887873411178589, + "learning_rate": 3.547047970479705e-06, + "loss": 1.4677, + "step": 886 + }, + { + "epoch": 1.10392034847542, + "grad_norm": 2.946845054626465, + "learning_rate": 3.5447416974169745e-06, + "loss": 1.4983, + "step": 887 + }, + { + "epoch": 1.105164903546982, + "grad_norm": 2.752552032470703, + "learning_rate": 3.5424354243542435e-06, + "loss": 1.4766, + "step": 888 + }, + { + "epoch": 1.1064094586185438, + "grad_norm": 2.788634777069092, + "learning_rate": 3.5401291512915133e-06, + "loss": 1.4747, + "step": 889 + }, + { + "epoch": 1.1076540136901059, + "grad_norm": 2.782792568206787, + "learning_rate": 3.5378228782287828e-06, + "loss": 1.5503, + "step": 890 + }, + { + "epoch": 1.1076540136901059, + "eval_loss": 1.6698333024978638, + "eval_runtime": 49.1373, + "eval_samples_per_second": 20.351, + "eval_steps_per_second": 0.855, + "step": 890 + }, + { + "epoch": 1.1088985687616677, + "grad_norm": 2.8351166248321533, + "learning_rate": 3.535516605166052e-06, + "loss": 1.4589, + "step": 891 + }, + { + "epoch": 1.1101431238332296, + "grad_norm": 2.798872709274292, + "learning_rate": 3.533210332103321e-06, + "loss": 1.4496, + "step": 892 + }, + { + "epoch": 1.1113876789047916, + "grad_norm": 3.0863239765167236, + "learning_rate": 3.5309040590405906e-06, + "loss": 1.4698, + "step": 893 + }, + { + "epoch": 1.1126322339763535, + "grad_norm": 2.853862762451172, + "learning_rate": 3.5285977859778605e-06, + "loss": 1.4429, + "step": 894 + }, + { + "epoch": 1.1138767890479153, + "grad_norm": 2.8946170806884766, + "learning_rate": 3.5262915129151295e-06, + "loss": 1.4697, + "step": 895 + }, + { + "epoch": 1.1151213441194774, + "grad_norm": 3.043823003768921, + "learning_rate": 3.523985239852399e-06, + "loss": 1.4666, + "step": 896 + }, + { + "epoch": 1.1163658991910392, + "grad_norm": 2.7822980880737305, + "learning_rate": 3.5216789667896683e-06, + "loss": 1.4523, + "step": 897 + }, + { + "epoch": 1.117610454262601, + "grad_norm": 2.9454171657562256, + "learning_rate": 3.5193726937269373e-06, + "loss": 1.5009, + "step": 898 + }, + { + "epoch": 1.1188550093341632, + "grad_norm": 2.8533740043640137, + "learning_rate": 3.5170664206642067e-06, + "loss": 1.4265, + "step": 899 + }, + { + "epoch": 1.120099564405725, + "grad_norm": 2.894937038421631, + "learning_rate": 3.5147601476014766e-06, + "loss": 1.4895, + "step": 900 + }, + { + "epoch": 1.120099564405725, + "eval_loss": 1.6701407432556152, + "eval_runtime": 45.477, + "eval_samples_per_second": 21.989, + "eval_steps_per_second": 0.924, + "step": 900 + }, + { + "epoch": 1.1213441194772868, + "grad_norm": 2.933821201324463, + "learning_rate": 3.5124538745387456e-06, + "loss": 1.468, + "step": 901 + }, + { + "epoch": 1.1225886745488487, + "grad_norm": 2.834265947341919, + "learning_rate": 3.510147601476015e-06, + "loss": 1.4621, + "step": 902 + }, + { + "epoch": 1.1238332296204108, + "grad_norm": 2.9064722061157227, + "learning_rate": 3.5078413284132844e-06, + "loss": 1.504, + "step": 903 + }, + { + "epoch": 1.1250777846919726, + "grad_norm": 2.8753886222839355, + "learning_rate": 3.505535055350554e-06, + "loss": 1.4643, + "step": 904 + }, + { + "epoch": 1.1263223397635345, + "grad_norm": 2.894659996032715, + "learning_rate": 3.503228782287823e-06, + "loss": 1.4909, + "step": 905 + }, + { + "epoch": 1.1275668948350965, + "grad_norm": 3.085977792739868, + "learning_rate": 3.5009225092250927e-06, + "loss": 1.459, + "step": 906 + }, + { + "epoch": 1.1288114499066584, + "grad_norm": 2.9958064556121826, + "learning_rate": 3.498616236162362e-06, + "loss": 1.4152, + "step": 907 + }, + { + "epoch": 1.1300560049782202, + "grad_norm": 2.9136762619018555, + "learning_rate": 3.496309963099631e-06, + "loss": 1.473, + "step": 908 + }, + { + "epoch": 1.1313005600497823, + "grad_norm": 2.956437826156616, + "learning_rate": 3.4940036900369005e-06, + "loss": 1.5167, + "step": 909 + }, + { + "epoch": 1.1325451151213441, + "grad_norm": 2.7354865074157715, + "learning_rate": 3.4916974169741704e-06, + "loss": 1.4413, + "step": 910 + }, + { + "epoch": 1.1325451151213441, + "eval_loss": 1.6629912853240967, + "eval_runtime": 41.5478, + "eval_samples_per_second": 24.069, + "eval_steps_per_second": 1.011, + "step": 910 + }, + { + "epoch": 1.133789670192906, + "grad_norm": 2.7443008422851562, + "learning_rate": 3.4893911439114394e-06, + "loss": 1.4475, + "step": 911 + }, + { + "epoch": 1.135034225264468, + "grad_norm": 2.7811238765716553, + "learning_rate": 3.487084870848709e-06, + "loss": 1.4257, + "step": 912 + }, + { + "epoch": 1.13627878033603, + "grad_norm": 2.7916479110717773, + "learning_rate": 3.4847785977859782e-06, + "loss": 1.4048, + "step": 913 + }, + { + "epoch": 1.1375233354075918, + "grad_norm": 2.7835633754730225, + "learning_rate": 3.4824723247232472e-06, + "loss": 1.3996, + "step": 914 + }, + { + "epoch": 1.1387678904791536, + "grad_norm": 2.8188705444335938, + "learning_rate": 3.4801660516605166e-06, + "loss": 1.4541, + "step": 915 + }, + { + "epoch": 1.1400124455507157, + "grad_norm": 3.0134100914001465, + "learning_rate": 3.4778597785977865e-06, + "loss": 1.4353, + "step": 916 + }, + { + "epoch": 1.1412570006222775, + "grad_norm": 2.6965606212615967, + "learning_rate": 3.475553505535056e-06, + "loss": 1.4308, + "step": 917 + }, + { + "epoch": 1.1425015556938394, + "grad_norm": 2.8179919719696045, + "learning_rate": 3.473247232472325e-06, + "loss": 1.5204, + "step": 918 + }, + { + "epoch": 1.1437461107654014, + "grad_norm": 2.7213902473449707, + "learning_rate": 3.4709409594095943e-06, + "loss": 1.4344, + "step": 919 + }, + { + "epoch": 1.1449906658369633, + "grad_norm": 2.850193738937378, + "learning_rate": 3.4686346863468638e-06, + "loss": 1.473, + "step": 920 + }, + { + "epoch": 1.1449906658369633, + "eval_loss": 1.663345217704773, + "eval_runtime": 43.8397, + "eval_samples_per_second": 22.81, + "eval_steps_per_second": 0.958, + "step": 920 + }, + { + "epoch": 1.1462352209085251, + "grad_norm": 2.780198574066162, + "learning_rate": 3.4663284132841328e-06, + "loss": 1.4472, + "step": 921 + }, + { + "epoch": 1.1474797759800872, + "grad_norm": 2.9373831748962402, + "learning_rate": 3.4640221402214026e-06, + "loss": 1.426, + "step": 922 + }, + { + "epoch": 1.148724331051649, + "grad_norm": 2.6986873149871826, + "learning_rate": 3.461715867158672e-06, + "loss": 1.3953, + "step": 923 + }, + { + "epoch": 1.149968886123211, + "grad_norm": 2.927441120147705, + "learning_rate": 3.459409594095941e-06, + "loss": 1.4578, + "step": 924 + }, + { + "epoch": 1.151213441194773, + "grad_norm": 2.8469648361206055, + "learning_rate": 3.4571033210332105e-06, + "loss": 1.4313, + "step": 925 + }, + { + "epoch": 1.1524579962663348, + "grad_norm": 2.8114013671875, + "learning_rate": 3.4547970479704803e-06, + "loss": 1.4267, + "step": 926 + }, + { + "epoch": 1.1537025513378967, + "grad_norm": 2.977605104446411, + "learning_rate": 3.4524907749077493e-06, + "loss": 1.4747, + "step": 927 + }, + { + "epoch": 1.1549471064094585, + "grad_norm": 2.9327852725982666, + "learning_rate": 3.4501845018450187e-06, + "loss": 1.4219, + "step": 928 + }, + { + "epoch": 1.1561916614810206, + "grad_norm": 2.844038963317871, + "learning_rate": 3.447878228782288e-06, + "loss": 1.5146, + "step": 929 + }, + { + "epoch": 1.1574362165525824, + "grad_norm": 2.9402201175689697, + "learning_rate": 3.4455719557195576e-06, + "loss": 1.45, + "step": 930 + }, + { + "epoch": 1.1574362165525824, + "eval_loss": 1.6620122194290161, + "eval_runtime": 46.1748, + "eval_samples_per_second": 21.657, + "eval_steps_per_second": 0.91, + "step": 930 + }, + { + "epoch": 1.1586807716241443, + "grad_norm": 2.842651128768921, + "learning_rate": 3.4432656826568266e-06, + "loss": 1.4415, + "step": 931 + }, + { + "epoch": 1.1599253266957064, + "grad_norm": 2.8909685611724854, + "learning_rate": 3.4409594095940964e-06, + "loss": 1.4529, + "step": 932 + }, + { + "epoch": 1.1611698817672682, + "grad_norm": 2.8025808334350586, + "learning_rate": 3.438653136531366e-06, + "loss": 1.4552, + "step": 933 + }, + { + "epoch": 1.16241443683883, + "grad_norm": 2.6995794773101807, + "learning_rate": 3.436346863468635e-06, + "loss": 1.4435, + "step": 934 + }, + { + "epoch": 1.1636589919103921, + "grad_norm": 2.7307121753692627, + "learning_rate": 3.4340405904059043e-06, + "loss": 1.4447, + "step": 935 + }, + { + "epoch": 1.164903546981954, + "grad_norm": 2.9532501697540283, + "learning_rate": 3.4317343173431737e-06, + "loss": 1.4747, + "step": 936 + }, + { + "epoch": 1.1661481020535158, + "grad_norm": 3.0135788917541504, + "learning_rate": 3.4294280442804427e-06, + "loss": 1.525, + "step": 937 + }, + { + "epoch": 1.167392657125078, + "grad_norm": 2.796355962753296, + "learning_rate": 3.4271217712177125e-06, + "loss": 1.4539, + "step": 938 + }, + { + "epoch": 1.1686372121966397, + "grad_norm": 2.9422309398651123, + "learning_rate": 3.424815498154982e-06, + "loss": 1.4664, + "step": 939 + }, + { + "epoch": 1.1698817672682016, + "grad_norm": 2.7680039405822754, + "learning_rate": 3.4225092250922514e-06, + "loss": 1.4837, + "step": 940 + }, + { + "epoch": 1.1698817672682016, + "eval_loss": 1.660908818244934, + "eval_runtime": 45.6998, + "eval_samples_per_second": 21.882, + "eval_steps_per_second": 0.919, + "step": 940 + }, + { + "epoch": 1.1711263223397634, + "grad_norm": 2.8888845443725586, + "learning_rate": 3.4202029520295204e-06, + "loss": 1.5315, + "step": 941 + }, + { + "epoch": 1.1723708774113255, + "grad_norm": 2.7124130725860596, + "learning_rate": 3.41789667896679e-06, + "loss": 1.4188, + "step": 942 + }, + { + "epoch": 1.1736154324828874, + "grad_norm": 2.75203013420105, + "learning_rate": 3.4155904059040596e-06, + "loss": 1.4533, + "step": 943 + }, + { + "epoch": 1.1748599875544492, + "grad_norm": 2.865480422973633, + "learning_rate": 3.4132841328413286e-06, + "loss": 1.4532, + "step": 944 + }, + { + "epoch": 1.1761045426260113, + "grad_norm": 2.8618321418762207, + "learning_rate": 3.410977859778598e-06, + "loss": 1.4386, + "step": 945 + }, + { + "epoch": 1.1773490976975731, + "grad_norm": 2.7656145095825195, + "learning_rate": 3.4086715867158675e-06, + "loss": 1.3996, + "step": 946 + }, + { + "epoch": 1.178593652769135, + "grad_norm": 2.76213002204895, + "learning_rate": 3.4063653136531365e-06, + "loss": 1.4979, + "step": 947 + }, + { + "epoch": 1.179838207840697, + "grad_norm": 2.8581957817077637, + "learning_rate": 3.4040590405904063e-06, + "loss": 1.4673, + "step": 948 + }, + { + "epoch": 1.1810827629122589, + "grad_norm": 2.7825586795806885, + "learning_rate": 3.4017527675276758e-06, + "loss": 1.4453, + "step": 949 + }, + { + "epoch": 1.1823273179838207, + "grad_norm": 2.9226131439208984, + "learning_rate": 3.3994464944649448e-06, + "loss": 1.4879, + "step": 950 + }, + { + "epoch": 1.1823273179838207, + "eval_loss": 1.6621263027191162, + "eval_runtime": 45.1222, + "eval_samples_per_second": 22.162, + "eval_steps_per_second": 0.931, + "step": 950 + }, + { + "epoch": 1.1835718730553828, + "grad_norm": 2.9997975826263428, + "learning_rate": 3.397140221402214e-06, + "loss": 1.505, + "step": 951 + }, + { + "epoch": 1.1848164281269447, + "grad_norm": 2.7225453853607178, + "learning_rate": 3.3948339483394836e-06, + "loss": 1.4695, + "step": 952 + }, + { + "epoch": 1.1860609831985065, + "grad_norm": 2.916473150253296, + "learning_rate": 3.3925276752767534e-06, + "loss": 1.4901, + "step": 953 + }, + { + "epoch": 1.1873055382700684, + "grad_norm": 2.8371644020080566, + "learning_rate": 3.3902214022140224e-06, + "loss": 1.4944, + "step": 954 + }, + { + "epoch": 1.1885500933416304, + "grad_norm": 2.813662528991699, + "learning_rate": 3.387915129151292e-06, + "loss": 1.4603, + "step": 955 + }, + { + "epoch": 1.1897946484131923, + "grad_norm": 2.991931676864624, + "learning_rate": 3.3856088560885613e-06, + "loss": 1.5496, + "step": 956 + }, + { + "epoch": 1.1910392034847541, + "grad_norm": 2.6252501010894775, + "learning_rate": 3.3833025830258303e-06, + "loss": 1.4405, + "step": 957 + }, + { + "epoch": 1.1922837585563162, + "grad_norm": 2.747725009918213, + "learning_rate": 3.3809963099630997e-06, + "loss": 1.4689, + "step": 958 + }, + { + "epoch": 1.193528313627878, + "grad_norm": 2.887763261795044, + "learning_rate": 3.3786900369003696e-06, + "loss": 1.4501, + "step": 959 + }, + { + "epoch": 1.1947728686994399, + "grad_norm": 2.9954354763031006, + "learning_rate": 3.3763837638376386e-06, + "loss": 1.4978, + "step": 960 + }, + { + "epoch": 1.1947728686994399, + "eval_loss": 1.664442539215088, + "eval_runtime": 49.1266, + "eval_samples_per_second": 20.356, + "eval_steps_per_second": 0.855, + "step": 960 + }, + { + "epoch": 1.196017423771002, + "grad_norm": 3.0174152851104736, + "learning_rate": 3.374077490774908e-06, + "loss": 1.4353, + "step": 961 + }, + { + "epoch": 1.1972619788425638, + "grad_norm": 2.9082882404327393, + "learning_rate": 3.3717712177121774e-06, + "loss": 1.4878, + "step": 962 + }, + { + "epoch": 1.1985065339141256, + "grad_norm": 2.897993326187134, + "learning_rate": 3.3694649446494464e-06, + "loss": 1.4975, + "step": 963 + }, + { + "epoch": 1.1997510889856877, + "grad_norm": 2.739470958709717, + "learning_rate": 3.3671586715867163e-06, + "loss": 1.4202, + "step": 964 + }, + { + "epoch": 1.2009956440572496, + "grad_norm": 2.817786693572998, + "learning_rate": 3.3648523985239857e-06, + "loss": 1.5195, + "step": 965 + }, + { + "epoch": 1.2022401991288114, + "grad_norm": 2.8476386070251465, + "learning_rate": 3.362546125461255e-06, + "loss": 1.444, + "step": 966 + }, + { + "epoch": 1.2034847542003733, + "grad_norm": 2.804047107696533, + "learning_rate": 3.360239852398524e-06, + "loss": 1.4442, + "step": 967 + }, + { + "epoch": 1.2047293092719353, + "grad_norm": 2.9285430908203125, + "learning_rate": 3.3579335793357935e-06, + "loss": 1.4877, + "step": 968 + }, + { + "epoch": 1.2059738643434972, + "grad_norm": 2.8814878463745117, + "learning_rate": 3.3556273062730634e-06, + "loss": 1.5078, + "step": 969 + }, + { + "epoch": 1.207218419415059, + "grad_norm": 2.970728635787964, + "learning_rate": 3.3533210332103324e-06, + "loss": 1.4166, + "step": 970 + }, + { + "epoch": 1.207218419415059, + "eval_loss": 1.6635315418243408, + "eval_runtime": 47.2321, + "eval_samples_per_second": 21.172, + "eval_steps_per_second": 0.889, + "step": 970 + }, + { + "epoch": 1.208462974486621, + "grad_norm": 2.9836292266845703, + "learning_rate": 3.3510147601476018e-06, + "loss": 1.4777, + "step": 971 + }, + { + "epoch": 1.209707529558183, + "grad_norm": 2.8495216369628906, + "learning_rate": 3.348708487084871e-06, + "loss": 1.4829, + "step": 972 + }, + { + "epoch": 1.2109520846297448, + "grad_norm": 2.7709312438964844, + "learning_rate": 3.34640221402214e-06, + "loss": 1.4242, + "step": 973 + }, + { + "epoch": 1.2121966397013069, + "grad_norm": 2.999462366104126, + "learning_rate": 3.3440959409594096e-06, + "loss": 1.4148, + "step": 974 + }, + { + "epoch": 1.2134411947728687, + "grad_norm": 2.8719074726104736, + "learning_rate": 3.3417896678966795e-06, + "loss": 1.4196, + "step": 975 + }, + { + "epoch": 1.2146857498444306, + "grad_norm": 2.8469338417053223, + "learning_rate": 3.3394833948339485e-06, + "loss": 1.4941, + "step": 976 + }, + { + "epoch": 1.2159303049159926, + "grad_norm": 2.9780938625335693, + "learning_rate": 3.337177121771218e-06, + "loss": 1.513, + "step": 977 + }, + { + "epoch": 1.2171748599875545, + "grad_norm": 2.814760684967041, + "learning_rate": 3.3348708487084873e-06, + "loss": 1.4204, + "step": 978 + }, + { + "epoch": 1.2184194150591163, + "grad_norm": 3.016261339187622, + "learning_rate": 3.332564575645757e-06, + "loss": 1.4389, + "step": 979 + }, + { + "epoch": 1.2196639701306782, + "grad_norm": 2.9385855197906494, + "learning_rate": 3.330258302583026e-06, + "loss": 1.4885, + "step": 980 + }, + { + "epoch": 1.2196639701306782, + "eval_loss": 1.6688873767852783, + "eval_runtime": 46.2942, + "eval_samples_per_second": 21.601, + "eval_steps_per_second": 0.907, + "step": 980 + }, + { + "epoch": 1.2209085252022402, + "grad_norm": 3.1434757709503174, + "learning_rate": 3.3279520295202956e-06, + "loss": 1.5157, + "step": 981 + }, + { + "epoch": 1.222153080273802, + "grad_norm": 3.0104827880859375, + "learning_rate": 3.325645756457565e-06, + "loss": 1.49, + "step": 982 + }, + { + "epoch": 1.223397635345364, + "grad_norm": 3.1913645267486572, + "learning_rate": 3.323339483394834e-06, + "loss": 1.4937, + "step": 983 + }, + { + "epoch": 1.224642190416926, + "grad_norm": 3.19496750831604, + "learning_rate": 3.3210332103321034e-06, + "loss": 1.5197, + "step": 984 + }, + { + "epoch": 1.2258867454884879, + "grad_norm": 3.046232223510742, + "learning_rate": 3.3187269372693733e-06, + "loss": 1.4497, + "step": 985 + }, + { + "epoch": 1.2271313005600497, + "grad_norm": 2.850675344467163, + "learning_rate": 3.3164206642066423e-06, + "loss": 1.4595, + "step": 986 + }, + { + "epoch": 1.2283758556316118, + "grad_norm": 2.7744576930999756, + "learning_rate": 3.3141143911439117e-06, + "loss": 1.4726, + "step": 987 + }, + { + "epoch": 1.2296204107031736, + "grad_norm": 3.017153024673462, + "learning_rate": 3.311808118081181e-06, + "loss": 1.493, + "step": 988 + }, + { + "epoch": 1.2308649657747355, + "grad_norm": 2.8821120262145996, + "learning_rate": 3.3095018450184506e-06, + "loss": 1.4734, + "step": 989 + }, + { + "epoch": 1.2321095208462975, + "grad_norm": 2.8805840015411377, + "learning_rate": 3.3071955719557196e-06, + "loss": 1.4562, + "step": 990 + }, + { + "epoch": 1.2321095208462975, + "eval_loss": 1.669345736503601, + "eval_runtime": 47.0893, + "eval_samples_per_second": 21.236, + "eval_steps_per_second": 0.892, + "step": 990 + }, + { + "epoch": 1.2333540759178594, + "grad_norm": 2.7599008083343506, + "learning_rate": 3.3048892988929894e-06, + "loss": 1.4218, + "step": 991 + }, + { + "epoch": 1.2345986309894212, + "grad_norm": 2.9835240840911865, + "learning_rate": 3.302583025830259e-06, + "loss": 1.4596, + "step": 992 + }, + { + "epoch": 1.235843186060983, + "grad_norm": 2.843358039855957, + "learning_rate": 3.300276752767528e-06, + "loss": 1.4561, + "step": 993 + }, + { + "epoch": 1.2370877411325452, + "grad_norm": 2.7386093139648438, + "learning_rate": 3.2979704797047972e-06, + "loss": 1.4475, + "step": 994 + }, + { + "epoch": 1.238332296204107, + "grad_norm": 2.668161392211914, + "learning_rate": 3.2956642066420667e-06, + "loss": 1.4661, + "step": 995 + }, + { + "epoch": 1.2395768512756689, + "grad_norm": 2.947235107421875, + "learning_rate": 3.2933579335793357e-06, + "loss": 1.4358, + "step": 996 + }, + { + "epoch": 1.240821406347231, + "grad_norm": 2.8159878253936768, + "learning_rate": 3.2910516605166055e-06, + "loss": 1.4432, + "step": 997 + }, + { + "epoch": 1.2420659614187928, + "grad_norm": 2.896597385406494, + "learning_rate": 3.288745387453875e-06, + "loss": 1.5585, + "step": 998 + }, + { + "epoch": 1.2433105164903546, + "grad_norm": 2.80241322517395, + "learning_rate": 3.286439114391144e-06, + "loss": 1.5076, + "step": 999 + }, + { + "epoch": 1.2445550715619167, + "grad_norm": 3.008939027786255, + "learning_rate": 3.2841328413284134e-06, + "loss": 1.4234, + "step": 1000 + }, + { + "epoch": 1.2445550715619167, + "eval_loss": 1.6621588468551636, + "eval_runtime": 50.4926, + "eval_samples_per_second": 19.805, + "eval_steps_per_second": 0.832, + "step": 1000 + }, + { + "epoch": 1.2457996266334785, + "grad_norm": 2.9872381687164307, + "learning_rate": 3.281826568265683e-06, + "loss": 1.453, + "step": 1001 + }, + { + "epoch": 1.2470441817050404, + "grad_norm": 2.704676628112793, + "learning_rate": 3.2795202952029526e-06, + "loss": 1.4711, + "step": 1002 + }, + { + "epoch": 1.2482887367766025, + "grad_norm": 2.8100757598876953, + "learning_rate": 3.2772140221402216e-06, + "loss": 1.3884, + "step": 1003 + }, + { + "epoch": 1.2495332918481643, + "grad_norm": 2.902953624725342, + "learning_rate": 3.274907749077491e-06, + "loss": 1.4986, + "step": 1004 + }, + { + "epoch": 1.2507778469197262, + "grad_norm": 2.980163097381592, + "learning_rate": 3.2726014760147605e-06, + "loss": 1.5067, + "step": 1005 + }, + { + "epoch": 1.252022401991288, + "grad_norm": 2.8820345401763916, + "learning_rate": 3.2702952029520295e-06, + "loss": 1.4995, + "step": 1006 + }, + { + "epoch": 1.25326695706285, + "grad_norm": 2.8219635486602783, + "learning_rate": 3.2679889298892993e-06, + "loss": 1.4881, + "step": 1007 + }, + { + "epoch": 1.254511512134412, + "grad_norm": 2.8593199253082275, + "learning_rate": 3.2656826568265687e-06, + "loss": 1.4469, + "step": 1008 + }, + { + "epoch": 1.255756067205974, + "grad_norm": 2.888357639312744, + "learning_rate": 3.2633763837638377e-06, + "loss": 1.4869, + "step": 1009 + }, + { + "epoch": 1.2570006222775358, + "grad_norm": 2.9985013008117676, + "learning_rate": 3.261070110701107e-06, + "loss": 1.4724, + "step": 1010 + }, + { + "epoch": 1.2570006222775358, + "eval_loss": 1.6492141485214233, + "eval_runtime": 45.2197, + "eval_samples_per_second": 22.114, + "eval_steps_per_second": 0.929, + "step": 1010 + }, + { + "epoch": 1.2582451773490977, + "grad_norm": 2.860780715942383, + "learning_rate": 3.2587638376383766e-06, + "loss": 1.4618, + "step": 1011 + }, + { + "epoch": 1.2594897324206595, + "grad_norm": 2.766479730606079, + "learning_rate": 3.2564575645756456e-06, + "loss": 1.4642, + "step": 1012 + }, + { + "epoch": 1.2607342874922216, + "grad_norm": 2.711759328842163, + "learning_rate": 3.2541512915129154e-06, + "loss": 1.4283, + "step": 1013 + }, + { + "epoch": 1.2619788425637835, + "grad_norm": 2.8365769386291504, + "learning_rate": 3.251845018450185e-06, + "loss": 1.4248, + "step": 1014 + }, + { + "epoch": 1.2632233976353453, + "grad_norm": 2.957620143890381, + "learning_rate": 3.2495387453874543e-06, + "loss": 1.4662, + "step": 1015 + }, + { + "epoch": 1.2644679527069074, + "grad_norm": 2.8066208362579346, + "learning_rate": 3.2472324723247233e-06, + "loss": 1.4447, + "step": 1016 + }, + { + "epoch": 1.2657125077784692, + "grad_norm": 2.902040958404541, + "learning_rate": 3.244926199261993e-06, + "loss": 1.4743, + "step": 1017 + }, + { + "epoch": 1.266957062850031, + "grad_norm": 2.7636687755584717, + "learning_rate": 3.2426199261992625e-06, + "loss": 1.4465, + "step": 1018 + }, + { + "epoch": 1.268201617921593, + "grad_norm": 2.95261549949646, + "learning_rate": 3.2403136531365315e-06, + "loss": 1.4769, + "step": 1019 + }, + { + "epoch": 1.269446172993155, + "grad_norm": 2.911128520965576, + "learning_rate": 3.238007380073801e-06, + "loss": 1.3843, + "step": 1020 + }, + { + "epoch": 1.269446172993155, + "eval_loss": 1.6526964902877808, + "eval_runtime": 48.2126, + "eval_samples_per_second": 20.741, + "eval_steps_per_second": 0.871, + "step": 1020 + }, + { + "epoch": 1.2706907280647168, + "grad_norm": 2.769951820373535, + "learning_rate": 3.2357011070110704e-06, + "loss": 1.4847, + "step": 1021 + }, + { + "epoch": 1.271935283136279, + "grad_norm": 3.0451674461364746, + "learning_rate": 3.2333948339483394e-06, + "loss": 1.4453, + "step": 1022 + }, + { + "epoch": 1.2731798382078408, + "grad_norm": 2.9543521404266357, + "learning_rate": 3.2310885608856092e-06, + "loss": 1.4789, + "step": 1023 + }, + { + "epoch": 1.2744243932794026, + "grad_norm": 2.9174394607543945, + "learning_rate": 3.2287822878228787e-06, + "loss": 1.4807, + "step": 1024 + }, + { + "epoch": 1.2756689483509644, + "grad_norm": 2.888258934020996, + "learning_rate": 3.2264760147601477e-06, + "loss": 1.4367, + "step": 1025 + }, + { + "epoch": 1.2769135034225265, + "grad_norm": 2.8346047401428223, + "learning_rate": 3.224169741697417e-06, + "loss": 1.4313, + "step": 1026 + }, + { + "epoch": 1.2781580584940884, + "grad_norm": 2.7531332969665527, + "learning_rate": 3.2218634686346865e-06, + "loss": 1.4018, + "step": 1027 + }, + { + "epoch": 1.2794026135656502, + "grad_norm": 3.136946439743042, + "learning_rate": 3.2195571955719564e-06, + "loss": 1.4417, + "step": 1028 + }, + { + "epoch": 1.2806471686372123, + "grad_norm": 2.9618165493011475, + "learning_rate": 3.2172509225092254e-06, + "loss": 1.5117, + "step": 1029 + }, + { + "epoch": 1.2818917237087741, + "grad_norm": 3.0740747451782227, + "learning_rate": 3.2149446494464948e-06, + "loss": 1.4641, + "step": 1030 + }, + { + "epoch": 1.2818917237087741, + "eval_loss": 1.6572761535644531, + "eval_runtime": 51.6696, + "eval_samples_per_second": 19.354, + "eval_steps_per_second": 0.813, + "step": 1030 + }, + { + "epoch": 1.283136278780336, + "grad_norm": 2.9225106239318848, + "learning_rate": 3.212638376383764e-06, + "loss": 1.4421, + "step": 1031 + }, + { + "epoch": 1.2843808338518978, + "grad_norm": 2.9530649185180664, + "learning_rate": 3.210332103321033e-06, + "loss": 1.5138, + "step": 1032 + }, + { + "epoch": 1.28562538892346, + "grad_norm": 2.8513238430023193, + "learning_rate": 3.208025830258303e-06, + "loss": 1.411, + "step": 1033 + }, + { + "epoch": 1.2868699439950217, + "grad_norm": 2.7440171241760254, + "learning_rate": 3.2057195571955725e-06, + "loss": 1.4182, + "step": 1034 + }, + { + "epoch": 1.2881144990665838, + "grad_norm": 2.8126630783081055, + "learning_rate": 3.2034132841328415e-06, + "loss": 1.4333, + "step": 1035 + }, + { + "epoch": 1.2893590541381457, + "grad_norm": 2.8598744869232178, + "learning_rate": 3.201107011070111e-06, + "loss": 1.439, + "step": 1036 + }, + { + "epoch": 1.2906036092097075, + "grad_norm": 2.901622772216797, + "learning_rate": 3.1988007380073803e-06, + "loss": 1.4461, + "step": 1037 + }, + { + "epoch": 1.2918481642812694, + "grad_norm": 2.7670180797576904, + "learning_rate": 3.1964944649446493e-06, + "loss": 1.4772, + "step": 1038 + }, + { + "epoch": 1.2930927193528314, + "grad_norm": 2.7860350608825684, + "learning_rate": 3.194188191881919e-06, + "loss": 1.4414, + "step": 1039 + }, + { + "epoch": 1.2943372744243933, + "grad_norm": 2.822052478790283, + "learning_rate": 3.1918819188191886e-06, + "loss": 1.4192, + "step": 1040 + }, + { + "epoch": 1.2943372744243933, + "eval_loss": 1.663284420967102, + "eval_runtime": 47.2822, + "eval_samples_per_second": 21.15, + "eval_steps_per_second": 0.888, + "step": 1040 + }, + { + "epoch": 1.2955818294959551, + "grad_norm": 2.909109115600586, + "learning_rate": 3.189575645756458e-06, + "loss": 1.4795, + "step": 1041 + }, + { + "epoch": 1.2968263845675172, + "grad_norm": 2.933673143386841, + "learning_rate": 3.187269372693727e-06, + "loss": 1.5038, + "step": 1042 + }, + { + "epoch": 1.298070939639079, + "grad_norm": 3.032518148422241, + "learning_rate": 3.1849630996309964e-06, + "loss": 1.5133, + "step": 1043 + }, + { + "epoch": 1.299315494710641, + "grad_norm": 2.826040506362915, + "learning_rate": 3.1826568265682663e-06, + "loss": 1.4395, + "step": 1044 + }, + { + "epoch": 1.3005600497822027, + "grad_norm": 2.8152658939361572, + "learning_rate": 3.1803505535055353e-06, + "loss": 1.4787, + "step": 1045 + }, + { + "epoch": 1.3018046048537648, + "grad_norm": 2.9410431385040283, + "learning_rate": 3.1780442804428047e-06, + "loss": 1.4375, + "step": 1046 + }, + { + "epoch": 1.3030491599253267, + "grad_norm": 2.948636531829834, + "learning_rate": 3.175738007380074e-06, + "loss": 1.4847, + "step": 1047 + }, + { + "epoch": 1.3042937149968887, + "grad_norm": 2.6287615299224854, + "learning_rate": 3.173431734317343e-06, + "loss": 1.4186, + "step": 1048 + }, + { + "epoch": 1.3055382700684506, + "grad_norm": 2.836029052734375, + "learning_rate": 3.1711254612546125e-06, + "loss": 1.4251, + "step": 1049 + }, + { + "epoch": 1.3067828251400124, + "grad_norm": 2.982081651687622, + "learning_rate": 3.1688191881918824e-06, + "loss": 1.4951, + "step": 1050 + }, + { + "epoch": 1.3067828251400124, + "eval_loss": 1.6578067541122437, + "eval_runtime": 47.5847, + "eval_samples_per_second": 21.015, + "eval_steps_per_second": 0.883, + "step": 1050 + }, + { + "epoch": 1.3080273802115743, + "grad_norm": 2.9297118186950684, + "learning_rate": 3.166512915129152e-06, + "loss": 1.4699, + "step": 1051 + }, + { + "epoch": 1.3092719352831363, + "grad_norm": 2.906435251235962, + "learning_rate": 3.164206642066421e-06, + "loss": 1.4562, + "step": 1052 + }, + { + "epoch": 1.3105164903546982, + "grad_norm": 2.8655896186828613, + "learning_rate": 3.1619003690036902e-06, + "loss": 1.4351, + "step": 1053 + }, + { + "epoch": 1.31176104542626, + "grad_norm": 2.779244899749756, + "learning_rate": 3.15959409594096e-06, + "loss": 1.4479, + "step": 1054 + }, + { + "epoch": 1.3130056004978221, + "grad_norm": 2.9863739013671875, + "learning_rate": 3.157287822878229e-06, + "loss": 1.4964, + "step": 1055 + }, + { + "epoch": 1.314250155569384, + "grad_norm": 2.821871042251587, + "learning_rate": 3.1549815498154985e-06, + "loss": 1.4173, + "step": 1056 + }, + { + "epoch": 1.3154947106409458, + "grad_norm": 2.8573882579803467, + "learning_rate": 3.152675276752768e-06, + "loss": 1.4877, + "step": 1057 + }, + { + "epoch": 1.3167392657125077, + "grad_norm": 2.9824137687683105, + "learning_rate": 3.150369003690037e-06, + "loss": 1.5124, + "step": 1058 + }, + { + "epoch": 1.3179838207840697, + "grad_norm": 2.881098747253418, + "learning_rate": 3.1480627306273063e-06, + "loss": 1.4346, + "step": 1059 + }, + { + "epoch": 1.3192283758556316, + "grad_norm": 2.918782949447632, + "learning_rate": 3.145756457564576e-06, + "loss": 1.4797, + "step": 1060 + }, + { + "epoch": 1.3192283758556316, + "eval_loss": 1.6510179042816162, + "eval_runtime": 53.6356, + "eval_samples_per_second": 18.644, + "eval_steps_per_second": 0.783, + "step": 1060 + }, + { + "epoch": 1.3204729309271936, + "grad_norm": 3.045776128768921, + "learning_rate": 3.143450184501845e-06, + "loss": 1.5039, + "step": 1061 + }, + { + "epoch": 1.3217174859987555, + "grad_norm": 3.041501760482788, + "learning_rate": 3.1411439114391146e-06, + "loss": 1.4878, + "step": 1062 + }, + { + "epoch": 1.3229620410703173, + "grad_norm": 2.917837142944336, + "learning_rate": 3.138837638376384e-06, + "loss": 1.4202, + "step": 1063 + }, + { + "epoch": 1.3242065961418792, + "grad_norm": 2.708070755004883, + "learning_rate": 3.1365313653136535e-06, + "loss": 1.4071, + "step": 1064 + }, + { + "epoch": 1.3254511512134413, + "grad_norm": 2.9146156311035156, + "learning_rate": 3.1342250922509225e-06, + "loss": 1.4428, + "step": 1065 + }, + { + "epoch": 1.326695706285003, + "grad_norm": 3.220919132232666, + "learning_rate": 3.1319188191881923e-06, + "loss": 1.4733, + "step": 1066 + }, + { + "epoch": 1.327940261356565, + "grad_norm": 2.9127554893493652, + "learning_rate": 3.1296125461254617e-06, + "loss": 1.4554, + "step": 1067 + }, + { + "epoch": 1.329184816428127, + "grad_norm": 3.053191661834717, + "learning_rate": 3.1273062730627307e-06, + "loss": 1.4044, + "step": 1068 + }, + { + "epoch": 1.3304293714996889, + "grad_norm": 3.2937021255493164, + "learning_rate": 3.125e-06, + "loss": 1.4949, + "step": 1069 + }, + { + "epoch": 1.3316739265712507, + "grad_norm": 2.976717948913574, + "learning_rate": 3.12269372693727e-06, + "loss": 1.446, + "step": 1070 + }, + { + "epoch": 1.3316739265712507, + "eval_loss": 1.6503105163574219, + "eval_runtime": 50.7579, + "eval_samples_per_second": 19.701, + "eval_steps_per_second": 0.827, + "step": 1070 + }, + { + "epoch": 1.3329184816428126, + "grad_norm": 3.0860936641693115, + "learning_rate": 3.120387453874539e-06, + "loss": 1.4055, + "step": 1071 + }, + { + "epoch": 1.3341630367143746, + "grad_norm": 3.0401217937469482, + "learning_rate": 3.1180811808118084e-06, + "loss": 1.4688, + "step": 1072 + }, + { + "epoch": 1.3354075917859365, + "grad_norm": 2.953253984451294, + "learning_rate": 3.115774907749078e-06, + "loss": 1.5001, + "step": 1073 + }, + { + "epoch": 1.3366521468574986, + "grad_norm": 3.0665059089660645, + "learning_rate": 3.113468634686347e-06, + "loss": 1.4217, + "step": 1074 + }, + { + "epoch": 1.3378967019290604, + "grad_norm": 3.121600389480591, + "learning_rate": 3.1111623616236163e-06, + "loss": 1.4508, + "step": 1075 + }, + { + "epoch": 1.3391412570006223, + "grad_norm": 2.8057339191436768, + "learning_rate": 3.108856088560886e-06, + "loss": 1.4241, + "step": 1076 + }, + { + "epoch": 1.340385812072184, + "grad_norm": 3.0147411823272705, + "learning_rate": 3.1065498154981555e-06, + "loss": 1.479, + "step": 1077 + }, + { + "epoch": 1.3416303671437462, + "grad_norm": 3.030010223388672, + "learning_rate": 3.1042435424354245e-06, + "loss": 1.4861, + "step": 1078 + }, + { + "epoch": 1.342874922215308, + "grad_norm": 3.027998208999634, + "learning_rate": 3.101937269372694e-06, + "loss": 1.4916, + "step": 1079 + }, + { + "epoch": 1.3441194772868699, + "grad_norm": 2.9252536296844482, + "learning_rate": 3.0996309963099634e-06, + "loss": 1.418, + "step": 1080 + }, + { + "epoch": 1.3441194772868699, + "eval_loss": 1.6490333080291748, + "eval_runtime": 47.4026, + "eval_samples_per_second": 21.096, + "eval_steps_per_second": 0.886, + "step": 1080 + }, + { + "epoch": 1.345364032358432, + "grad_norm": 3.110847234725952, + "learning_rate": 3.0973247232472324e-06, + "loss": 1.4127, + "step": 1081 + }, + { + "epoch": 1.3466085874299938, + "grad_norm": 2.846491575241089, + "learning_rate": 3.0950184501845022e-06, + "loss": 1.444, + "step": 1082 + }, + { + "epoch": 1.3478531425015556, + "grad_norm": 2.950533151626587, + "learning_rate": 3.0927121771217716e-06, + "loss": 1.4456, + "step": 1083 + }, + { + "epoch": 1.3490976975731175, + "grad_norm": 2.875964403152466, + "learning_rate": 3.0904059040590406e-06, + "loss": 1.4485, + "step": 1084 + }, + { + "epoch": 1.3503422526446796, + "grad_norm": 2.9612326622009277, + "learning_rate": 3.08809963099631e-06, + "loss": 1.4446, + "step": 1085 + }, + { + "epoch": 1.3515868077162414, + "grad_norm": 2.925297975540161, + "learning_rate": 3.0857933579335795e-06, + "loss": 1.4847, + "step": 1086 + }, + { + "epoch": 1.3528313627878035, + "grad_norm": 2.991616725921631, + "learning_rate": 3.0834870848708485e-06, + "loss": 1.5151, + "step": 1087 + }, + { + "epoch": 1.3540759178593653, + "grad_norm": 2.96329927444458, + "learning_rate": 3.0811808118081183e-06, + "loss": 1.4574, + "step": 1088 + }, + { + "epoch": 1.3553204729309272, + "grad_norm": 2.948702573776245, + "learning_rate": 3.0788745387453878e-06, + "loss": 1.4585, + "step": 1089 + }, + { + "epoch": 1.356565028002489, + "grad_norm": 3.218463659286499, + "learning_rate": 3.076568265682657e-06, + "loss": 1.4439, + "step": 1090 + }, + { + "epoch": 1.356565028002489, + "eval_loss": 1.645838975906372, + "eval_runtime": 47.048, + "eval_samples_per_second": 21.255, + "eval_steps_per_second": 0.893, + "step": 1090 + }, + { + "epoch": 1.357809583074051, + "grad_norm": 2.9799578189849854, + "learning_rate": 3.074261992619926e-06, + "loss": 1.4883, + "step": 1091 + }, + { + "epoch": 1.359054138145613, + "grad_norm": 2.7660505771636963, + "learning_rate": 3.071955719557196e-06, + "loss": 1.4349, + "step": 1092 + }, + { + "epoch": 1.3602986932171748, + "grad_norm": 3.0201916694641113, + "learning_rate": 3.0696494464944655e-06, + "loss": 1.418, + "step": 1093 + }, + { + "epoch": 1.3615432482887369, + "grad_norm": 2.844207763671875, + "learning_rate": 3.0673431734317345e-06, + "loss": 1.4769, + "step": 1094 + }, + { + "epoch": 1.3627878033602987, + "grad_norm": 2.8306753635406494, + "learning_rate": 3.065036900369004e-06, + "loss": 1.4653, + "step": 1095 + }, + { + "epoch": 1.3640323584318605, + "grad_norm": 2.805023431777954, + "learning_rate": 3.0627306273062733e-06, + "loss": 1.4771, + "step": 1096 + }, + { + "epoch": 1.3652769135034224, + "grad_norm": 3.020883083343506, + "learning_rate": 3.0604243542435423e-06, + "loss": 1.4237, + "step": 1097 + }, + { + "epoch": 1.3665214685749845, + "grad_norm": 2.8395731449127197, + "learning_rate": 3.058118081180812e-06, + "loss": 1.4647, + "step": 1098 + }, + { + "epoch": 1.3677660236465463, + "grad_norm": 2.871156692504883, + "learning_rate": 3.0558118081180816e-06, + "loss": 1.4196, + "step": 1099 + }, + { + "epoch": 1.3690105787181084, + "grad_norm": 2.974905014038086, + "learning_rate": 3.053505535055351e-06, + "loss": 1.4779, + "step": 1100 + }, + { + "epoch": 1.3690105787181084, + "eval_loss": 1.651503562927246, + "eval_runtime": 54.2713, + "eval_samples_per_second": 18.426, + "eval_steps_per_second": 0.774, + "step": 1100 + }, + { + "epoch": 1.3702551337896702, + "grad_norm": 2.7591328620910645, + "learning_rate": 3.05119926199262e-06, + "loss": 1.3899, + "step": 1101 + }, + { + "epoch": 1.371499688861232, + "grad_norm": 2.8575668334960938, + "learning_rate": 3.0488929889298894e-06, + "loss": 1.4482, + "step": 1102 + }, + { + "epoch": 1.372744243932794, + "grad_norm": 2.9049339294433594, + "learning_rate": 3.0465867158671593e-06, + "loss": 1.4785, + "step": 1103 + }, + { + "epoch": 1.373988799004356, + "grad_norm": 2.816453695297241, + "learning_rate": 3.0442804428044283e-06, + "loss": 1.4601, + "step": 1104 + }, + { + "epoch": 1.3752333540759178, + "grad_norm": 2.8844189643859863, + "learning_rate": 3.0419741697416977e-06, + "loss": 1.496, + "step": 1105 + }, + { + "epoch": 1.3764779091474797, + "grad_norm": 3.014744281768799, + "learning_rate": 3.039667896678967e-06, + "loss": 1.4098, + "step": 1106 + }, + { + "epoch": 1.3777224642190418, + "grad_norm": 2.8243560791015625, + "learning_rate": 3.037361623616236e-06, + "loss": 1.4086, + "step": 1107 + }, + { + "epoch": 1.3789670192906036, + "grad_norm": 2.8333253860473633, + "learning_rate": 3.035055350553506e-06, + "loss": 1.3516, + "step": 1108 + }, + { + "epoch": 1.3802115743621655, + "grad_norm": 2.926455020904541, + "learning_rate": 3.0327490774907754e-06, + "loss": 1.3976, + "step": 1109 + }, + { + "epoch": 1.3814561294337273, + "grad_norm": 2.900937080383301, + "learning_rate": 3.0304428044280444e-06, + "loss": 1.4336, + "step": 1110 + }, + { + "epoch": 1.3814561294337273, + "eval_loss": 1.647048830986023, + "eval_runtime": 42.3955, + "eval_samples_per_second": 23.587, + "eval_steps_per_second": 0.991, + "step": 1110 + }, + { + "epoch": 1.3827006845052894, + "grad_norm": 3.0756969451904297, + "learning_rate": 3.028136531365314e-06, + "loss": 1.4674, + "step": 1111 + }, + { + "epoch": 1.3839452395768512, + "grad_norm": 2.7754578590393066, + "learning_rate": 3.0258302583025832e-06, + "loss": 1.4435, + "step": 1112 + }, + { + "epoch": 1.3851897946484133, + "grad_norm": 2.8011393547058105, + "learning_rate": 3.023523985239853e-06, + "loss": 1.4212, + "step": 1113 + }, + { + "epoch": 1.3864343497199751, + "grad_norm": 2.7063710689544678, + "learning_rate": 3.021217712177122e-06, + "loss": 1.3769, + "step": 1114 + }, + { + "epoch": 1.387678904791537, + "grad_norm": 2.830411434173584, + "learning_rate": 3.0189114391143915e-06, + "loss": 1.4832, + "step": 1115 + }, + { + "epoch": 1.3889234598630988, + "grad_norm": 2.9075911045074463, + "learning_rate": 3.016605166051661e-06, + "loss": 1.486, + "step": 1116 + }, + { + "epoch": 1.390168014934661, + "grad_norm": 2.847822666168213, + "learning_rate": 3.01429889298893e-06, + "loss": 1.424, + "step": 1117 + }, + { + "epoch": 1.3914125700062228, + "grad_norm": 2.8822033405303955, + "learning_rate": 3.0119926199261993e-06, + "loss": 1.4218, + "step": 1118 + }, + { + "epoch": 1.3926571250777848, + "grad_norm": 2.8944740295410156, + "learning_rate": 3.009686346863469e-06, + "loss": 1.4283, + "step": 1119 + }, + { + "epoch": 1.3939016801493467, + "grad_norm": 2.765190839767456, + "learning_rate": 3.007380073800738e-06, + "loss": 1.4622, + "step": 1120 + }, + { + "epoch": 1.3939016801493467, + "eval_loss": 1.6451724767684937, + "eval_runtime": 43.7449, + "eval_samples_per_second": 22.86, + "eval_steps_per_second": 0.96, + "step": 1120 + }, + { + "epoch": 1.3951462352209085, + "grad_norm": 2.877002716064453, + "learning_rate": 3.0050738007380076e-06, + "loss": 1.4145, + "step": 1121 + }, + { + "epoch": 1.3963907902924704, + "grad_norm": 2.870389699935913, + "learning_rate": 3.002767527675277e-06, + "loss": 1.4401, + "step": 1122 + }, + { + "epoch": 1.3976353453640322, + "grad_norm": 2.7788920402526855, + "learning_rate": 3.000461254612546e-06, + "loss": 1.4514, + "step": 1123 + }, + { + "epoch": 1.3988799004355943, + "grad_norm": 2.9269332885742188, + "learning_rate": 2.998154981549816e-06, + "loss": 1.3961, + "step": 1124 + }, + { + "epoch": 1.4001244555071561, + "grad_norm": 3.013291835784912, + "learning_rate": 2.9958487084870853e-06, + "loss": 1.4597, + "step": 1125 + }, + { + "epoch": 1.4013690105787182, + "grad_norm": 2.9063880443573, + "learning_rate": 2.9935424354243547e-06, + "loss": 1.4668, + "step": 1126 + }, + { + "epoch": 1.40261356565028, + "grad_norm": 2.8926970958709717, + "learning_rate": 2.9912361623616237e-06, + "loss": 1.4332, + "step": 1127 + }, + { + "epoch": 1.403858120721842, + "grad_norm": 2.8563647270202637, + "learning_rate": 2.988929889298893e-06, + "loss": 1.407, + "step": 1128 + }, + { + "epoch": 1.4051026757934038, + "grad_norm": 2.8725359439849854, + "learning_rate": 2.986623616236163e-06, + "loss": 1.4434, + "step": 1129 + }, + { + "epoch": 1.4063472308649658, + "grad_norm": 2.906898260116577, + "learning_rate": 2.984317343173432e-06, + "loss": 1.4745, + "step": 1130 + }, + { + "epoch": 1.4063472308649658, + "eval_loss": 1.647884726524353, + "eval_runtime": 44.9221, + "eval_samples_per_second": 22.261, + "eval_steps_per_second": 0.935, + "step": 1130 + }, + { + "epoch": 1.4075917859365277, + "grad_norm": 2.8297719955444336, + "learning_rate": 2.9820110701107014e-06, + "loss": 1.4542, + "step": 1131 + }, + { + "epoch": 1.4088363410080897, + "grad_norm": 2.841031551361084, + "learning_rate": 2.979704797047971e-06, + "loss": 1.4105, + "step": 1132 + }, + { + "epoch": 1.4100808960796516, + "grad_norm": 2.867791175842285, + "learning_rate": 2.97739852398524e-06, + "loss": 1.4409, + "step": 1133 + }, + { + "epoch": 1.4113254511512134, + "grad_norm": 2.8156039714813232, + "learning_rate": 2.9750922509225093e-06, + "loss": 1.4284, + "step": 1134 + }, + { + "epoch": 1.4125700062227753, + "grad_norm": 2.825985908508301, + "learning_rate": 2.972785977859779e-06, + "loss": 1.4594, + "step": 1135 + }, + { + "epoch": 1.4138145612943374, + "grad_norm": 3.0134711265563965, + "learning_rate": 2.970479704797048e-06, + "loss": 1.4743, + "step": 1136 + }, + { + "epoch": 1.4150591163658992, + "grad_norm": 2.9829142093658447, + "learning_rate": 2.9681734317343175e-06, + "loss": 1.4935, + "step": 1137 + }, + { + "epoch": 1.416303671437461, + "grad_norm": 3.067620277404785, + "learning_rate": 2.965867158671587e-06, + "loss": 1.4266, + "step": 1138 + }, + { + "epoch": 1.4175482265090231, + "grad_norm": 2.8969919681549072, + "learning_rate": 2.9635608856088564e-06, + "loss": 1.451, + "step": 1139 + }, + { + "epoch": 1.418792781580585, + "grad_norm": 2.8706412315368652, + "learning_rate": 2.9612546125461254e-06, + "loss": 1.4834, + "step": 1140 + }, + { + "epoch": 1.418792781580585, + "eval_loss": 1.643480658531189, + "eval_runtime": 44.743, + "eval_samples_per_second": 22.35, + "eval_steps_per_second": 0.939, + "step": 1140 + }, + { + "epoch": 1.4200373366521468, + "grad_norm": 2.8801848888397217, + "learning_rate": 2.958948339483395e-06, + "loss": 1.4258, + "step": 1141 + }, + { + "epoch": 1.4212818917237087, + "grad_norm": 2.774635076522827, + "learning_rate": 2.9566420664206646e-06, + "loss": 1.416, + "step": 1142 + }, + { + "epoch": 1.4225264467952707, + "grad_norm": 2.8460140228271484, + "learning_rate": 2.9543357933579336e-06, + "loss": 1.4213, + "step": 1143 + }, + { + "epoch": 1.4237710018668326, + "grad_norm": 2.907888889312744, + "learning_rate": 2.952029520295203e-06, + "loss": 1.4507, + "step": 1144 + }, + { + "epoch": 1.4250155569383947, + "grad_norm": 2.789041757583618, + "learning_rate": 2.949723247232473e-06, + "loss": 1.4257, + "step": 1145 + }, + { + "epoch": 1.4262601120099565, + "grad_norm": 2.78859806060791, + "learning_rate": 2.947416974169742e-06, + "loss": 1.4377, + "step": 1146 + }, + { + "epoch": 1.4275046670815184, + "grad_norm": 2.840303421020508, + "learning_rate": 2.9451107011070113e-06, + "loss": 1.4684, + "step": 1147 + }, + { + "epoch": 1.4287492221530802, + "grad_norm": 2.8800530433654785, + "learning_rate": 2.9428044280442807e-06, + "loss": 1.4912, + "step": 1148 + }, + { + "epoch": 1.4299937772246423, + "grad_norm": 2.841092824935913, + "learning_rate": 2.9404981549815497e-06, + "loss": 1.4234, + "step": 1149 + }, + { + "epoch": 1.4312383322962041, + "grad_norm": 2.8226537704467773, + "learning_rate": 2.938191881918819e-06, + "loss": 1.4578, + "step": 1150 + }, + { + "epoch": 1.4312383322962041, + "eval_loss": 1.643249273300171, + "eval_runtime": 43.5586, + "eval_samples_per_second": 22.958, + "eval_steps_per_second": 0.964, + "step": 1150 + }, + { + "epoch": 1.432482887367766, + "grad_norm": 2.869736909866333, + "learning_rate": 2.935885608856089e-06, + "loss": 1.4701, + "step": 1151 + }, + { + "epoch": 1.433727442439328, + "grad_norm": 2.689211130142212, + "learning_rate": 2.9335793357933584e-06, + "loss": 1.4483, + "step": 1152 + }, + { + "epoch": 1.4349719975108899, + "grad_norm": 2.9267077445983887, + "learning_rate": 2.9312730627306274e-06, + "loss": 1.4446, + "step": 1153 + }, + { + "epoch": 1.4362165525824517, + "grad_norm": 2.906862497329712, + "learning_rate": 2.928966789667897e-06, + "loss": 1.4524, + "step": 1154 + }, + { + "epoch": 1.4374611076540136, + "grad_norm": 2.8249800205230713, + "learning_rate": 2.9266605166051663e-06, + "loss": 1.4208, + "step": 1155 + }, + { + "epoch": 1.4387056627255757, + "grad_norm": 2.9836032390594482, + "learning_rate": 2.9243542435424353e-06, + "loss": 1.4581, + "step": 1156 + }, + { + "epoch": 1.4399502177971375, + "grad_norm": 2.9595284461975098, + "learning_rate": 2.922047970479705e-06, + "loss": 1.4141, + "step": 1157 + }, + { + "epoch": 1.4411947728686996, + "grad_norm": 3.0363423824310303, + "learning_rate": 2.9197416974169746e-06, + "loss": 1.4859, + "step": 1158 + }, + { + "epoch": 1.4424393279402614, + "grad_norm": 2.911201000213623, + "learning_rate": 2.9174354243542436e-06, + "loss": 1.4346, + "step": 1159 + }, + { + "epoch": 1.4436838830118233, + "grad_norm": 2.8617193698883057, + "learning_rate": 2.915129151291513e-06, + "loss": 1.433, + "step": 1160 + }, + { + "epoch": 1.4436838830118233, + "eval_loss": 1.6437232494354248, + "eval_runtime": 43.961, + "eval_samples_per_second": 22.747, + "eval_steps_per_second": 0.955, + "step": 1160 + }, + { + "epoch": 1.4449284380833851, + "grad_norm": 2.9584295749664307, + "learning_rate": 2.912822878228783e-06, + "loss": 1.4847, + "step": 1161 + }, + { + "epoch": 1.4461729931549472, + "grad_norm": 2.8209400177001953, + "learning_rate": 2.9105166051660522e-06, + "loss": 1.421, + "step": 1162 + }, + { + "epoch": 1.447417548226509, + "grad_norm": 2.847637414932251, + "learning_rate": 2.9082103321033212e-06, + "loss": 1.4326, + "step": 1163 + }, + { + "epoch": 1.4486621032980709, + "grad_norm": 2.836228132247925, + "learning_rate": 2.9059040590405907e-06, + "loss": 1.4459, + "step": 1164 + }, + { + "epoch": 1.449906658369633, + "grad_norm": 2.8327207565307617, + "learning_rate": 2.90359778597786e-06, + "loss": 1.4189, + "step": 1165 + }, + { + "epoch": 1.4511512134411948, + "grad_norm": 2.880643606185913, + "learning_rate": 2.901291512915129e-06, + "loss": 1.4214, + "step": 1166 + }, + { + "epoch": 1.4523957685127566, + "grad_norm": 2.9637348651885986, + "learning_rate": 2.898985239852399e-06, + "loss": 1.4471, + "step": 1167 + }, + { + "epoch": 1.4536403235843185, + "grad_norm": 3.006145477294922, + "learning_rate": 2.8966789667896684e-06, + "loss": 1.4228, + "step": 1168 + }, + { + "epoch": 1.4548848786558806, + "grad_norm": 2.8547627925872803, + "learning_rate": 2.8943726937269374e-06, + "loss": 1.4707, + "step": 1169 + }, + { + "epoch": 1.4561294337274424, + "grad_norm": 2.973092794418335, + "learning_rate": 2.8920664206642068e-06, + "loss": 1.4804, + "step": 1170 + }, + { + "epoch": 1.4561294337274424, + "eval_loss": 1.6340677738189697, + "eval_runtime": 44.6094, + "eval_samples_per_second": 22.417, + "eval_steps_per_second": 0.942, + "step": 1170 + }, + { + "epoch": 1.4573739887990045, + "grad_norm": 2.9433603286743164, + "learning_rate": 2.889760147601476e-06, + "loss": 1.4354, + "step": 1171 + }, + { + "epoch": 1.4586185438705663, + "grad_norm": 2.872236728668213, + "learning_rate": 2.887453874538745e-06, + "loss": 1.418, + "step": 1172 + }, + { + "epoch": 1.4598630989421282, + "grad_norm": 2.847557783126831, + "learning_rate": 2.885147601476015e-06, + "loss": 1.4306, + "step": 1173 + }, + { + "epoch": 1.46110765401369, + "grad_norm": 2.7554969787597656, + "learning_rate": 2.8828413284132845e-06, + "loss": 1.4295, + "step": 1174 + }, + { + "epoch": 1.462352209085252, + "grad_norm": 2.8754842281341553, + "learning_rate": 2.880535055350554e-06, + "loss": 1.4482, + "step": 1175 + }, + { + "epoch": 1.463596764156814, + "grad_norm": 3.0115292072296143, + "learning_rate": 2.878228782287823e-06, + "loss": 1.4428, + "step": 1176 + }, + { + "epoch": 1.4648413192283758, + "grad_norm": 2.8976168632507324, + "learning_rate": 2.8759225092250923e-06, + "loss": 1.4707, + "step": 1177 + }, + { + "epoch": 1.4660858742999379, + "grad_norm": 2.7981812953948975, + "learning_rate": 2.873616236162362e-06, + "loss": 1.4693, + "step": 1178 + }, + { + "epoch": 1.4673304293714997, + "grad_norm": 2.8562371730804443, + "learning_rate": 2.871309963099631e-06, + "loss": 1.4242, + "step": 1179 + }, + { + "epoch": 1.4685749844430616, + "grad_norm": 2.8705570697784424, + "learning_rate": 2.8690036900369006e-06, + "loss": 1.4725, + "step": 1180 + }, + { + "epoch": 1.4685749844430616, + "eval_loss": 1.6338286399841309, + "eval_runtime": 44.7171, + "eval_samples_per_second": 22.363, + "eval_steps_per_second": 0.939, + "step": 1180 + }, + { + "epoch": 1.4698195395146234, + "grad_norm": 2.8025710582733154, + "learning_rate": 2.86669741697417e-06, + "loss": 1.427, + "step": 1181 + }, + { + "epoch": 1.4710640945861855, + "grad_norm": 2.845151901245117, + "learning_rate": 2.864391143911439e-06, + "loss": 1.4402, + "step": 1182 + }, + { + "epoch": 1.4723086496577473, + "grad_norm": 2.943373680114746, + "learning_rate": 2.862084870848709e-06, + "loss": 1.5194, + "step": 1183 + }, + { + "epoch": 1.4735532047293094, + "grad_norm": 2.896955966949463, + "learning_rate": 2.8597785977859783e-06, + "loss": 1.4242, + "step": 1184 + }, + { + "epoch": 1.4747977598008712, + "grad_norm": 3.131305456161499, + "learning_rate": 2.8574723247232473e-06, + "loss": 1.5008, + "step": 1185 + }, + { + "epoch": 1.476042314872433, + "grad_norm": 2.8605735301971436, + "learning_rate": 2.8551660516605167e-06, + "loss": 1.3533, + "step": 1186 + }, + { + "epoch": 1.477286869943995, + "grad_norm": 2.9158811569213867, + "learning_rate": 2.852859778597786e-06, + "loss": 1.4404, + "step": 1187 + }, + { + "epoch": 1.478531425015557, + "grad_norm": 2.7795321941375732, + "learning_rate": 2.850553505535056e-06, + "loss": 1.3977, + "step": 1188 + }, + { + "epoch": 1.4797759800871189, + "grad_norm": 2.8194212913513184, + "learning_rate": 2.848247232472325e-06, + "loss": 1.4111, + "step": 1189 + }, + { + "epoch": 1.4810205351586807, + "grad_norm": 2.983750104904175, + "learning_rate": 2.8459409594095944e-06, + "loss": 1.4061, + "step": 1190 + }, + { + "epoch": 1.4810205351586807, + "eval_loss": 1.6375409364700317, + "eval_runtime": 45.4026, + "eval_samples_per_second": 22.025, + "eval_steps_per_second": 0.925, + "step": 1190 + }, + { + "epoch": 1.4822650902302428, + "grad_norm": 3.0246665477752686, + "learning_rate": 2.843634686346864e-06, + "loss": 1.4532, + "step": 1191 + }, + { + "epoch": 1.4835096453018046, + "grad_norm": 2.8856449127197266, + "learning_rate": 2.841328413284133e-06, + "loss": 1.4528, + "step": 1192 + }, + { + "epoch": 1.4847542003733665, + "grad_norm": 2.848987102508545, + "learning_rate": 2.8390221402214022e-06, + "loss": 1.4369, + "step": 1193 + }, + { + "epoch": 1.4859987554449283, + "grad_norm": 2.970419406890869, + "learning_rate": 2.836715867158672e-06, + "loss": 1.4605, + "step": 1194 + }, + { + "epoch": 1.4872433105164904, + "grad_norm": 3.156613826751709, + "learning_rate": 2.834409594095941e-06, + "loss": 1.4489, + "step": 1195 + }, + { + "epoch": 1.4884878655880522, + "grad_norm": 3.095349073410034, + "learning_rate": 2.8321033210332105e-06, + "loss": 1.4604, + "step": 1196 + }, + { + "epoch": 1.4897324206596143, + "grad_norm": 2.870742082595825, + "learning_rate": 2.82979704797048e-06, + "loss": 1.4284, + "step": 1197 + }, + { + "epoch": 1.4909769757311762, + "grad_norm": 2.9415056705474854, + "learning_rate": 2.827490774907749e-06, + "loss": 1.4715, + "step": 1198 + }, + { + "epoch": 1.492221530802738, + "grad_norm": 2.8297767639160156, + "learning_rate": 2.8251845018450188e-06, + "loss": 1.4508, + "step": 1199 + }, + { + "epoch": 1.4934660858742999, + "grad_norm": 2.872086524963379, + "learning_rate": 2.822878228782288e-06, + "loss": 1.5098, + "step": 1200 + }, + { + "epoch": 1.4934660858742999, + "eval_loss": 1.6357526779174805, + "eval_runtime": 51.422, + "eval_samples_per_second": 19.447, + "eval_steps_per_second": 0.817, + "step": 1200 + }, + { + "epoch": 1.494710640945862, + "grad_norm": 2.7152082920074463, + "learning_rate": 2.8205719557195576e-06, + "loss": 1.4329, + "step": 1201 + }, + { + "epoch": 1.4959551960174238, + "grad_norm": 2.866170883178711, + "learning_rate": 2.8182656826568266e-06, + "loss": 1.4455, + "step": 1202 + }, + { + "epoch": 1.4971997510889856, + "grad_norm": 2.9010684490203857, + "learning_rate": 2.815959409594096e-06, + "loss": 1.423, + "step": 1203 + }, + { + "epoch": 1.4984443061605477, + "grad_norm": 2.8588502407073975, + "learning_rate": 2.813653136531366e-06, + "loss": 1.4041, + "step": 1204 + }, + { + "epoch": 1.4996888612321095, + "grad_norm": 2.932544231414795, + "learning_rate": 2.811346863468635e-06, + "loss": 1.4559, + "step": 1205 + }, + { + "epoch": 1.5009334163036714, + "grad_norm": 2.7486014366149902, + "learning_rate": 2.8090405904059043e-06, + "loss": 1.4337, + "step": 1206 + }, + { + "epoch": 1.5021779713752332, + "grad_norm": 3.0866589546203613, + "learning_rate": 2.8067343173431737e-06, + "loss": 1.4186, + "step": 1207 + }, + { + "epoch": 1.5034225264467953, + "grad_norm": 2.9710276126861572, + "learning_rate": 2.8044280442804427e-06, + "loss": 1.419, + "step": 1208 + }, + { + "epoch": 1.5046670815183572, + "grad_norm": 2.8379950523376465, + "learning_rate": 2.802121771217712e-06, + "loss": 1.4922, + "step": 1209 + }, + { + "epoch": 1.5059116365899192, + "grad_norm": 2.939629554748535, + "learning_rate": 2.799815498154982e-06, + "loss": 1.4402, + "step": 1210 + }, + { + "epoch": 1.5059116365899192, + "eval_loss": 1.6419442892074585, + "eval_runtime": 43.1202, + "eval_samples_per_second": 23.191, + "eval_steps_per_second": 0.974, + "step": 1210 + }, + { + "epoch": 1.507156191661481, + "grad_norm": 3.041389226913452, + "learning_rate": 2.7975092250922514e-06, + "loss": 1.4873, + "step": 1211 + }, + { + "epoch": 1.508400746733043, + "grad_norm": 2.9776995182037354, + "learning_rate": 2.7952029520295204e-06, + "loss": 1.4422, + "step": 1212 + }, + { + "epoch": 1.5096453018046048, + "grad_norm": 2.8798792362213135, + "learning_rate": 2.79289667896679e-06, + "loss": 1.4535, + "step": 1213 + }, + { + "epoch": 1.5108898568761666, + "grad_norm": 2.9016385078430176, + "learning_rate": 2.7905904059040597e-06, + "loss": 1.4462, + "step": 1214 + }, + { + "epoch": 1.5121344119477287, + "grad_norm": 2.8987581729888916, + "learning_rate": 2.7882841328413287e-06, + "loss": 1.4477, + "step": 1215 + }, + { + "epoch": 1.5133789670192908, + "grad_norm": 2.960266351699829, + "learning_rate": 2.785977859778598e-06, + "loss": 1.4811, + "step": 1216 + }, + { + "epoch": 1.5146235220908526, + "grad_norm": 2.8965611457824707, + "learning_rate": 2.7836715867158675e-06, + "loss": 1.4366, + "step": 1217 + }, + { + "epoch": 1.5158680771624145, + "grad_norm": 3.0667364597320557, + "learning_rate": 2.7813653136531365e-06, + "loss": 1.3828, + "step": 1218 + }, + { + "epoch": 1.5171126322339763, + "grad_norm": 2.952362537384033, + "learning_rate": 2.779059040590406e-06, + "loss": 1.3903, + "step": 1219 + }, + { + "epoch": 1.5183571873055381, + "grad_norm": 2.9899795055389404, + "learning_rate": 2.776752767527676e-06, + "loss": 1.4312, + "step": 1220 + }, + { + "epoch": 1.5183571873055381, + "eval_loss": 1.640383005142212, + "eval_runtime": 45.4128, + "eval_samples_per_second": 22.02, + "eval_steps_per_second": 0.925, + "step": 1220 + }, + { + "epoch": 1.5196017423771002, + "grad_norm": 3.2202725410461426, + "learning_rate": 2.774446494464945e-06, + "loss": 1.4344, + "step": 1221 + }, + { + "epoch": 1.520846297448662, + "grad_norm": 3.2202725410461426, + "learning_rate": 2.774446494464945e-06, + "loss": 1.4386, + "step": 1222 + }, + { + "epoch": 1.5220908525202241, + "grad_norm": 2.883223295211792, + "learning_rate": 2.7721402214022142e-06, + "loss": 1.4401, + "step": 1223 + }, + { + "epoch": 1.523335407591786, + "grad_norm": 2.8346424102783203, + "learning_rate": 2.7698339483394837e-06, + "loss": 1.3926, + "step": 1224 + }, + { + "epoch": 1.5245799626633478, + "grad_norm": 2.8898870944976807, + "learning_rate": 2.767527675276753e-06, + "loss": 1.4128, + "step": 1225 + }, + { + "epoch": 1.5258245177349097, + "grad_norm": 2.828594923019409, + "learning_rate": 2.765221402214022e-06, + "loss": 1.4103, + "step": 1226 + }, + { + "epoch": 1.5270690728064715, + "grad_norm": 2.882558822631836, + "learning_rate": 2.762915129151292e-06, + "loss": 1.4186, + "step": 1227 + }, + { + "epoch": 1.5283136278780336, + "grad_norm": 2.955760955810547, + "learning_rate": 2.7606088560885613e-06, + "loss": 1.3705, + "step": 1228 + }, + { + "epoch": 1.5295581829495957, + "grad_norm": 2.862934112548828, + "learning_rate": 2.7583025830258303e-06, + "loss": 1.5012, + "step": 1229 + }, + { + "epoch": 1.5308027380211575, + "grad_norm": 2.926501989364624, + "learning_rate": 2.7559963099630998e-06, + "loss": 1.3875, + "step": 1230 + }, + { + "epoch": 1.5308027380211575, + "eval_loss": 1.6376254558563232, + "eval_runtime": 47.9535, + "eval_samples_per_second": 20.854, + "eval_steps_per_second": 0.876, + "step": 1230 + }, + { + "epoch": 1.5320472930927194, + "grad_norm": 2.7648823261260986, + "learning_rate": 2.753690036900369e-06, + "loss": 1.3781, + "step": 1231 + }, + { + "epoch": 1.5332918481642812, + "grad_norm": 2.927103281021118, + "learning_rate": 2.751383763837638e-06, + "loss": 1.4091, + "step": 1232 + }, + { + "epoch": 1.534536403235843, + "grad_norm": 2.964823007583618, + "learning_rate": 2.749077490774908e-06, + "loss": 1.5075, + "step": 1233 + }, + { + "epoch": 1.5357809583074051, + "grad_norm": 2.7944016456604004, + "learning_rate": 2.7467712177121775e-06, + "loss": 1.3888, + "step": 1234 + }, + { + "epoch": 1.537025513378967, + "grad_norm": 2.891592025756836, + "learning_rate": 2.7444649446494465e-06, + "loss": 1.4825, + "step": 1235 + }, + { + "epoch": 1.538270068450529, + "grad_norm": 2.8120031356811523, + "learning_rate": 2.742158671586716e-06, + "loss": 1.4246, + "step": 1236 + }, + { + "epoch": 1.539514623522091, + "grad_norm": 2.9384255409240723, + "learning_rate": 2.7398523985239857e-06, + "loss": 1.4657, + "step": 1237 + }, + { + "epoch": 1.5407591785936527, + "grad_norm": 2.73111891746521, + "learning_rate": 2.737546125461255e-06, + "loss": 1.4689, + "step": 1238 + }, + { + "epoch": 1.5420037336652146, + "grad_norm": 2.858963966369629, + "learning_rate": 2.735239852398524e-06, + "loss": 1.4626, + "step": 1239 + }, + { + "epoch": 1.5432482887367764, + "grad_norm": 2.9148507118225098, + "learning_rate": 2.7329335793357936e-06, + "loss": 1.4787, + "step": 1240 + }, + { + "epoch": 1.5432482887367764, + "eval_loss": 1.6436412334442139, + "eval_runtime": 47.3139, + "eval_samples_per_second": 21.135, + "eval_steps_per_second": 0.888, + "step": 1240 + }, + { + "epoch": 1.5444928438083385, + "grad_norm": 2.8731327056884766, + "learning_rate": 2.730627306273063e-06, + "loss": 1.4748, + "step": 1241 + }, + { + "epoch": 1.5457373988799006, + "grad_norm": 2.990265369415283, + "learning_rate": 2.728321033210332e-06, + "loss": 1.4375, + "step": 1242 + }, + { + "epoch": 1.5469819539514624, + "grad_norm": 2.874321460723877, + "learning_rate": 2.726014760147602e-06, + "loss": 1.4154, + "step": 1243 + }, + { + "epoch": 1.5482265090230243, + "grad_norm": 2.9316225051879883, + "learning_rate": 2.7237084870848713e-06, + "loss": 1.4285, + "step": 1244 + }, + { + "epoch": 1.5494710640945861, + "grad_norm": 2.783966302871704, + "learning_rate": 2.7214022140221403e-06, + "loss": 1.409, + "step": 1245 + }, + { + "epoch": 1.550715619166148, + "grad_norm": 2.9578707218170166, + "learning_rate": 2.7190959409594097e-06, + "loss": 1.4325, + "step": 1246 + }, + { + "epoch": 1.55196017423771, + "grad_norm": 2.869893789291382, + "learning_rate": 2.716789667896679e-06, + "loss": 1.3989, + "step": 1247 + }, + { + "epoch": 1.553204729309272, + "grad_norm": 2.962944269180298, + "learning_rate": 2.714483394833948e-06, + "loss": 1.4302, + "step": 1248 + }, + { + "epoch": 1.554449284380834, + "grad_norm": 2.974494695663452, + "learning_rate": 2.712177121771218e-06, + "loss": 1.46, + "step": 1249 + }, + { + "epoch": 1.5556938394523958, + "grad_norm": 2.9154603481292725, + "learning_rate": 2.7098708487084874e-06, + "loss": 1.4388, + "step": 1250 + }, + { + "epoch": 1.5556938394523958, + "eval_loss": 1.6357773542404175, + "eval_runtime": 47.6085, + "eval_samples_per_second": 21.005, + "eval_steps_per_second": 0.882, + "step": 1250 + }, + { + "epoch": 1.5569383945239577, + "grad_norm": 3.0333547592163086, + "learning_rate": 2.707564575645757e-06, + "loss": 1.4846, + "step": 1251 + }, + { + "epoch": 1.5581829495955195, + "grad_norm": 3.06097674369812, + "learning_rate": 2.705258302583026e-06, + "loss": 1.4355, + "step": 1252 + }, + { + "epoch": 1.5594275046670814, + "grad_norm": 2.872680425643921, + "learning_rate": 2.7029520295202956e-06, + "loss": 1.4219, + "step": 1253 + }, + { + "epoch": 1.5606720597386434, + "grad_norm": 2.9066731929779053, + "learning_rate": 2.700645756457565e-06, + "loss": 1.384, + "step": 1254 + }, + { + "epoch": 1.5619166148102055, + "grad_norm": 3.0570194721221924, + "learning_rate": 2.698339483394834e-06, + "loss": 1.4687, + "step": 1255 + }, + { + "epoch": 1.5631611698817673, + "grad_norm": 3.1955857276916504, + "learning_rate": 2.6960332103321035e-06, + "loss": 1.4736, + "step": 1256 + }, + { + "epoch": 1.5644057249533292, + "grad_norm": 2.9686198234558105, + "learning_rate": 2.693726937269373e-06, + "loss": 1.4176, + "step": 1257 + }, + { + "epoch": 1.565650280024891, + "grad_norm": 3.1161272525787354, + "learning_rate": 2.691420664206642e-06, + "loss": 1.443, + "step": 1258 + }, + { + "epoch": 1.5668948350964529, + "grad_norm": 2.9031193256378174, + "learning_rate": 2.6891143911439118e-06, + "loss": 1.3871, + "step": 1259 + }, + { + "epoch": 1.568139390168015, + "grad_norm": 2.9120476245880127, + "learning_rate": 2.686808118081181e-06, + "loss": 1.4406, + "step": 1260 + }, + { + "epoch": 1.568139390168015, + "eval_loss": 1.6291446685791016, + "eval_runtime": 49.7006, + "eval_samples_per_second": 20.12, + "eval_steps_per_second": 0.845, + "step": 1260 + }, + { + "epoch": 1.5693839452395768, + "grad_norm": 3.3181824684143066, + "learning_rate": 2.6845018450184506e-06, + "loss": 1.5278, + "step": 1261 + }, + { + "epoch": 1.5706285003111389, + "grad_norm": 2.9628148078918457, + "learning_rate": 2.6821955719557196e-06, + "loss": 1.4206, + "step": 1262 + }, + { + "epoch": 1.5718730553827007, + "grad_norm": 2.9828474521636963, + "learning_rate": 2.679889298892989e-06, + "loss": 1.5478, + "step": 1263 + }, + { + "epoch": 1.5731176104542626, + "grad_norm": 3.018711805343628, + "learning_rate": 2.677583025830259e-06, + "loss": 1.4442, + "step": 1264 + }, + { + "epoch": 1.5743621655258244, + "grad_norm": 2.8596792221069336, + "learning_rate": 2.675276752767528e-06, + "loss": 1.4325, + "step": 1265 + }, + { + "epoch": 1.5756067205973865, + "grad_norm": 2.9592769145965576, + "learning_rate": 2.6729704797047973e-06, + "loss": 1.4059, + "step": 1266 + }, + { + "epoch": 1.5768512756689483, + "grad_norm": 2.773589611053467, + "learning_rate": 2.6706642066420667e-06, + "loss": 1.3608, + "step": 1267 + }, + { + "epoch": 1.5780958307405104, + "grad_norm": 3.313255548477173, + "learning_rate": 2.6683579335793357e-06, + "loss": 1.4552, + "step": 1268 + }, + { + "epoch": 1.5793403858120723, + "grad_norm": 2.977649211883545, + "learning_rate": 2.6660516605166056e-06, + "loss": 1.4221, + "step": 1269 + }, + { + "epoch": 1.580584940883634, + "grad_norm": 3.0672051906585693, + "learning_rate": 2.663745387453875e-06, + "loss": 1.4568, + "step": 1270 + }, + { + "epoch": 1.580584940883634, + "eval_loss": 1.6317907571792603, + "eval_runtime": 47.2354, + "eval_samples_per_second": 21.171, + "eval_steps_per_second": 0.889, + "step": 1270 + }, + { + "epoch": 1.581829495955196, + "grad_norm": 3.029320240020752, + "learning_rate": 2.661439114391144e-06, + "loss": 1.4432, + "step": 1271 + }, + { + "epoch": 1.5830740510267578, + "grad_norm": 3.0183846950531006, + "learning_rate": 2.6591328413284134e-06, + "loss": 1.4332, + "step": 1272 + }, + { + "epoch": 1.5843186060983199, + "grad_norm": 2.856551170349121, + "learning_rate": 2.656826568265683e-06, + "loss": 1.4291, + "step": 1273 + }, + { + "epoch": 1.5855631611698817, + "grad_norm": 3.124971389770508, + "learning_rate": 2.6545202952029527e-06, + "loss": 1.4363, + "step": 1274 + }, + { + "epoch": 1.5868077162414438, + "grad_norm": 2.9470791816711426, + "learning_rate": 2.6522140221402217e-06, + "loss": 1.4502, + "step": 1275 + }, + { + "epoch": 1.5880522713130056, + "grad_norm": 2.9116358757019043, + "learning_rate": 2.649907749077491e-06, + "loss": 1.4314, + "step": 1276 + }, + { + "epoch": 1.5892968263845675, + "grad_norm": 3.2575323581695557, + "learning_rate": 2.6476014760147605e-06, + "loss": 1.4749, + "step": 1277 + }, + { + "epoch": 1.5905413814561293, + "grad_norm": 2.935518503189087, + "learning_rate": 2.6452952029520295e-06, + "loss": 1.4265, + "step": 1278 + }, + { + "epoch": 1.5917859365276914, + "grad_norm": 2.9750072956085205, + "learning_rate": 2.642988929889299e-06, + "loss": 1.4501, + "step": 1279 + }, + { + "epoch": 1.5930304915992533, + "grad_norm": 2.8086845874786377, + "learning_rate": 2.640682656826569e-06, + "loss": 1.4557, + "step": 1280 + }, + { + "epoch": 1.5930304915992533, + "eval_loss": 1.6302106380462646, + "eval_runtime": 52.3962, + "eval_samples_per_second": 19.085, + "eval_steps_per_second": 0.802, + "step": 1280 + }, + { + "epoch": 1.5942750466708153, + "grad_norm": 2.741849422454834, + "learning_rate": 2.638376383763838e-06, + "loss": 1.3918, + "step": 1281 + }, + { + "epoch": 1.5955196017423772, + "grad_norm": 2.9414806365966797, + "learning_rate": 2.6360701107011072e-06, + "loss": 1.3882, + "step": 1282 + }, + { + "epoch": 1.596764156813939, + "grad_norm": 2.901514768600464, + "learning_rate": 2.6337638376383766e-06, + "loss": 1.4331, + "step": 1283 + }, + { + "epoch": 1.5980087118855009, + "grad_norm": 2.8159122467041016, + "learning_rate": 2.6314575645756456e-06, + "loss": 1.4064, + "step": 1284 + }, + { + "epoch": 1.5992532669570627, + "grad_norm": 2.900561571121216, + "learning_rate": 2.629151291512915e-06, + "loss": 1.3731, + "step": 1285 + }, + { + "epoch": 1.6004978220286248, + "grad_norm": 3.019540548324585, + "learning_rate": 2.626845018450185e-06, + "loss": 1.4506, + "step": 1286 + }, + { + "epoch": 1.6017423771001866, + "grad_norm": 2.9263620376586914, + "learning_rate": 2.6245387453874543e-06, + "loss": 1.4754, + "step": 1287 + }, + { + "epoch": 1.6029869321717487, + "grad_norm": 2.8681881427764893, + "learning_rate": 2.6222324723247233e-06, + "loss": 1.4124, + "step": 1288 + }, + { + "epoch": 1.6042314872433105, + "grad_norm": 2.922945261001587, + "learning_rate": 2.6199261992619928e-06, + "loss": 1.5101, + "step": 1289 + }, + { + "epoch": 1.6054760423148724, + "grad_norm": 2.9548606872558594, + "learning_rate": 2.6176199261992626e-06, + "loss": 1.41, + "step": 1290 + }, + { + "epoch": 1.6054760423148724, + "eval_loss": 1.6357502937316895, + "eval_runtime": 46.9547, + "eval_samples_per_second": 21.297, + "eval_steps_per_second": 0.894, + "step": 1290 + }, + { + "epoch": 1.6067205973864342, + "grad_norm": 2.837184190750122, + "learning_rate": 2.6153136531365316e-06, + "loss": 1.4354, + "step": 1291 + }, + { + "epoch": 1.6079651524579963, + "grad_norm": 2.818990468978882, + "learning_rate": 2.613007380073801e-06, + "loss": 1.4474, + "step": 1292 + }, + { + "epoch": 1.6092097075295582, + "grad_norm": 2.7880146503448486, + "learning_rate": 2.6107011070110704e-06, + "loss": 1.4156, + "step": 1293 + }, + { + "epoch": 1.6104542626011202, + "grad_norm": 2.790971040725708, + "learning_rate": 2.6083948339483394e-06, + "loss": 1.4671, + "step": 1294 + }, + { + "epoch": 1.611698817672682, + "grad_norm": 2.943129777908325, + "learning_rate": 2.606088560885609e-06, + "loss": 1.4801, + "step": 1295 + }, + { + "epoch": 1.612943372744244, + "grad_norm": 2.8589110374450684, + "learning_rate": 2.6037822878228787e-06, + "loss": 1.4894, + "step": 1296 + }, + { + "epoch": 1.6141879278158058, + "grad_norm": 2.8929262161254883, + "learning_rate": 2.6014760147601477e-06, + "loss": 1.4022, + "step": 1297 + }, + { + "epoch": 1.6154324828873676, + "grad_norm": 2.970099925994873, + "learning_rate": 2.599169741697417e-06, + "loss": 1.5136, + "step": 1298 + }, + { + "epoch": 1.6166770379589297, + "grad_norm": 2.928466796875, + "learning_rate": 2.5968634686346866e-06, + "loss": 1.4743, + "step": 1299 + }, + { + "epoch": 1.6179215930304915, + "grad_norm": 2.941580057144165, + "learning_rate": 2.594557195571956e-06, + "loss": 1.4397, + "step": 1300 + }, + { + "epoch": 1.6179215930304915, + "eval_loss": 1.6348339319229126, + "eval_runtime": 50.433, + "eval_samples_per_second": 19.828, + "eval_steps_per_second": 0.833, + "step": 1300 + }, + { + "epoch": 1.6191661481020536, + "grad_norm": 2.903212308883667, + "learning_rate": 2.592250922509225e-06, + "loss": 1.4221, + "step": 1301 + }, + { + "epoch": 1.6204107031736155, + "grad_norm": 2.948651075363159, + "learning_rate": 2.589944649446495e-06, + "loss": 1.4363, + "step": 1302 + }, + { + "epoch": 1.6216552582451773, + "grad_norm": 2.8940505981445312, + "learning_rate": 2.5876383763837643e-06, + "loss": 1.398, + "step": 1303 + }, + { + "epoch": 1.6228998133167392, + "grad_norm": 2.950056552886963, + "learning_rate": 2.5853321033210333e-06, + "loss": 1.4425, + "step": 1304 + }, + { + "epoch": 1.6241443683883012, + "grad_norm": 2.9674088954925537, + "learning_rate": 2.5830258302583027e-06, + "loss": 1.4473, + "step": 1305 + }, + { + "epoch": 1.625388923459863, + "grad_norm": 2.7541685104370117, + "learning_rate": 2.5807195571955725e-06, + "loss": 1.4483, + "step": 1306 + }, + { + "epoch": 1.6266334785314251, + "grad_norm": 2.8897035121917725, + "learning_rate": 2.5784132841328415e-06, + "loss": 1.4572, + "step": 1307 + }, + { + "epoch": 1.627878033602987, + "grad_norm": 3.021559238433838, + "learning_rate": 2.576107011070111e-06, + "loss": 1.5164, + "step": 1308 + }, + { + "epoch": 1.6291225886745488, + "grad_norm": 2.884009838104248, + "learning_rate": 2.5738007380073804e-06, + "loss": 1.4196, + "step": 1309 + }, + { + "epoch": 1.6303671437461107, + "grad_norm": 2.749497890472412, + "learning_rate": 2.5714944649446494e-06, + "loss": 1.4324, + "step": 1310 + }, + { + "epoch": 1.6303671437461107, + "eval_loss": 1.6314265727996826, + "eval_runtime": 42.4647, + "eval_samples_per_second": 23.549, + "eval_steps_per_second": 0.989, + "step": 1310 + }, + { + "epoch": 1.6316116988176725, + "grad_norm": 2.8908045291900635, + "learning_rate": 2.5691881918819188e-06, + "loss": 1.4388, + "step": 1311 + }, + { + "epoch": 1.6328562538892346, + "grad_norm": 2.832808256149292, + "learning_rate": 2.5668819188191886e-06, + "loss": 1.4368, + "step": 1312 + }, + { + "epoch": 1.6341008089607967, + "grad_norm": 2.9047906398773193, + "learning_rate": 2.564575645756458e-06, + "loss": 1.4186, + "step": 1313 + }, + { + "epoch": 1.6353453640323585, + "grad_norm": 2.9271419048309326, + "learning_rate": 2.562269372693727e-06, + "loss": 1.3776, + "step": 1314 + }, + { + "epoch": 1.6365899191039204, + "grad_norm": 2.760314702987671, + "learning_rate": 2.5599630996309965e-06, + "loss": 1.386, + "step": 1315 + }, + { + "epoch": 1.6378344741754822, + "grad_norm": 2.9694864749908447, + "learning_rate": 2.557656826568266e-06, + "loss": 1.4542, + "step": 1316 + }, + { + "epoch": 1.639079029247044, + "grad_norm": 2.9950435161590576, + "learning_rate": 2.555350553505535e-06, + "loss": 1.4242, + "step": 1317 + }, + { + "epoch": 1.6403235843186061, + "grad_norm": 2.9291114807128906, + "learning_rate": 2.5530442804428047e-06, + "loss": 1.4289, + "step": 1318 + }, + { + "epoch": 1.641568139390168, + "grad_norm": 2.981961488723755, + "learning_rate": 2.550738007380074e-06, + "loss": 1.4283, + "step": 1319 + }, + { + "epoch": 1.64281269446173, + "grad_norm": 2.734123706817627, + "learning_rate": 2.548431734317343e-06, + "loss": 1.4144, + "step": 1320 + }, + { + "epoch": 1.64281269446173, + "eval_loss": 1.6249239444732666, + "eval_runtime": 48.8116, + "eval_samples_per_second": 20.487, + "eval_steps_per_second": 0.86, + "step": 1320 + }, + { + "epoch": 1.644057249533292, + "grad_norm": 2.9752919673919678, + "learning_rate": 2.5461254612546126e-06, + "loss": 1.3641, + "step": 1321 + }, + { + "epoch": 1.6453018046048538, + "grad_norm": 2.972463607788086, + "learning_rate": 2.543819188191882e-06, + "loss": 1.3797, + "step": 1322 + }, + { + "epoch": 1.6465463596764156, + "grad_norm": 2.905499219894409, + "learning_rate": 2.541512915129152e-06, + "loss": 1.4221, + "step": 1323 + }, + { + "epoch": 1.6477909147479775, + "grad_norm": 2.7921793460845947, + "learning_rate": 2.539206642066421e-06, + "loss": 1.4118, + "step": 1324 + }, + { + "epoch": 1.6490354698195395, + "grad_norm": 3.064558267593384, + "learning_rate": 2.5369003690036903e-06, + "loss": 1.4611, + "step": 1325 + }, + { + "epoch": 1.6502800248911016, + "grad_norm": 2.9690096378326416, + "learning_rate": 2.5345940959409597e-06, + "loss": 1.461, + "step": 1326 + }, + { + "epoch": 1.6515245799626634, + "grad_norm": 3.0379040241241455, + "learning_rate": 2.5322878228782287e-06, + "loss": 1.451, + "step": 1327 + }, + { + "epoch": 1.6527691350342253, + "grad_norm": 2.9426493644714355, + "learning_rate": 2.5299815498154986e-06, + "loss": 1.4671, + "step": 1328 + }, + { + "epoch": 1.6540136901057871, + "grad_norm": 2.915076732635498, + "learning_rate": 2.527675276752768e-06, + "loss": 1.4159, + "step": 1329 + }, + { + "epoch": 1.655258245177349, + "grad_norm": 2.8588879108428955, + "learning_rate": 2.525369003690037e-06, + "loss": 1.4978, + "step": 1330 + }, + { + "epoch": 1.655258245177349, + "eval_loss": 1.624009132385254, + "eval_runtime": 47.0461, + "eval_samples_per_second": 21.256, + "eval_steps_per_second": 0.893, + "step": 1330 + }, + { + "epoch": 1.656502800248911, + "grad_norm": 2.9942209720611572, + "learning_rate": 2.5230627306273064e-06, + "loss": 1.3872, + "step": 1331 + }, + { + "epoch": 1.657747355320473, + "grad_norm": 2.8424410820007324, + "learning_rate": 2.520756457564576e-06, + "loss": 1.4181, + "step": 1332 + }, + { + "epoch": 1.658991910392035, + "grad_norm": 2.9234843254089355, + "learning_rate": 2.518450184501845e-06, + "loss": 1.4516, + "step": 1333 + }, + { + "epoch": 1.6602364654635968, + "grad_norm": 2.8367815017700195, + "learning_rate": 2.5161439114391147e-06, + "loss": 1.4214, + "step": 1334 + }, + { + "epoch": 1.6614810205351587, + "grad_norm": 2.819149971008301, + "learning_rate": 2.513837638376384e-06, + "loss": 1.3883, + "step": 1335 + }, + { + "epoch": 1.6627255756067205, + "grad_norm": 3.024693250656128, + "learning_rate": 2.5115313653136535e-06, + "loss": 1.3902, + "step": 1336 + }, + { + "epoch": 1.6639701306782824, + "grad_norm": 2.903599262237549, + "learning_rate": 2.5092250922509225e-06, + "loss": 1.4238, + "step": 1337 + }, + { + "epoch": 1.6652146857498444, + "grad_norm": 2.8296589851379395, + "learning_rate": 2.506918819188192e-06, + "loss": 1.3993, + "step": 1338 + }, + { + "epoch": 1.6664592408214065, + "grad_norm": 2.873272657394409, + "learning_rate": 2.5046125461254618e-06, + "loss": 1.4186, + "step": 1339 + }, + { + "epoch": 1.6677037958929684, + "grad_norm": 2.9268789291381836, + "learning_rate": 2.5023062730627308e-06, + "loss": 1.4242, + "step": 1340 + }, + { + "epoch": 1.6677037958929684, + "eval_loss": 1.6260672807693481, + "eval_runtime": 47.5895, + "eval_samples_per_second": 21.013, + "eval_steps_per_second": 0.883, + "step": 1340 + }, + { + "epoch": 1.6689483509645302, + "grad_norm": 3.057659387588501, + "learning_rate": 2.5e-06, + "loss": 1.4077, + "step": 1341 + }, + { + "epoch": 1.670192906036092, + "grad_norm": 2.875095844268799, + "learning_rate": 2.4976937269372696e-06, + "loss": 1.394, + "step": 1342 + }, + { + "epoch": 1.671437461107654, + "grad_norm": 2.7579872608184814, + "learning_rate": 2.495387453874539e-06, + "loss": 1.4029, + "step": 1343 + }, + { + "epoch": 1.672682016179216, + "grad_norm": 2.8981001377105713, + "learning_rate": 2.4930811808118085e-06, + "loss": 1.4494, + "step": 1344 + }, + { + "epoch": 1.6739265712507778, + "grad_norm": 2.9696900844573975, + "learning_rate": 2.490774907749078e-06, + "loss": 1.4692, + "step": 1345 + }, + { + "epoch": 1.6751711263223399, + "grad_norm": 2.8094892501831055, + "learning_rate": 2.488468634686347e-06, + "loss": 1.441, + "step": 1346 + }, + { + "epoch": 1.6764156813939017, + "grad_norm": 2.8619563579559326, + "learning_rate": 2.4861623616236163e-06, + "loss": 1.4885, + "step": 1347 + }, + { + "epoch": 1.6776602364654636, + "grad_norm": 2.854151487350464, + "learning_rate": 2.4838560885608857e-06, + "loss": 1.4338, + "step": 1348 + }, + { + "epoch": 1.6789047915370254, + "grad_norm": 2.9707868099212646, + "learning_rate": 2.481549815498155e-06, + "loss": 1.4529, + "step": 1349 + }, + { + "epoch": 1.6801493466085873, + "grad_norm": 3.0289969444274902, + "learning_rate": 2.4792435424354246e-06, + "loss": 1.4855, + "step": 1350 + }, + { + "epoch": 1.6801493466085873, + "eval_loss": 1.6256393194198608, + "eval_runtime": 45.9524, + "eval_samples_per_second": 21.762, + "eval_steps_per_second": 0.914, + "step": 1350 + }, + { + "epoch": 1.6813939016801493, + "grad_norm": 2.8407845497131348, + "learning_rate": 2.476937269372694e-06, + "loss": 1.3876, + "step": 1351 + }, + { + "epoch": 1.6826384567517114, + "grad_norm": 2.9876599311828613, + "learning_rate": 2.4746309963099634e-06, + "loss": 1.3931, + "step": 1352 + }, + { + "epoch": 1.6838830118232733, + "grad_norm": 3.008683919906616, + "learning_rate": 2.472324723247233e-06, + "loss": 1.4482, + "step": 1353 + }, + { + "epoch": 1.6851275668948351, + "grad_norm": 2.8110079765319824, + "learning_rate": 2.470018450184502e-06, + "loss": 1.3851, + "step": 1354 + }, + { + "epoch": 1.686372121966397, + "grad_norm": 2.9402263164520264, + "learning_rate": 2.4677121771217713e-06, + "loss": 1.4009, + "step": 1355 + }, + { + "epoch": 1.6876166770379588, + "grad_norm": 2.9846489429473877, + "learning_rate": 2.4654059040590407e-06, + "loss": 1.3975, + "step": 1356 + }, + { + "epoch": 1.6888612321095209, + "grad_norm": 2.835801362991333, + "learning_rate": 2.46309963099631e-06, + "loss": 1.3421, + "step": 1357 + }, + { + "epoch": 1.6901057871810827, + "grad_norm": 2.9438889026641846, + "learning_rate": 2.4607933579335795e-06, + "loss": 1.4422, + "step": 1358 + }, + { + "epoch": 1.6913503422526448, + "grad_norm": 3.168829917907715, + "learning_rate": 2.458487084870849e-06, + "loss": 1.4596, + "step": 1359 + }, + { + "epoch": 1.6925948973242066, + "grad_norm": 2.8412039279937744, + "learning_rate": 2.4561808118081184e-06, + "loss": 1.4263, + "step": 1360 + }, + { + "epoch": 1.6925948973242066, + "eval_loss": 1.6265885829925537, + "eval_runtime": 50.3402, + "eval_samples_per_second": 19.865, + "eval_steps_per_second": 0.834, + "step": 1360 + }, + { + "epoch": 1.6938394523957685, + "grad_norm": 3.0540060997009277, + "learning_rate": 2.453874538745388e-06, + "loss": 1.4123, + "step": 1361 + }, + { + "epoch": 1.6950840074673303, + "grad_norm": 3.0544989109039307, + "learning_rate": 2.451568265682657e-06, + "loss": 1.412, + "step": 1362 + }, + { + "epoch": 1.6963285625388922, + "grad_norm": 3.114126205444336, + "learning_rate": 2.4492619926199267e-06, + "loss": 1.4952, + "step": 1363 + }, + { + "epoch": 1.6975731176104543, + "grad_norm": 2.9366016387939453, + "learning_rate": 2.4469557195571957e-06, + "loss": 1.5155, + "step": 1364 + }, + { + "epoch": 1.6988176726820163, + "grad_norm": 2.910677671432495, + "learning_rate": 2.444649446494465e-06, + "loss": 1.4393, + "step": 1365 + }, + { + "epoch": 1.7000622277535782, + "grad_norm": 2.8373727798461914, + "learning_rate": 2.4423431734317345e-06, + "loss": 1.4223, + "step": 1366 + }, + { + "epoch": 1.70130678282514, + "grad_norm": 2.865161895751953, + "learning_rate": 2.440036900369004e-06, + "loss": 1.4605, + "step": 1367 + }, + { + "epoch": 1.7025513378967019, + "grad_norm": 2.7929177284240723, + "learning_rate": 2.437730627306273e-06, + "loss": 1.4695, + "step": 1368 + }, + { + "epoch": 1.7037958929682637, + "grad_norm": 2.867953300476074, + "learning_rate": 2.4354243542435428e-06, + "loss": 1.3524, + "step": 1369 + }, + { + "epoch": 1.7050404480398258, + "grad_norm": 2.8980207443237305, + "learning_rate": 2.4331180811808118e-06, + "loss": 1.4568, + "step": 1370 + }, + { + "epoch": 1.7050404480398258, + "eval_loss": 1.6297556161880493, + "eval_runtime": 51.447, + "eval_samples_per_second": 19.437, + "eval_steps_per_second": 0.816, + "step": 1370 + }, + { + "epoch": 1.7062850031113876, + "grad_norm": 2.954972505569458, + "learning_rate": 2.4308118081180816e-06, + "loss": 1.4, + "step": 1371 + }, + { + "epoch": 1.7075295581829497, + "grad_norm": 2.973191738128662, + "learning_rate": 2.4285055350553506e-06, + "loss": 1.4062, + "step": 1372 + }, + { + "epoch": 1.7087741132545116, + "grad_norm": 2.951444387435913, + "learning_rate": 2.42619926199262e-06, + "loss": 1.4523, + "step": 1373 + }, + { + "epoch": 1.7100186683260734, + "grad_norm": 2.850445508956909, + "learning_rate": 2.4238929889298895e-06, + "loss": 1.4282, + "step": 1374 + }, + { + "epoch": 1.7112632233976353, + "grad_norm": 2.7643561363220215, + "learning_rate": 2.421586715867159e-06, + "loss": 1.3807, + "step": 1375 + }, + { + "epoch": 1.712507778469197, + "grad_norm": 2.8872134685516357, + "learning_rate": 2.4192804428044283e-06, + "loss": 1.4801, + "step": 1376 + }, + { + "epoch": 1.7137523335407592, + "grad_norm": 2.8748483657836914, + "learning_rate": 2.4169741697416977e-06, + "loss": 1.381, + "step": 1377 + }, + { + "epoch": 1.7149968886123212, + "grad_norm": 2.7833831310272217, + "learning_rate": 2.4146678966789667e-06, + "loss": 1.389, + "step": 1378 + }, + { + "epoch": 1.716241443683883, + "grad_norm": 2.994715690612793, + "learning_rate": 2.4123616236162366e-06, + "loss": 1.3917, + "step": 1379 + }, + { + "epoch": 1.717485998755445, + "grad_norm": 2.887026786804199, + "learning_rate": 2.4100553505535056e-06, + "loss": 1.3725, + "step": 1380 + }, + { + "epoch": 1.717485998755445, + "eval_loss": 1.6283434629440308, + "eval_runtime": 50.596, + "eval_samples_per_second": 19.764, + "eval_steps_per_second": 0.83, + "step": 1380 + }, + { + "epoch": 1.7187305538270068, + "grad_norm": 3.2046546936035156, + "learning_rate": 2.407749077490775e-06, + "loss": 1.4422, + "step": 1381 + }, + { + "epoch": 1.7199751088985686, + "grad_norm": 3.0105698108673096, + "learning_rate": 2.4054428044280444e-06, + "loss": 1.4494, + "step": 1382 + }, + { + "epoch": 1.7212196639701307, + "grad_norm": 2.9917800426483154, + "learning_rate": 2.403136531365314e-06, + "loss": 1.4407, + "step": 1383 + }, + { + "epoch": 1.7224642190416926, + "grad_norm": 2.844325065612793, + "learning_rate": 2.4008302583025833e-06, + "loss": 1.4067, + "step": 1384 + }, + { + "epoch": 1.7237087741132546, + "grad_norm": 2.919576406478882, + "learning_rate": 2.3985239852398527e-06, + "loss": 1.4352, + "step": 1385 + }, + { + "epoch": 1.7249533291848165, + "grad_norm": 2.8357744216918945, + "learning_rate": 2.3962177121771217e-06, + "loss": 1.3978, + "step": 1386 + }, + { + "epoch": 1.7261978842563783, + "grad_norm": 2.82425856590271, + "learning_rate": 2.3939114391143915e-06, + "loss": 1.4439, + "step": 1387 + }, + { + "epoch": 1.7274424393279402, + "grad_norm": 2.9155638217926025, + "learning_rate": 2.3916051660516605e-06, + "loss": 1.4326, + "step": 1388 + }, + { + "epoch": 1.728686994399502, + "grad_norm": 2.862804651260376, + "learning_rate": 2.3892988929889304e-06, + "loss": 1.3766, + "step": 1389 + }, + { + "epoch": 1.729931549471064, + "grad_norm": 3.012230634689331, + "learning_rate": 2.3869926199261994e-06, + "loss": 1.4569, + "step": 1390 + }, + { + "epoch": 1.729931549471064, + "eval_loss": 1.6268510818481445, + "eval_runtime": 52.5438, + "eval_samples_per_second": 19.032, + "eval_steps_per_second": 0.799, + "step": 1390 + }, + { + "epoch": 1.7311761045426262, + "grad_norm": 2.9922425746917725, + "learning_rate": 2.384686346863469e-06, + "loss": 1.4066, + "step": 1391 + }, + { + "epoch": 1.732420659614188, + "grad_norm": 2.960145950317383, + "learning_rate": 2.3823800738007382e-06, + "loss": 1.4514, + "step": 1392 + }, + { + "epoch": 1.7336652146857499, + "grad_norm": 2.834998846054077, + "learning_rate": 2.3800738007380077e-06, + "loss": 1.4714, + "step": 1393 + }, + { + "epoch": 1.7349097697573117, + "grad_norm": 2.9646077156066895, + "learning_rate": 2.377767527675277e-06, + "loss": 1.4368, + "step": 1394 + }, + { + "epoch": 1.7361543248288736, + "grad_norm": 2.8233699798583984, + "learning_rate": 2.3754612546125465e-06, + "loss": 1.4598, + "step": 1395 + }, + { + "epoch": 1.7373988799004356, + "grad_norm": 2.895472764968872, + "learning_rate": 2.3731549815498155e-06, + "loss": 1.4551, + "step": 1396 + }, + { + "epoch": 1.7386434349719975, + "grad_norm": 2.7917752265930176, + "learning_rate": 2.3708487084870853e-06, + "loss": 1.4759, + "step": 1397 + }, + { + "epoch": 1.7398879900435595, + "grad_norm": 2.820672035217285, + "learning_rate": 2.3685424354243543e-06, + "loss": 1.3711, + "step": 1398 + }, + { + "epoch": 1.7411325451151214, + "grad_norm": 2.804952621459961, + "learning_rate": 2.3662361623616238e-06, + "loss": 1.4227, + "step": 1399 + }, + { + "epoch": 1.7423771001866832, + "grad_norm": 2.981553316116333, + "learning_rate": 2.363929889298893e-06, + "loss": 1.4639, + "step": 1400 + }, + { + "epoch": 1.7423771001866832, + "eval_loss": 1.6229957342147827, + "eval_runtime": 49.1507, + "eval_samples_per_second": 20.346, + "eval_steps_per_second": 0.855, + "step": 1400 + }, + { + "epoch": 1.743621655258245, + "grad_norm": 2.8649392127990723, + "learning_rate": 2.3616236162361626e-06, + "loss": 1.4283, + "step": 1401 + }, + { + "epoch": 1.744866210329807, + "grad_norm": 2.9438977241516113, + "learning_rate": 2.359317343173432e-06, + "loss": 1.444, + "step": 1402 + }, + { + "epoch": 1.746110765401369, + "grad_norm": 2.9270176887512207, + "learning_rate": 2.3570110701107015e-06, + "loss": 1.4044, + "step": 1403 + }, + { + "epoch": 1.747355320472931, + "grad_norm": 2.791527271270752, + "learning_rate": 2.3547047970479705e-06, + "loss": 1.4195, + "step": 1404 + }, + { + "epoch": 1.748599875544493, + "grad_norm": 2.8519608974456787, + "learning_rate": 2.3523985239852403e-06, + "loss": 1.4031, + "step": 1405 + }, + { + "epoch": 1.7498444306160548, + "grad_norm": 2.90872859954834, + "learning_rate": 2.3500922509225093e-06, + "loss": 1.4616, + "step": 1406 + }, + { + "epoch": 1.7510889856876166, + "grad_norm": 2.9190874099731445, + "learning_rate": 2.3477859778597787e-06, + "loss": 1.3917, + "step": 1407 + }, + { + "epoch": 1.7523335407591785, + "grad_norm": 2.868323802947998, + "learning_rate": 2.345479704797048e-06, + "loss": 1.4053, + "step": 1408 + }, + { + "epoch": 1.7535780958307405, + "grad_norm": 2.8638579845428467, + "learning_rate": 2.3431734317343176e-06, + "loss": 1.4042, + "step": 1409 + }, + { + "epoch": 1.7548226509023024, + "grad_norm": 3.1759817600250244, + "learning_rate": 2.340867158671587e-06, + "loss": 1.4937, + "step": 1410 + }, + { + "epoch": 1.7548226509023024, + "eval_loss": 1.6252617835998535, + "eval_runtime": 44.3041, + "eval_samples_per_second": 22.571, + "eval_steps_per_second": 0.948, + "step": 1410 + }, + { + "epoch": 1.7560672059738645, + "grad_norm": 2.8265459537506104, + "learning_rate": 2.3385608856088564e-06, + "loss": 1.406, + "step": 1411 + }, + { + "epoch": 1.7573117610454263, + "grad_norm": 2.856229305267334, + "learning_rate": 2.336254612546126e-06, + "loss": 1.4395, + "step": 1412 + }, + { + "epoch": 1.7585563161169881, + "grad_norm": 2.9112775325775146, + "learning_rate": 2.333948339483395e-06, + "loss": 1.427, + "step": 1413 + }, + { + "epoch": 1.75980087118855, + "grad_norm": 3.098846435546875, + "learning_rate": 2.3316420664206643e-06, + "loss": 1.4222, + "step": 1414 + }, + { + "epoch": 1.7610454262601118, + "grad_norm": 2.758100986480713, + "learning_rate": 2.3293357933579337e-06, + "loss": 1.3844, + "step": 1415 + }, + { + "epoch": 1.762289981331674, + "grad_norm": 2.8422274589538574, + "learning_rate": 2.327029520295203e-06, + "loss": 1.4033, + "step": 1416 + }, + { + "epoch": 1.763534536403236, + "grad_norm": 2.9257912635803223, + "learning_rate": 2.3247232472324725e-06, + "loss": 1.3961, + "step": 1417 + }, + { + "epoch": 1.7647790914747978, + "grad_norm": 2.9431657791137695, + "learning_rate": 2.322416974169742e-06, + "loss": 1.4451, + "step": 1418 + }, + { + "epoch": 1.7660236465463597, + "grad_norm": 2.8598814010620117, + "learning_rate": 2.3201107011070114e-06, + "loss": 1.3816, + "step": 1419 + }, + { + "epoch": 1.7672682016179215, + "grad_norm": 2.9868006706237793, + "learning_rate": 2.317804428044281e-06, + "loss": 1.4037, + "step": 1420 + }, + { + "epoch": 1.7672682016179215, + "eval_loss": 1.6231273412704468, + "eval_runtime": 48.8067, + "eval_samples_per_second": 20.489, + "eval_steps_per_second": 0.861, + "step": 1420 + }, + { + "epoch": 1.7685127566894834, + "grad_norm": 2.888206720352173, + "learning_rate": 2.31549815498155e-06, + "loss": 1.4087, + "step": 1421 + }, + { + "epoch": 1.7697573117610454, + "grad_norm": 2.9240078926086426, + "learning_rate": 2.3131918819188192e-06, + "loss": 1.3909, + "step": 1422 + }, + { + "epoch": 1.7710018668326073, + "grad_norm": 3.0401012897491455, + "learning_rate": 2.3108856088560886e-06, + "loss": 1.423, + "step": 1423 + }, + { + "epoch": 1.7722464219041694, + "grad_norm": 2.9484200477600098, + "learning_rate": 2.308579335793358e-06, + "loss": 1.449, + "step": 1424 + }, + { + "epoch": 1.7734909769757312, + "grad_norm": 2.863267660140991, + "learning_rate": 2.3062730627306275e-06, + "loss": 1.4215, + "step": 1425 + }, + { + "epoch": 1.774735532047293, + "grad_norm": 2.9615612030029297, + "learning_rate": 2.303966789667897e-06, + "loss": 1.4391, + "step": 1426 + }, + { + "epoch": 1.775980087118855, + "grad_norm": 2.7958569526672363, + "learning_rate": 2.3016605166051663e-06, + "loss": 1.3662, + "step": 1427 + }, + { + "epoch": 1.777224642190417, + "grad_norm": 2.898787021636963, + "learning_rate": 2.2993542435424358e-06, + "loss": 1.362, + "step": 1428 + }, + { + "epoch": 1.7784691972619788, + "grad_norm": 2.883049726486206, + "learning_rate": 2.2970479704797048e-06, + "loss": 1.4339, + "step": 1429 + }, + { + "epoch": 1.779713752333541, + "grad_norm": 3.0204827785491943, + "learning_rate": 2.294741697416974e-06, + "loss": 1.5174, + "step": 1430 + }, + { + "epoch": 1.779713752333541, + "eval_loss": 1.6246228218078613, + "eval_runtime": 47.5611, + "eval_samples_per_second": 21.026, + "eval_steps_per_second": 0.883, + "step": 1430 + }, + { + "epoch": 1.7809583074051027, + "grad_norm": 2.9354310035705566, + "learning_rate": 2.2924354243542436e-06, + "loss": 1.4115, + "step": 1431 + }, + { + "epoch": 1.7822028624766646, + "grad_norm": 2.9498608112335205, + "learning_rate": 2.290129151291513e-06, + "loss": 1.4058, + "step": 1432 + }, + { + "epoch": 1.7834474175482264, + "grad_norm": 2.9706921577453613, + "learning_rate": 2.2878228782287825e-06, + "loss": 1.426, + "step": 1433 + }, + { + "epoch": 1.7846919726197883, + "grad_norm": 2.8945024013519287, + "learning_rate": 2.285516605166052e-06, + "loss": 1.4011, + "step": 1434 + }, + { + "epoch": 1.7859365276913504, + "grad_norm": 2.8987081050872803, + "learning_rate": 2.2832103321033213e-06, + "loss": 1.4163, + "step": 1435 + }, + { + "epoch": 1.7871810827629122, + "grad_norm": 2.9797592163085938, + "learning_rate": 2.2809040590405907e-06, + "loss": 1.4367, + "step": 1436 + }, + { + "epoch": 1.7884256378344743, + "grad_norm": 2.8872063159942627, + "learning_rate": 2.2785977859778597e-06, + "loss": 1.4226, + "step": 1437 + }, + { + "epoch": 1.7896701929060361, + "grad_norm": 2.942655563354492, + "learning_rate": 2.2762915129151296e-06, + "loss": 1.4132, + "step": 1438 + }, + { + "epoch": 1.790914747977598, + "grad_norm": 3.027392625808716, + "learning_rate": 2.2739852398523986e-06, + "loss": 1.4148, + "step": 1439 + }, + { + "epoch": 1.7921593030491598, + "grad_norm": 2.9475181102752686, + "learning_rate": 2.271678966789668e-06, + "loss": 1.4585, + "step": 1440 + }, + { + "epoch": 1.7921593030491598, + "eval_loss": 1.6234897375106812, + "eval_runtime": 53.3737, + "eval_samples_per_second": 18.736, + "eval_steps_per_second": 0.787, + "step": 1440 + }, + { + "epoch": 1.793403858120722, + "grad_norm": 2.846304416656494, + "learning_rate": 2.2693726937269374e-06, + "loss": 1.3667, + "step": 1441 + }, + { + "epoch": 1.7946484131922837, + "grad_norm": 2.8925700187683105, + "learning_rate": 2.267066420664207e-06, + "loss": 1.4318, + "step": 1442 + }, + { + "epoch": 1.7958929682638458, + "grad_norm": 2.884025812149048, + "learning_rate": 2.2647601476014763e-06, + "loss": 1.4395, + "step": 1443 + }, + { + "epoch": 1.7971375233354077, + "grad_norm": 2.9763753414154053, + "learning_rate": 2.2624538745387457e-06, + "loss": 1.3742, + "step": 1444 + }, + { + "epoch": 1.7983820784069695, + "grad_norm": 3.0834898948669434, + "learning_rate": 2.2601476014760147e-06, + "loss": 1.4651, + "step": 1445 + }, + { + "epoch": 1.7996266334785314, + "grad_norm": 3.0673978328704834, + "learning_rate": 2.2578413284132845e-06, + "loss": 1.475, + "step": 1446 + }, + { + "epoch": 1.8008711885500932, + "grad_norm": 2.931521415710449, + "learning_rate": 2.2555350553505535e-06, + "loss": 1.4506, + "step": 1447 + }, + { + "epoch": 1.8021157436216553, + "grad_norm": 2.953301429748535, + "learning_rate": 2.253228782287823e-06, + "loss": 1.4224, + "step": 1448 + }, + { + "epoch": 1.8033602986932171, + "grad_norm": 3.0210883617401123, + "learning_rate": 2.2509225092250924e-06, + "loss": 1.4387, + "step": 1449 + }, + { + "epoch": 1.8046048537647792, + "grad_norm": 2.975741386413574, + "learning_rate": 2.248616236162362e-06, + "loss": 1.4318, + "step": 1450 + }, + { + "epoch": 1.8046048537647792, + "eval_loss": 1.618930697441101, + "eval_runtime": 49.6692, + "eval_samples_per_second": 20.133, + "eval_steps_per_second": 0.846, + "step": 1450 + }, + { + "epoch": 1.805849408836341, + "grad_norm": 2.9083876609802246, + "learning_rate": 2.2463099630996312e-06, + "loss": 1.3837, + "step": 1451 + }, + { + "epoch": 1.807093963907903, + "grad_norm": 3.364248275756836, + "learning_rate": 2.2440036900369006e-06, + "loss": 1.4589, + "step": 1452 + }, + { + "epoch": 1.8083385189794647, + "grad_norm": 2.935478925704956, + "learning_rate": 2.2416974169741696e-06, + "loss": 1.4119, + "step": 1453 + }, + { + "epoch": 1.8095830740510268, + "grad_norm": 3.1742911338806152, + "learning_rate": 2.2393911439114395e-06, + "loss": 1.4203, + "step": 1454 + }, + { + "epoch": 1.8108276291225887, + "grad_norm": 2.96966290473938, + "learning_rate": 2.2370848708487085e-06, + "loss": 1.4054, + "step": 1455 + }, + { + "epoch": 1.8120721841941507, + "grad_norm": 3.0919055938720703, + "learning_rate": 2.2347785977859783e-06, + "loss": 1.4289, + "step": 1456 + }, + { + "epoch": 1.8133167392657126, + "grad_norm": 2.9061055183410645, + "learning_rate": 2.2324723247232473e-06, + "loss": 1.4958, + "step": 1457 + }, + { + "epoch": 1.8145612943372744, + "grad_norm": 2.968348503112793, + "learning_rate": 2.2301660516605168e-06, + "loss": 1.4033, + "step": 1458 + }, + { + "epoch": 1.8158058494088363, + "grad_norm": 3.0435378551483154, + "learning_rate": 2.227859778597786e-06, + "loss": 1.414, + "step": 1459 + }, + { + "epoch": 1.8170504044803981, + "grad_norm": 2.9067916870117188, + "learning_rate": 2.2255535055350556e-06, + "loss": 1.3954, + "step": 1460 + }, + { + "epoch": 1.8170504044803981, + "eval_loss": 1.6240814924240112, + "eval_runtime": 51.7164, + "eval_samples_per_second": 19.336, + "eval_steps_per_second": 0.812, + "step": 1460 + }, + { + "epoch": 1.8182949595519602, + "grad_norm": 3.156384229660034, + "learning_rate": 2.2232472324723246e-06, + "loss": 1.4658, + "step": 1461 + }, + { + "epoch": 1.819539514623522, + "grad_norm": 2.9885313510894775, + "learning_rate": 2.2209409594095944e-06, + "loss": 1.4123, + "step": 1462 + }, + { + "epoch": 1.820784069695084, + "grad_norm": 3.227780818939209, + "learning_rate": 2.2186346863468634e-06, + "loss": 1.3587, + "step": 1463 + }, + { + "epoch": 1.822028624766646, + "grad_norm": 3.0375192165374756, + "learning_rate": 2.2163284132841333e-06, + "loss": 1.4015, + "step": 1464 + }, + { + "epoch": 1.8232731798382078, + "grad_norm": 3.0200846195220947, + "learning_rate": 2.2140221402214023e-06, + "loss": 1.419, + "step": 1465 + }, + { + "epoch": 1.8245177349097697, + "grad_norm": 2.9456615447998047, + "learning_rate": 2.2117158671586717e-06, + "loss": 1.4075, + "step": 1466 + }, + { + "epoch": 1.8257622899813317, + "grad_norm": 3.3129165172576904, + "learning_rate": 2.209409594095941e-06, + "loss": 1.4241, + "step": 1467 + }, + { + "epoch": 1.8270068450528936, + "grad_norm": 3.013141393661499, + "learning_rate": 2.2071033210332106e-06, + "loss": 1.4066, + "step": 1468 + }, + { + "epoch": 1.8282514001244556, + "grad_norm": 2.8414366245269775, + "learning_rate": 2.20479704797048e-06, + "loss": 1.4009, + "step": 1469 + }, + { + "epoch": 1.8294959551960175, + "grad_norm": 2.8920438289642334, + "learning_rate": 2.2024907749077494e-06, + "loss": 1.3916, + "step": 1470 + }, + { + "epoch": 1.8294959551960175, + "eval_loss": 1.6220412254333496, + "eval_runtime": 52.6199, + "eval_samples_per_second": 19.004, + "eval_steps_per_second": 0.798, + "step": 1470 + }, + { + "epoch": 1.8307405102675793, + "grad_norm": 3.032353639602661, + "learning_rate": 2.2001845018450184e-06, + "loss": 1.402, + "step": 1471 + }, + { + "epoch": 1.8319850653391412, + "grad_norm": 2.9512081146240234, + "learning_rate": 2.1978782287822883e-06, + "loss": 1.429, + "step": 1472 + }, + { + "epoch": 1.833229620410703, + "grad_norm": 2.846081018447876, + "learning_rate": 2.1955719557195573e-06, + "loss": 1.4203, + "step": 1473 + }, + { + "epoch": 1.834474175482265, + "grad_norm": 3.093378782272339, + "learning_rate": 2.1932656826568267e-06, + "loss": 1.4307, + "step": 1474 + }, + { + "epoch": 1.8357187305538272, + "grad_norm": 2.808326005935669, + "learning_rate": 2.190959409594096e-06, + "loss": 1.3753, + "step": 1475 + }, + { + "epoch": 1.836963285625389, + "grad_norm": 3.0328383445739746, + "learning_rate": 2.1886531365313655e-06, + "loss": 1.4534, + "step": 1476 + }, + { + "epoch": 1.8382078406969509, + "grad_norm": 3.045897960662842, + "learning_rate": 2.186346863468635e-06, + "loss": 1.3259, + "step": 1477 + }, + { + "epoch": 1.8394523957685127, + "grad_norm": 3.171846628189087, + "learning_rate": 2.1840405904059044e-06, + "loss": 1.397, + "step": 1478 + }, + { + "epoch": 1.8406969508400746, + "grad_norm": 2.893061637878418, + "learning_rate": 2.1817343173431734e-06, + "loss": 1.4201, + "step": 1479 + }, + { + "epoch": 1.8419415059116366, + "grad_norm": 3.0151467323303223, + "learning_rate": 2.1794280442804432e-06, + "loss": 1.4266, + "step": 1480 + }, + { + "epoch": 1.8419415059116366, + "eval_loss": 1.6166906356811523, + "eval_runtime": 51.8266, + "eval_samples_per_second": 19.295, + "eval_steps_per_second": 0.81, + "step": 1480 + }, + { + "epoch": 1.8431860609831985, + "grad_norm": 2.942431688308716, + "learning_rate": 2.177121771217712e-06, + "loss": 1.3744, + "step": 1481 + }, + { + "epoch": 1.8444306160547606, + "grad_norm": 2.8482232093811035, + "learning_rate": 2.1748154981549816e-06, + "loss": 1.3914, + "step": 1482 + }, + { + "epoch": 1.8456751711263224, + "grad_norm": 2.9315359592437744, + "learning_rate": 2.172509225092251e-06, + "loss": 1.419, + "step": 1483 + }, + { + "epoch": 1.8469197261978842, + "grad_norm": 3.191800594329834, + "learning_rate": 2.1702029520295205e-06, + "loss": 1.3524, + "step": 1484 + }, + { + "epoch": 1.848164281269446, + "grad_norm": 2.9308040142059326, + "learning_rate": 2.16789667896679e-06, + "loss": 1.3742, + "step": 1485 + }, + { + "epoch": 1.849408836341008, + "grad_norm": 2.7678964138031006, + "learning_rate": 2.1655904059040593e-06, + "loss": 1.3786, + "step": 1486 + }, + { + "epoch": 1.85065339141257, + "grad_norm": 3.0300636291503906, + "learning_rate": 2.1632841328413287e-06, + "loss": 1.4469, + "step": 1487 + }, + { + "epoch": 1.851897946484132, + "grad_norm": 3.142781972885132, + "learning_rate": 2.160977859778598e-06, + "loss": 1.4565, + "step": 1488 + }, + { + "epoch": 1.853142501555694, + "grad_norm": 2.954916000366211, + "learning_rate": 2.158671586715867e-06, + "loss": 1.4356, + "step": 1489 + }, + { + "epoch": 1.8543870566272558, + "grad_norm": 2.864875316619873, + "learning_rate": 2.1563653136531366e-06, + "loss": 1.4521, + "step": 1490 + }, + { + "epoch": 1.8543870566272558, + "eval_loss": 1.6207302808761597, + "eval_runtime": 49.2215, + "eval_samples_per_second": 20.316, + "eval_steps_per_second": 0.853, + "step": 1490 + }, + { + "epoch": 1.8556316116988176, + "grad_norm": 2.8908467292785645, + "learning_rate": 2.154059040590406e-06, + "loss": 1.4304, + "step": 1491 + }, + { + "epoch": 1.8568761667703795, + "grad_norm": 2.9439549446105957, + "learning_rate": 2.1517527675276754e-06, + "loss": 1.398, + "step": 1492 + }, + { + "epoch": 1.8581207218419415, + "grad_norm": 2.967728853225708, + "learning_rate": 2.149446494464945e-06, + "loss": 1.5038, + "step": 1493 + }, + { + "epoch": 1.8593652769135034, + "grad_norm": 2.938123941421509, + "learning_rate": 2.1471402214022143e-06, + "loss": 1.4395, + "step": 1494 + }, + { + "epoch": 1.8606098319850655, + "grad_norm": 2.93609881401062, + "learning_rate": 2.1448339483394837e-06, + "loss": 1.3939, + "step": 1495 + }, + { + "epoch": 1.8618543870566273, + "grad_norm": 2.9093775749206543, + "learning_rate": 2.142527675276753e-06, + "loss": 1.3942, + "step": 1496 + }, + { + "epoch": 1.8630989421281892, + "grad_norm": 2.773960590362549, + "learning_rate": 2.140221402214022e-06, + "loss": 1.3886, + "step": 1497 + }, + { + "epoch": 1.864343497199751, + "grad_norm": 2.9858205318450928, + "learning_rate": 2.1379151291512916e-06, + "loss": 1.3814, + "step": 1498 + }, + { + "epoch": 1.8655880522713129, + "grad_norm": 3.1233417987823486, + "learning_rate": 2.135608856088561e-06, + "loss": 1.4313, + "step": 1499 + }, + { + "epoch": 1.866832607342875, + "grad_norm": 2.9569480419158936, + "learning_rate": 2.1333025830258304e-06, + "loss": 1.4302, + "step": 1500 + }, + { + "epoch": 1.866832607342875, + "eval_loss": 1.6130231618881226, + "eval_runtime": 46.2834, + "eval_samples_per_second": 21.606, + "eval_steps_per_second": 0.907, + "step": 1500 + }, + { + "epoch": 1.868077162414437, + "grad_norm": 2.926203727722168, + "learning_rate": 2.1309963099631e-06, + "loss": 1.4236, + "step": 1501 + }, + { + "epoch": 1.8693217174859988, + "grad_norm": 2.9496357440948486, + "learning_rate": 2.1286900369003692e-06, + "loss": 1.4014, + "step": 1502 + }, + { + "epoch": 1.8705662725575607, + "grad_norm": 2.9314279556274414, + "learning_rate": 2.1263837638376387e-06, + "loss": 1.4414, + "step": 1503 + }, + { + "epoch": 1.8718108276291225, + "grad_norm": 2.889460802078247, + "learning_rate": 2.124077490774908e-06, + "loss": 1.3883, + "step": 1504 + }, + { + "epoch": 1.8730553827006844, + "grad_norm": 2.9859671592712402, + "learning_rate": 2.1217712177121775e-06, + "loss": 1.4551, + "step": 1505 + }, + { + "epoch": 1.8742999377722465, + "grad_norm": 2.9330570697784424, + "learning_rate": 2.1194649446494465e-06, + "loss": 1.3895, + "step": 1506 + }, + { + "epoch": 1.8755444928438083, + "grad_norm": 2.9062387943267822, + "learning_rate": 2.117158671586716e-06, + "loss": 1.4492, + "step": 1507 + }, + { + "epoch": 1.8767890479153704, + "grad_norm": 3.0337791442871094, + "learning_rate": 2.1148523985239854e-06, + "loss": 1.4682, + "step": 1508 + }, + { + "epoch": 1.8780336029869322, + "grad_norm": 3.0448431968688965, + "learning_rate": 2.1125461254612548e-06, + "loss": 1.4836, + "step": 1509 + }, + { + "epoch": 1.879278158058494, + "grad_norm": 2.912764310836792, + "learning_rate": 2.110239852398524e-06, + "loss": 1.4644, + "step": 1510 + }, + { + "epoch": 1.879278158058494, + "eval_loss": 1.6124104261398315, + "eval_runtime": 41.9713, + "eval_samples_per_second": 23.826, + "eval_steps_per_second": 1.001, + "step": 1510 + }, + { + "epoch": 1.880522713130056, + "grad_norm": 3.0399203300476074, + "learning_rate": 2.1079335793357936e-06, + "loss": 1.4391, + "step": 1511 + }, + { + "epoch": 1.8817672682016178, + "grad_norm": 2.9852235317230225, + "learning_rate": 2.1056273062730626e-06, + "loss": 1.346, + "step": 1512 + }, + { + "epoch": 1.8830118232731798, + "grad_norm": 2.904437780380249, + "learning_rate": 2.1033210332103325e-06, + "loss": 1.4523, + "step": 1513 + }, + { + "epoch": 1.884256378344742, + "grad_norm": 2.8988049030303955, + "learning_rate": 2.1010147601476015e-06, + "loss": 1.4001, + "step": 1514 + }, + { + "epoch": 1.8855009334163038, + "grad_norm": 2.9890260696411133, + "learning_rate": 2.098708487084871e-06, + "loss": 1.4074, + "step": 1515 + }, + { + "epoch": 1.8867454884878656, + "grad_norm": 3.0486819744110107, + "learning_rate": 2.0964022140221403e-06, + "loss": 1.461, + "step": 1516 + }, + { + "epoch": 1.8879900435594275, + "grad_norm": 2.8746016025543213, + "learning_rate": 2.0940959409594097e-06, + "loss": 1.4013, + "step": 1517 + }, + { + "epoch": 1.8892345986309893, + "grad_norm": 2.979480266571045, + "learning_rate": 2.091789667896679e-06, + "loss": 1.4567, + "step": 1518 + }, + { + "epoch": 1.8904791537025514, + "grad_norm": 2.8351497650146484, + "learning_rate": 2.0894833948339486e-06, + "loss": 1.3655, + "step": 1519 + }, + { + "epoch": 1.8917237087741132, + "grad_norm": 2.9061646461486816, + "learning_rate": 2.0871771217712176e-06, + "loss": 1.4165, + "step": 1520 + }, + { + "epoch": 1.8917237087741132, + "eval_loss": 1.6172568798065186, + "eval_runtime": 42.998, + "eval_samples_per_second": 23.257, + "eval_steps_per_second": 0.977, + "step": 1520 + }, + { + "epoch": 1.8929682638456753, + "grad_norm": 2.8751838207244873, + "learning_rate": 2.0848708487084874e-06, + "loss": 1.4133, + "step": 1521 + }, + { + "epoch": 1.8942128189172371, + "grad_norm": 2.7513587474823, + "learning_rate": 2.0825645756457564e-06, + "loss": 1.3734, + "step": 1522 + }, + { + "epoch": 1.895457373988799, + "grad_norm": 2.9244418144226074, + "learning_rate": 2.0802583025830263e-06, + "loss": 1.4181, + "step": 1523 + }, + { + "epoch": 1.8967019290603608, + "grad_norm": 2.877495765686035, + "learning_rate": 2.0779520295202953e-06, + "loss": 1.4009, + "step": 1524 + }, + { + "epoch": 1.8979464841319227, + "grad_norm": 2.9316184520721436, + "learning_rate": 2.0756457564575647e-06, + "loss": 1.3796, + "step": 1525 + }, + { + "epoch": 1.8991910392034848, + "grad_norm": 3.0403976440429688, + "learning_rate": 2.073339483394834e-06, + "loss": 1.4101, + "step": 1526 + }, + { + "epoch": 1.9004355942750468, + "grad_norm": 3.0020930767059326, + "learning_rate": 2.0710332103321035e-06, + "loss": 1.4166, + "step": 1527 + }, + { + "epoch": 1.9016801493466087, + "grad_norm": 3.0937304496765137, + "learning_rate": 2.0687269372693725e-06, + "loss": 1.4599, + "step": 1528 + }, + { + "epoch": 1.9029247044181705, + "grad_norm": 2.901540517807007, + "learning_rate": 2.0664206642066424e-06, + "loss": 1.3642, + "step": 1529 + }, + { + "epoch": 1.9041692594897324, + "grad_norm": 3.0517938137054443, + "learning_rate": 2.0641143911439114e-06, + "loss": 1.451, + "step": 1530 + }, + { + "epoch": 1.9041692594897324, + "eval_loss": 1.6178728342056274, + "eval_runtime": 43.9067, + "eval_samples_per_second": 22.776, + "eval_steps_per_second": 0.957, + "step": 1530 + }, + { + "epoch": 1.9054138145612942, + "grad_norm": 3.12484073638916, + "learning_rate": 2.0618081180811812e-06, + "loss": 1.4522, + "step": 1531 + }, + { + "epoch": 1.9066583696328563, + "grad_norm": 2.9447414875030518, + "learning_rate": 2.0595018450184502e-06, + "loss": 1.4154, + "step": 1532 + }, + { + "epoch": 1.9079029247044181, + "grad_norm": 2.909738779067993, + "learning_rate": 2.0571955719557197e-06, + "loss": 1.4133, + "step": 1533 + }, + { + "epoch": 1.9091474797759802, + "grad_norm": 3.1522421836853027, + "learning_rate": 2.054889298892989e-06, + "loss": 1.4777, + "step": 1534 + }, + { + "epoch": 1.910392034847542, + "grad_norm": 3.041447639465332, + "learning_rate": 2.0525830258302585e-06, + "loss": 1.4104, + "step": 1535 + }, + { + "epoch": 1.911636589919104, + "grad_norm": 3.01129412651062, + "learning_rate": 2.050276752767528e-06, + "loss": 1.3852, + "step": 1536 + }, + { + "epoch": 1.9128811449906657, + "grad_norm": 3.0697543621063232, + "learning_rate": 2.0479704797047974e-06, + "loss": 1.4389, + "step": 1537 + }, + { + "epoch": 1.9141257000622276, + "grad_norm": 2.9391236305236816, + "learning_rate": 2.0456642066420664e-06, + "loss": 1.4239, + "step": 1538 + }, + { + "epoch": 1.9153702551337897, + "grad_norm": 2.978236675262451, + "learning_rate": 2.043357933579336e-06, + "loss": 1.462, + "step": 1539 + }, + { + "epoch": 1.9166148102053517, + "grad_norm": 2.8124167919158936, + "learning_rate": 2.041051660516605e-06, + "loss": 1.3186, + "step": 1540 + }, + { + "epoch": 1.9166148102053517, + "eval_loss": 1.6120468378067017, + "eval_runtime": 43.3787, + "eval_samples_per_second": 23.053, + "eval_steps_per_second": 0.968, + "step": 1540 + }, + { + "epoch": 1.9178593652769136, + "grad_norm": 3.029939889907837, + "learning_rate": 2.0387453874538746e-06, + "loss": 1.4481, + "step": 1541 + }, + { + "epoch": 1.9191039203484754, + "grad_norm": 2.8749356269836426, + "learning_rate": 2.036439114391144e-06, + "loss": 1.3735, + "step": 1542 + }, + { + "epoch": 1.9203484754200373, + "grad_norm": 2.9181101322174072, + "learning_rate": 2.0341328413284135e-06, + "loss": 1.4085, + "step": 1543 + }, + { + "epoch": 1.9215930304915991, + "grad_norm": 2.936319589614868, + "learning_rate": 2.031826568265683e-06, + "loss": 1.3827, + "step": 1544 + }, + { + "epoch": 1.9228375855631612, + "grad_norm": 2.9286844730377197, + "learning_rate": 2.0295202952029523e-06, + "loss": 1.4563, + "step": 1545 + }, + { + "epoch": 1.924082140634723, + "grad_norm": 2.790752649307251, + "learning_rate": 2.0272140221402213e-06, + "loss": 1.3935, + "step": 1546 + }, + { + "epoch": 1.9253266957062851, + "grad_norm": 2.878164529800415, + "learning_rate": 2.024907749077491e-06, + "loss": 1.4159, + "step": 1547 + }, + { + "epoch": 1.926571250777847, + "grad_norm": 3.0583767890930176, + "learning_rate": 2.02260147601476e-06, + "loss": 1.4036, + "step": 1548 + }, + { + "epoch": 1.9278158058494088, + "grad_norm": 2.943686008453369, + "learning_rate": 2.02029520295203e-06, + "loss": 1.4178, + "step": 1549 + }, + { + "epoch": 1.9290603609209707, + "grad_norm": 2.983553886413574, + "learning_rate": 2.017988929889299e-06, + "loss": 1.3959, + "step": 1550 + }, + { + "epoch": 1.9290603609209707, + "eval_loss": 1.610864520072937, + "eval_runtime": 44.5262, + "eval_samples_per_second": 22.459, + "eval_steps_per_second": 0.943, + "step": 1550 + }, + { + "epoch": 1.9303049159925325, + "grad_norm": 3.0323703289031982, + "learning_rate": 2.0156826568265684e-06, + "loss": 1.4362, + "step": 1551 + }, + { + "epoch": 1.9315494710640946, + "grad_norm": 3.036627769470215, + "learning_rate": 2.013376383763838e-06, + "loss": 1.4132, + "step": 1552 + }, + { + "epoch": 1.9327940261356567, + "grad_norm": 2.901071310043335, + "learning_rate": 2.0110701107011073e-06, + "loss": 1.3846, + "step": 1553 + }, + { + "epoch": 1.9340385812072185, + "grad_norm": 3.002803325653076, + "learning_rate": 2.0087638376383767e-06, + "loss": 1.4402, + "step": 1554 + }, + { + "epoch": 1.9352831362787803, + "grad_norm": 2.969228982925415, + "learning_rate": 2.006457564575646e-06, + "loss": 1.4387, + "step": 1555 + }, + { + "epoch": 1.9365276913503422, + "grad_norm": 2.9773998260498047, + "learning_rate": 2.004151291512915e-06, + "loss": 1.4149, + "step": 1556 + }, + { + "epoch": 1.937772246421904, + "grad_norm": 2.831655502319336, + "learning_rate": 2.0018450184501845e-06, + "loss": 1.3753, + "step": 1557 + }, + { + "epoch": 1.9390168014934661, + "grad_norm": 2.835994005203247, + "learning_rate": 1.999538745387454e-06, + "loss": 1.345, + "step": 1558 + }, + { + "epoch": 1.940261356565028, + "grad_norm": 3.0328211784362793, + "learning_rate": 1.9972324723247234e-06, + "loss": 1.3473, + "step": 1559 + }, + { + "epoch": 1.94150591163659, + "grad_norm": 2.9441378116607666, + "learning_rate": 1.994926199261993e-06, + "loss": 1.3913, + "step": 1560 + }, + { + "epoch": 1.94150591163659, + "eval_loss": 1.6193854808807373, + "eval_runtime": 44.6194, + "eval_samples_per_second": 22.412, + "eval_steps_per_second": 0.941, + "step": 1560 + }, + { + "epoch": 1.9427504667081519, + "grad_norm": 2.905486822128296, + "learning_rate": 1.9926199261992622e-06, + "loss": 1.4043, + "step": 1561 + }, + { + "epoch": 1.9439950217797137, + "grad_norm": 2.7758026123046875, + "learning_rate": 1.9903136531365317e-06, + "loss": 1.3974, + "step": 1562 + }, + { + "epoch": 1.9452395768512756, + "grad_norm": 2.7883355617523193, + "learning_rate": 1.988007380073801e-06, + "loss": 1.4246, + "step": 1563 + }, + { + "epoch": 1.9464841319228374, + "grad_norm": 2.8322808742523193, + "learning_rate": 1.98570110701107e-06, + "loss": 1.4002, + "step": 1564 + }, + { + "epoch": 1.9477286869943995, + "grad_norm": 2.948850631713867, + "learning_rate": 1.9833948339483395e-06, + "loss": 1.3866, + "step": 1565 + }, + { + "epoch": 1.9489732420659616, + "grad_norm": 2.9772586822509766, + "learning_rate": 1.981088560885609e-06, + "loss": 1.4473, + "step": 1566 + }, + { + "epoch": 1.9502177971375234, + "grad_norm": 2.9183740615844727, + "learning_rate": 1.9787822878228783e-06, + "loss": 1.3943, + "step": 1567 + }, + { + "epoch": 1.9514623522090853, + "grad_norm": 3.10422682762146, + "learning_rate": 1.9764760147601478e-06, + "loss": 1.465, + "step": 1568 + }, + { + "epoch": 1.952706907280647, + "grad_norm": 3.001533031463623, + "learning_rate": 1.974169741697417e-06, + "loss": 1.4133, + "step": 1569 + }, + { + "epoch": 1.953951462352209, + "grad_norm": 2.9521870613098145, + "learning_rate": 1.9718634686346866e-06, + "loss": 1.4652, + "step": 1570 + }, + { + "epoch": 1.953951462352209, + "eval_loss": 1.617977499961853, + "eval_runtime": 45.7189, + "eval_samples_per_second": 21.873, + "eval_steps_per_second": 0.919, + "step": 1570 + }, + { + "epoch": 1.955196017423771, + "grad_norm": 2.9504826068878174, + "learning_rate": 1.969557195571956e-06, + "loss": 1.4117, + "step": 1571 + }, + { + "epoch": 1.9564405724953329, + "grad_norm": 2.9383559226989746, + "learning_rate": 1.9672509225092255e-06, + "loss": 1.3439, + "step": 1572 + }, + { + "epoch": 1.957685127566895, + "grad_norm": 2.850644588470459, + "learning_rate": 1.9649446494464945e-06, + "loss": 1.3521, + "step": 1573 + }, + { + "epoch": 1.9589296826384568, + "grad_norm": 2.8787410259246826, + "learning_rate": 1.962638376383764e-06, + "loss": 1.4268, + "step": 1574 + }, + { + "epoch": 1.9601742377100186, + "grad_norm": 2.966080665588379, + "learning_rate": 1.9603321033210333e-06, + "loss": 1.3367, + "step": 1575 + }, + { + "epoch": 1.9614187927815805, + "grad_norm": 2.858222007751465, + "learning_rate": 1.9580258302583027e-06, + "loss": 1.3842, + "step": 1576 + }, + { + "epoch": 1.9626633478531423, + "grad_norm": 2.924525737762451, + "learning_rate": 1.955719557195572e-06, + "loss": 1.4561, + "step": 1577 + }, + { + "epoch": 1.9639079029247044, + "grad_norm": 2.9281182289123535, + "learning_rate": 1.9534132841328416e-06, + "loss": 1.4179, + "step": 1578 + }, + { + "epoch": 1.9651524579962665, + "grad_norm": 2.982023000717163, + "learning_rate": 1.951107011070111e-06, + "loss": 1.4413, + "step": 1579 + }, + { + "epoch": 1.9663970130678283, + "grad_norm": 2.7912652492523193, + "learning_rate": 1.9488007380073804e-06, + "loss": 1.3577, + "step": 1580 + }, + { + "epoch": 1.9663970130678283, + "eval_loss": 1.6062211990356445, + "eval_runtime": 46.7931, + "eval_samples_per_second": 21.371, + "eval_steps_per_second": 0.898, + "step": 1580 + }, + { + "epoch": 1.9676415681393902, + "grad_norm": 2.9585583209991455, + "learning_rate": 1.9464944649446494e-06, + "loss": 1.4084, + "step": 1581 + }, + { + "epoch": 1.968886123210952, + "grad_norm": 2.879201889038086, + "learning_rate": 1.944188191881919e-06, + "loss": 1.4164, + "step": 1582 + }, + { + "epoch": 1.9701306782825139, + "grad_norm": 2.9510116577148438, + "learning_rate": 1.9418819188191883e-06, + "loss": 1.4211, + "step": 1583 + }, + { + "epoch": 1.971375233354076, + "grad_norm": 2.973844528198242, + "learning_rate": 1.9395756457564577e-06, + "loss": 1.4358, + "step": 1584 + }, + { + "epoch": 1.9726197884256378, + "grad_norm": 2.939131736755371, + "learning_rate": 1.937269372693727e-06, + "loss": 1.407, + "step": 1585 + }, + { + "epoch": 1.9738643434971999, + "grad_norm": 2.956646680831909, + "learning_rate": 1.9349630996309965e-06, + "loss": 1.3872, + "step": 1586 + }, + { + "epoch": 1.9751088985687617, + "grad_norm": 3.070354461669922, + "learning_rate": 1.932656826568266e-06, + "loss": 1.412, + "step": 1587 + }, + { + "epoch": 1.9763534536403236, + "grad_norm": 2.8143749237060547, + "learning_rate": 1.9303505535055354e-06, + "loss": 1.3522, + "step": 1588 + }, + { + "epoch": 1.9775980087118854, + "grad_norm": 2.8610002994537354, + "learning_rate": 1.9280442804428044e-06, + "loss": 1.3988, + "step": 1589 + }, + { + "epoch": 1.9788425637834475, + "grad_norm": 2.951023817062378, + "learning_rate": 1.925738007380074e-06, + "loss": 1.3855, + "step": 1590 + }, + { + "epoch": 1.9788425637834475, + "eval_loss": 1.613527536392212, + "eval_runtime": 46.5242, + "eval_samples_per_second": 21.494, + "eval_steps_per_second": 0.903, + "step": 1590 + }, + { + "epoch": 1.9800871188550093, + "grad_norm": 2.9333510398864746, + "learning_rate": 1.9234317343173432e-06, + "loss": 1.4518, + "step": 1591 + }, + { + "epoch": 1.9813316739265714, + "grad_norm": 2.847015619277954, + "learning_rate": 1.9211254612546126e-06, + "loss": 1.3999, + "step": 1592 + }, + { + "epoch": 1.9825762289981332, + "grad_norm": 2.969320058822632, + "learning_rate": 1.918819188191882e-06, + "loss": 1.4241, + "step": 1593 + }, + { + "epoch": 1.983820784069695, + "grad_norm": 2.862797498703003, + "learning_rate": 1.9165129151291515e-06, + "loss": 1.3929, + "step": 1594 + }, + { + "epoch": 1.985065339141257, + "grad_norm": 2.90384840965271, + "learning_rate": 1.914206642066421e-06, + "loss": 1.4127, + "step": 1595 + }, + { + "epoch": 1.9863098942128188, + "grad_norm": 2.8059256076812744, + "learning_rate": 1.9119003690036903e-06, + "loss": 1.4155, + "step": 1596 + }, + { + "epoch": 1.9875544492843809, + "grad_norm": 2.9068455696105957, + "learning_rate": 1.9095940959409593e-06, + "loss": 1.3924, + "step": 1597 + }, + { + "epoch": 1.9887990043559427, + "grad_norm": 2.903542995452881, + "learning_rate": 1.907287822878229e-06, + "loss": 1.4039, + "step": 1598 + }, + { + "epoch": 1.9900435594275048, + "grad_norm": 2.8566174507141113, + "learning_rate": 1.9049815498154984e-06, + "loss": 1.4276, + "step": 1599 + }, + { + "epoch": 1.9912881144990666, + "grad_norm": 2.8891515731811523, + "learning_rate": 1.9026752767527676e-06, + "loss": 1.4555, + "step": 1600 + }, + { + "epoch": 1.9912881144990666, + "eval_loss": 1.6086686849594116, + "eval_runtime": 46.2785, + "eval_samples_per_second": 21.608, + "eval_steps_per_second": 0.908, + "step": 1600 + } + ], + "logging_steps": 1, + "max_steps": 2409, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 100, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 4.987695054979072e+17, + "train_batch_size": 3, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/training_args.bin b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..6cdf93817e8f6d23b60d79888b226cc66ed2e88b --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a50e4918f3c49370326e14a7ed2f73b48b05e521f4b3399b53438fac6abee926 +size 6011 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/zero_to_fp32.py b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..24cc342e78d1a006c782b3a4cd68d9ce786d8fd8 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/zero_to_fp32.py @@ -0,0 +1,604 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: python zero_to_fp32.py . pytorch_model.bin + +import argparse +import torch +import glob +import math +import os +import re +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + + total_files = len(files) + state_dicts = [] + for f in files: + state_dict = torch.load(f, map_location=device) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + if zero_stage <= 2: + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + elif zero_stage == 3: + # if there is more than one param group, there will be multiple flattened tensors - one + # flattened tensor per group - for simplicity merge them into a single tensor + # + # XXX: could make the script more memory efficient for when there are multiple groups - it + # will require matching the sub-lists of param_shapes for each param group flattened tensor + + fp32_flat_groups = [ + torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts)) + ] + + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = fp32_flat_groups[0].numel() * world_size + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + for name, shape in param_shapes.items(): + + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # XXX: memory usage doubles here + state_dict[name] = torch.cat( + tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)), + 0).narrow(0, 0, unpartitioned_numel).view(shape) + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + + Returns: + - pytorch ``state_dict`` + + Note: this approach may not work if your application doesn't have sufficient free CPU memory and + you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters) + print(f"Saving fp32 state dict to {output_file}") + torch.save(state_dict, output_file) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info(f"Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info(f"Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument( + "output_file", + type=str, + help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_file, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fb0f9a53a507ffcdb60410deb71d60ef801bf350 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/config.json @@ -0,0 +1,36 @@ +{ + "_name_or_path": "meta-llama/Llama-3.2-3B", + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "eos_token_id": 128001, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 3072, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 24, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "torch_dtype": "float16", + "transformers_version": "4.45.1", + "use_cache": true, + "vocab_size": 128256 +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/generation_config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9c389b841a9a29ddd904e02d1eb0e08dcce82ad9 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": 128001, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.45.1" +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/latest b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/latest new file mode 100644 index 0000000000000000000000000000000000000000..cb24338af451be6d3ccddc18a950b58a70589761 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/latest @@ -0,0 +1 @@ +global_step1700 \ No newline at end of file diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/model.safetensors.index.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..ed64de846d720b9a7859dc20575fea8e8ca51940 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/model.safetensors.index.json @@ -0,0 +1,262 @@ +{ + "metadata": { + "total_size": 7213504512 + }, + "weight_map": { + "lm_head.weight": "model-00002-of-00002.safetensors", + "model.embed_tokens.weight": "model-00001-of-00002.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.norm.weight": "model-00002-of-00002.safetensors" + } +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/rng_state_0.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..730ab64da6759d278cc4d392a46a4e478bc2db94 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:772a3144cbd9fbfdcc2a816e3bfc9ba739a564939cc672ef5db80b8a6b8716ad +size 16567 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/rng_state_1.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..911ce6b22fd2dd279a884a7de1c42f2c9c06d01f --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:376340f42c3a329b4fa09ef6e859175de32ddb6c1c3f29007b22d619a322ed44 +size 16567 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/rng_state_2.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..583834e36ff694999bd58aa81e74493bdcf05ce0 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f5a3470801c87b94055fac7d653742f4609d2d396ed360f956ffe6523739152 +size 16567 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/scheduler.pt b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..e4f4ac5cc99d8c4d4a27f8b873eff267fa6d8f4c --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae3e07f9d999b52e0c48c324100ffdc1dbae20a129a817bfeb6bd37a6af2e020 +size 627 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/special_tokens_map.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..6949c5975ee0e961ef61cf31010dce04df0a03f8 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "[PAD]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/tokenizer.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f28ecaeab53ae07feed29ccf8624d2b0a8344df9 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19fb2e1e3cdd6f7433d89fd6d62c82042599dd4984f342efe7fec6e159e6a8f6 +size 17210734 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/tokenizer_config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8991b49e9c2a43fc527dab9e09ad8171f0cc5943 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/tokenizer_config.json @@ -0,0 +1,2086 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "[PAD]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128257": { + "content": "🅂", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "128258": { + "content": "🄿", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "[PAD]", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/trainer_state.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..4a79b1f77af87a48cfa88d0d493c5055a51cd3c5 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/trainer_state.json @@ -0,0 +1,13293 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.115743621655258, + "eval_steps": 10, + "global_step": 1700, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0012445550715619166, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8368, + "step": 1 + }, + { + "epoch": 0.002489110143123833, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.86, + "step": 2 + }, + { + "epoch": 0.00373366521468575, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8265, + "step": 3 + }, + { + "epoch": 0.004978220286247666, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8271, + "step": 4 + }, + { + "epoch": 0.006222775357809583, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8768, + "step": 5 + }, + { + "epoch": 0.0074673304293715, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8402, + "step": 6 + }, + { + "epoch": 0.008711885500933417, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.785, + "step": 7 + }, + { + "epoch": 0.009956440572495333, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8466, + "step": 8 + }, + { + "epoch": 0.01120099564405725, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.7673, + "step": 9 + }, + { + "epoch": 0.012445550715619166, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8074, + "step": 10 + }, + { + "epoch": 0.012445550715619166, + "eval_loss": 2.8535053730010986, + "eval_runtime": 42.446, + "eval_samples_per_second": 23.559, + "eval_steps_per_second": 0.989, + "step": 10 + }, + { + "epoch": 0.013690105787181083, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8592, + "step": 11 + }, + { + "epoch": 0.014934660858743, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.804, + "step": 12 + }, + { + "epoch": 0.016179215930304917, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8076, + "step": 13 + }, + { + "epoch": 0.017423771001866834, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8219, + "step": 14 + }, + { + "epoch": 0.018668326073428748, + "grad_norm": 5.105710506439209, + "learning_rate": 2.0746887966804982e-08, + "loss": 2.7995, + "step": 15 + }, + { + "epoch": 0.019912881144990666, + "grad_norm": 5.156588077545166, + "learning_rate": 4.1493775933609963e-08, + "loss": 2.8577, + "step": 16 + }, + { + "epoch": 0.021157436216552583, + "grad_norm": 5.709373950958252, + "learning_rate": 6.224066390041494e-08, + "loss": 2.8433, + "step": 17 + }, + { + "epoch": 0.0224019912881145, + "grad_norm": 5.64658784866333, + "learning_rate": 8.298755186721993e-08, + "loss": 2.8428, + "step": 18 + }, + { + "epoch": 0.023646546359676415, + "grad_norm": 5.541937351226807, + "learning_rate": 1.037344398340249e-07, + "loss": 2.7785, + "step": 19 + }, + { + "epoch": 0.024891101431238332, + "grad_norm": 5.233374118804932, + "learning_rate": 1.2448132780082988e-07, + "loss": 2.8599, + "step": 20 + }, + { + "epoch": 0.024891101431238332, + "eval_loss": 2.8521182537078857, + "eval_runtime": 43.5664, + "eval_samples_per_second": 22.953, + "eval_steps_per_second": 0.964, + "step": 20 + }, + { + "epoch": 0.02613565650280025, + "grad_norm": 5.407598972320557, + "learning_rate": 1.4522821576763488e-07, + "loss": 2.8081, + "step": 21 + }, + { + "epoch": 0.027380211574362167, + "grad_norm": 5.0483317375183105, + "learning_rate": 1.6597510373443985e-07, + "loss": 2.8489, + "step": 22 + }, + { + "epoch": 0.02862476664592408, + "grad_norm": 4.985588550567627, + "learning_rate": 1.8672199170124483e-07, + "loss": 2.8375, + "step": 23 + }, + { + "epoch": 0.029869321717486, + "grad_norm": 5.201700687408447, + "learning_rate": 2.074688796680498e-07, + "loss": 2.8355, + "step": 24 + }, + { + "epoch": 0.031113876789047916, + "grad_norm": 5.009235382080078, + "learning_rate": 2.2821576763485478e-07, + "loss": 2.7774, + "step": 25 + }, + { + "epoch": 0.03235843186060983, + "grad_norm": 4.509210109710693, + "learning_rate": 2.4896265560165975e-07, + "loss": 2.7992, + "step": 26 + }, + { + "epoch": 0.03360298693217175, + "grad_norm": 4.529262065887451, + "learning_rate": 2.6970954356846476e-07, + "loss": 2.836, + "step": 27 + }, + { + "epoch": 0.03484754200373367, + "grad_norm": 4.526787281036377, + "learning_rate": 2.9045643153526976e-07, + "loss": 2.8128, + "step": 28 + }, + { + "epoch": 0.03609209707529558, + "grad_norm": 4.016609191894531, + "learning_rate": 3.112033195020747e-07, + "loss": 2.7947, + "step": 29 + }, + { + "epoch": 0.037336652146857496, + "grad_norm": 3.4784562587738037, + "learning_rate": 3.319502074688797e-07, + "loss": 2.7854, + "step": 30 + }, + { + "epoch": 0.037336652146857496, + "eval_loss": 2.8020100593566895, + "eval_runtime": 44.8464, + "eval_samples_per_second": 22.298, + "eval_steps_per_second": 0.937, + "step": 30 + }, + { + "epoch": 0.038581207218419414, + "grad_norm": 3.4887945652008057, + "learning_rate": 3.5269709543568466e-07, + "loss": 2.7908, + "step": 31 + }, + { + "epoch": 0.03982576228998133, + "grad_norm": 3.4533374309539795, + "learning_rate": 3.7344398340248966e-07, + "loss": 2.7358, + "step": 32 + }, + { + "epoch": 0.04107031736154325, + "grad_norm": 3.270954132080078, + "learning_rate": 3.941908713692946e-07, + "loss": 2.8013, + "step": 33 + }, + { + "epoch": 0.042314872433105166, + "grad_norm": 3.61322283744812, + "learning_rate": 4.149377593360996e-07, + "loss": 2.7521, + "step": 34 + }, + { + "epoch": 0.043559427504667084, + "grad_norm": 3.158719062805176, + "learning_rate": 4.3568464730290456e-07, + "loss": 2.7573, + "step": 35 + }, + { + "epoch": 0.044803982576229, + "grad_norm": 2.888746500015259, + "learning_rate": 4.5643153526970956e-07, + "loss": 2.7795, + "step": 36 + }, + { + "epoch": 0.04604853764779091, + "grad_norm": 3.022629499435425, + "learning_rate": 4.771784232365145e-07, + "loss": 2.7891, + "step": 37 + }, + { + "epoch": 0.04729309271935283, + "grad_norm": 3.038097381591797, + "learning_rate": 4.979253112033195e-07, + "loss": 2.7376, + "step": 38 + }, + { + "epoch": 0.04853764779091475, + "grad_norm": 2.9992239475250244, + "learning_rate": 5.186721991701245e-07, + "loss": 2.7279, + "step": 39 + }, + { + "epoch": 0.049782202862476664, + "grad_norm": 2.908842086791992, + "learning_rate": 5.394190871369295e-07, + "loss": 2.6582, + "step": 40 + }, + { + "epoch": 0.049782202862476664, + "eval_loss": 2.725968837738037, + "eval_runtime": 44.9498, + "eval_samples_per_second": 22.247, + "eval_steps_per_second": 0.934, + "step": 40 + }, + { + "epoch": 0.05102675793403858, + "grad_norm": 2.8025174140930176, + "learning_rate": 5.601659751037345e-07, + "loss": 2.6809, + "step": 41 + }, + { + "epoch": 0.0522713130056005, + "grad_norm": 2.7555699348449707, + "learning_rate": 5.809128630705395e-07, + "loss": 2.6954, + "step": 42 + }, + { + "epoch": 0.053515868077162417, + "grad_norm": 2.4732933044433594, + "learning_rate": 6.016597510373444e-07, + "loss": 2.7307, + "step": 43 + }, + { + "epoch": 0.054760423148724334, + "grad_norm": 2.451366424560547, + "learning_rate": 6.224066390041494e-07, + "loss": 2.7064, + "step": 44 + }, + { + "epoch": 0.056004978220286245, + "grad_norm": 2.5768678188323975, + "learning_rate": 6.431535269709543e-07, + "loss": 2.6466, + "step": 45 + }, + { + "epoch": 0.05724953329184816, + "grad_norm": 2.579332113265991, + "learning_rate": 6.639004149377594e-07, + "loss": 2.6127, + "step": 46 + }, + { + "epoch": 0.05849408836341008, + "grad_norm": 2.231207847595215, + "learning_rate": 6.846473029045644e-07, + "loss": 2.6429, + "step": 47 + }, + { + "epoch": 0.059738643434972, + "grad_norm": 2.134375810623169, + "learning_rate": 7.053941908713693e-07, + "loss": 2.6423, + "step": 48 + }, + { + "epoch": 0.060983198506533914, + "grad_norm": 2.0485494136810303, + "learning_rate": 7.261410788381744e-07, + "loss": 2.581, + "step": 49 + }, + { + "epoch": 0.06222775357809583, + "grad_norm": 2.02909517288208, + "learning_rate": 7.468879668049793e-07, + "loss": 2.6357, + "step": 50 + }, + { + "epoch": 0.06222775357809583, + "eval_loss": 2.632822036743164, + "eval_runtime": 47.0104, + "eval_samples_per_second": 21.272, + "eval_steps_per_second": 0.893, + "step": 50 + }, + { + "epoch": 0.06347230864965775, + "grad_norm": 1.892500877380371, + "learning_rate": 7.676348547717843e-07, + "loss": 2.5945, + "step": 51 + }, + { + "epoch": 0.06471686372121967, + "grad_norm": 2.107921600341797, + "learning_rate": 7.883817427385892e-07, + "loss": 2.5867, + "step": 52 + }, + { + "epoch": 0.06596141879278158, + "grad_norm": 1.8287527561187744, + "learning_rate": 8.091286307053943e-07, + "loss": 2.5862, + "step": 53 + }, + { + "epoch": 0.0672059738643435, + "grad_norm": 1.7342065572738647, + "learning_rate": 8.298755186721992e-07, + "loss": 2.5735, + "step": 54 + }, + { + "epoch": 0.06845052893590542, + "grad_norm": 1.7905038595199585, + "learning_rate": 8.506224066390042e-07, + "loss": 2.572, + "step": 55 + }, + { + "epoch": 0.06969508400746734, + "grad_norm": 1.9715144634246826, + "learning_rate": 8.713692946058091e-07, + "loss": 2.5797, + "step": 56 + }, + { + "epoch": 0.07093963907902924, + "grad_norm": 1.788966417312622, + "learning_rate": 8.921161825726142e-07, + "loss": 2.5567, + "step": 57 + }, + { + "epoch": 0.07218419415059116, + "grad_norm": 1.8978915214538574, + "learning_rate": 9.128630705394191e-07, + "loss": 2.5776, + "step": 58 + }, + { + "epoch": 0.07342874922215308, + "grad_norm": 1.6401689052581787, + "learning_rate": 9.336099585062241e-07, + "loss": 2.5662, + "step": 59 + }, + { + "epoch": 0.07467330429371499, + "grad_norm": 1.8200798034667969, + "learning_rate": 9.54356846473029e-07, + "loss": 2.5001, + "step": 60 + }, + { + "epoch": 0.07467330429371499, + "eval_loss": 2.569110870361328, + "eval_runtime": 47.3292, + "eval_samples_per_second": 21.129, + "eval_steps_per_second": 0.887, + "step": 60 + }, + { + "epoch": 0.07591785936527691, + "grad_norm": 1.5468674898147583, + "learning_rate": 9.751037344398341e-07, + "loss": 2.5308, + "step": 61 + }, + { + "epoch": 0.07716241443683883, + "grad_norm": 1.723684310913086, + "learning_rate": 9.95850622406639e-07, + "loss": 2.4907, + "step": 62 + }, + { + "epoch": 0.07840696950840075, + "grad_norm": 1.6346293687820435, + "learning_rate": 1.0165975103734441e-06, + "loss": 2.514, + "step": 63 + }, + { + "epoch": 0.07965152457996266, + "grad_norm": 1.8902088403701782, + "learning_rate": 1.037344398340249e-06, + "loss": 2.4859, + "step": 64 + }, + { + "epoch": 0.08089607965152458, + "grad_norm": 2.092611312866211, + "learning_rate": 1.058091286307054e-06, + "loss": 2.4888, + "step": 65 + }, + { + "epoch": 0.0821406347230865, + "grad_norm": 1.758482575416565, + "learning_rate": 1.078838174273859e-06, + "loss": 2.4537, + "step": 66 + }, + { + "epoch": 0.08338518979464841, + "grad_norm": 1.8941714763641357, + "learning_rate": 1.099585062240664e-06, + "loss": 2.5416, + "step": 67 + }, + { + "epoch": 0.08462974486621033, + "grad_norm": 1.7030234336853027, + "learning_rate": 1.120331950207469e-06, + "loss": 2.4791, + "step": 68 + }, + { + "epoch": 0.08587429993777225, + "grad_norm": 1.7602269649505615, + "learning_rate": 1.141078838174274e-06, + "loss": 2.4316, + "step": 69 + }, + { + "epoch": 0.08711885500933417, + "grad_norm": 1.7635431289672852, + "learning_rate": 1.161825726141079e-06, + "loss": 2.4674, + "step": 70 + }, + { + "epoch": 0.08711885500933417, + "eval_loss": 2.4922080039978027, + "eval_runtime": 45.3028, + "eval_samples_per_second": 22.074, + "eval_steps_per_second": 0.927, + "step": 70 + }, + { + "epoch": 0.08836341008089608, + "grad_norm": 2.0913803577423096, + "learning_rate": 1.182572614107884e-06, + "loss": 2.4341, + "step": 71 + }, + { + "epoch": 0.089607965152458, + "grad_norm": 1.8292183876037598, + "learning_rate": 1.2033195020746888e-06, + "loss": 2.4291, + "step": 72 + }, + { + "epoch": 0.09085252022401992, + "grad_norm": 2.206770181655884, + "learning_rate": 1.224066390041494e-06, + "loss": 2.3764, + "step": 73 + }, + { + "epoch": 0.09209707529558182, + "grad_norm": 1.8263559341430664, + "learning_rate": 1.2448132780082988e-06, + "loss": 2.4082, + "step": 74 + }, + { + "epoch": 0.09334163036714374, + "grad_norm": 1.9405455589294434, + "learning_rate": 1.2655601659751037e-06, + "loss": 2.4673, + "step": 75 + }, + { + "epoch": 0.09458618543870566, + "grad_norm": 2.138108015060425, + "learning_rate": 1.2863070539419086e-06, + "loss": 2.3879, + "step": 76 + }, + { + "epoch": 0.09583074051026758, + "grad_norm": 1.9024745225906372, + "learning_rate": 1.307053941908714e-06, + "loss": 2.4344, + "step": 77 + }, + { + "epoch": 0.0970752955818295, + "grad_norm": 1.9136689901351929, + "learning_rate": 1.3278008298755188e-06, + "loss": 2.474, + "step": 78 + }, + { + "epoch": 0.09831985065339141, + "grad_norm": 2.4752795696258545, + "learning_rate": 1.3485477178423237e-06, + "loss": 2.3412, + "step": 79 + }, + { + "epoch": 0.09956440572495333, + "grad_norm": 1.8720006942749023, + "learning_rate": 1.3692946058091288e-06, + "loss": 2.3438, + "step": 80 + }, + { + "epoch": 0.09956440572495333, + "eval_loss": 2.3980512619018555, + "eval_runtime": 48.2788, + "eval_samples_per_second": 20.713, + "eval_steps_per_second": 0.87, + "step": 80 + }, + { + "epoch": 0.10080896079651525, + "grad_norm": 2.671691656112671, + "learning_rate": 1.3900414937759337e-06, + "loss": 2.3336, + "step": 81 + }, + { + "epoch": 0.10205351586807716, + "grad_norm": 2.2953391075134277, + "learning_rate": 1.4107883817427386e-06, + "loss": 2.377, + "step": 82 + }, + { + "epoch": 0.10329807093963908, + "grad_norm": 3.009018898010254, + "learning_rate": 1.4315352697095435e-06, + "loss": 2.2977, + "step": 83 + }, + { + "epoch": 0.104542626011201, + "grad_norm": 2.664454936981201, + "learning_rate": 1.4522821576763488e-06, + "loss": 2.3271, + "step": 84 + }, + { + "epoch": 0.10578718108276292, + "grad_norm": 3.017303705215454, + "learning_rate": 1.4730290456431537e-06, + "loss": 2.3251, + "step": 85 + }, + { + "epoch": 0.10703173615432483, + "grad_norm": 2.634716510772705, + "learning_rate": 1.4937759336099586e-06, + "loss": 2.332, + "step": 86 + }, + { + "epoch": 0.10827629122588675, + "grad_norm": 3.059644937515259, + "learning_rate": 1.5145228215767635e-06, + "loss": 2.3478, + "step": 87 + }, + { + "epoch": 0.10952084629744867, + "grad_norm": 2.6962637901306152, + "learning_rate": 1.5352697095435686e-06, + "loss": 2.2792, + "step": 88 + }, + { + "epoch": 0.11076540136901059, + "grad_norm": 3.419729709625244, + "learning_rate": 1.5560165975103735e-06, + "loss": 2.2571, + "step": 89 + }, + { + "epoch": 0.11200995644057249, + "grad_norm": 2.740781545639038, + "learning_rate": 1.5767634854771784e-06, + "loss": 2.2875, + "step": 90 + }, + { + "epoch": 0.11200995644057249, + "eval_loss": 2.30843186378479, + "eval_runtime": 49.9264, + "eval_samples_per_second": 20.029, + "eval_steps_per_second": 0.841, + "step": 90 + }, + { + "epoch": 0.1132545115121344, + "grad_norm": 2.5608789920806885, + "learning_rate": 1.5975103734439833e-06, + "loss": 2.2592, + "step": 91 + }, + { + "epoch": 0.11449906658369632, + "grad_norm": 2.687999963760376, + "learning_rate": 1.6182572614107886e-06, + "loss": 2.2546, + "step": 92 + }, + { + "epoch": 0.11574362165525824, + "grad_norm": 2.695909023284912, + "learning_rate": 1.6390041493775935e-06, + "loss": 2.2525, + "step": 93 + }, + { + "epoch": 0.11698817672682016, + "grad_norm": 2.818357467651367, + "learning_rate": 1.6597510373443984e-06, + "loss": 2.216, + "step": 94 + }, + { + "epoch": 0.11823273179838208, + "grad_norm": 2.884119987487793, + "learning_rate": 1.6804979253112035e-06, + "loss": 2.2321, + "step": 95 + }, + { + "epoch": 0.119477286869944, + "grad_norm": 2.52104115486145, + "learning_rate": 1.7012448132780084e-06, + "loss": 2.199, + "step": 96 + }, + { + "epoch": 0.12072184194150591, + "grad_norm": 2.420313596725464, + "learning_rate": 1.7219917012448133e-06, + "loss": 2.1862, + "step": 97 + }, + { + "epoch": 0.12196639701306783, + "grad_norm": 2.8047542572021484, + "learning_rate": 1.7427385892116182e-06, + "loss": 2.1793, + "step": 98 + }, + { + "epoch": 0.12321095208462975, + "grad_norm": 2.836482286453247, + "learning_rate": 1.7634854771784235e-06, + "loss": 2.2271, + "step": 99 + }, + { + "epoch": 0.12445550715619166, + "grad_norm": 2.5282301902770996, + "learning_rate": 1.7842323651452284e-06, + "loss": 2.1768, + "step": 100 + }, + { + "epoch": 0.12445550715619166, + "eval_loss": 2.2312686443328857, + "eval_runtime": 49.2382, + "eval_samples_per_second": 20.309, + "eval_steps_per_second": 0.853, + "step": 100 + }, + { + "epoch": 0.12570006222775357, + "grad_norm": 3.3407280445098877, + "learning_rate": 1.8049792531120333e-06, + "loss": 2.1666, + "step": 101 + }, + { + "epoch": 0.1269446172993155, + "grad_norm": 2.4754133224487305, + "learning_rate": 1.8257261410788382e-06, + "loss": 2.1768, + "step": 102 + }, + { + "epoch": 0.1281891723708774, + "grad_norm": 3.430889129638672, + "learning_rate": 1.8464730290456433e-06, + "loss": 2.1953, + "step": 103 + }, + { + "epoch": 0.12943372744243933, + "grad_norm": 2.835294246673584, + "learning_rate": 1.8672199170124482e-06, + "loss": 2.146, + "step": 104 + }, + { + "epoch": 0.13067828251400124, + "grad_norm": 3.1532323360443115, + "learning_rate": 1.8879668049792531e-06, + "loss": 2.1729, + "step": 105 + }, + { + "epoch": 0.13192283758556317, + "grad_norm": 3.2278342247009277, + "learning_rate": 1.908713692946058e-06, + "loss": 2.1336, + "step": 106 + }, + { + "epoch": 0.13316739265712507, + "grad_norm": 2.7892515659332275, + "learning_rate": 1.929460580912863e-06, + "loss": 2.141, + "step": 107 + }, + { + "epoch": 0.134411947728687, + "grad_norm": 3.0179977416992188, + "learning_rate": 1.9502074688796682e-06, + "loss": 2.1255, + "step": 108 + }, + { + "epoch": 0.1356565028002489, + "grad_norm": 2.977935552597046, + "learning_rate": 1.970954356846473e-06, + "loss": 2.0986, + "step": 109 + }, + { + "epoch": 0.13690105787181084, + "grad_norm": 3.3042492866516113, + "learning_rate": 1.991701244813278e-06, + "loss": 2.0992, + "step": 110 + }, + { + "epoch": 0.13690105787181084, + "eval_loss": 2.171299695968628, + "eval_runtime": 42.9511, + "eval_samples_per_second": 23.282, + "eval_steps_per_second": 0.978, + "step": 110 + }, + { + "epoch": 0.13814561294337274, + "grad_norm": 2.9750890731811523, + "learning_rate": 2.012448132780083e-06, + "loss": 2.1116, + "step": 111 + }, + { + "epoch": 0.13939016801493467, + "grad_norm": 2.8199286460876465, + "learning_rate": 2.0331950207468883e-06, + "loss": 2.0822, + "step": 112 + }, + { + "epoch": 0.14063472308649658, + "grad_norm": 2.930532217025757, + "learning_rate": 2.053941908713693e-06, + "loss": 2.161, + "step": 113 + }, + { + "epoch": 0.14187927815805848, + "grad_norm": 3.3400321006774902, + "learning_rate": 2.074688796680498e-06, + "loss": 2.0953, + "step": 114 + }, + { + "epoch": 0.1431238332296204, + "grad_norm": 2.716564655303955, + "learning_rate": 2.095435684647303e-06, + "loss": 2.0757, + "step": 115 + }, + { + "epoch": 0.14436838830118232, + "grad_norm": 2.7586236000061035, + "learning_rate": 2.116182572614108e-06, + "loss": 2.0979, + "step": 116 + }, + { + "epoch": 0.14561294337274425, + "grad_norm": 2.622126340866089, + "learning_rate": 2.136929460580913e-06, + "loss": 2.0748, + "step": 117 + }, + { + "epoch": 0.14685749844430615, + "grad_norm": 2.646477460861206, + "learning_rate": 2.157676348547718e-06, + "loss": 2.0605, + "step": 118 + }, + { + "epoch": 0.14810205351586808, + "grad_norm": 2.8882410526275635, + "learning_rate": 2.178423236514523e-06, + "loss": 2.0501, + "step": 119 + }, + { + "epoch": 0.14934660858742999, + "grad_norm": 2.8302247524261475, + "learning_rate": 2.199170124481328e-06, + "loss": 2.0592, + "step": 120 + }, + { + "epoch": 0.14934660858742999, + "eval_loss": 2.1291966438293457, + "eval_runtime": 47.5399, + "eval_samples_per_second": 21.035, + "eval_steps_per_second": 0.883, + "step": 120 + }, + { + "epoch": 0.15059116365899192, + "grad_norm": 3.26979660987854, + "learning_rate": 2.219917012448133e-06, + "loss": 2.0422, + "step": 121 + }, + { + "epoch": 0.15183571873055382, + "grad_norm": 2.7483913898468018, + "learning_rate": 2.240663900414938e-06, + "loss": 2.0872, + "step": 122 + }, + { + "epoch": 0.15308027380211575, + "grad_norm": 3.1252024173736572, + "learning_rate": 2.2614107883817427e-06, + "loss": 2.0684, + "step": 123 + }, + { + "epoch": 0.15432482887367766, + "grad_norm": 2.9175283908843994, + "learning_rate": 2.282157676348548e-06, + "loss": 2.0522, + "step": 124 + }, + { + "epoch": 0.1555693839452396, + "grad_norm": 2.7950754165649414, + "learning_rate": 2.302904564315353e-06, + "loss": 2.065, + "step": 125 + }, + { + "epoch": 0.1568139390168015, + "grad_norm": 2.84049654006958, + "learning_rate": 2.323651452282158e-06, + "loss": 2.0781, + "step": 126 + }, + { + "epoch": 0.15805849408836342, + "grad_norm": 2.7396647930145264, + "learning_rate": 2.3443983402489627e-06, + "loss": 2.1758, + "step": 127 + }, + { + "epoch": 0.15930304915992533, + "grad_norm": 3.0765926837921143, + "learning_rate": 2.365145228215768e-06, + "loss": 2.0172, + "step": 128 + }, + { + "epoch": 0.16054760423148726, + "grad_norm": 3.217189073562622, + "learning_rate": 2.385892116182573e-06, + "loss": 2.0286, + "step": 129 + }, + { + "epoch": 0.16179215930304916, + "grad_norm": 3.141545295715332, + "learning_rate": 2.4066390041493776e-06, + "loss": 2.0583, + "step": 130 + }, + { + "epoch": 0.16179215930304916, + "eval_loss": 2.0946149826049805, + "eval_runtime": 54.92, + "eval_samples_per_second": 18.208, + "eval_steps_per_second": 0.765, + "step": 130 + }, + { + "epoch": 0.16303671437461106, + "grad_norm": 2.850052833557129, + "learning_rate": 2.4273858921161828e-06, + "loss": 2.0746, + "step": 131 + }, + { + "epoch": 0.164281269446173, + "grad_norm": 3.28913950920105, + "learning_rate": 2.448132780082988e-06, + "loss": 2.0539, + "step": 132 + }, + { + "epoch": 0.1655258245177349, + "grad_norm": 2.8819124698638916, + "learning_rate": 2.468879668049793e-06, + "loss": 1.9843, + "step": 133 + }, + { + "epoch": 0.16677037958929683, + "grad_norm": 3.6254632472991943, + "learning_rate": 2.4896265560165977e-06, + "loss": 2.0233, + "step": 134 + }, + { + "epoch": 0.16801493466085873, + "grad_norm": 2.7385146617889404, + "learning_rate": 2.5103734439834028e-06, + "loss": 1.9667, + "step": 135 + }, + { + "epoch": 0.16925948973242066, + "grad_norm": 2.9722647666931152, + "learning_rate": 2.5311203319502074e-06, + "loss": 1.9527, + "step": 136 + }, + { + "epoch": 0.17050404480398257, + "grad_norm": 3.139526605606079, + "learning_rate": 2.5518672199170125e-06, + "loss": 1.9967, + "step": 137 + }, + { + "epoch": 0.1717485998755445, + "grad_norm": 3.399920701980591, + "learning_rate": 2.5726141078838172e-06, + "loss": 1.9798, + "step": 138 + }, + { + "epoch": 0.1729931549471064, + "grad_norm": 2.8017327785491943, + "learning_rate": 2.5933609958506228e-06, + "loss": 1.983, + "step": 139 + }, + { + "epoch": 0.17423771001866833, + "grad_norm": 3.9967198371887207, + "learning_rate": 2.614107883817428e-06, + "loss": 1.977, + "step": 140 + }, + { + "epoch": 0.17423771001866833, + "eval_loss": 2.059037685394287, + "eval_runtime": 52.6232, + "eval_samples_per_second": 19.003, + "eval_steps_per_second": 0.798, + "step": 140 + }, + { + "epoch": 0.17548226509023024, + "grad_norm": 2.893092393875122, + "learning_rate": 2.6348547717842326e-06, + "loss": 1.9888, + "step": 141 + }, + { + "epoch": 0.17672682016179217, + "grad_norm": 3.634352207183838, + "learning_rate": 2.6556016597510377e-06, + "loss": 2.0726, + "step": 142 + }, + { + "epoch": 0.17797137523335407, + "grad_norm": 3.3651444911956787, + "learning_rate": 2.6763485477178423e-06, + "loss": 1.973, + "step": 143 + }, + { + "epoch": 0.179215930304916, + "grad_norm": 3.968986988067627, + "learning_rate": 2.6970954356846475e-06, + "loss": 1.9948, + "step": 144 + }, + { + "epoch": 0.1804604853764779, + "grad_norm": 3.20105242729187, + "learning_rate": 2.717842323651452e-06, + "loss": 1.9238, + "step": 145 + }, + { + "epoch": 0.18170504044803984, + "grad_norm": 3.648339033126831, + "learning_rate": 2.7385892116182577e-06, + "loss": 1.9923, + "step": 146 + }, + { + "epoch": 0.18294959551960174, + "grad_norm": 3.0856316089630127, + "learning_rate": 2.7593360995850628e-06, + "loss": 1.9749, + "step": 147 + }, + { + "epoch": 0.18419415059116365, + "grad_norm": 3.2681071758270264, + "learning_rate": 2.7800829875518675e-06, + "loss": 1.9304, + "step": 148 + }, + { + "epoch": 0.18543870566272558, + "grad_norm": 2.634958267211914, + "learning_rate": 2.8008298755186726e-06, + "loss": 1.9237, + "step": 149 + }, + { + "epoch": 0.18668326073428748, + "grad_norm": 2.769491672515869, + "learning_rate": 2.8215767634854773e-06, + "loss": 1.8963, + "step": 150 + }, + { + "epoch": 0.18668326073428748, + "eval_loss": 2.0394654273986816, + "eval_runtime": 55.2136, + "eval_samples_per_second": 18.111, + "eval_steps_per_second": 0.761, + "step": 150 + }, + { + "epoch": 0.1879278158058494, + "grad_norm": 3.4345781803131104, + "learning_rate": 2.8423236514522824e-06, + "loss": 1.9836, + "step": 151 + }, + { + "epoch": 0.18917237087741132, + "grad_norm": 3.115727424621582, + "learning_rate": 2.863070539419087e-06, + "loss": 1.9352, + "step": 152 + }, + { + "epoch": 0.19041692594897325, + "grad_norm": 3.050652503967285, + "learning_rate": 2.883817427385892e-06, + "loss": 1.9031, + "step": 153 + }, + { + "epoch": 0.19166148102053515, + "grad_norm": 2.9404428005218506, + "learning_rate": 2.9045643153526977e-06, + "loss": 1.9196, + "step": 154 + }, + { + "epoch": 0.19290603609209708, + "grad_norm": 3.080810785293579, + "learning_rate": 2.9253112033195024e-06, + "loss": 1.9405, + "step": 155 + }, + { + "epoch": 0.194150591163659, + "grad_norm": 3.058558702468872, + "learning_rate": 2.9460580912863075e-06, + "loss": 1.9052, + "step": 156 + }, + { + "epoch": 0.19539514623522092, + "grad_norm": 3.307955026626587, + "learning_rate": 2.966804979253112e-06, + "loss": 1.9245, + "step": 157 + }, + { + "epoch": 0.19663970130678282, + "grad_norm": 2.845506191253662, + "learning_rate": 2.9875518672199173e-06, + "loss": 1.9461, + "step": 158 + }, + { + "epoch": 0.19788425637834475, + "grad_norm": 2.758654832839966, + "learning_rate": 3.008298755186722e-06, + "loss": 1.8885, + "step": 159 + }, + { + "epoch": 0.19912881144990666, + "grad_norm": 3.161252737045288, + "learning_rate": 3.029045643153527e-06, + "loss": 1.8535, + "step": 160 + }, + { + "epoch": 0.19912881144990666, + "eval_loss": 2.018101453781128, + "eval_runtime": 51.0744, + "eval_samples_per_second": 19.579, + "eval_steps_per_second": 0.822, + "step": 160 + }, + { + "epoch": 0.2003733665214686, + "grad_norm": 3.196847677230835, + "learning_rate": 3.0497925311203326e-06, + "loss": 1.8853, + "step": 161 + }, + { + "epoch": 0.2016179215930305, + "grad_norm": 2.8364663124084473, + "learning_rate": 3.0705394190871373e-06, + "loss": 1.9463, + "step": 162 + }, + { + "epoch": 0.2028624766645924, + "grad_norm": 3.0874054431915283, + "learning_rate": 3.0912863070539424e-06, + "loss": 1.9254, + "step": 163 + }, + { + "epoch": 0.20410703173615433, + "grad_norm": 2.7914493083953857, + "learning_rate": 3.112033195020747e-06, + "loss": 1.9213, + "step": 164 + }, + { + "epoch": 0.20535158680771623, + "grad_norm": 3.3871428966522217, + "learning_rate": 3.132780082987552e-06, + "loss": 1.8993, + "step": 165 + }, + { + "epoch": 0.20659614187927816, + "grad_norm": 3.096653461456299, + "learning_rate": 3.153526970954357e-06, + "loss": 1.8827, + "step": 166 + }, + { + "epoch": 0.20784069695084006, + "grad_norm": 2.8276076316833496, + "learning_rate": 3.174273858921162e-06, + "loss": 1.912, + "step": 167 + }, + { + "epoch": 0.209085252022402, + "grad_norm": 3.3058435916900635, + "learning_rate": 3.1950207468879666e-06, + "loss": 1.8381, + "step": 168 + }, + { + "epoch": 0.2103298070939639, + "grad_norm": 3.5017333030700684, + "learning_rate": 3.215767634854772e-06, + "loss": 1.9544, + "step": 169 + }, + { + "epoch": 0.21157436216552583, + "grad_norm": 3.457296133041382, + "learning_rate": 3.2365145228215773e-06, + "loss": 1.922, + "step": 170 + }, + { + "epoch": 0.21157436216552583, + "eval_loss": 1.988455891609192, + "eval_runtime": 56.2099, + "eval_samples_per_second": 17.79, + "eval_steps_per_second": 0.747, + "step": 170 + }, + { + "epoch": 0.21281891723708773, + "grad_norm": 3.1964566707611084, + "learning_rate": 3.257261410788382e-06, + "loss": 1.8925, + "step": 171 + }, + { + "epoch": 0.21406347230864967, + "grad_norm": 3.234652042388916, + "learning_rate": 3.278008298755187e-06, + "loss": 1.884, + "step": 172 + }, + { + "epoch": 0.21530802738021157, + "grad_norm": 3.14414119720459, + "learning_rate": 3.2987551867219918e-06, + "loss": 1.8905, + "step": 173 + }, + { + "epoch": 0.2165525824517735, + "grad_norm": 3.606379508972168, + "learning_rate": 3.319502074688797e-06, + "loss": 1.946, + "step": 174 + }, + { + "epoch": 0.2177971375233354, + "grad_norm": 2.986646890640259, + "learning_rate": 3.3402489626556016e-06, + "loss": 1.8691, + "step": 175 + }, + { + "epoch": 0.21904169259489734, + "grad_norm": 3.7298269271850586, + "learning_rate": 3.360995850622407e-06, + "loss": 1.8717, + "step": 176 + }, + { + "epoch": 0.22028624766645924, + "grad_norm": 3.422295093536377, + "learning_rate": 3.381742738589212e-06, + "loss": 1.8568, + "step": 177 + }, + { + "epoch": 0.22153080273802117, + "grad_norm": 3.2293782234191895, + "learning_rate": 3.402489626556017e-06, + "loss": 1.8471, + "step": 178 + }, + { + "epoch": 0.22277535780958307, + "grad_norm": 3.2293782234191895, + "learning_rate": 3.402489626556017e-06, + "loss": 1.9303, + "step": 179 + }, + { + "epoch": 0.22401991288114498, + "grad_norm": 3.506223440170288, + "learning_rate": 3.423236514522822e-06, + "loss": 1.8237, + "step": 180 + }, + { + "epoch": 0.22401991288114498, + "eval_loss": 1.9734643697738647, + "eval_runtime": 52.2815, + "eval_samples_per_second": 19.127, + "eval_steps_per_second": 0.803, + "step": 180 + }, + { + "epoch": 0.2252644679527069, + "grad_norm": 3.0156619548797607, + "learning_rate": 3.4439834024896267e-06, + "loss": 1.908, + "step": 181 + }, + { + "epoch": 0.2265090230242688, + "grad_norm": 3.55517315864563, + "learning_rate": 3.4647302904564318e-06, + "loss": 1.9104, + "step": 182 + }, + { + "epoch": 0.22775357809583074, + "grad_norm": 3.144984006881714, + "learning_rate": 3.4854771784232365e-06, + "loss": 1.8198, + "step": 183 + }, + { + "epoch": 0.22899813316739265, + "grad_norm": 4.302074432373047, + "learning_rate": 3.5062240663900416e-06, + "loss": 1.8237, + "step": 184 + }, + { + "epoch": 0.23024268823895458, + "grad_norm": 3.0522446632385254, + "learning_rate": 3.526970954356847e-06, + "loss": 1.8409, + "step": 185 + }, + { + "epoch": 0.23148724331051648, + "grad_norm": 3.8607394695281982, + "learning_rate": 3.5477178423236518e-06, + "loss": 1.8807, + "step": 186 + }, + { + "epoch": 0.23273179838207841, + "grad_norm": 2.9236302375793457, + "learning_rate": 3.568464730290457e-06, + "loss": 1.7873, + "step": 187 + }, + { + "epoch": 0.23397635345364032, + "grad_norm": 4.013780117034912, + "learning_rate": 3.5892116182572616e-06, + "loss": 1.7909, + "step": 188 + }, + { + "epoch": 0.23522090852520225, + "grad_norm": 3.0933122634887695, + "learning_rate": 3.6099585062240667e-06, + "loss": 1.8469, + "step": 189 + }, + { + "epoch": 0.23646546359676415, + "grad_norm": 3.487816095352173, + "learning_rate": 3.6307053941908714e-06, + "loss": 1.8469, + "step": 190 + }, + { + "epoch": 0.23646546359676415, + "eval_loss": 1.9604240655899048, + "eval_runtime": 53.4604, + "eval_samples_per_second": 18.705, + "eval_steps_per_second": 0.786, + "step": 190 + }, + { + "epoch": 0.23771001866832608, + "grad_norm": 2.829219341278076, + "learning_rate": 3.6514522821576765e-06, + "loss": 1.8639, + "step": 191 + }, + { + "epoch": 0.238954573739888, + "grad_norm": 3.595534324645996, + "learning_rate": 3.672199170124482e-06, + "loss": 1.8988, + "step": 192 + }, + { + "epoch": 0.24019912881144992, + "grad_norm": 2.9022483825683594, + "learning_rate": 3.6929460580912867e-06, + "loss": 1.8972, + "step": 193 + }, + { + "epoch": 0.24144368388301182, + "grad_norm": 3.0667994022369385, + "learning_rate": 3.713692946058092e-06, + "loss": 1.8499, + "step": 194 + }, + { + "epoch": 0.24268823895457373, + "grad_norm": 3.0498485565185547, + "learning_rate": 3.7344398340248965e-06, + "loss": 1.8341, + "step": 195 + }, + { + "epoch": 0.24393279402613566, + "grad_norm": 4.058084964752197, + "learning_rate": 3.7551867219917016e-06, + "loss": 1.8641, + "step": 196 + }, + { + "epoch": 0.24517734909769756, + "grad_norm": 2.9995028972625732, + "learning_rate": 3.7759336099585063e-06, + "loss": 1.7885, + "step": 197 + }, + { + "epoch": 0.2464219041692595, + "grad_norm": 3.254704475402832, + "learning_rate": 3.7966804979253114e-06, + "loss": 1.7544, + "step": 198 + }, + { + "epoch": 0.2476664592408214, + "grad_norm": 2.9126744270324707, + "learning_rate": 3.817427385892116e-06, + "loss": 1.8461, + "step": 199 + }, + { + "epoch": 0.24891101431238333, + "grad_norm": 3.475247621536255, + "learning_rate": 3.838174273858922e-06, + "loss": 1.8095, + "step": 200 + }, + { + "epoch": 0.24891101431238333, + "eval_loss": 1.9373760223388672, + "eval_runtime": 51.8733, + "eval_samples_per_second": 19.278, + "eval_steps_per_second": 0.81, + "step": 200 + }, + { + "epoch": 0.25015556938394523, + "grad_norm": 3.2663731575012207, + "learning_rate": 3.858921161825726e-06, + "loss": 1.8076, + "step": 201 + }, + { + "epoch": 0.25140012445550713, + "grad_norm": 3.1151726245880127, + "learning_rate": 3.879668049792531e-06, + "loss": 1.8143, + "step": 202 + }, + { + "epoch": 0.2526446795270691, + "grad_norm": 3.102038860321045, + "learning_rate": 3.9004149377593365e-06, + "loss": 1.8279, + "step": 203 + }, + { + "epoch": 0.253889234598631, + "grad_norm": 3.370642900466919, + "learning_rate": 3.921161825726142e-06, + "loss": 1.8522, + "step": 204 + }, + { + "epoch": 0.2551337896701929, + "grad_norm": 3.116128921508789, + "learning_rate": 3.941908713692946e-06, + "loss": 1.8104, + "step": 205 + }, + { + "epoch": 0.2563783447417548, + "grad_norm": 3.12028431892395, + "learning_rate": 3.962655601659751e-06, + "loss": 1.7723, + "step": 206 + }, + { + "epoch": 0.25762289981331676, + "grad_norm": 3.1251418590545654, + "learning_rate": 3.983402489626556e-06, + "loss": 1.863, + "step": 207 + }, + { + "epoch": 0.25886745488487867, + "grad_norm": 3.073702812194824, + "learning_rate": 4.004149377593361e-06, + "loss": 1.805, + "step": 208 + }, + { + "epoch": 0.26011200995644057, + "grad_norm": 2.9772469997406006, + "learning_rate": 4.024896265560166e-06, + "loss": 1.8251, + "step": 209 + }, + { + "epoch": 0.2613565650280025, + "grad_norm": 2.9970037937164307, + "learning_rate": 4.045643153526971e-06, + "loss": 1.8386, + "step": 210 + }, + { + "epoch": 0.2613565650280025, + "eval_loss": 1.9167065620422363, + "eval_runtime": 42.3328, + "eval_samples_per_second": 23.622, + "eval_steps_per_second": 0.992, + "step": 210 + }, + { + "epoch": 0.26260112009956443, + "grad_norm": 3.1391561031341553, + "learning_rate": 4.0663900414937765e-06, + "loss": 1.8109, + "step": 211 + }, + { + "epoch": 0.26384567517112634, + "grad_norm": 3.161538600921631, + "learning_rate": 4.087136929460581e-06, + "loss": 1.8441, + "step": 212 + }, + { + "epoch": 0.26509023024268824, + "grad_norm": 3.12412166595459, + "learning_rate": 4.107883817427386e-06, + "loss": 1.7736, + "step": 213 + }, + { + "epoch": 0.26633478531425014, + "grad_norm": 3.3241145610809326, + "learning_rate": 4.128630705394191e-06, + "loss": 1.8067, + "step": 214 + }, + { + "epoch": 0.26757934038581205, + "grad_norm": 3.503307819366455, + "learning_rate": 4.149377593360996e-06, + "loss": 1.7858, + "step": 215 + }, + { + "epoch": 0.268823895457374, + "grad_norm": 3.1009578704833984, + "learning_rate": 4.170124481327801e-06, + "loss": 1.7542, + "step": 216 + }, + { + "epoch": 0.2700684505289359, + "grad_norm": 3.2236011028289795, + "learning_rate": 4.190871369294606e-06, + "loss": 1.8006, + "step": 217 + }, + { + "epoch": 0.2713130056004978, + "grad_norm": 3.048935651779175, + "learning_rate": 4.211618257261411e-06, + "loss": 1.7706, + "step": 218 + }, + { + "epoch": 0.2725575606720597, + "grad_norm": 3.3416147232055664, + "learning_rate": 4.232365145228216e-06, + "loss": 1.8108, + "step": 219 + }, + { + "epoch": 0.2738021157436217, + "grad_norm": 3.2010738849639893, + "learning_rate": 4.253112033195021e-06, + "loss": 1.8561, + "step": 220 + }, + { + "epoch": 0.2738021157436217, + "eval_loss": 1.9111930131912231, + "eval_runtime": 50.4077, + "eval_samples_per_second": 19.838, + "eval_steps_per_second": 0.833, + "step": 220 + }, + { + "epoch": 0.2750466708151836, + "grad_norm": 3.4033374786376953, + "learning_rate": 4.273858921161826e-06, + "loss": 1.8218, + "step": 221 + }, + { + "epoch": 0.2762912258867455, + "grad_norm": 3.2086002826690674, + "learning_rate": 4.294605809128631e-06, + "loss": 1.8213, + "step": 222 + }, + { + "epoch": 0.2775357809583074, + "grad_norm": 4.090956211090088, + "learning_rate": 4.315352697095436e-06, + "loss": 1.8455, + "step": 223 + }, + { + "epoch": 0.27878033602986935, + "grad_norm": 3.1142985820770264, + "learning_rate": 4.336099585062241e-06, + "loss": 1.7737, + "step": 224 + }, + { + "epoch": 0.28002489110143125, + "grad_norm": 3.369669198989868, + "learning_rate": 4.356846473029046e-06, + "loss": 1.824, + "step": 225 + }, + { + "epoch": 0.28126944617299315, + "grad_norm": 3.165672779083252, + "learning_rate": 4.3775933609958506e-06, + "loss": 1.7768, + "step": 226 + }, + { + "epoch": 0.28251400124455506, + "grad_norm": 3.3343470096588135, + "learning_rate": 4.398340248962656e-06, + "loss": 1.7278, + "step": 227 + }, + { + "epoch": 0.28375855631611696, + "grad_norm": 3.2574994564056396, + "learning_rate": 4.419087136929461e-06, + "loss": 1.8439, + "step": 228 + }, + { + "epoch": 0.2850031113876789, + "grad_norm": 3.043928623199463, + "learning_rate": 4.439834024896266e-06, + "loss": 1.818, + "step": 229 + }, + { + "epoch": 0.2862476664592408, + "grad_norm": 2.9703125953674316, + "learning_rate": 4.460580912863071e-06, + "loss": 1.773, + "step": 230 + }, + { + "epoch": 0.2862476664592408, + "eval_loss": 1.898109793663025, + "eval_runtime": 45.6023, + "eval_samples_per_second": 21.929, + "eval_steps_per_second": 0.921, + "step": 230 + }, + { + "epoch": 0.2874922215308027, + "grad_norm": 2.9997220039367676, + "learning_rate": 4.481327800829876e-06, + "loss": 1.7727, + "step": 231 + }, + { + "epoch": 0.28873677660236463, + "grad_norm": 3.1358482837677, + "learning_rate": 4.502074688796681e-06, + "loss": 1.7764, + "step": 232 + }, + { + "epoch": 0.2899813316739266, + "grad_norm": 3.395747661590576, + "learning_rate": 4.5228215767634855e-06, + "loss": 1.7826, + "step": 233 + }, + { + "epoch": 0.2912258867454885, + "grad_norm": 3.7494754791259766, + "learning_rate": 4.543568464730291e-06, + "loss": 1.7181, + "step": 234 + }, + { + "epoch": 0.2924704418170504, + "grad_norm": 3.42293381690979, + "learning_rate": 4.564315352697096e-06, + "loss": 1.7773, + "step": 235 + }, + { + "epoch": 0.2937149968886123, + "grad_norm": 3.2524514198303223, + "learning_rate": 4.585062240663901e-06, + "loss": 1.7294, + "step": 236 + }, + { + "epoch": 0.29495955196017426, + "grad_norm": 3.657869577407837, + "learning_rate": 4.605809128630706e-06, + "loss": 1.762, + "step": 237 + }, + { + "epoch": 0.29620410703173616, + "grad_norm": 3.127372980117798, + "learning_rate": 4.626556016597511e-06, + "loss": 1.7623, + "step": 238 + }, + { + "epoch": 0.29744866210329807, + "grad_norm": 4.166962146759033, + "learning_rate": 4.647302904564316e-06, + "loss": 1.6995, + "step": 239 + }, + { + "epoch": 0.29869321717485997, + "grad_norm": 3.094264030456543, + "learning_rate": 4.66804979253112e-06, + "loss": 1.7469, + "step": 240 + }, + { + "epoch": 0.29869321717485997, + "eval_loss": 1.8908178806304932, + "eval_runtime": 44.6332, + "eval_samples_per_second": 22.405, + "eval_steps_per_second": 0.941, + "step": 240 + }, + { + "epoch": 0.29993777224642193, + "grad_norm": 3.8644745349884033, + "learning_rate": 4.6887966804979255e-06, + "loss": 1.7644, + "step": 241 + }, + { + "epoch": 0.30118232731798383, + "grad_norm": 3.1488852500915527, + "learning_rate": 4.709543568464731e-06, + "loss": 1.772, + "step": 242 + }, + { + "epoch": 0.30242688238954574, + "grad_norm": 3.3179638385772705, + "learning_rate": 4.730290456431536e-06, + "loss": 1.7588, + "step": 243 + }, + { + "epoch": 0.30367143746110764, + "grad_norm": 3.328355550765991, + "learning_rate": 4.751037344398341e-06, + "loss": 1.7417, + "step": 244 + }, + { + "epoch": 0.30491599253266954, + "grad_norm": 3.2690482139587402, + "learning_rate": 4.771784232365146e-06, + "loss": 1.7562, + "step": 245 + }, + { + "epoch": 0.3061605476042315, + "grad_norm": 3.2759454250335693, + "learning_rate": 4.792531120331951e-06, + "loss": 1.7536, + "step": 246 + }, + { + "epoch": 0.3074051026757934, + "grad_norm": 2.937964916229248, + "learning_rate": 4.813278008298755e-06, + "loss": 1.7518, + "step": 247 + }, + { + "epoch": 0.3086496577473553, + "grad_norm": 3.0995302200317383, + "learning_rate": 4.83402489626556e-06, + "loss": 1.7593, + "step": 248 + }, + { + "epoch": 0.3098942128189172, + "grad_norm": 3.230459213256836, + "learning_rate": 4.8547717842323655e-06, + "loss": 1.6962, + "step": 249 + }, + { + "epoch": 0.3111387678904792, + "grad_norm": 3.242576837539673, + "learning_rate": 4.875518672199171e-06, + "loss": 1.7307, + "step": 250 + }, + { + "epoch": 0.3111387678904792, + "eval_loss": 1.8805371522903442, + "eval_runtime": 45.9886, + "eval_samples_per_second": 21.745, + "eval_steps_per_second": 0.913, + "step": 250 + }, + { + "epoch": 0.3123833229620411, + "grad_norm": 3.097045421600342, + "learning_rate": 4.896265560165976e-06, + "loss": 1.7438, + "step": 251 + }, + { + "epoch": 0.313627878033603, + "grad_norm": 3.2428948879241943, + "learning_rate": 4.91701244813278e-06, + "loss": 1.7179, + "step": 252 + }, + { + "epoch": 0.3148724331051649, + "grad_norm": 3.196274518966675, + "learning_rate": 4.937759336099586e-06, + "loss": 1.7855, + "step": 253 + }, + { + "epoch": 0.31611698817672684, + "grad_norm": 2.978203535079956, + "learning_rate": 4.95850622406639e-06, + "loss": 1.7144, + "step": 254 + }, + { + "epoch": 0.31736154324828875, + "grad_norm": 3.2641701698303223, + "learning_rate": 4.979253112033195e-06, + "loss": 1.7427, + "step": 255 + }, + { + "epoch": 0.31860609831985065, + "grad_norm": 2.7441232204437256, + "learning_rate": 5e-06, + "loss": 1.7619, + "step": 256 + }, + { + "epoch": 0.31985065339141255, + "grad_norm": 3.0723495483398438, + "learning_rate": 4.99769372693727e-06, + "loss": 1.8017, + "step": 257 + }, + { + "epoch": 0.3210952084629745, + "grad_norm": 3.104752540588379, + "learning_rate": 4.995387453874539e-06, + "loss": 1.7761, + "step": 258 + }, + { + "epoch": 0.3223397635345364, + "grad_norm": 3.138627052307129, + "learning_rate": 4.993081180811809e-06, + "loss": 1.7879, + "step": 259 + }, + { + "epoch": 0.3235843186060983, + "grad_norm": 2.8567333221435547, + "learning_rate": 4.990774907749078e-06, + "loss": 1.7561, + "step": 260 + }, + { + "epoch": 0.3235843186060983, + "eval_loss": 1.8606494665145874, + "eval_runtime": 46.453, + "eval_samples_per_second": 21.527, + "eval_steps_per_second": 0.904, + "step": 260 + }, + { + "epoch": 0.3248288736776602, + "grad_norm": 3.2696540355682373, + "learning_rate": 4.988468634686347e-06, + "loss": 1.7201, + "step": 261 + }, + { + "epoch": 0.3260734287492221, + "grad_norm": 2.6692731380462646, + "learning_rate": 4.986162361623617e-06, + "loss": 1.7264, + "step": 262 + }, + { + "epoch": 0.3273179838207841, + "grad_norm": 3.047549247741699, + "learning_rate": 4.983856088560886e-06, + "loss": 1.7362, + "step": 263 + }, + { + "epoch": 0.328562538892346, + "grad_norm": 3.0667457580566406, + "learning_rate": 4.981549815498156e-06, + "loss": 1.7321, + "step": 264 + }, + { + "epoch": 0.3298070939639079, + "grad_norm": 2.560047149658203, + "learning_rate": 4.979243542435424e-06, + "loss": 1.7508, + "step": 265 + }, + { + "epoch": 0.3310516490354698, + "grad_norm": 3.26595401763916, + "learning_rate": 4.976937269372694e-06, + "loss": 1.7248, + "step": 266 + }, + { + "epoch": 0.33229620410703176, + "grad_norm": 2.929210662841797, + "learning_rate": 4.974630996309964e-06, + "loss": 1.697, + "step": 267 + }, + { + "epoch": 0.33354075917859366, + "grad_norm": 3.1526286602020264, + "learning_rate": 4.972324723247233e-06, + "loss": 1.7385, + "step": 268 + }, + { + "epoch": 0.33478531425015556, + "grad_norm": 3.001619577407837, + "learning_rate": 4.970018450184502e-06, + "loss": 1.7072, + "step": 269 + }, + { + "epoch": 0.33602986932171747, + "grad_norm": 3.2464189529418945, + "learning_rate": 4.9677121771217715e-06, + "loss": 1.7361, + "step": 270 + }, + { + "epoch": 0.33602986932171747, + "eval_loss": 1.860226035118103, + "eval_runtime": 46.6881, + "eval_samples_per_second": 21.419, + "eval_steps_per_second": 0.9, + "step": 270 + }, + { + "epoch": 0.3372744243932794, + "grad_norm": 3.2104530334472656, + "learning_rate": 4.965405904059041e-06, + "loss": 1.7171, + "step": 271 + }, + { + "epoch": 0.33851897946484133, + "grad_norm": 3.146847724914551, + "learning_rate": 4.96309963099631e-06, + "loss": 1.7311, + "step": 272 + }, + { + "epoch": 0.33976353453640323, + "grad_norm": 3.1431286334991455, + "learning_rate": 4.96079335793358e-06, + "loss": 1.7345, + "step": 273 + }, + { + "epoch": 0.34100808960796514, + "grad_norm": 2.847163677215576, + "learning_rate": 4.958487084870849e-06, + "loss": 1.7262, + "step": 274 + }, + { + "epoch": 0.3422526446795271, + "grad_norm": 3.1987810134887695, + "learning_rate": 4.956180811808119e-06, + "loss": 1.7945, + "step": 275 + }, + { + "epoch": 0.343497199751089, + "grad_norm": 2.7475385665893555, + "learning_rate": 4.953874538745388e-06, + "loss": 1.76, + "step": 276 + }, + { + "epoch": 0.3447417548226509, + "grad_norm": 3.0427663326263428, + "learning_rate": 4.9515682656826574e-06, + "loss": 1.7324, + "step": 277 + }, + { + "epoch": 0.3459863098942128, + "grad_norm": 3.605212688446045, + "learning_rate": 4.949261992619927e-06, + "loss": 1.7712, + "step": 278 + }, + { + "epoch": 0.3472308649657747, + "grad_norm": 3.0564935207366943, + "learning_rate": 4.946955719557196e-06, + "loss": 1.6792, + "step": 279 + }, + { + "epoch": 0.34847542003733667, + "grad_norm": 3.1069741249084473, + "learning_rate": 4.944649446494466e-06, + "loss": 1.7145, + "step": 280 + }, + { + "epoch": 0.34847542003733667, + "eval_loss": 1.8537051677703857, + "eval_runtime": 47.4813, + "eval_samples_per_second": 21.061, + "eval_steps_per_second": 0.885, + "step": 280 + }, + { + "epoch": 0.3497199751088986, + "grad_norm": 2.8021512031555176, + "learning_rate": 4.942343173431734e-06, + "loss": 1.7145, + "step": 281 + }, + { + "epoch": 0.3509645301804605, + "grad_norm": 3.5549023151397705, + "learning_rate": 4.940036900369004e-06, + "loss": 1.7642, + "step": 282 + }, + { + "epoch": 0.3522090852520224, + "grad_norm": 2.7648985385894775, + "learning_rate": 4.937730627306274e-06, + "loss": 1.7255, + "step": 283 + }, + { + "epoch": 0.35345364032358434, + "grad_norm": 3.0815863609313965, + "learning_rate": 4.9354243542435426e-06, + "loss": 1.7055, + "step": 284 + }, + { + "epoch": 0.35469819539514624, + "grad_norm": 2.9009227752685547, + "learning_rate": 4.933118081180812e-06, + "loss": 1.7019, + "step": 285 + }, + { + "epoch": 0.35594275046670815, + "grad_norm": 2.618429660797119, + "learning_rate": 4.930811808118081e-06, + "loss": 1.6318, + "step": 286 + }, + { + "epoch": 0.35718730553827005, + "grad_norm": 3.368230104446411, + "learning_rate": 4.928505535055351e-06, + "loss": 1.7304, + "step": 287 + }, + { + "epoch": 0.358431860609832, + "grad_norm": 3.256889820098877, + "learning_rate": 4.92619926199262e-06, + "loss": 1.7019, + "step": 288 + }, + { + "epoch": 0.3596764156813939, + "grad_norm": 2.9366304874420166, + "learning_rate": 4.92389298892989e-06, + "loss": 1.734, + "step": 289 + }, + { + "epoch": 0.3609209707529558, + "grad_norm": 3.0393142700195312, + "learning_rate": 4.921586715867159e-06, + "loss": 1.7504, + "step": 290 + }, + { + "epoch": 0.3609209707529558, + "eval_loss": 1.8359886407852173, + "eval_runtime": 49.7622, + "eval_samples_per_second": 20.096, + "eval_steps_per_second": 0.844, + "step": 290 + }, + { + "epoch": 0.3621655258245177, + "grad_norm": 2.7288589477539062, + "learning_rate": 4.9192804428044285e-06, + "loss": 1.7004, + "step": 291 + }, + { + "epoch": 0.3634100808960797, + "grad_norm": 3.2793378829956055, + "learning_rate": 4.916974169741698e-06, + "loss": 1.7028, + "step": 292 + }, + { + "epoch": 0.3646546359676416, + "grad_norm": 2.960880756378174, + "learning_rate": 4.914667896678967e-06, + "loss": 1.6759, + "step": 293 + }, + { + "epoch": 0.3658991910392035, + "grad_norm": 2.836421012878418, + "learning_rate": 4.912361623616237e-06, + "loss": 1.6475, + "step": 294 + }, + { + "epoch": 0.3671437461107654, + "grad_norm": 2.9495465755462646, + "learning_rate": 4.910055350553506e-06, + "loss": 1.7076, + "step": 295 + }, + { + "epoch": 0.3683883011823273, + "grad_norm": 2.954730272293091, + "learning_rate": 4.907749077490776e-06, + "loss": 1.7261, + "step": 296 + }, + { + "epoch": 0.36963285625388925, + "grad_norm": 2.8167543411254883, + "learning_rate": 4.905442804428044e-06, + "loss": 1.6939, + "step": 297 + }, + { + "epoch": 0.37087741132545116, + "grad_norm": 2.857316017150879, + "learning_rate": 4.903136531365314e-06, + "loss": 1.7362, + "step": 298 + }, + { + "epoch": 0.37212196639701306, + "grad_norm": 2.8612918853759766, + "learning_rate": 4.900830258302584e-06, + "loss": 1.6489, + "step": 299 + }, + { + "epoch": 0.37336652146857496, + "grad_norm": 2.9618914127349854, + "learning_rate": 4.898523985239853e-06, + "loss": 1.718, + "step": 300 + }, + { + "epoch": 0.37336652146857496, + "eval_loss": 1.827571988105774, + "eval_runtime": 46.4552, + "eval_samples_per_second": 21.526, + "eval_steps_per_second": 0.904, + "step": 300 + }, + { + "epoch": 0.3746110765401369, + "grad_norm": 2.8706953525543213, + "learning_rate": 4.896217712177122e-06, + "loss": 1.6925, + "step": 301 + }, + { + "epoch": 0.3758556316116988, + "grad_norm": 3.2565090656280518, + "learning_rate": 4.893911439114391e-06, + "loss": 1.7369, + "step": 302 + }, + { + "epoch": 0.37710018668326073, + "grad_norm": 2.928858995437622, + "learning_rate": 4.891605166051661e-06, + "loss": 1.6882, + "step": 303 + }, + { + "epoch": 0.37834474175482263, + "grad_norm": 3.042314052581787, + "learning_rate": 4.88929889298893e-06, + "loss": 1.6466, + "step": 304 + }, + { + "epoch": 0.3795892968263846, + "grad_norm": 2.7509143352508545, + "learning_rate": 4.8869926199262e-06, + "loss": 1.685, + "step": 305 + }, + { + "epoch": 0.3808338518979465, + "grad_norm": 2.9601588249206543, + "learning_rate": 4.884686346863469e-06, + "loss": 1.6772, + "step": 306 + }, + { + "epoch": 0.3820784069695084, + "grad_norm": 2.618608236312866, + "learning_rate": 4.8823800738007384e-06, + "loss": 1.7127, + "step": 307 + }, + { + "epoch": 0.3833229620410703, + "grad_norm": 3.047581195831299, + "learning_rate": 4.880073800738008e-06, + "loss": 1.7174, + "step": 308 + }, + { + "epoch": 0.3845675171126322, + "grad_norm": 2.87735652923584, + "learning_rate": 4.877767527675277e-06, + "loss": 1.6677, + "step": 309 + }, + { + "epoch": 0.38581207218419417, + "grad_norm": 2.7169721126556396, + "learning_rate": 4.875461254612546e-06, + "loss": 1.7486, + "step": 310 + }, + { + "epoch": 0.38581207218419417, + "eval_loss": 1.8221794366836548, + "eval_runtime": 43.1681, + "eval_samples_per_second": 23.165, + "eval_steps_per_second": 0.973, + "step": 310 + }, + { + "epoch": 0.38705662725575607, + "grad_norm": 2.7094991207122803, + "learning_rate": 4.873154981549816e-06, + "loss": 1.7348, + "step": 311 + }, + { + "epoch": 0.388301182327318, + "grad_norm": 2.8989078998565674, + "learning_rate": 4.8708487084870856e-06, + "loss": 1.7352, + "step": 312 + }, + { + "epoch": 0.3895457373988799, + "grad_norm": 2.9565601348876953, + "learning_rate": 4.868542435424355e-06, + "loss": 1.7063, + "step": 313 + }, + { + "epoch": 0.39079029247044184, + "grad_norm": 2.8354082107543945, + "learning_rate": 4.8662361623616235e-06, + "loss": 1.6808, + "step": 314 + }, + { + "epoch": 0.39203484754200374, + "grad_norm": 2.8287479877471924, + "learning_rate": 4.863929889298894e-06, + "loss": 1.6731, + "step": 315 + }, + { + "epoch": 0.39327940261356564, + "grad_norm": 2.6537821292877197, + "learning_rate": 4.861623616236163e-06, + "loss": 1.695, + "step": 316 + }, + { + "epoch": 0.39452395768512755, + "grad_norm": 2.7135190963745117, + "learning_rate": 4.859317343173432e-06, + "loss": 1.6897, + "step": 317 + }, + { + "epoch": 0.3957685127566895, + "grad_norm": 3.1019129753112793, + "learning_rate": 4.857011070110701e-06, + "loss": 1.7219, + "step": 318 + }, + { + "epoch": 0.3970130678282514, + "grad_norm": 2.945737838745117, + "learning_rate": 4.854704797047971e-06, + "loss": 1.7316, + "step": 319 + }, + { + "epoch": 0.3982576228998133, + "grad_norm": 2.7668251991271973, + "learning_rate": 4.85239852398524e-06, + "loss": 1.6461, + "step": 320 + }, + { + "epoch": 0.3982576228998133, + "eval_loss": 1.8210570812225342, + "eval_runtime": 46.2552, + "eval_samples_per_second": 21.619, + "eval_steps_per_second": 0.908, + "step": 320 + }, + { + "epoch": 0.3995021779713752, + "grad_norm": 2.738109588623047, + "learning_rate": 4.8500922509225095e-06, + "loss": 1.6907, + "step": 321 + }, + { + "epoch": 0.4007467330429372, + "grad_norm": 3.0195062160491943, + "learning_rate": 4.847785977859779e-06, + "loss": 1.7099, + "step": 322 + }, + { + "epoch": 0.4019912881144991, + "grad_norm": 2.8672404289245605, + "learning_rate": 4.845479704797048e-06, + "loss": 1.6509, + "step": 323 + }, + { + "epoch": 0.403235843186061, + "grad_norm": 2.9041008949279785, + "learning_rate": 4.843173431734318e-06, + "loss": 1.5818, + "step": 324 + }, + { + "epoch": 0.4044803982576229, + "grad_norm": 2.9105985164642334, + "learning_rate": 4.840867158671587e-06, + "loss": 1.7002, + "step": 325 + }, + { + "epoch": 0.4057249533291848, + "grad_norm": 2.749009847640991, + "learning_rate": 4.838560885608857e-06, + "loss": 1.7021, + "step": 326 + }, + { + "epoch": 0.40696950840074675, + "grad_norm": 2.6998822689056396, + "learning_rate": 4.836254612546126e-06, + "loss": 1.6853, + "step": 327 + }, + { + "epoch": 0.40821406347230865, + "grad_norm": 2.678201675415039, + "learning_rate": 4.8339483394833955e-06, + "loss": 1.6934, + "step": 328 + }, + { + "epoch": 0.40945861854387056, + "grad_norm": 2.774845600128174, + "learning_rate": 4.831642066420665e-06, + "loss": 1.657, + "step": 329 + }, + { + "epoch": 0.41070317361543246, + "grad_norm": 2.6363000869750977, + "learning_rate": 4.8293357933579335e-06, + "loss": 1.6555, + "step": 330 + }, + { + "epoch": 0.41070317361543246, + "eval_loss": 1.8126581907272339, + "eval_runtime": 52.7717, + "eval_samples_per_second": 18.95, + "eval_steps_per_second": 0.796, + "step": 330 + }, + { + "epoch": 0.4119477286869944, + "grad_norm": 3.011659622192383, + "learning_rate": 4.827029520295204e-06, + "loss": 1.7202, + "step": 331 + }, + { + "epoch": 0.4131922837585563, + "grad_norm": 2.8322594165802, + "learning_rate": 4.824723247232473e-06, + "loss": 1.6736, + "step": 332 + }, + { + "epoch": 0.4144368388301182, + "grad_norm": 2.9360146522521973, + "learning_rate": 4.822416974169742e-06, + "loss": 1.7213, + "step": 333 + }, + { + "epoch": 0.41568139390168013, + "grad_norm": 2.810206413269043, + "learning_rate": 4.820110701107011e-06, + "loss": 1.6867, + "step": 334 + }, + { + "epoch": 0.4169259489732421, + "grad_norm": 2.6864097118377686, + "learning_rate": 4.817804428044281e-06, + "loss": 1.6854, + "step": 335 + }, + { + "epoch": 0.418170504044804, + "grad_norm": 2.7396061420440674, + "learning_rate": 4.81549815498155e-06, + "loss": 1.68, + "step": 336 + }, + { + "epoch": 0.4194150591163659, + "grad_norm": 2.602771520614624, + "learning_rate": 4.8131918819188194e-06, + "loss": 1.6269, + "step": 337 + }, + { + "epoch": 0.4206596141879278, + "grad_norm": 2.8174757957458496, + "learning_rate": 4.810885608856089e-06, + "loss": 1.6981, + "step": 338 + }, + { + "epoch": 0.42190416925948976, + "grad_norm": 3.2398130893707275, + "learning_rate": 4.808579335793358e-06, + "loss": 1.7061, + "step": 339 + }, + { + "epoch": 0.42314872433105166, + "grad_norm": 2.7119736671447754, + "learning_rate": 4.806273062730628e-06, + "loss": 1.6875, + "step": 340 + }, + { + "epoch": 0.42314872433105166, + "eval_loss": 1.7973068952560425, + "eval_runtime": 53.5791, + "eval_samples_per_second": 18.664, + "eval_steps_per_second": 0.784, + "step": 340 + }, + { + "epoch": 0.42439327940261357, + "grad_norm": 2.868533134460449, + "learning_rate": 4.803966789667897e-06, + "loss": 1.6967, + "step": 341 + }, + { + "epoch": 0.42563783447417547, + "grad_norm": 2.528083562850952, + "learning_rate": 4.8016605166051665e-06, + "loss": 1.6195, + "step": 342 + }, + { + "epoch": 0.4268823895457374, + "grad_norm": 2.7885115146636963, + "learning_rate": 4.799354243542436e-06, + "loss": 1.6197, + "step": 343 + }, + { + "epoch": 0.42812694461729933, + "grad_norm": 2.768247604370117, + "learning_rate": 4.797047970479705e-06, + "loss": 1.6271, + "step": 344 + }, + { + "epoch": 0.42937149968886124, + "grad_norm": 2.7710275650024414, + "learning_rate": 4.794741697416975e-06, + "loss": 1.6768, + "step": 345 + }, + { + "epoch": 0.43061605476042314, + "grad_norm": 2.968337297439575, + "learning_rate": 4.792435424354243e-06, + "loss": 1.6433, + "step": 346 + }, + { + "epoch": 0.43186060983198504, + "grad_norm": 2.5739808082580566, + "learning_rate": 4.790129151291514e-06, + "loss": 1.6327, + "step": 347 + }, + { + "epoch": 0.433105164903547, + "grad_norm": 2.7676408290863037, + "learning_rate": 4.787822878228783e-06, + "loss": 1.6178, + "step": 348 + }, + { + "epoch": 0.4343497199751089, + "grad_norm": 2.959059476852417, + "learning_rate": 4.7855166051660525e-06, + "loss": 1.6809, + "step": 349 + }, + { + "epoch": 0.4355942750466708, + "grad_norm": 2.559967279434204, + "learning_rate": 4.783210332103321e-06, + "loss": 1.6704, + "step": 350 + }, + { + "epoch": 0.4355942750466708, + "eval_loss": 1.8013949394226074, + "eval_runtime": 50.4649, + "eval_samples_per_second": 19.816, + "eval_steps_per_second": 0.832, + "step": 350 + }, + { + "epoch": 0.4368388301182327, + "grad_norm": 2.817901611328125, + "learning_rate": 4.7809040590405905e-06, + "loss": 1.6296, + "step": 351 + }, + { + "epoch": 0.43808338518979467, + "grad_norm": 2.7116055488586426, + "learning_rate": 4.778597785977861e-06, + "loss": 1.6652, + "step": 352 + }, + { + "epoch": 0.4393279402613566, + "grad_norm": 2.544750452041626, + "learning_rate": 4.776291512915129e-06, + "loss": 1.5668, + "step": 353 + }, + { + "epoch": 0.4405724953329185, + "grad_norm": 2.720534086227417, + "learning_rate": 4.773985239852399e-06, + "loss": 1.6152, + "step": 354 + }, + { + "epoch": 0.4418170504044804, + "grad_norm": 2.7270126342773438, + "learning_rate": 4.771678966789668e-06, + "loss": 1.5624, + "step": 355 + }, + { + "epoch": 0.44306160547604234, + "grad_norm": 2.7986624240875244, + "learning_rate": 4.769372693726938e-06, + "loss": 1.6621, + "step": 356 + }, + { + "epoch": 0.44430616054760425, + "grad_norm": 2.943107843399048, + "learning_rate": 4.767066420664207e-06, + "loss": 1.6713, + "step": 357 + }, + { + "epoch": 0.44555071561916615, + "grad_norm": 2.739898681640625, + "learning_rate": 4.7647601476014765e-06, + "loss": 1.6363, + "step": 358 + }, + { + "epoch": 0.44679527069072805, + "grad_norm": 2.8729329109191895, + "learning_rate": 4.762453874538746e-06, + "loss": 1.6605, + "step": 359 + }, + { + "epoch": 0.44803982576228996, + "grad_norm": 2.909029483795166, + "learning_rate": 4.760147601476015e-06, + "loss": 1.6277, + "step": 360 + }, + { + "epoch": 0.44803982576228996, + "eval_loss": 1.8110179901123047, + "eval_runtime": 47.924, + "eval_samples_per_second": 20.866, + "eval_steps_per_second": 0.876, + "step": 360 + }, + { + "epoch": 0.4492843808338519, + "grad_norm": 3.3826682567596436, + "learning_rate": 4.757841328413285e-06, + "loss": 1.7155, + "step": 361 + }, + { + "epoch": 0.4505289359054138, + "grad_norm": 3.0586729049682617, + "learning_rate": 4.755535055350554e-06, + "loss": 1.5991, + "step": 362 + }, + { + "epoch": 0.4517734909769757, + "grad_norm": 2.645576238632202, + "learning_rate": 4.753228782287823e-06, + "loss": 1.6504, + "step": 363 + }, + { + "epoch": 0.4530180460485376, + "grad_norm": 2.705967664718628, + "learning_rate": 4.750922509225093e-06, + "loss": 1.6761, + "step": 364 + }, + { + "epoch": 0.4542626011200996, + "grad_norm": 2.7651922702789307, + "learning_rate": 4.748616236162362e-06, + "loss": 1.61, + "step": 365 + }, + { + "epoch": 0.4555071561916615, + "grad_norm": 2.5654966831207275, + "learning_rate": 4.746309963099631e-06, + "loss": 1.6335, + "step": 366 + }, + { + "epoch": 0.4567517112632234, + "grad_norm": 2.6606173515319824, + "learning_rate": 4.7440036900369e-06, + "loss": 1.6207, + "step": 367 + }, + { + "epoch": 0.4579962663347853, + "grad_norm": 2.724755048751831, + "learning_rate": 4.741697416974171e-06, + "loss": 1.6329, + "step": 368 + }, + { + "epoch": 0.45924082140634725, + "grad_norm": 2.7019615173339844, + "learning_rate": 4.739391143911439e-06, + "loss": 1.6763, + "step": 369 + }, + { + "epoch": 0.46048537647790916, + "grad_norm": 2.6594254970550537, + "learning_rate": 4.737084870848709e-06, + "loss": 1.6174, + "step": 370 + }, + { + "epoch": 0.46048537647790916, + "eval_loss": 1.7924141883850098, + "eval_runtime": 53.0349, + "eval_samples_per_second": 18.855, + "eval_steps_per_second": 0.792, + "step": 370 + }, + { + "epoch": 0.46172993154947106, + "grad_norm": 2.668421745300293, + "learning_rate": 4.734778597785978e-06, + "loss": 1.6421, + "step": 371 + }, + { + "epoch": 0.46297448662103297, + "grad_norm": 2.7148995399475098, + "learning_rate": 4.7324723247232475e-06, + "loss": 1.6635, + "step": 372 + }, + { + "epoch": 0.4642190416925949, + "grad_norm": 3.156646490097046, + "learning_rate": 4.730166051660517e-06, + "loss": 1.6679, + "step": 373 + }, + { + "epoch": 0.46546359676415683, + "grad_norm": 2.8170645236968994, + "learning_rate": 4.727859778597786e-06, + "loss": 1.6492, + "step": 374 + }, + { + "epoch": 0.46670815183571873, + "grad_norm": 2.9796457290649414, + "learning_rate": 4.725553505535056e-06, + "loss": 1.6025, + "step": 375 + }, + { + "epoch": 0.46795270690728064, + "grad_norm": 2.6493406295776367, + "learning_rate": 4.723247232472325e-06, + "loss": 1.6398, + "step": 376 + }, + { + "epoch": 0.46919726197884254, + "grad_norm": 3.2643542289733887, + "learning_rate": 4.720940959409595e-06, + "loss": 1.6413, + "step": 377 + }, + { + "epoch": 0.4704418170504045, + "grad_norm": 2.7613179683685303, + "learning_rate": 4.718634686346864e-06, + "loss": 1.6251, + "step": 378 + }, + { + "epoch": 0.4716863721219664, + "grad_norm": 3.1216518878936768, + "learning_rate": 4.716328413284133e-06, + "loss": 1.6268, + "step": 379 + }, + { + "epoch": 0.4729309271935283, + "grad_norm": 2.571647882461548, + "learning_rate": 4.714022140221403e-06, + "loss": 1.6245, + "step": 380 + }, + { + "epoch": 0.4729309271935283, + "eval_loss": 1.7875893115997314, + "eval_runtime": 49.3058, + "eval_samples_per_second": 20.282, + "eval_steps_per_second": 0.852, + "step": 380 + }, + { + "epoch": 0.4741754822650902, + "grad_norm": 3.443906545639038, + "learning_rate": 4.711715867158672e-06, + "loss": 1.6799, + "step": 381 + }, + { + "epoch": 0.47542003733665217, + "grad_norm": 2.7887957096099854, + "learning_rate": 4.709409594095941e-06, + "loss": 1.6589, + "step": 382 + }, + { + "epoch": 0.47666459240821407, + "grad_norm": 3.2833151817321777, + "learning_rate": 4.70710332103321e-06, + "loss": 1.6064, + "step": 383 + }, + { + "epoch": 0.477909147479776, + "grad_norm": 2.6975057125091553, + "learning_rate": 4.704797047970481e-06, + "loss": 1.6029, + "step": 384 + }, + { + "epoch": 0.4791537025513379, + "grad_norm": 2.745246171951294, + "learning_rate": 4.702490774907749e-06, + "loss": 1.7127, + "step": 385 + }, + { + "epoch": 0.48039825762289984, + "grad_norm": 3.0522775650024414, + "learning_rate": 4.700184501845019e-06, + "loss": 1.6508, + "step": 386 + }, + { + "epoch": 0.48164281269446174, + "grad_norm": 2.689218044281006, + "learning_rate": 4.697878228782288e-06, + "loss": 1.6971, + "step": 387 + }, + { + "epoch": 0.48288736776602365, + "grad_norm": 3.060361862182617, + "learning_rate": 4.6955719557195575e-06, + "loss": 1.6464, + "step": 388 + }, + { + "epoch": 0.48413192283758555, + "grad_norm": 2.872262477874756, + "learning_rate": 4.693265682656827e-06, + "loss": 1.6293, + "step": 389 + }, + { + "epoch": 0.48537647790914745, + "grad_norm": 2.980050563812256, + "learning_rate": 4.690959409594096e-06, + "loss": 1.6435, + "step": 390 + }, + { + "epoch": 0.48537647790914745, + "eval_loss": 1.7791496515274048, + "eval_runtime": 50.0299, + "eval_samples_per_second": 19.988, + "eval_steps_per_second": 0.839, + "step": 390 + }, + { + "epoch": 0.4866210329807094, + "grad_norm": 2.764827251434326, + "learning_rate": 4.688653136531366e-06, + "loss": 1.6279, + "step": 391 + }, + { + "epoch": 0.4878655880522713, + "grad_norm": 2.6912333965301514, + "learning_rate": 4.686346863468635e-06, + "loss": 1.7276, + "step": 392 + }, + { + "epoch": 0.4891101431238332, + "grad_norm": 2.8358330726623535, + "learning_rate": 4.6840405904059046e-06, + "loss": 1.6482, + "step": 393 + }, + { + "epoch": 0.4903546981953951, + "grad_norm": 2.6581575870513916, + "learning_rate": 4.681734317343174e-06, + "loss": 1.6853, + "step": 394 + }, + { + "epoch": 0.4915992532669571, + "grad_norm": 2.941425323486328, + "learning_rate": 4.6794280442804426e-06, + "loss": 1.6075, + "step": 395 + }, + { + "epoch": 0.492843808338519, + "grad_norm": 3.2334814071655273, + "learning_rate": 4.677121771217713e-06, + "loss": 1.6157, + "step": 396 + }, + { + "epoch": 0.4940883634100809, + "grad_norm": 2.984410524368286, + "learning_rate": 4.674815498154982e-06, + "loss": 1.6257, + "step": 397 + }, + { + "epoch": 0.4953329184816428, + "grad_norm": 3.0650789737701416, + "learning_rate": 4.672509225092252e-06, + "loss": 1.6573, + "step": 398 + }, + { + "epoch": 0.49657747355320475, + "grad_norm": 2.798664093017578, + "learning_rate": 4.67020295202952e-06, + "loss": 1.638, + "step": 399 + }, + { + "epoch": 0.49782202862476665, + "grad_norm": 2.8266706466674805, + "learning_rate": 4.66789667896679e-06, + "loss": 1.6484, + "step": 400 + }, + { + "epoch": 0.49782202862476665, + "eval_loss": 1.7714862823486328, + "eval_runtime": 48.0657, + "eval_samples_per_second": 20.805, + "eval_steps_per_second": 0.874, + "step": 400 + }, + { + "epoch": 0.49906658369632856, + "grad_norm": 2.9928388595581055, + "learning_rate": 4.66559040590406e-06, + "loss": 1.6721, + "step": 401 + }, + { + "epoch": 0.5003111387678905, + "grad_norm": 2.573514461517334, + "learning_rate": 4.6632841328413285e-06, + "loss": 1.679, + "step": 402 + }, + { + "epoch": 0.5015556938394524, + "grad_norm": 3.1736996173858643, + "learning_rate": 4.660977859778598e-06, + "loss": 1.7124, + "step": 403 + }, + { + "epoch": 0.5028002489110143, + "grad_norm": 2.858849287033081, + "learning_rate": 4.658671586715867e-06, + "loss": 1.6622, + "step": 404 + }, + { + "epoch": 0.5040448039825762, + "grad_norm": 2.825698137283325, + "learning_rate": 4.656365313653137e-06, + "loss": 1.6464, + "step": 405 + }, + { + "epoch": 0.5052893590541382, + "grad_norm": 3.783891439437866, + "learning_rate": 4.654059040590406e-06, + "loss": 1.6665, + "step": 406 + }, + { + "epoch": 0.5065339141257, + "grad_norm": 2.789813756942749, + "learning_rate": 4.651752767527676e-06, + "loss": 1.6064, + "step": 407 + }, + { + "epoch": 0.507778469197262, + "grad_norm": 3.3106348514556885, + "learning_rate": 4.649446494464945e-06, + "loss": 1.6542, + "step": 408 + }, + { + "epoch": 0.5090230242688238, + "grad_norm": 2.8805112838745117, + "learning_rate": 4.6471402214022145e-06, + "loss": 1.6058, + "step": 409 + }, + { + "epoch": 0.5102675793403858, + "grad_norm": 3.30898118019104, + "learning_rate": 4.644833948339484e-06, + "loss": 1.5933, + "step": 410 + }, + { + "epoch": 0.5102675793403858, + "eval_loss": 1.7787818908691406, + "eval_runtime": 44.0876, + "eval_samples_per_second": 22.682, + "eval_steps_per_second": 0.953, + "step": 410 + }, + { + "epoch": 0.5115121344119478, + "grad_norm": 3.3563625812530518, + "learning_rate": 4.642527675276753e-06, + "loss": 1.6751, + "step": 411 + }, + { + "epoch": 0.5127566894835096, + "grad_norm": 2.968120574951172, + "learning_rate": 4.640221402214023e-06, + "loss": 1.663, + "step": 412 + }, + { + "epoch": 0.5140012445550716, + "grad_norm": 3.310248374938965, + "learning_rate": 4.637915129151292e-06, + "loss": 1.658, + "step": 413 + }, + { + "epoch": 0.5152457996266335, + "grad_norm": 2.7011170387268066, + "learning_rate": 4.635608856088562e-06, + "loss": 1.6894, + "step": 414 + }, + { + "epoch": 0.5164903546981954, + "grad_norm": 2.804901361465454, + "learning_rate": 4.63330258302583e-06, + "loss": 1.602, + "step": 415 + }, + { + "epoch": 0.5177349097697573, + "grad_norm": 3.2412750720977783, + "learning_rate": 4.6309963099631e-06, + "loss": 1.6158, + "step": 416 + }, + { + "epoch": 0.5189794648413192, + "grad_norm": 2.699909210205078, + "learning_rate": 4.62869003690037e-06, + "loss": 1.5842, + "step": 417 + }, + { + "epoch": 0.5202240199128811, + "grad_norm": 3.065929412841797, + "learning_rate": 4.6263837638376384e-06, + "loss": 1.6434, + "step": 418 + }, + { + "epoch": 0.5214685749844431, + "grad_norm": 2.895305633544922, + "learning_rate": 4.624077490774908e-06, + "loss": 1.5969, + "step": 419 + }, + { + "epoch": 0.522713130056005, + "grad_norm": 3.1590664386749268, + "learning_rate": 4.621771217712177e-06, + "loss": 1.6211, + "step": 420 + }, + { + "epoch": 0.522713130056005, + "eval_loss": 1.7674189805984497, + "eval_runtime": 53.328, + "eval_samples_per_second": 18.752, + "eval_steps_per_second": 0.788, + "step": 420 + }, + { + "epoch": 0.5239576851275669, + "grad_norm": 2.7126312255859375, + "learning_rate": 4.619464944649447e-06, + "loss": 1.6246, + "step": 421 + }, + { + "epoch": 0.5252022401991289, + "grad_norm": 2.918508768081665, + "learning_rate": 4.617158671586716e-06, + "loss": 1.6937, + "step": 422 + }, + { + "epoch": 0.5264467952706907, + "grad_norm": 3.008610725402832, + "learning_rate": 4.6148523985239856e-06, + "loss": 1.6864, + "step": 423 + }, + { + "epoch": 0.5276913503422527, + "grad_norm": 2.6612043380737305, + "learning_rate": 4.612546125461255e-06, + "loss": 1.5785, + "step": 424 + }, + { + "epoch": 0.5289359054138145, + "grad_norm": 2.850679874420166, + "learning_rate": 4.610239852398524e-06, + "loss": 1.5899, + "step": 425 + }, + { + "epoch": 0.5301804604853765, + "grad_norm": 2.9205031394958496, + "learning_rate": 4.607933579335794e-06, + "loss": 1.5867, + "step": 426 + }, + { + "epoch": 0.5314250155569384, + "grad_norm": 2.929532527923584, + "learning_rate": 4.605627306273063e-06, + "loss": 1.6182, + "step": 427 + }, + { + "epoch": 0.5326695706285003, + "grad_norm": 2.874936103820801, + "learning_rate": 4.603321033210333e-06, + "loss": 1.5789, + "step": 428 + }, + { + "epoch": 0.5339141257000622, + "grad_norm": 2.8703525066375732, + "learning_rate": 4.601014760147602e-06, + "loss": 1.5859, + "step": 429 + }, + { + "epoch": 0.5351586807716241, + "grad_norm": 2.7076902389526367, + "learning_rate": 4.5987084870848715e-06, + "loss": 1.644, + "step": 430 + }, + { + "epoch": 0.5351586807716241, + "eval_loss": 1.7592095136642456, + "eval_runtime": 47.9438, + "eval_samples_per_second": 20.858, + "eval_steps_per_second": 0.876, + "step": 430 + }, + { + "epoch": 0.536403235843186, + "grad_norm": 2.839160680770874, + "learning_rate": 4.59640221402214e-06, + "loss": 1.6357, + "step": 431 + }, + { + "epoch": 0.537647790914748, + "grad_norm": 2.674025535583496, + "learning_rate": 4.5940959409594095e-06, + "loss": 1.676, + "step": 432 + }, + { + "epoch": 0.5388923459863099, + "grad_norm": 3.0619537830352783, + "learning_rate": 4.59178966789668e-06, + "loss": 1.5617, + "step": 433 + }, + { + "epoch": 0.5401369010578718, + "grad_norm": 3.058418035507202, + "learning_rate": 4.589483394833948e-06, + "loss": 1.5951, + "step": 434 + }, + { + "epoch": 0.5413814561294338, + "grad_norm": 3.0046396255493164, + "learning_rate": 4.587177121771218e-06, + "loss": 1.6473, + "step": 435 + }, + { + "epoch": 0.5426260112009956, + "grad_norm": 2.7736752033233643, + "learning_rate": 4.584870848708487e-06, + "loss": 1.6262, + "step": 436 + }, + { + "epoch": 0.5438705662725576, + "grad_norm": 2.7804994583129883, + "learning_rate": 4.5825645756457575e-06, + "loss": 1.6886, + "step": 437 + }, + { + "epoch": 0.5451151213441194, + "grad_norm": 3.0717954635620117, + "learning_rate": 4.580258302583026e-06, + "loss": 1.6287, + "step": 438 + }, + { + "epoch": 0.5463596764156814, + "grad_norm": 2.800243854522705, + "learning_rate": 4.5779520295202955e-06, + "loss": 1.6169, + "step": 439 + }, + { + "epoch": 0.5476042314872434, + "grad_norm": 2.9581644535064697, + "learning_rate": 4.575645756457565e-06, + "loss": 1.6617, + "step": 440 + }, + { + "epoch": 0.5476042314872434, + "eval_loss": 1.7642868757247925, + "eval_runtime": 51.2646, + "eval_samples_per_second": 19.507, + "eval_steps_per_second": 0.819, + "step": 440 + }, + { + "epoch": 0.5488487865588052, + "grad_norm": 3.0067460536956787, + "learning_rate": 4.573339483394834e-06, + "loss": 1.6482, + "step": 441 + }, + { + "epoch": 0.5500933416303672, + "grad_norm": 2.9075019359588623, + "learning_rate": 4.571033210332104e-06, + "loss": 1.6595, + "step": 442 + }, + { + "epoch": 0.551337896701929, + "grad_norm": 2.7788755893707275, + "learning_rate": 4.568726937269373e-06, + "loss": 1.5733, + "step": 443 + }, + { + "epoch": 0.552582451773491, + "grad_norm": 2.7775425910949707, + "learning_rate": 4.566420664206643e-06, + "loss": 1.6925, + "step": 444 + }, + { + "epoch": 0.5538270068450529, + "grad_norm": 2.6949567794799805, + "learning_rate": 4.564114391143912e-06, + "loss": 1.5738, + "step": 445 + }, + { + "epoch": 0.5550715619166148, + "grad_norm": 2.7093007564544678, + "learning_rate": 4.5618081180811814e-06, + "loss": 1.5848, + "step": 446 + }, + { + "epoch": 0.5563161169881767, + "grad_norm": 2.6920173168182373, + "learning_rate": 4.559501845018451e-06, + "loss": 1.6092, + "step": 447 + }, + { + "epoch": 0.5575606720597387, + "grad_norm": 2.733731985092163, + "learning_rate": 4.5571955719557194e-06, + "loss": 1.6056, + "step": 448 + }, + { + "epoch": 0.5588052271313005, + "grad_norm": 2.9156274795532227, + "learning_rate": 4.55488929889299e-06, + "loss": 1.6436, + "step": 449 + }, + { + "epoch": 0.5600497822028625, + "grad_norm": 2.8802616596221924, + "learning_rate": 4.552583025830259e-06, + "loss": 1.6309, + "step": 450 + }, + { + "epoch": 0.5600497822028625, + "eval_loss": 1.7486340999603271, + "eval_runtime": 47.2219, + "eval_samples_per_second": 21.177, + "eval_steps_per_second": 0.889, + "step": 450 + }, + { + "epoch": 0.5612943372744243, + "grad_norm": 2.729743242263794, + "learning_rate": 4.550276752767528e-06, + "loss": 1.7171, + "step": 451 + }, + { + "epoch": 0.5625388923459863, + "grad_norm": 2.800049066543579, + "learning_rate": 4.547970479704797e-06, + "loss": 1.6098, + "step": 452 + }, + { + "epoch": 0.5637834474175483, + "grad_norm": 2.9659311771392822, + "learning_rate": 4.5456642066420666e-06, + "loss": 1.6646, + "step": 453 + }, + { + "epoch": 0.5650280024891101, + "grad_norm": 2.5417754650115967, + "learning_rate": 4.543357933579336e-06, + "loss": 1.5847, + "step": 454 + }, + { + "epoch": 0.5662725575606721, + "grad_norm": 2.8471338748931885, + "learning_rate": 4.541051660516605e-06, + "loss": 1.6235, + "step": 455 + }, + { + "epoch": 0.5675171126322339, + "grad_norm": 2.7039637565612793, + "learning_rate": 4.538745387453875e-06, + "loss": 1.5613, + "step": 456 + }, + { + "epoch": 0.5687616677037959, + "grad_norm": 3.0383341312408447, + "learning_rate": 4.536439114391144e-06, + "loss": 1.6485, + "step": 457 + }, + { + "epoch": 0.5700062227753578, + "grad_norm": 2.528388261795044, + "learning_rate": 4.534132841328414e-06, + "loss": 1.6117, + "step": 458 + }, + { + "epoch": 0.5712507778469197, + "grad_norm": 2.7623119354248047, + "learning_rate": 4.531826568265683e-06, + "loss": 1.6017, + "step": 459 + }, + { + "epoch": 0.5724953329184816, + "grad_norm": 2.9213945865631104, + "learning_rate": 4.5295202952029525e-06, + "loss": 1.643, + "step": 460 + }, + { + "epoch": 0.5724953329184816, + "eval_loss": 1.747575044631958, + "eval_runtime": 45.2844, + "eval_samples_per_second": 22.083, + "eval_steps_per_second": 0.927, + "step": 460 + }, + { + "epoch": 0.5737398879900436, + "grad_norm": 2.664275884628296, + "learning_rate": 4.527214022140222e-06, + "loss": 1.6154, + "step": 461 + }, + { + "epoch": 0.5749844430616055, + "grad_norm": 2.5891788005828857, + "learning_rate": 4.524907749077491e-06, + "loss": 1.5975, + "step": 462 + }, + { + "epoch": 0.5762289981331674, + "grad_norm": 2.8126487731933594, + "learning_rate": 4.522601476014761e-06, + "loss": 1.6306, + "step": 463 + }, + { + "epoch": 0.5774735532047293, + "grad_norm": 2.799391031265259, + "learning_rate": 4.520295202952029e-06, + "loss": 1.6139, + "step": 464 + }, + { + "epoch": 0.5787181082762912, + "grad_norm": 2.7323718070983887, + "learning_rate": 4.5179889298893e-06, + "loss": 1.632, + "step": 465 + }, + { + "epoch": 0.5799626633478532, + "grad_norm": 2.601386785507202, + "learning_rate": 4.515682656826569e-06, + "loss": 1.6139, + "step": 466 + }, + { + "epoch": 0.581207218419415, + "grad_norm": 2.562162399291992, + "learning_rate": 4.513376383763838e-06, + "loss": 1.5168, + "step": 467 + }, + { + "epoch": 0.582451773490977, + "grad_norm": 2.6912002563476562, + "learning_rate": 4.511070110701107e-06, + "loss": 1.6249, + "step": 468 + }, + { + "epoch": 0.583696328562539, + "grad_norm": 2.6971495151519775, + "learning_rate": 4.5087638376383765e-06, + "loss": 1.616, + "step": 469 + }, + { + "epoch": 0.5849408836341008, + "grad_norm": 2.6898910999298096, + "learning_rate": 4.506457564575646e-06, + "loss": 1.614, + "step": 470 + }, + { + "epoch": 0.5849408836341008, + "eval_loss": 1.7460769414901733, + "eval_runtime": 45.3055, + "eval_samples_per_second": 22.072, + "eval_steps_per_second": 0.927, + "step": 470 + }, + { + "epoch": 0.5861854387056628, + "grad_norm": 2.698180675506592, + "learning_rate": 4.504151291512915e-06, + "loss": 1.5993, + "step": 471 + }, + { + "epoch": 0.5874299937772246, + "grad_norm": 2.7125210762023926, + "learning_rate": 4.501845018450185e-06, + "loss": 1.6113, + "step": 472 + }, + { + "epoch": 0.5886745488487866, + "grad_norm": 2.754445791244507, + "learning_rate": 4.499538745387454e-06, + "loss": 1.6719, + "step": 473 + }, + { + "epoch": 0.5899191039203485, + "grad_norm": 2.6979362964630127, + "learning_rate": 4.497232472324724e-06, + "loss": 1.608, + "step": 474 + }, + { + "epoch": 0.5911636589919104, + "grad_norm": 2.7694528102874756, + "learning_rate": 4.494926199261993e-06, + "loss": 1.5695, + "step": 475 + }, + { + "epoch": 0.5924082140634723, + "grad_norm": 2.653353214263916, + "learning_rate": 4.4926199261992624e-06, + "loss": 1.5523, + "step": 476 + }, + { + "epoch": 0.5936527691350342, + "grad_norm": 2.6632070541381836, + "learning_rate": 4.490313653136532e-06, + "loss": 1.5675, + "step": 477 + }, + { + "epoch": 0.5948973242065961, + "grad_norm": 3.038543939590454, + "learning_rate": 4.488007380073801e-06, + "loss": 1.5692, + "step": 478 + }, + { + "epoch": 0.5961418792781581, + "grad_norm": 2.8123953342437744, + "learning_rate": 4.485701107011071e-06, + "loss": 1.6223, + "step": 479 + }, + { + "epoch": 0.5973864343497199, + "grad_norm": 2.5845773220062256, + "learning_rate": 4.483394833948339e-06, + "loss": 1.5773, + "step": 480 + }, + { + "epoch": 0.5973864343497199, + "eval_loss": 1.7419319152832031, + "eval_runtime": 44.6842, + "eval_samples_per_second": 22.379, + "eval_steps_per_second": 0.94, + "step": 480 + }, + { + "epoch": 0.5986309894212819, + "grad_norm": 3.069936752319336, + "learning_rate": 4.4810885608856096e-06, + "loss": 1.6323, + "step": 481 + }, + { + "epoch": 0.5998755444928439, + "grad_norm": 2.9085781574249268, + "learning_rate": 4.478782287822879e-06, + "loss": 1.5845, + "step": 482 + }, + { + "epoch": 0.6011200995644057, + "grad_norm": 2.8568010330200195, + "learning_rate": 4.4764760147601476e-06, + "loss": 1.5898, + "step": 483 + }, + { + "epoch": 0.6023646546359677, + "grad_norm": 3.089081048965454, + "learning_rate": 4.474169741697417e-06, + "loss": 1.6622, + "step": 484 + }, + { + "epoch": 0.6036092097075295, + "grad_norm": 2.8515470027923584, + "learning_rate": 4.471863468634686e-06, + "loss": 1.5859, + "step": 485 + }, + { + "epoch": 0.6048537647790915, + "grad_norm": 2.647765874862671, + "learning_rate": 4.469557195571957e-06, + "loss": 1.5926, + "step": 486 + }, + { + "epoch": 0.6060983198506534, + "grad_norm": 2.61676025390625, + "learning_rate": 4.467250922509225e-06, + "loss": 1.5684, + "step": 487 + }, + { + "epoch": 0.6073428749222153, + "grad_norm": 2.7127039432525635, + "learning_rate": 4.464944649446495e-06, + "loss": 1.5702, + "step": 488 + }, + { + "epoch": 0.6085874299937772, + "grad_norm": 2.9932055473327637, + "learning_rate": 4.462638376383764e-06, + "loss": 1.5976, + "step": 489 + }, + { + "epoch": 0.6098319850653391, + "grad_norm": 2.6508774757385254, + "learning_rate": 4.4603321033210335e-06, + "loss": 1.586, + "step": 490 + }, + { + "epoch": 0.6098319850653391, + "eval_loss": 1.7357326745986938, + "eval_runtime": 45.393, + "eval_samples_per_second": 22.03, + "eval_steps_per_second": 0.925, + "step": 490 + }, + { + "epoch": 0.611076540136901, + "grad_norm": 2.7626044750213623, + "learning_rate": 4.458025830258303e-06, + "loss": 1.5773, + "step": 491 + }, + { + "epoch": 0.612321095208463, + "grad_norm": 2.595003604888916, + "learning_rate": 4.455719557195572e-06, + "loss": 1.601, + "step": 492 + }, + { + "epoch": 0.6135656502800249, + "grad_norm": 2.595767021179199, + "learning_rate": 4.453413284132842e-06, + "loss": 1.6287, + "step": 493 + }, + { + "epoch": 0.6148102053515868, + "grad_norm": 2.755845308303833, + "learning_rate": 4.451107011070111e-06, + "loss": 1.593, + "step": 494 + }, + { + "epoch": 0.6160547604231488, + "grad_norm": 2.7302653789520264, + "learning_rate": 4.448800738007381e-06, + "loss": 1.5768, + "step": 495 + }, + { + "epoch": 0.6172993154947106, + "grad_norm": 2.6493024826049805, + "learning_rate": 4.446494464944649e-06, + "loss": 1.6059, + "step": 496 + }, + { + "epoch": 0.6185438705662726, + "grad_norm": 2.8633735179901123, + "learning_rate": 4.4441881918819195e-06, + "loss": 1.598, + "step": 497 + }, + { + "epoch": 0.6197884256378344, + "grad_norm": 2.65639066696167, + "learning_rate": 4.441881918819189e-06, + "loss": 1.6036, + "step": 498 + }, + { + "epoch": 0.6210329807093964, + "grad_norm": 2.6867759227752686, + "learning_rate": 4.439575645756458e-06, + "loss": 1.5676, + "step": 499 + }, + { + "epoch": 0.6222775357809583, + "grad_norm": 2.7986443042755127, + "learning_rate": 4.437269372693727e-06, + "loss": 1.524, + "step": 500 + }, + { + "epoch": 0.6222775357809583, + "eval_loss": 1.7449125051498413, + "eval_runtime": 43.9329, + "eval_samples_per_second": 22.762, + "eval_steps_per_second": 0.956, + "step": 500 + }, + { + "epoch": 0.6235220908525202, + "grad_norm": 2.665905714035034, + "learning_rate": 4.434963099630996e-06, + "loss": 1.5635, + "step": 501 + }, + { + "epoch": 0.6247666459240822, + "grad_norm": 2.902435541152954, + "learning_rate": 4.432656826568267e-06, + "loss": 1.6377, + "step": 502 + }, + { + "epoch": 0.6260112009956441, + "grad_norm": 2.6872262954711914, + "learning_rate": 4.430350553505535e-06, + "loss": 1.5553, + "step": 503 + }, + { + "epoch": 0.627255756067206, + "grad_norm": 2.676621913909912, + "learning_rate": 4.428044280442805e-06, + "loss": 1.6101, + "step": 504 + }, + { + "epoch": 0.6285003111387679, + "grad_norm": 2.756347179412842, + "learning_rate": 4.425738007380074e-06, + "loss": 1.5884, + "step": 505 + }, + { + "epoch": 0.6297448662103298, + "grad_norm": 2.856882333755493, + "learning_rate": 4.4234317343173434e-06, + "loss": 1.5635, + "step": 506 + }, + { + "epoch": 0.6309894212818917, + "grad_norm": 2.7708330154418945, + "learning_rate": 4.421125461254613e-06, + "loss": 1.6028, + "step": 507 + }, + { + "epoch": 0.6322339763534537, + "grad_norm": 2.8167600631713867, + "learning_rate": 4.418819188191882e-06, + "loss": 1.6026, + "step": 508 + }, + { + "epoch": 0.6334785314250155, + "grad_norm": 2.54194974899292, + "learning_rate": 4.416512915129152e-06, + "loss": 1.6001, + "step": 509 + }, + { + "epoch": 0.6347230864965775, + "grad_norm": 2.683037519454956, + "learning_rate": 4.414206642066421e-06, + "loss": 1.6084, + "step": 510 + }, + { + "epoch": 0.6347230864965775, + "eval_loss": 1.7376186847686768, + "eval_runtime": 42.3026, + "eval_samples_per_second": 23.639, + "eval_steps_per_second": 0.993, + "step": 510 + }, + { + "epoch": 0.6359676415681393, + "grad_norm": 2.82627010345459, + "learning_rate": 4.4119003690036905e-06, + "loss": 1.6258, + "step": 511 + }, + { + "epoch": 0.6372121966397013, + "grad_norm": 2.6139848232269287, + "learning_rate": 4.40959409594096e-06, + "loss": 1.6029, + "step": 512 + }, + { + "epoch": 0.6384567517112633, + "grad_norm": 2.646712303161621, + "learning_rate": 4.407287822878229e-06, + "loss": 1.5602, + "step": 513 + }, + { + "epoch": 0.6397013067828251, + "grad_norm": 2.910935163497925, + "learning_rate": 4.404981549815499e-06, + "loss": 1.5757, + "step": 514 + }, + { + "epoch": 0.6409458618543871, + "grad_norm": 2.6001148223876953, + "learning_rate": 4.402675276752768e-06, + "loss": 1.5857, + "step": 515 + }, + { + "epoch": 0.642190416925949, + "grad_norm": 2.8525235652923584, + "learning_rate": 4.400369003690037e-06, + "loss": 1.6092, + "step": 516 + }, + { + "epoch": 0.6434349719975109, + "grad_norm": 2.5335488319396973, + "learning_rate": 4.398062730627306e-06, + "loss": 1.5849, + "step": 517 + }, + { + "epoch": 0.6446795270690728, + "grad_norm": 2.5787103176116943, + "learning_rate": 4.3957564575645765e-06, + "loss": 1.634, + "step": 518 + }, + { + "epoch": 0.6459240821406347, + "grad_norm": 2.6188197135925293, + "learning_rate": 4.393450184501845e-06, + "loss": 1.5477, + "step": 519 + }, + { + "epoch": 0.6471686372121966, + "grad_norm": 2.6548666954040527, + "learning_rate": 4.3911439114391145e-06, + "loss": 1.6133, + "step": 520 + }, + { + "epoch": 0.6471686372121966, + "eval_loss": 1.7393039464950562, + "eval_runtime": 45.9234, + "eval_samples_per_second": 21.775, + "eval_steps_per_second": 0.915, + "step": 520 + }, + { + "epoch": 0.6484131922837586, + "grad_norm": 2.907928228378296, + "learning_rate": 4.388837638376384e-06, + "loss": 1.587, + "step": 521 + }, + { + "epoch": 0.6496577473553204, + "grad_norm": 2.7899692058563232, + "learning_rate": 4.386531365313653e-06, + "loss": 1.6126, + "step": 522 + }, + { + "epoch": 0.6509023024268824, + "grad_norm": 2.680147886276245, + "learning_rate": 4.384225092250923e-06, + "loss": 1.565, + "step": 523 + }, + { + "epoch": 0.6521468574984443, + "grad_norm": 2.6590754985809326, + "learning_rate": 4.381918819188192e-06, + "loss": 1.6127, + "step": 524 + }, + { + "epoch": 0.6533914125700062, + "grad_norm": 2.6638906002044678, + "learning_rate": 4.379612546125462e-06, + "loss": 1.5615, + "step": 525 + }, + { + "epoch": 0.6546359676415682, + "grad_norm": 2.6518194675445557, + "learning_rate": 4.377306273062731e-06, + "loss": 1.6202, + "step": 526 + }, + { + "epoch": 0.65588052271313, + "grad_norm": 2.5323619842529297, + "learning_rate": 4.3750000000000005e-06, + "loss": 1.5305, + "step": 527 + }, + { + "epoch": 0.657125077784692, + "grad_norm": 2.6871442794799805, + "learning_rate": 4.37269372693727e-06, + "loss": 1.6043, + "step": 528 + }, + { + "epoch": 0.6583696328562539, + "grad_norm": 2.760746717453003, + "learning_rate": 4.370387453874539e-06, + "loss": 1.6059, + "step": 529 + }, + { + "epoch": 0.6596141879278158, + "grad_norm": 2.780482769012451, + "learning_rate": 4.368081180811809e-06, + "loss": 1.6082, + "step": 530 + }, + { + "epoch": 0.6596141879278158, + "eval_loss": 1.7219713926315308, + "eval_runtime": 51.8478, + "eval_samples_per_second": 19.287, + "eval_steps_per_second": 0.81, + "step": 530 + }, + { + "epoch": 0.6608587429993777, + "grad_norm": 2.605890989303589, + "learning_rate": 4.365774907749078e-06, + "loss": 1.5554, + "step": 531 + }, + { + "epoch": 0.6621032980709396, + "grad_norm": 2.731555461883545, + "learning_rate": 4.363468634686347e-06, + "loss": 1.606, + "step": 532 + }, + { + "epoch": 0.6633478531425016, + "grad_norm": 2.8356943130493164, + "learning_rate": 4.361162361623616e-06, + "loss": 1.5601, + "step": 533 + }, + { + "epoch": 0.6645924082140635, + "grad_norm": 2.7196593284606934, + "learning_rate": 4.3588560885608864e-06, + "loss": 1.5722, + "step": 534 + }, + { + "epoch": 0.6658369632856254, + "grad_norm": 2.621371269226074, + "learning_rate": 4.356549815498156e-06, + "loss": 1.6194, + "step": 535 + }, + { + "epoch": 0.6670815183571873, + "grad_norm": 2.653916120529175, + "learning_rate": 4.354243542435424e-06, + "loss": 1.5592, + "step": 536 + }, + { + "epoch": 0.6683260734287492, + "grad_norm": 2.88431453704834, + "learning_rate": 4.351937269372694e-06, + "loss": 1.6296, + "step": 537 + }, + { + "epoch": 0.6695706285003111, + "grad_norm": 2.667130470275879, + "learning_rate": 4.349630996309963e-06, + "loss": 1.5624, + "step": 538 + }, + { + "epoch": 0.6708151835718731, + "grad_norm": 2.6453566551208496, + "learning_rate": 4.347324723247233e-06, + "loss": 1.6025, + "step": 539 + }, + { + "epoch": 0.6720597386434349, + "grad_norm": 3.032271146774292, + "learning_rate": 4.345018450184502e-06, + "loss": 1.6771, + "step": 540 + }, + { + "epoch": 0.6720597386434349, + "eval_loss": 1.721895456314087, + "eval_runtime": 50.62, + "eval_samples_per_second": 19.755, + "eval_steps_per_second": 0.83, + "step": 540 + }, + { + "epoch": 0.6733042937149969, + "grad_norm": 2.8476362228393555, + "learning_rate": 4.3427121771217715e-06, + "loss": 1.5663, + "step": 541 + }, + { + "epoch": 0.6745488487865589, + "grad_norm": 2.615602970123291, + "learning_rate": 4.340405904059041e-06, + "loss": 1.5815, + "step": 542 + }, + { + "epoch": 0.6757934038581207, + "grad_norm": 3.221571683883667, + "learning_rate": 4.33809963099631e-06, + "loss": 1.5955, + "step": 543 + }, + { + "epoch": 0.6770379589296827, + "grad_norm": 2.602994203567505, + "learning_rate": 4.33579335793358e-06, + "loss": 1.5667, + "step": 544 + }, + { + "epoch": 0.6782825140012445, + "grad_norm": 2.9380433559417725, + "learning_rate": 4.333487084870848e-06, + "loss": 1.5651, + "step": 545 + }, + { + "epoch": 0.6795270690728065, + "grad_norm": 3.0689425468444824, + "learning_rate": 4.331180811808119e-06, + "loss": 1.6003, + "step": 546 + }, + { + "epoch": 0.6807716241443684, + "grad_norm": 2.770317316055298, + "learning_rate": 4.328874538745388e-06, + "loss": 1.5576, + "step": 547 + }, + { + "epoch": 0.6820161792159303, + "grad_norm": 2.8631815910339355, + "learning_rate": 4.3265682656826575e-06, + "loss": 1.5308, + "step": 548 + }, + { + "epoch": 0.6832607342874922, + "grad_norm": 2.785576820373535, + "learning_rate": 4.324261992619926e-06, + "loss": 1.6113, + "step": 549 + }, + { + "epoch": 0.6845052893590542, + "grad_norm": 2.723919630050659, + "learning_rate": 4.321955719557196e-06, + "loss": 1.5606, + "step": 550 + }, + { + "epoch": 0.6845052893590542, + "eval_loss": 1.721803069114685, + "eval_runtime": 51.6036, + "eval_samples_per_second": 19.378, + "eval_steps_per_second": 0.814, + "step": 550 + }, + { + "epoch": 0.685749844430616, + "grad_norm": 3.081198215484619, + "learning_rate": 4.319649446494466e-06, + "loss": 1.5643, + "step": 551 + }, + { + "epoch": 0.686994399502178, + "grad_norm": 2.757392168045044, + "learning_rate": 4.317343173431734e-06, + "loss": 1.5909, + "step": 552 + }, + { + "epoch": 0.6882389545737398, + "grad_norm": 3.038753032684326, + "learning_rate": 4.315036900369004e-06, + "loss": 1.6023, + "step": 553 + }, + { + "epoch": 0.6894835096453018, + "grad_norm": 2.8246452808380127, + "learning_rate": 4.312730627306273e-06, + "loss": 1.5548, + "step": 554 + }, + { + "epoch": 0.6907280647168638, + "grad_norm": 2.589320182800293, + "learning_rate": 4.310424354243543e-06, + "loss": 1.5424, + "step": 555 + }, + { + "epoch": 0.6919726197884256, + "grad_norm": 2.6413373947143555, + "learning_rate": 4.308118081180812e-06, + "loss": 1.6243, + "step": 556 + }, + { + "epoch": 0.6932171748599876, + "grad_norm": 2.764784336090088, + "learning_rate": 4.3058118081180815e-06, + "loss": 1.589, + "step": 557 + }, + { + "epoch": 0.6944617299315494, + "grad_norm": 2.806821823120117, + "learning_rate": 4.303505535055351e-06, + "loss": 1.621, + "step": 558 + }, + { + "epoch": 0.6957062850031114, + "grad_norm": 2.718017578125, + "learning_rate": 4.30119926199262e-06, + "loss": 1.5773, + "step": 559 + }, + { + "epoch": 0.6969508400746733, + "grad_norm": 2.7620160579681396, + "learning_rate": 4.29889298892989e-06, + "loss": 1.5868, + "step": 560 + }, + { + "epoch": 0.6969508400746733, + "eval_loss": 1.7152249813079834, + "eval_runtime": 52.0463, + "eval_samples_per_second": 19.214, + "eval_steps_per_second": 0.807, + "step": 560 + }, + { + "epoch": 0.6981953951462352, + "grad_norm": 2.66686749458313, + "learning_rate": 4.296586715867159e-06, + "loss": 1.534, + "step": 561 + }, + { + "epoch": 0.6994399502177971, + "grad_norm": 2.812577247619629, + "learning_rate": 4.2942804428044286e-06, + "loss": 1.5501, + "step": 562 + }, + { + "epoch": 0.7006845052893591, + "grad_norm": 2.578508138656616, + "learning_rate": 4.291974169741698e-06, + "loss": 1.6286, + "step": 563 + }, + { + "epoch": 0.701929060360921, + "grad_norm": 2.524249315261841, + "learning_rate": 4.289667896678967e-06, + "loss": 1.5719, + "step": 564 + }, + { + "epoch": 0.7031736154324829, + "grad_norm": 2.827235460281372, + "learning_rate": 4.287361623616236e-06, + "loss": 1.5333, + "step": 565 + }, + { + "epoch": 0.7044181705040448, + "grad_norm": 2.6359963417053223, + "learning_rate": 4.285055350553506e-06, + "loss": 1.5849, + "step": 566 + }, + { + "epoch": 0.7056627255756067, + "grad_norm": 2.930530071258545, + "learning_rate": 4.282749077490776e-06, + "loss": 1.5672, + "step": 567 + }, + { + "epoch": 0.7069072806471687, + "grad_norm": 2.750102996826172, + "learning_rate": 4.280442804428044e-06, + "loss": 1.5696, + "step": 568 + }, + { + "epoch": 0.7081518357187305, + "grad_norm": 2.869690418243408, + "learning_rate": 4.278136531365314e-06, + "loss": 1.5689, + "step": 569 + }, + { + "epoch": 0.7093963907902925, + "grad_norm": 2.954852819442749, + "learning_rate": 4.275830258302583e-06, + "loss": 1.5931, + "step": 570 + }, + { + "epoch": 0.7093963907902925, + "eval_loss": 1.7143300771713257, + "eval_runtime": 52.7293, + "eval_samples_per_second": 18.965, + "eval_steps_per_second": 0.797, + "step": 570 + }, + { + "epoch": 0.7106409458618543, + "grad_norm": 2.702223539352417, + "learning_rate": 4.273523985239853e-06, + "loss": 1.5635, + "step": 571 + }, + { + "epoch": 0.7118855009334163, + "grad_norm": 2.689995050430298, + "learning_rate": 4.271217712177122e-06, + "loss": 1.5545, + "step": 572 + }, + { + "epoch": 0.7131300560049783, + "grad_norm": 2.68979811668396, + "learning_rate": 4.268911439114391e-06, + "loss": 1.5404, + "step": 573 + }, + { + "epoch": 0.7143746110765401, + "grad_norm": 2.7477986812591553, + "learning_rate": 4.266605166051661e-06, + "loss": 1.5719, + "step": 574 + }, + { + "epoch": 0.7156191661481021, + "grad_norm": 2.975778341293335, + "learning_rate": 4.26429889298893e-06, + "loss": 1.5587, + "step": 575 + }, + { + "epoch": 0.716863721219664, + "grad_norm": 2.658170700073242, + "learning_rate": 4.2619926199262e-06, + "loss": 1.5746, + "step": 576 + }, + { + "epoch": 0.7181082762912259, + "grad_norm": 2.8825011253356934, + "learning_rate": 4.259686346863469e-06, + "loss": 1.5246, + "step": 577 + }, + { + "epoch": 0.7193528313627878, + "grad_norm": 2.845280170440674, + "learning_rate": 4.2573800738007385e-06, + "loss": 1.5703, + "step": 578 + }, + { + "epoch": 0.7205973864343497, + "grad_norm": 2.660616636276245, + "learning_rate": 4.255073800738008e-06, + "loss": 1.5373, + "step": 579 + }, + { + "epoch": 0.7218419415059116, + "grad_norm": 2.749447822570801, + "learning_rate": 4.252767527675277e-06, + "loss": 1.59, + "step": 580 + }, + { + "epoch": 0.7218419415059116, + "eval_loss": 1.7149444818496704, + "eval_runtime": 52.2099, + "eval_samples_per_second": 19.153, + "eval_steps_per_second": 0.804, + "step": 580 + }, + { + "epoch": 0.7230864965774736, + "grad_norm": 2.813328742980957, + "learning_rate": 4.250461254612546e-06, + "loss": 1.5694, + "step": 581 + }, + { + "epoch": 0.7243310516490354, + "grad_norm": 2.823866844177246, + "learning_rate": 4.248154981549816e-06, + "loss": 1.5143, + "step": 582 + }, + { + "epoch": 0.7255756067205974, + "grad_norm": 2.71337890625, + "learning_rate": 4.245848708487086e-06, + "loss": 1.5915, + "step": 583 + }, + { + "epoch": 0.7268201617921594, + "grad_norm": 2.718085765838623, + "learning_rate": 4.243542435424355e-06, + "loss": 1.604, + "step": 584 + }, + { + "epoch": 0.7280647168637212, + "grad_norm": 2.5741796493530273, + "learning_rate": 4.241236162361624e-06, + "loss": 1.5398, + "step": 585 + }, + { + "epoch": 0.7293092719352832, + "grad_norm": 2.727114200592041, + "learning_rate": 4.238929889298893e-06, + "loss": 1.5886, + "step": 586 + }, + { + "epoch": 0.730553827006845, + "grad_norm": 2.6703338623046875, + "learning_rate": 4.236623616236163e-06, + "loss": 1.5959, + "step": 587 + }, + { + "epoch": 0.731798382078407, + "grad_norm": 2.750326633453369, + "learning_rate": 4.234317343173432e-06, + "loss": 1.5916, + "step": 588 + }, + { + "epoch": 0.7330429371499689, + "grad_norm": 2.6680715084075928, + "learning_rate": 4.232011070110701e-06, + "loss": 1.562, + "step": 589 + }, + { + "epoch": 0.7342874922215308, + "grad_norm": 2.6603832244873047, + "learning_rate": 4.229704797047971e-06, + "loss": 1.6121, + "step": 590 + }, + { + "epoch": 0.7342874922215308, + "eval_loss": 1.7144734859466553, + "eval_runtime": 50.5317, + "eval_samples_per_second": 19.79, + "eval_steps_per_second": 0.831, + "step": 590 + }, + { + "epoch": 0.7355320472930927, + "grad_norm": 2.7868523597717285, + "learning_rate": 4.22739852398524e-06, + "loss": 1.5488, + "step": 591 + }, + { + "epoch": 0.7367766023646546, + "grad_norm": 2.755270004272461, + "learning_rate": 4.2250922509225096e-06, + "loss": 1.5801, + "step": 592 + }, + { + "epoch": 0.7380211574362165, + "grad_norm": 2.7614126205444336, + "learning_rate": 4.222785977859779e-06, + "loss": 1.5507, + "step": 593 + }, + { + "epoch": 0.7392657125077785, + "grad_norm": 2.8674862384796143, + "learning_rate": 4.220479704797048e-06, + "loss": 1.5807, + "step": 594 + }, + { + "epoch": 0.7405102675793404, + "grad_norm": 2.7366859912872314, + "learning_rate": 4.218173431734318e-06, + "loss": 1.6594, + "step": 595 + }, + { + "epoch": 0.7417548226509023, + "grad_norm": 2.6231045722961426, + "learning_rate": 4.215867158671587e-06, + "loss": 1.5609, + "step": 596 + }, + { + "epoch": 0.7429993777224643, + "grad_norm": 2.7694146633148193, + "learning_rate": 4.213560885608857e-06, + "loss": 1.5889, + "step": 597 + }, + { + "epoch": 0.7442439327940261, + "grad_norm": 2.6275386810302734, + "learning_rate": 4.211254612546125e-06, + "loss": 1.575, + "step": 598 + }, + { + "epoch": 0.7454884878655881, + "grad_norm": 2.9413866996765137, + "learning_rate": 4.2089483394833955e-06, + "loss": 1.589, + "step": 599 + }, + { + "epoch": 0.7467330429371499, + "grad_norm": 2.768606185913086, + "learning_rate": 4.206642066420665e-06, + "loss": 1.5738, + "step": 600 + }, + { + "epoch": 0.7467330429371499, + "eval_loss": 1.7148027420043945, + "eval_runtime": 54.7031, + "eval_samples_per_second": 18.28, + "eval_steps_per_second": 0.768, + "step": 600 + }, + { + "epoch": 0.7479775980087119, + "grad_norm": 2.7917706966400146, + "learning_rate": 4.2043357933579335e-06, + "loss": 1.499, + "step": 601 + }, + { + "epoch": 0.7492221530802738, + "grad_norm": 2.5982208251953125, + "learning_rate": 4.202029520295203e-06, + "loss": 1.5113, + "step": 602 + }, + { + "epoch": 0.7504667081518357, + "grad_norm": 2.692032814025879, + "learning_rate": 4.199723247232473e-06, + "loss": 1.4721, + "step": 603 + }, + { + "epoch": 0.7517112632233977, + "grad_norm": 2.975860834121704, + "learning_rate": 4.197416974169742e-06, + "loss": 1.5147, + "step": 604 + }, + { + "epoch": 0.7529558182949595, + "grad_norm": 2.758610248565674, + "learning_rate": 4.195110701107011e-06, + "loss": 1.5674, + "step": 605 + }, + { + "epoch": 0.7542003733665215, + "grad_norm": 2.7620184421539307, + "learning_rate": 4.192804428044281e-06, + "loss": 1.5669, + "step": 606 + }, + { + "epoch": 0.7554449284380834, + "grad_norm": 3.043940305709839, + "learning_rate": 4.19049815498155e-06, + "loss": 1.587, + "step": 607 + }, + { + "epoch": 0.7566894835096453, + "grad_norm": 2.7482151985168457, + "learning_rate": 4.1881918819188195e-06, + "loss": 1.5451, + "step": 608 + }, + { + "epoch": 0.7579340385812072, + "grad_norm": 2.7259294986724854, + "learning_rate": 4.185885608856089e-06, + "loss": 1.5295, + "step": 609 + }, + { + "epoch": 0.7591785936527692, + "grad_norm": 2.6340832710266113, + "learning_rate": 4.183579335793358e-06, + "loss": 1.5335, + "step": 610 + }, + { + "epoch": 0.7591785936527692, + "eval_loss": 1.7038393020629883, + "eval_runtime": 44.0792, + "eval_samples_per_second": 22.686, + "eval_steps_per_second": 0.953, + "step": 610 + }, + { + "epoch": 0.760423148724331, + "grad_norm": 2.6144909858703613, + "learning_rate": 4.181273062730628e-06, + "loss": 1.5907, + "step": 611 + }, + { + "epoch": 0.761667703795893, + "grad_norm": 2.7754175662994385, + "learning_rate": 4.178966789667897e-06, + "loss": 1.5773, + "step": 612 + }, + { + "epoch": 0.7629122588674548, + "grad_norm": 2.6313252449035645, + "learning_rate": 4.176660516605167e-06, + "loss": 1.5675, + "step": 613 + }, + { + "epoch": 0.7641568139390168, + "grad_norm": 2.549074649810791, + "learning_rate": 4.174354243542435e-06, + "loss": 1.595, + "step": 614 + }, + { + "epoch": 0.7654013690105788, + "grad_norm": 2.611804246902466, + "learning_rate": 4.1720479704797054e-06, + "loss": 1.5416, + "step": 615 + }, + { + "epoch": 0.7666459240821406, + "grad_norm": 2.6322927474975586, + "learning_rate": 4.169741697416975e-06, + "loss": 1.604, + "step": 616 + }, + { + "epoch": 0.7678904791537026, + "grad_norm": 2.5792219638824463, + "learning_rate": 4.1674354243542434e-06, + "loss": 1.5976, + "step": 617 + }, + { + "epoch": 0.7691350342252644, + "grad_norm": 2.845416307449341, + "learning_rate": 4.165129151291513e-06, + "loss": 1.5945, + "step": 618 + }, + { + "epoch": 0.7703795892968264, + "grad_norm": 2.8647871017456055, + "learning_rate": 4.162822878228783e-06, + "loss": 1.5637, + "step": 619 + }, + { + "epoch": 0.7716241443683883, + "grad_norm": 2.590719699859619, + "learning_rate": 4.1605166051660526e-06, + "loss": 1.556, + "step": 620 + }, + { + "epoch": 0.7716241443683883, + "eval_loss": 1.7115222215652466, + "eval_runtime": 50.3905, + "eval_samples_per_second": 19.845, + "eval_steps_per_second": 0.833, + "step": 620 + }, + { + "epoch": 0.7728686994399502, + "grad_norm": 2.626723527908325, + "learning_rate": 4.158210332103321e-06, + "loss": 1.5486, + "step": 621 + }, + { + "epoch": 0.7741132545115121, + "grad_norm": 2.713069438934326, + "learning_rate": 4.1559040590405906e-06, + "loss": 1.5474, + "step": 622 + }, + { + "epoch": 0.7753578095830741, + "grad_norm": 2.685563087463379, + "learning_rate": 4.15359778597786e-06, + "loss": 1.5529, + "step": 623 + }, + { + "epoch": 0.776602364654636, + "grad_norm": 2.690586805343628, + "learning_rate": 4.151291512915129e-06, + "loss": 1.5293, + "step": 624 + }, + { + "epoch": 0.7778469197261979, + "grad_norm": 2.865345001220703, + "learning_rate": 4.148985239852399e-06, + "loss": 1.5095, + "step": 625 + }, + { + "epoch": 0.7790914747977598, + "grad_norm": 2.92393159866333, + "learning_rate": 4.146678966789668e-06, + "loss": 1.5561, + "step": 626 + }, + { + "epoch": 0.7803360298693217, + "grad_norm": 2.554480791091919, + "learning_rate": 4.144372693726938e-06, + "loss": 1.4876, + "step": 627 + }, + { + "epoch": 0.7815805849408837, + "grad_norm": 2.6128833293914795, + "learning_rate": 4.142066420664207e-06, + "loss": 1.5378, + "step": 628 + }, + { + "epoch": 0.7828251400124455, + "grad_norm": 3.0110888481140137, + "learning_rate": 4.1397601476014765e-06, + "loss": 1.5859, + "step": 629 + }, + { + "epoch": 0.7840696950840075, + "grad_norm": 2.8152706623077393, + "learning_rate": 4.137453874538745e-06, + "loss": 1.5841, + "step": 630 + }, + { + "epoch": 0.7840696950840075, + "eval_loss": 1.702181339263916, + "eval_runtime": 53.6926, + "eval_samples_per_second": 18.625, + "eval_steps_per_second": 0.782, + "step": 630 + }, + { + "epoch": 0.7853142501555694, + "grad_norm": 2.8634233474731445, + "learning_rate": 4.135147601476015e-06, + "loss": 1.564, + "step": 631 + }, + { + "epoch": 0.7865588052271313, + "grad_norm": 2.8939666748046875, + "learning_rate": 4.132841328413285e-06, + "loss": 1.5859, + "step": 632 + }, + { + "epoch": 0.7878033602986932, + "grad_norm": 2.6560444831848145, + "learning_rate": 4.130535055350554e-06, + "loss": 1.5199, + "step": 633 + }, + { + "epoch": 0.7890479153702551, + "grad_norm": 2.964721202850342, + "learning_rate": 4.128228782287823e-06, + "loss": 1.5888, + "step": 634 + }, + { + "epoch": 0.790292470441817, + "grad_norm": 2.74668288230896, + "learning_rate": 4.125922509225092e-06, + "loss": 1.5847, + "step": 635 + }, + { + "epoch": 0.791537025513379, + "grad_norm": 2.723123550415039, + "learning_rate": 4.1236162361623625e-06, + "loss": 1.5166, + "step": 636 + }, + { + "epoch": 0.7927815805849409, + "grad_norm": 2.7265713214874268, + "learning_rate": 4.121309963099631e-06, + "loss": 1.5954, + "step": 637 + }, + { + "epoch": 0.7940261356565028, + "grad_norm": 2.979126214981079, + "learning_rate": 4.1190036900369005e-06, + "loss": 1.5583, + "step": 638 + }, + { + "epoch": 0.7952706907280647, + "grad_norm": 2.844376802444458, + "learning_rate": 4.11669741697417e-06, + "loss": 1.5564, + "step": 639 + }, + { + "epoch": 0.7965152457996266, + "grad_norm": 2.7583703994750977, + "learning_rate": 4.114391143911439e-06, + "loss": 1.5094, + "step": 640 + }, + { + "epoch": 0.7965152457996266, + "eval_loss": 1.6986565589904785, + "eval_runtime": 47.3549, + "eval_samples_per_second": 21.117, + "eval_steps_per_second": 0.887, + "step": 640 + }, + { + "epoch": 0.7977598008711886, + "grad_norm": 2.8942224979400635, + "learning_rate": 4.112084870848709e-06, + "loss": 1.5504, + "step": 641 + }, + { + "epoch": 0.7990043559427504, + "grad_norm": 2.6590495109558105, + "learning_rate": 4.109778597785978e-06, + "loss": 1.5227, + "step": 642 + }, + { + "epoch": 0.8002489110143124, + "grad_norm": 2.5988378524780273, + "learning_rate": 4.107472324723248e-06, + "loss": 1.5054, + "step": 643 + }, + { + "epoch": 0.8014934660858744, + "grad_norm": 2.787335157394409, + "learning_rate": 4.105166051660517e-06, + "loss": 1.5844, + "step": 644 + }, + { + "epoch": 0.8027380211574362, + "grad_norm": 2.8202896118164062, + "learning_rate": 4.1028597785977864e-06, + "loss": 1.5596, + "step": 645 + }, + { + "epoch": 0.8039825762289982, + "grad_norm": 2.65376615524292, + "learning_rate": 4.100553505535056e-06, + "loss": 1.5893, + "step": 646 + }, + { + "epoch": 0.80522713130056, + "grad_norm": 2.5933308601379395, + "learning_rate": 4.098247232472325e-06, + "loss": 1.5403, + "step": 647 + }, + { + "epoch": 0.806471686372122, + "grad_norm": 2.777070999145508, + "learning_rate": 4.095940959409595e-06, + "loss": 1.5231, + "step": 648 + }, + { + "epoch": 0.8077162414436839, + "grad_norm": 2.6427664756774902, + "learning_rate": 4.093634686346864e-06, + "loss": 1.5937, + "step": 649 + }, + { + "epoch": 0.8089607965152458, + "grad_norm": 2.651561737060547, + "learning_rate": 4.091328413284133e-06, + "loss": 1.4847, + "step": 650 + }, + { + "epoch": 0.8089607965152458, + "eval_loss": 1.7089996337890625, + "eval_runtime": 47.4346, + "eval_samples_per_second": 21.082, + "eval_steps_per_second": 0.885, + "step": 650 + }, + { + "epoch": 0.8102053515868077, + "grad_norm": 2.844067335128784, + "learning_rate": 4.089022140221402e-06, + "loss": 1.488, + "step": 651 + }, + { + "epoch": 0.8114499066583696, + "grad_norm": 2.603480577468872, + "learning_rate": 4.086715867158672e-06, + "loss": 1.5248, + "step": 652 + }, + { + "epoch": 0.8126944617299315, + "grad_norm": 2.789322853088379, + "learning_rate": 4.084409594095941e-06, + "loss": 1.5119, + "step": 653 + }, + { + "epoch": 0.8139390168014935, + "grad_norm": 2.7183773517608643, + "learning_rate": 4.08210332103321e-06, + "loss": 1.5123, + "step": 654 + }, + { + "epoch": 0.8151835718730553, + "grad_norm": 2.80613374710083, + "learning_rate": 4.07979704797048e-06, + "loss": 1.5526, + "step": 655 + }, + { + "epoch": 0.8164281269446173, + "grad_norm": 2.641035795211792, + "learning_rate": 4.077490774907749e-06, + "loss": 1.6127, + "step": 656 + }, + { + "epoch": 0.8176726820161793, + "grad_norm": 2.773684024810791, + "learning_rate": 4.075184501845019e-06, + "loss": 1.5404, + "step": 657 + }, + { + "epoch": 0.8189172370877411, + "grad_norm": 2.6951382160186768, + "learning_rate": 4.072878228782288e-06, + "loss": 1.4928, + "step": 658 + }, + { + "epoch": 0.8201617921593031, + "grad_norm": 2.931771755218506, + "learning_rate": 4.0705719557195575e-06, + "loss": 1.5949, + "step": 659 + }, + { + "epoch": 0.8214063472308649, + "grad_norm": 2.8786075115203857, + "learning_rate": 4.068265682656827e-06, + "loss": 1.5582, + "step": 660 + }, + { + "epoch": 0.8214063472308649, + "eval_loss": 1.698564052581787, + "eval_runtime": 46.6299, + "eval_samples_per_second": 21.445, + "eval_steps_per_second": 0.901, + "step": 660 + }, + { + "epoch": 0.8226509023024269, + "grad_norm": 2.7402451038360596, + "learning_rate": 4.065959409594096e-06, + "loss": 1.5562, + "step": 661 + }, + { + "epoch": 0.8238954573739888, + "grad_norm": 2.7968692779541016, + "learning_rate": 4.063653136531366e-06, + "loss": 1.556, + "step": 662 + }, + { + "epoch": 0.8251400124455507, + "grad_norm": 2.8233132362365723, + "learning_rate": 4.061346863468635e-06, + "loss": 1.5527, + "step": 663 + }, + { + "epoch": 0.8263845675171126, + "grad_norm": 2.713390827178955, + "learning_rate": 4.059040590405905e-06, + "loss": 1.6012, + "step": 664 + }, + { + "epoch": 0.8276291225886746, + "grad_norm": 2.7783989906311035, + "learning_rate": 4.056734317343174e-06, + "loss": 1.5247, + "step": 665 + }, + { + "epoch": 0.8288736776602365, + "grad_norm": 2.808554172515869, + "learning_rate": 4.054428044280443e-06, + "loss": 1.5412, + "step": 666 + }, + { + "epoch": 0.8301182327317984, + "grad_norm": 2.8476831912994385, + "learning_rate": 4.052121771217712e-06, + "loss": 1.5307, + "step": 667 + }, + { + "epoch": 0.8313627878033603, + "grad_norm": 2.8287601470947266, + "learning_rate": 4.049815498154982e-06, + "loss": 1.6021, + "step": 668 + }, + { + "epoch": 0.8326073428749222, + "grad_norm": 2.975510835647583, + "learning_rate": 4.047509225092252e-06, + "loss": 1.5474, + "step": 669 + }, + { + "epoch": 0.8338518979464842, + "grad_norm": 2.723236560821533, + "learning_rate": 4.04520295202952e-06, + "loss": 1.5242, + "step": 670 + }, + { + "epoch": 0.8338518979464842, + "eval_loss": 1.7005239725112915, + "eval_runtime": 49.4504, + "eval_samples_per_second": 20.222, + "eval_steps_per_second": 0.849, + "step": 670 + }, + { + "epoch": 0.835096453018046, + "grad_norm": 2.7521748542785645, + "learning_rate": 4.04289667896679e-06, + "loss": 1.5467, + "step": 671 + }, + { + "epoch": 0.836341008089608, + "grad_norm": 2.7770044803619385, + "learning_rate": 4.04059040590406e-06, + "loss": 1.5407, + "step": 672 + }, + { + "epoch": 0.8375855631611698, + "grad_norm": 2.744323492050171, + "learning_rate": 4.038284132841329e-06, + "loss": 1.5422, + "step": 673 + }, + { + "epoch": 0.8388301182327318, + "grad_norm": 2.6699817180633545, + "learning_rate": 4.035977859778598e-06, + "loss": 1.5508, + "step": 674 + }, + { + "epoch": 0.8400746733042938, + "grad_norm": 2.705273151397705, + "learning_rate": 4.0336715867158674e-06, + "loss": 1.5099, + "step": 675 + }, + { + "epoch": 0.8413192283758556, + "grad_norm": 2.9485747814178467, + "learning_rate": 4.031365313653137e-06, + "loss": 1.5529, + "step": 676 + }, + { + "epoch": 0.8425637834474176, + "grad_norm": 3.1750423908233643, + "learning_rate": 4.029059040590406e-06, + "loss": 1.5894, + "step": 677 + }, + { + "epoch": 0.8438083385189795, + "grad_norm": 2.76448655128479, + "learning_rate": 4.026752767527676e-06, + "loss": 1.5752, + "step": 678 + }, + { + "epoch": 0.8450528935905414, + "grad_norm": 2.676708698272705, + "learning_rate": 4.024446494464945e-06, + "loss": 1.5487, + "step": 679 + }, + { + "epoch": 0.8462974486621033, + "grad_norm": 2.6832051277160645, + "learning_rate": 4.0221402214022145e-06, + "loss": 1.5055, + "step": 680 + }, + { + "epoch": 0.8462974486621033, + "eval_loss": 1.6945017576217651, + "eval_runtime": 43.9369, + "eval_samples_per_second": 22.76, + "eval_steps_per_second": 0.956, + "step": 680 + }, + { + "epoch": 0.8475420037336652, + "grad_norm": 2.6812336444854736, + "learning_rate": 4.019833948339484e-06, + "loss": 1.5315, + "step": 681 + }, + { + "epoch": 0.8487865588052271, + "grad_norm": 2.9394326210021973, + "learning_rate": 4.017527675276753e-06, + "loss": 1.5636, + "step": 682 + }, + { + "epoch": 0.8500311138767891, + "grad_norm": 2.944952964782715, + "learning_rate": 4.015221402214022e-06, + "loss": 1.56, + "step": 683 + }, + { + "epoch": 0.8512756689483509, + "grad_norm": 2.7475314140319824, + "learning_rate": 4.012915129151292e-06, + "loss": 1.5442, + "step": 684 + }, + { + "epoch": 0.8525202240199129, + "grad_norm": 3.0572879314422607, + "learning_rate": 4.010608856088562e-06, + "loss": 1.5023, + "step": 685 + }, + { + "epoch": 0.8537647790914747, + "grad_norm": 2.75365948677063, + "learning_rate": 4.00830258302583e-06, + "loss": 1.5414, + "step": 686 + }, + { + "epoch": 0.8550093341630367, + "grad_norm": 2.5714449882507324, + "learning_rate": 4.0059963099631e-06, + "loss": 1.5027, + "step": 687 + }, + { + "epoch": 0.8562538892345987, + "grad_norm": 2.755167007446289, + "learning_rate": 4.003690036900369e-06, + "loss": 1.5788, + "step": 688 + }, + { + "epoch": 0.8574984443061605, + "grad_norm": 2.798967123031616, + "learning_rate": 4.0013837638376385e-06, + "loss": 1.5162, + "step": 689 + }, + { + "epoch": 0.8587429993777225, + "grad_norm": 2.803614854812622, + "learning_rate": 3.999077490774908e-06, + "loss": 1.5079, + "step": 690 + }, + { + "epoch": 0.8587429993777225, + "eval_loss": 1.6924962997436523, + "eval_runtime": 44.47, + "eval_samples_per_second": 22.487, + "eval_steps_per_second": 0.944, + "step": 690 + }, + { + "epoch": 0.8599875544492844, + "grad_norm": 2.6898066997528076, + "learning_rate": 3.996771217712177e-06, + "loss": 1.5478, + "step": 691 + }, + { + "epoch": 0.8612321095208463, + "grad_norm": 2.6376144886016846, + "learning_rate": 3.994464944649447e-06, + "loss": 1.5362, + "step": 692 + }, + { + "epoch": 0.8624766645924082, + "grad_norm": 2.7240405082702637, + "learning_rate": 3.992158671586716e-06, + "loss": 1.5502, + "step": 693 + }, + { + "epoch": 0.8637212196639701, + "grad_norm": 2.929445266723633, + "learning_rate": 3.989852398523986e-06, + "loss": 1.5799, + "step": 694 + }, + { + "epoch": 0.864965774735532, + "grad_norm": 2.593223810195923, + "learning_rate": 3.987546125461255e-06, + "loss": 1.5352, + "step": 695 + }, + { + "epoch": 0.866210329807094, + "grad_norm": 2.7710154056549072, + "learning_rate": 3.9852398523985245e-06, + "loss": 1.5048, + "step": 696 + }, + { + "epoch": 0.8674548848786559, + "grad_norm": 2.7370848655700684, + "learning_rate": 3.982933579335794e-06, + "loss": 1.5233, + "step": 697 + }, + { + "epoch": 0.8686994399502178, + "grad_norm": 2.8632168769836426, + "learning_rate": 3.980627306273063e-06, + "loss": 1.5435, + "step": 698 + }, + { + "epoch": 0.8699439950217797, + "grad_norm": 2.752298593521118, + "learning_rate": 3.978321033210332e-06, + "loss": 1.5728, + "step": 699 + }, + { + "epoch": 0.8711885500933416, + "grad_norm": 2.8345625400543213, + "learning_rate": 3.976014760147602e-06, + "loss": 1.5727, + "step": 700 + }, + { + "epoch": 0.8711885500933416, + "eval_loss": 1.6855305433273315, + "eval_runtime": 43.3683, + "eval_samples_per_second": 23.058, + "eval_steps_per_second": 0.968, + "step": 700 + }, + { + "epoch": 0.8724331051649036, + "grad_norm": 2.8135335445404053, + "learning_rate": 3.973708487084872e-06, + "loss": 1.5086, + "step": 701 + }, + { + "epoch": 0.8736776602364654, + "grad_norm": 2.7416603565216064, + "learning_rate": 3.97140221402214e-06, + "loss": 1.545, + "step": 702 + }, + { + "epoch": 0.8749222153080274, + "grad_norm": 2.6902289390563965, + "learning_rate": 3.96909594095941e-06, + "loss": 1.5509, + "step": 703 + }, + { + "epoch": 0.8761667703795893, + "grad_norm": 2.6537084579467773, + "learning_rate": 3.966789667896679e-06, + "loss": 1.562, + "step": 704 + }, + { + "epoch": 0.8774113254511512, + "grad_norm": 2.6979284286499023, + "learning_rate": 3.964483394833948e-06, + "loss": 1.4829, + "step": 705 + }, + { + "epoch": 0.8786558805227132, + "grad_norm": 2.5795764923095703, + "learning_rate": 3.962177121771218e-06, + "loss": 1.4677, + "step": 706 + }, + { + "epoch": 0.879900435594275, + "grad_norm": 3.0185086727142334, + "learning_rate": 3.959870848708487e-06, + "loss": 1.5608, + "step": 707 + }, + { + "epoch": 0.881144990665837, + "grad_norm": 2.6458113193511963, + "learning_rate": 3.957564575645757e-06, + "loss": 1.5342, + "step": 708 + }, + { + "epoch": 0.8823895457373989, + "grad_norm": 2.7985665798187256, + "learning_rate": 3.955258302583026e-06, + "loss": 1.527, + "step": 709 + }, + { + "epoch": 0.8836341008089608, + "grad_norm": 2.9069082736968994, + "learning_rate": 3.9529520295202955e-06, + "loss": 1.5156, + "step": 710 + }, + { + "epoch": 0.8836341008089608, + "eval_loss": 1.6898822784423828, + "eval_runtime": 42.8749, + "eval_samples_per_second": 23.324, + "eval_steps_per_second": 0.98, + "step": 710 + }, + { + "epoch": 0.8848786558805227, + "grad_norm": 2.760143995285034, + "learning_rate": 3.950645756457565e-06, + "loss": 1.5124, + "step": 711 + }, + { + "epoch": 0.8861232109520847, + "grad_norm": 2.8085529804229736, + "learning_rate": 3.948339483394834e-06, + "loss": 1.5862, + "step": 712 + }, + { + "epoch": 0.8873677660236465, + "grad_norm": 2.909905195236206, + "learning_rate": 3.946033210332104e-06, + "loss": 1.5557, + "step": 713 + }, + { + "epoch": 0.8886123210952085, + "grad_norm": 2.826899290084839, + "learning_rate": 3.943726937269373e-06, + "loss": 1.5277, + "step": 714 + }, + { + "epoch": 0.8898568761667703, + "grad_norm": 2.6269052028656006, + "learning_rate": 3.941420664206642e-06, + "loss": 1.4819, + "step": 715 + }, + { + "epoch": 0.8911014312383323, + "grad_norm": 2.6687698364257812, + "learning_rate": 3.939114391143912e-06, + "loss": 1.5156, + "step": 716 + }, + { + "epoch": 0.8923459863098943, + "grad_norm": 2.9605629444122314, + "learning_rate": 3.9368081180811815e-06, + "loss": 1.5358, + "step": 717 + }, + { + "epoch": 0.8935905413814561, + "grad_norm": 3.0367166996002197, + "learning_rate": 3.934501845018451e-06, + "loss": 1.5847, + "step": 718 + }, + { + "epoch": 0.8948350964530181, + "grad_norm": 2.528796434402466, + "learning_rate": 3.9321955719557195e-06, + "loss": 1.4851, + "step": 719 + }, + { + "epoch": 0.8960796515245799, + "grad_norm": 2.7116613388061523, + "learning_rate": 3.929889298892989e-06, + "loss": 1.4706, + "step": 720 + }, + { + "epoch": 0.8960796515245799, + "eval_loss": 1.6845883131027222, + "eval_runtime": 44.4734, + "eval_samples_per_second": 22.485, + "eval_steps_per_second": 0.944, + "step": 720 + }, + { + "epoch": 0.8973242065961419, + "grad_norm": 2.771864414215088, + "learning_rate": 3.927583025830259e-06, + "loss": 1.5444, + "step": 721 + }, + { + "epoch": 0.8985687616677038, + "grad_norm": 2.773746967315674, + "learning_rate": 3.925276752767528e-06, + "loss": 1.564, + "step": 722 + }, + { + "epoch": 0.8998133167392657, + "grad_norm": 2.7069854736328125, + "learning_rate": 3.922970479704797e-06, + "loss": 1.5601, + "step": 723 + }, + { + "epoch": 0.9010578718108276, + "grad_norm": 2.840261697769165, + "learning_rate": 3.920664206642067e-06, + "loss": 1.5733, + "step": 724 + }, + { + "epoch": 0.9023024268823896, + "grad_norm": 2.709897041320801, + "learning_rate": 3.918357933579336e-06, + "loss": 1.528, + "step": 725 + }, + { + "epoch": 0.9035469819539514, + "grad_norm": 2.662367343902588, + "learning_rate": 3.9160516605166055e-06, + "loss": 1.5532, + "step": 726 + }, + { + "epoch": 0.9047915370255134, + "grad_norm": 2.6775717735290527, + "learning_rate": 3.913745387453875e-06, + "loss": 1.5464, + "step": 727 + }, + { + "epoch": 0.9060360920970753, + "grad_norm": 2.617842674255371, + "learning_rate": 3.911439114391144e-06, + "loss": 1.4539, + "step": 728 + }, + { + "epoch": 0.9072806471686372, + "grad_norm": 2.668649435043335, + "learning_rate": 3.909132841328414e-06, + "loss": 1.5627, + "step": 729 + }, + { + "epoch": 0.9085252022401992, + "grad_norm": 2.5331950187683105, + "learning_rate": 3.906826568265683e-06, + "loss": 1.5165, + "step": 730 + }, + { + "epoch": 0.9085252022401992, + "eval_loss": 1.6860331296920776, + "eval_runtime": 46.094, + "eval_samples_per_second": 21.695, + "eval_steps_per_second": 0.911, + "step": 730 + }, + { + "epoch": 0.909769757311761, + "grad_norm": 2.602492332458496, + "learning_rate": 3.9045202952029526e-06, + "loss": 1.5145, + "step": 731 + }, + { + "epoch": 0.911014312383323, + "grad_norm": 2.632782459259033, + "learning_rate": 3.902214022140222e-06, + "loss": 1.4413, + "step": 732 + }, + { + "epoch": 0.9122588674548848, + "grad_norm": 2.6025912761688232, + "learning_rate": 3.899907749077491e-06, + "loss": 1.5263, + "step": 733 + }, + { + "epoch": 0.9135034225264468, + "grad_norm": 2.770116090774536, + "learning_rate": 3.897601476014761e-06, + "loss": 1.5514, + "step": 734 + }, + { + "epoch": 0.9147479775980087, + "grad_norm": 2.7822961807250977, + "learning_rate": 3.895295202952029e-06, + "loss": 1.5296, + "step": 735 + }, + { + "epoch": 0.9159925326695706, + "grad_norm": 2.6689720153808594, + "learning_rate": 3.892988929889299e-06, + "loss": 1.4552, + "step": 736 + }, + { + "epoch": 0.9172370877411326, + "grad_norm": 2.8021798133850098, + "learning_rate": 3.890682656826569e-06, + "loss": 1.5308, + "step": 737 + }, + { + "epoch": 0.9184816428126945, + "grad_norm": 2.575313091278076, + "learning_rate": 3.888376383763838e-06, + "loss": 1.4807, + "step": 738 + }, + { + "epoch": 0.9197261978842564, + "grad_norm": 2.5955779552459717, + "learning_rate": 3.886070110701107e-06, + "loss": 1.4222, + "step": 739 + }, + { + "epoch": 0.9209707529558183, + "grad_norm": 2.647939920425415, + "learning_rate": 3.8837638376383765e-06, + "loss": 1.5285, + "step": 740 + }, + { + "epoch": 0.9209707529558183, + "eval_loss": 1.6907480955123901, + "eval_runtime": 45.7678, + "eval_samples_per_second": 21.849, + "eval_steps_per_second": 0.918, + "step": 740 + }, + { + "epoch": 0.9222153080273802, + "grad_norm": 2.7664523124694824, + "learning_rate": 3.881457564575646e-06, + "loss": 1.5349, + "step": 741 + }, + { + "epoch": 0.9234598630989421, + "grad_norm": 2.5770998001098633, + "learning_rate": 3.879151291512915e-06, + "loss": 1.5071, + "step": 742 + }, + { + "epoch": 0.9247044181705041, + "grad_norm": 2.502567768096924, + "learning_rate": 3.876845018450185e-06, + "loss": 1.492, + "step": 743 + }, + { + "epoch": 0.9259489732420659, + "grad_norm": 2.6531119346618652, + "learning_rate": 3.874538745387454e-06, + "loss": 1.5045, + "step": 744 + }, + { + "epoch": 0.9271935283136279, + "grad_norm": 2.7343876361846924, + "learning_rate": 3.872232472324724e-06, + "loss": 1.5194, + "step": 745 + }, + { + "epoch": 0.9284380833851898, + "grad_norm": 2.842313528060913, + "learning_rate": 3.869926199261993e-06, + "loss": 1.5588, + "step": 746 + }, + { + "epoch": 0.9296826384567517, + "grad_norm": 2.718245267868042, + "learning_rate": 3.8676199261992625e-06, + "loss": 1.5252, + "step": 747 + }, + { + "epoch": 0.9309271935283137, + "grad_norm": 2.695392370223999, + "learning_rate": 3.865313653136532e-06, + "loss": 1.5596, + "step": 748 + }, + { + "epoch": 0.9321717485998755, + "grad_norm": 2.7859959602355957, + "learning_rate": 3.863007380073801e-06, + "loss": 1.5151, + "step": 749 + }, + { + "epoch": 0.9334163036714375, + "grad_norm": 2.5199174880981445, + "learning_rate": 3.860701107011071e-06, + "loss": 1.4948, + "step": 750 + }, + { + "epoch": 0.9334163036714375, + "eval_loss": 1.6868674755096436, + "eval_runtime": 46.4115, + "eval_samples_per_second": 21.546, + "eval_steps_per_second": 0.905, + "step": 750 + }, + { + "epoch": 0.9346608587429994, + "grad_norm": 2.8194046020507812, + "learning_rate": 3.858394833948339e-06, + "loss": 1.5144, + "step": 751 + }, + { + "epoch": 0.9359054138145613, + "grad_norm": 2.810380697250366, + "learning_rate": 3.856088560885609e-06, + "loss": 1.4998, + "step": 752 + }, + { + "epoch": 0.9371499688861232, + "grad_norm": 2.6906750202178955, + "learning_rate": 3.853782287822879e-06, + "loss": 1.4748, + "step": 753 + }, + { + "epoch": 0.9383945239576851, + "grad_norm": 2.9030277729034424, + "learning_rate": 3.851476014760148e-06, + "loss": 1.615, + "step": 754 + }, + { + "epoch": 0.939639079029247, + "grad_norm": 2.8690176010131836, + "learning_rate": 3.849169741697417e-06, + "loss": 1.5083, + "step": 755 + }, + { + "epoch": 0.940883634100809, + "grad_norm": 2.669646978378296, + "learning_rate": 3.8468634686346865e-06, + "loss": 1.5572, + "step": 756 + }, + { + "epoch": 0.9421281891723708, + "grad_norm": 2.6517696380615234, + "learning_rate": 3.844557195571956e-06, + "loss": 1.5142, + "step": 757 + }, + { + "epoch": 0.9433727442439328, + "grad_norm": 2.5524444580078125, + "learning_rate": 3.842250922509225e-06, + "loss": 1.4924, + "step": 758 + }, + { + "epoch": 0.9446172993154948, + "grad_norm": 2.6532633304595947, + "learning_rate": 3.839944649446495e-06, + "loss": 1.484, + "step": 759 + }, + { + "epoch": 0.9458618543870566, + "grad_norm": 2.7779057025909424, + "learning_rate": 3.837638376383764e-06, + "loss": 1.5106, + "step": 760 + }, + { + "epoch": 0.9458618543870566, + "eval_loss": 1.6845752000808716, + "eval_runtime": 50.3415, + "eval_samples_per_second": 19.864, + "eval_steps_per_second": 0.834, + "step": 760 + }, + { + "epoch": 0.9471064094586186, + "grad_norm": 2.6192541122436523, + "learning_rate": 3.8353321033210336e-06, + "loss": 1.5454, + "step": 761 + }, + { + "epoch": 0.9483509645301804, + "grad_norm": 2.561861991882324, + "learning_rate": 3.833025830258303e-06, + "loss": 1.5141, + "step": 762 + }, + { + "epoch": 0.9495955196017424, + "grad_norm": 2.661829948425293, + "learning_rate": 3.830719557195572e-06, + "loss": 1.4997, + "step": 763 + }, + { + "epoch": 0.9508400746733043, + "grad_norm": 2.725275754928589, + "learning_rate": 3.828413284132842e-06, + "loss": 1.5224, + "step": 764 + }, + { + "epoch": 0.9520846297448662, + "grad_norm": 2.5664193630218506, + "learning_rate": 3.826107011070111e-06, + "loss": 1.5073, + "step": 765 + }, + { + "epoch": 0.9533291848164281, + "grad_norm": 2.595189332962036, + "learning_rate": 3.823800738007381e-06, + "loss": 1.5379, + "step": 766 + }, + { + "epoch": 0.95457373988799, + "grad_norm": 2.738060235977173, + "learning_rate": 3.821494464944649e-06, + "loss": 1.4814, + "step": 767 + }, + { + "epoch": 0.955818294959552, + "grad_norm": 2.601071357727051, + "learning_rate": 3.819188191881919e-06, + "loss": 1.4812, + "step": 768 + }, + { + "epoch": 0.9570628500311139, + "grad_norm": 2.657212257385254, + "learning_rate": 3.816881918819189e-06, + "loss": 1.4748, + "step": 769 + }, + { + "epoch": 0.9583074051026758, + "grad_norm": 2.7673351764678955, + "learning_rate": 3.814575645756458e-06, + "loss": 1.5535, + "step": 770 + }, + { + "epoch": 0.9583074051026758, + "eval_loss": 1.677243947982788, + "eval_runtime": 51.5537, + "eval_samples_per_second": 19.397, + "eval_steps_per_second": 0.815, + "step": 770 + }, + { + "epoch": 0.9595519601742377, + "grad_norm": 2.760890007019043, + "learning_rate": 3.812269372693727e-06, + "loss": 1.5654, + "step": 771 + }, + { + "epoch": 0.9607965152457997, + "grad_norm": 2.6934309005737305, + "learning_rate": 3.809963099630997e-06, + "loss": 1.5149, + "step": 772 + }, + { + "epoch": 0.9620410703173615, + "grad_norm": 2.729950428009033, + "learning_rate": 3.8076568265682662e-06, + "loss": 1.5579, + "step": 773 + }, + { + "epoch": 0.9632856253889235, + "grad_norm": 2.732926607131958, + "learning_rate": 3.8053505535055352e-06, + "loss": 1.5179, + "step": 774 + }, + { + "epoch": 0.9645301804604853, + "grad_norm": 2.745391368865967, + "learning_rate": 3.8030442804428046e-06, + "loss": 1.5124, + "step": 775 + }, + { + "epoch": 0.9657747355320473, + "grad_norm": 2.8050146102905273, + "learning_rate": 3.800738007380074e-06, + "loss": 1.519, + "step": 776 + }, + { + "epoch": 0.9670192906036092, + "grad_norm": 2.697171449661255, + "learning_rate": 3.798431734317343e-06, + "loss": 1.5219, + "step": 777 + }, + { + "epoch": 0.9682638456751711, + "grad_norm": 2.6375980377197266, + "learning_rate": 3.796125461254613e-06, + "loss": 1.5345, + "step": 778 + }, + { + "epoch": 0.9695084007467331, + "grad_norm": 2.586636781692505, + "learning_rate": 3.7938191881918823e-06, + "loss": 1.5182, + "step": 779 + }, + { + "epoch": 0.9707529558182949, + "grad_norm": 2.626453399658203, + "learning_rate": 3.7915129151291518e-06, + "loss": 1.4817, + "step": 780 + }, + { + "epoch": 0.9707529558182949, + "eval_loss": 1.6734713315963745, + "eval_runtime": 48.1436, + "eval_samples_per_second": 20.771, + "eval_steps_per_second": 0.872, + "step": 780 + }, + { + "epoch": 0.9719975108898569, + "grad_norm": 2.579970121383667, + "learning_rate": 3.7892066420664208e-06, + "loss": 1.4981, + "step": 781 + }, + { + "epoch": 0.9732420659614188, + "grad_norm": 2.6834843158721924, + "learning_rate": 3.7869003690036906e-06, + "loss": 1.5044, + "step": 782 + }, + { + "epoch": 0.9744866210329807, + "grad_norm": 2.586982488632202, + "learning_rate": 3.78459409594096e-06, + "loss": 1.5079, + "step": 783 + }, + { + "epoch": 0.9757311761045426, + "grad_norm": 2.910027503967285, + "learning_rate": 3.782287822878229e-06, + "loss": 1.5695, + "step": 784 + }, + { + "epoch": 0.9769757311761046, + "grad_norm": 2.7088494300842285, + "learning_rate": 3.7799815498154984e-06, + "loss": 1.4986, + "step": 785 + }, + { + "epoch": 0.9782202862476664, + "grad_norm": 2.581325054168701, + "learning_rate": 3.777675276752768e-06, + "loss": 1.5137, + "step": 786 + }, + { + "epoch": 0.9794648413192284, + "grad_norm": 2.700709819793701, + "learning_rate": 3.775369003690037e-06, + "loss": 1.4718, + "step": 787 + }, + { + "epoch": 0.9807093963907902, + "grad_norm": 2.6712708473205566, + "learning_rate": 3.7730627306273067e-06, + "loss": 1.5208, + "step": 788 + }, + { + "epoch": 0.9819539514623522, + "grad_norm": 2.6799817085266113, + "learning_rate": 3.770756457564576e-06, + "loss": 1.5374, + "step": 789 + }, + { + "epoch": 0.9831985065339142, + "grad_norm": 2.618988037109375, + "learning_rate": 3.768450184501845e-06, + "loss": 1.4997, + "step": 790 + }, + { + "epoch": 0.9831985065339142, + "eval_loss": 1.6860820055007935, + "eval_runtime": 51.7654, + "eval_samples_per_second": 19.318, + "eval_steps_per_second": 0.811, + "step": 790 + }, + { + "epoch": 0.984443061605476, + "grad_norm": 2.6899526119232178, + "learning_rate": 3.7661439114391146e-06, + "loss": 1.5216, + "step": 791 + }, + { + "epoch": 0.985687616677038, + "grad_norm": 2.8003487586975098, + "learning_rate": 3.763837638376384e-06, + "loss": 1.557, + "step": 792 + }, + { + "epoch": 0.9869321717485999, + "grad_norm": 2.744536876678467, + "learning_rate": 3.761531365313654e-06, + "loss": 1.5355, + "step": 793 + }, + { + "epoch": 0.9881767268201618, + "grad_norm": 2.587250232696533, + "learning_rate": 3.759225092250923e-06, + "loss": 1.5259, + "step": 794 + }, + { + "epoch": 0.9894212818917237, + "grad_norm": 2.616291046142578, + "learning_rate": 3.7569188191881922e-06, + "loss": 1.5176, + "step": 795 + }, + { + "epoch": 0.9906658369632856, + "grad_norm": 2.6410577297210693, + "learning_rate": 3.7546125461254617e-06, + "loss": 1.511, + "step": 796 + }, + { + "epoch": 0.9919103920348475, + "grad_norm": 2.577373504638672, + "learning_rate": 3.7523062730627307e-06, + "loss": 1.4715, + "step": 797 + }, + { + "epoch": 0.9931549471064095, + "grad_norm": 2.67305326461792, + "learning_rate": 3.7500000000000005e-06, + "loss": 1.4828, + "step": 798 + }, + { + "epoch": 0.9943995021779714, + "grad_norm": 3.1631500720977783, + "learning_rate": 3.74769372693727e-06, + "loss": 1.5602, + "step": 799 + }, + { + "epoch": 0.9956440572495333, + "grad_norm": 2.9222350120544434, + "learning_rate": 3.745387453874539e-06, + "loss": 1.4814, + "step": 800 + }, + { + "epoch": 0.9956440572495333, + "eval_loss": 1.6810544729232788, + "eval_runtime": 51.1262, + "eval_samples_per_second": 19.559, + "eval_steps_per_second": 0.821, + "step": 800 + }, + { + "epoch": 0.9968886123210952, + "grad_norm": 2.884312629699707, + "learning_rate": 3.7430811808118084e-06, + "loss": 1.4553, + "step": 801 + }, + { + "epoch": 0.9981331673926571, + "grad_norm": 2.712655782699585, + "learning_rate": 3.7407749077490778e-06, + "loss": 1.5275, + "step": 802 + }, + { + "epoch": 0.9993777224642191, + "grad_norm": 2.8700571060180664, + "learning_rate": 3.7384686346863468e-06, + "loss": 1.4923, + "step": 803 + }, + { + "epoch": 1.000622277535781, + "grad_norm": 2.7779245376586914, + "learning_rate": 3.7361623616236166e-06, + "loss": 1.5224, + "step": 804 + }, + { + "epoch": 1.0018668326073428, + "grad_norm": 2.7679810523986816, + "learning_rate": 3.733856088560886e-06, + "loss": 1.4977, + "step": 805 + }, + { + "epoch": 1.0031113876789048, + "grad_norm": 2.797023296356201, + "learning_rate": 3.7315498154981555e-06, + "loss": 1.4396, + "step": 806 + }, + { + "epoch": 1.0043559427504667, + "grad_norm": 3.007962465286255, + "learning_rate": 3.7292435424354245e-06, + "loss": 1.5015, + "step": 807 + }, + { + "epoch": 1.0056004978220285, + "grad_norm": 3.127639055252075, + "learning_rate": 3.726937269372694e-06, + "loss": 1.5176, + "step": 808 + }, + { + "epoch": 1.0068450528935906, + "grad_norm": 2.9001357555389404, + "learning_rate": 3.7246309963099637e-06, + "loss": 1.4735, + "step": 809 + }, + { + "epoch": 1.0080896079651525, + "grad_norm": 2.8245413303375244, + "learning_rate": 3.7223247232472327e-06, + "loss": 1.486, + "step": 810 + }, + { + "epoch": 1.0080896079651525, + "eval_loss": 1.6835130453109741, + "eval_runtime": 44.5682, + "eval_samples_per_second": 22.438, + "eval_steps_per_second": 0.942, + "step": 810 + }, + { + "epoch": 1.0093341630367143, + "grad_norm": 2.780172824859619, + "learning_rate": 3.720018450184502e-06, + "loss": 1.4437, + "step": 811 + }, + { + "epoch": 1.0105787181082764, + "grad_norm": 2.6013996601104736, + "learning_rate": 3.7177121771217716e-06, + "loss": 1.4529, + "step": 812 + }, + { + "epoch": 1.0118232731798382, + "grad_norm": 2.831015110015869, + "learning_rate": 3.7154059040590406e-06, + "loss": 1.4825, + "step": 813 + }, + { + "epoch": 1.0130678282514, + "grad_norm": 2.8341829776763916, + "learning_rate": 3.71309963099631e-06, + "loss": 1.4736, + "step": 814 + }, + { + "epoch": 1.0143123833229621, + "grad_norm": 2.769540309906006, + "learning_rate": 3.71079335793358e-06, + "loss": 1.4724, + "step": 815 + }, + { + "epoch": 1.015556938394524, + "grad_norm": 3.0083069801330566, + "learning_rate": 3.708487084870849e-06, + "loss": 1.4985, + "step": 816 + }, + { + "epoch": 1.0168014934660858, + "grad_norm": 3.052168607711792, + "learning_rate": 3.7061808118081183e-06, + "loss": 1.5193, + "step": 817 + }, + { + "epoch": 1.0180460485376477, + "grad_norm": 2.924877405166626, + "learning_rate": 3.7038745387453877e-06, + "loss": 1.5219, + "step": 818 + }, + { + "epoch": 1.0192906036092098, + "grad_norm": 3.150123357772827, + "learning_rate": 3.7015682656826576e-06, + "loss": 1.5026, + "step": 819 + }, + { + "epoch": 1.0205351586807716, + "grad_norm": 2.881655216217041, + "learning_rate": 3.6992619926199266e-06, + "loss": 1.5011, + "step": 820 + }, + { + "epoch": 1.0205351586807716, + "eval_loss": 1.6745303869247437, + "eval_runtime": 50.6791, + "eval_samples_per_second": 19.732, + "eval_steps_per_second": 0.829, + "step": 820 + }, + { + "epoch": 1.0217797137523335, + "grad_norm": 2.824249505996704, + "learning_rate": 3.696955719557196e-06, + "loss": 1.4642, + "step": 821 + }, + { + "epoch": 1.0230242688238955, + "grad_norm": 2.859144926071167, + "learning_rate": 3.6946494464944654e-06, + "loss": 1.4681, + "step": 822 + }, + { + "epoch": 1.0242688238954574, + "grad_norm": 2.90312123298645, + "learning_rate": 3.6923431734317344e-06, + "loss": 1.4926, + "step": 823 + }, + { + "epoch": 1.0255133789670192, + "grad_norm": 2.8340659141540527, + "learning_rate": 3.690036900369004e-06, + "loss": 1.4834, + "step": 824 + }, + { + "epoch": 1.0267579340385813, + "grad_norm": 2.9743151664733887, + "learning_rate": 3.6877306273062737e-06, + "loss": 1.4574, + "step": 825 + }, + { + "epoch": 1.0280024891101431, + "grad_norm": 3.032179594039917, + "learning_rate": 3.6854243542435427e-06, + "loss": 1.5571, + "step": 826 + }, + { + "epoch": 1.029247044181705, + "grad_norm": 2.816826343536377, + "learning_rate": 3.683118081180812e-06, + "loss": 1.455, + "step": 827 + }, + { + "epoch": 1.030491599253267, + "grad_norm": 3.046696186065674, + "learning_rate": 3.6808118081180815e-06, + "loss": 1.4871, + "step": 828 + }, + { + "epoch": 1.031736154324829, + "grad_norm": 2.8517391681671143, + "learning_rate": 3.678505535055351e-06, + "loss": 1.5292, + "step": 829 + }, + { + "epoch": 1.0329807093963908, + "grad_norm": 2.752811908721924, + "learning_rate": 3.67619926199262e-06, + "loss": 1.5091, + "step": 830 + }, + { + "epoch": 1.0329807093963908, + "eval_loss": 1.6692287921905518, + "eval_runtime": 50.8347, + "eval_samples_per_second": 19.672, + "eval_steps_per_second": 0.826, + "step": 830 + }, + { + "epoch": 1.0342252644679526, + "grad_norm": 2.9220356941223145, + "learning_rate": 3.6738929889298898e-06, + "loss": 1.4247, + "step": 831 + }, + { + "epoch": 1.0354698195395147, + "grad_norm": 2.8903002738952637, + "learning_rate": 3.671586715867159e-06, + "loss": 1.5595, + "step": 832 + }, + { + "epoch": 1.0367143746110765, + "grad_norm": 2.6495542526245117, + "learning_rate": 3.669280442804428e-06, + "loss": 1.4076, + "step": 833 + }, + { + "epoch": 1.0379589296826384, + "grad_norm": 2.880809783935547, + "learning_rate": 3.6669741697416976e-06, + "loss": 1.5285, + "step": 834 + }, + { + "epoch": 1.0392034847542004, + "grad_norm": 2.919142007827759, + "learning_rate": 3.6646678966789675e-06, + "loss": 1.5165, + "step": 835 + }, + { + "epoch": 1.0404480398257623, + "grad_norm": 3.1180522441864014, + "learning_rate": 3.6623616236162365e-06, + "loss": 1.5224, + "step": 836 + }, + { + "epoch": 1.0416925948973241, + "grad_norm": 2.767709255218506, + "learning_rate": 3.660055350553506e-06, + "loss": 1.4859, + "step": 837 + }, + { + "epoch": 1.0429371499688862, + "grad_norm": 2.708828926086426, + "learning_rate": 3.6577490774907753e-06, + "loss": 1.4803, + "step": 838 + }, + { + "epoch": 1.044181705040448, + "grad_norm": 2.688359260559082, + "learning_rate": 3.6554428044280443e-06, + "loss": 1.4571, + "step": 839 + }, + { + "epoch": 1.04542626011201, + "grad_norm": 2.691812753677368, + "learning_rate": 3.6531365313653137e-06, + "loss": 1.5043, + "step": 840 + }, + { + "epoch": 1.04542626011201, + "eval_loss": 1.6769559383392334, + "eval_runtime": 49.6233, + "eval_samples_per_second": 20.152, + "eval_steps_per_second": 0.846, + "step": 840 + }, + { + "epoch": 1.046670815183572, + "grad_norm": 2.686372995376587, + "learning_rate": 3.6508302583025836e-06, + "loss": 1.5221, + "step": 841 + }, + { + "epoch": 1.0479153702551338, + "grad_norm": 2.742859125137329, + "learning_rate": 3.648523985239853e-06, + "loss": 1.4685, + "step": 842 + }, + { + "epoch": 1.0491599253266957, + "grad_norm": 2.874018907546997, + "learning_rate": 3.646217712177122e-06, + "loss": 1.4813, + "step": 843 + }, + { + "epoch": 1.0504044803982575, + "grad_norm": 2.8159990310668945, + "learning_rate": 3.6439114391143914e-06, + "loss": 1.4421, + "step": 844 + }, + { + "epoch": 1.0516490354698196, + "grad_norm": 2.714787006378174, + "learning_rate": 3.641605166051661e-06, + "loss": 1.4879, + "step": 845 + }, + { + "epoch": 1.0528935905413814, + "grad_norm": 2.885148525238037, + "learning_rate": 3.63929889298893e-06, + "loss": 1.4725, + "step": 846 + }, + { + "epoch": 1.0541381456129433, + "grad_norm": 2.7272610664367676, + "learning_rate": 3.6369926199261997e-06, + "loss": 1.4276, + "step": 847 + }, + { + "epoch": 1.0553827006845053, + "grad_norm": 2.9114978313446045, + "learning_rate": 3.634686346863469e-06, + "loss": 1.4194, + "step": 848 + }, + { + "epoch": 1.0566272557560672, + "grad_norm": 3.019313335418701, + "learning_rate": 3.632380073800738e-06, + "loss": 1.5266, + "step": 849 + }, + { + "epoch": 1.057871810827629, + "grad_norm": 2.714751958847046, + "learning_rate": 3.6300738007380075e-06, + "loss": 1.4593, + "step": 850 + }, + { + "epoch": 1.057871810827629, + "eval_loss": 1.6760298013687134, + "eval_runtime": 52.1681, + "eval_samples_per_second": 19.169, + "eval_steps_per_second": 0.805, + "step": 850 + }, + { + "epoch": 1.0591163658991911, + "grad_norm": 3.0507936477661133, + "learning_rate": 3.627767527675277e-06, + "loss": 1.5064, + "step": 851 + }, + { + "epoch": 1.060360920970753, + "grad_norm": 2.8116295337677, + "learning_rate": 3.625461254612546e-06, + "loss": 1.5218, + "step": 852 + }, + { + "epoch": 1.0616054760423148, + "grad_norm": 2.8801465034484863, + "learning_rate": 3.623154981549816e-06, + "loss": 1.4641, + "step": 853 + }, + { + "epoch": 1.0628500311138769, + "grad_norm": 3.0581862926483154, + "learning_rate": 3.6208487084870852e-06, + "loss": 1.483, + "step": 854 + }, + { + "epoch": 1.0640945861854387, + "grad_norm": 2.898780584335327, + "learning_rate": 3.6185424354243547e-06, + "loss": 1.4623, + "step": 855 + }, + { + "epoch": 1.0653391412570006, + "grad_norm": 2.971820116043091, + "learning_rate": 3.6162361623616237e-06, + "loss": 1.4455, + "step": 856 + }, + { + "epoch": 1.0665836963285624, + "grad_norm": 3.0021913051605225, + "learning_rate": 3.6139298892988935e-06, + "loss": 1.4878, + "step": 857 + }, + { + "epoch": 1.0678282514001245, + "grad_norm": 3.0697457790374756, + "learning_rate": 3.611623616236163e-06, + "loss": 1.5008, + "step": 858 + }, + { + "epoch": 1.0690728064716863, + "grad_norm": 3.0787923336029053, + "learning_rate": 3.609317343173432e-06, + "loss": 1.4612, + "step": 859 + }, + { + "epoch": 1.0703173615432482, + "grad_norm": 2.789339780807495, + "learning_rate": 3.6070110701107014e-06, + "loss": 1.4287, + "step": 860 + }, + { + "epoch": 1.0703173615432482, + "eval_loss": 1.6747506856918335, + "eval_runtime": 51.4912, + "eval_samples_per_second": 19.421, + "eval_steps_per_second": 0.816, + "step": 860 + }, + { + "epoch": 1.0715619166148103, + "grad_norm": 2.9622788429260254, + "learning_rate": 3.6047047970479708e-06, + "loss": 1.4905, + "step": 861 + }, + { + "epoch": 1.072806471686372, + "grad_norm": 3.0675268173217773, + "learning_rate": 3.6023985239852398e-06, + "loss": 1.461, + "step": 862 + }, + { + "epoch": 1.074051026757934, + "grad_norm": 2.820996046066284, + "learning_rate": 3.6000922509225096e-06, + "loss": 1.4948, + "step": 863 + }, + { + "epoch": 1.075295581829496, + "grad_norm": 2.833761215209961, + "learning_rate": 3.597785977859779e-06, + "loss": 1.4683, + "step": 864 + }, + { + "epoch": 1.0765401369010579, + "grad_norm": 2.917313814163208, + "learning_rate": 3.595479704797048e-06, + "loss": 1.5191, + "step": 865 + }, + { + "epoch": 1.0777846919726197, + "grad_norm": 2.876166820526123, + "learning_rate": 3.5931734317343175e-06, + "loss": 1.4854, + "step": 866 + }, + { + "epoch": 1.0790292470441818, + "grad_norm": 3.139049768447876, + "learning_rate": 3.590867158671587e-06, + "loss": 1.5134, + "step": 867 + }, + { + "epoch": 1.0802738021157436, + "grad_norm": 2.9956753253936768, + "learning_rate": 3.5885608856088567e-06, + "loss": 1.5003, + "step": 868 + }, + { + "epoch": 1.0815183571873055, + "grad_norm": 2.957181930541992, + "learning_rate": 3.5862546125461257e-06, + "loss": 1.4588, + "step": 869 + }, + { + "epoch": 1.0827629122588673, + "grad_norm": 2.8576438426971436, + "learning_rate": 3.583948339483395e-06, + "loss": 1.48, + "step": 870 + }, + { + "epoch": 1.0827629122588673, + "eval_loss": 1.6724848747253418, + "eval_runtime": 50.9669, + "eval_samples_per_second": 19.621, + "eval_steps_per_second": 0.824, + "step": 870 + }, + { + "epoch": 1.0840074673304294, + "grad_norm": 2.8183560371398926, + "learning_rate": 3.5816420664206646e-06, + "loss": 1.4818, + "step": 871 + }, + { + "epoch": 1.0852520224019913, + "grad_norm": 2.7531797885894775, + "learning_rate": 3.5793357933579336e-06, + "loss": 1.4712, + "step": 872 + }, + { + "epoch": 1.086496577473553, + "grad_norm": 2.923962354660034, + "learning_rate": 3.5770295202952034e-06, + "loss": 1.4864, + "step": 873 + }, + { + "epoch": 1.0877411325451152, + "grad_norm": 2.851724147796631, + "learning_rate": 3.574723247232473e-06, + "loss": 1.4771, + "step": 874 + }, + { + "epoch": 1.088985687616677, + "grad_norm": 3.024634838104248, + "learning_rate": 3.572416974169742e-06, + "loss": 1.4643, + "step": 875 + }, + { + "epoch": 1.0902302426882389, + "grad_norm": 2.7100844383239746, + "learning_rate": 3.5701107011070113e-06, + "loss": 1.4773, + "step": 876 + }, + { + "epoch": 1.091474797759801, + "grad_norm": 2.730502128601074, + "learning_rate": 3.5678044280442807e-06, + "loss": 1.5051, + "step": 877 + }, + { + "epoch": 1.0927193528313628, + "grad_norm": 2.747082233428955, + "learning_rate": 3.5654981549815497e-06, + "loss": 1.5207, + "step": 878 + }, + { + "epoch": 1.0939639079029246, + "grad_norm": 2.718358039855957, + "learning_rate": 3.5631918819188195e-06, + "loss": 1.5067, + "step": 879 + }, + { + "epoch": 1.0952084629744867, + "grad_norm": 2.7794573307037354, + "learning_rate": 3.560885608856089e-06, + "loss": 1.4616, + "step": 880 + }, + { + "epoch": 1.0952084629744867, + "eval_loss": 1.6676132678985596, + "eval_runtime": 51.6157, + "eval_samples_per_second": 19.374, + "eval_steps_per_second": 0.814, + "step": 880 + }, + { + "epoch": 1.0964530180460486, + "grad_norm": 2.727416753768921, + "learning_rate": 3.5585793357933584e-06, + "loss": 1.4043, + "step": 881 + }, + { + "epoch": 1.0976975731176104, + "grad_norm": 2.856900930404663, + "learning_rate": 3.5562730627306274e-06, + "loss": 1.5198, + "step": 882 + }, + { + "epoch": 1.0989421281891725, + "grad_norm": 2.872823715209961, + "learning_rate": 3.553966789667897e-06, + "loss": 1.4981, + "step": 883 + }, + { + "epoch": 1.1001866832607343, + "grad_norm": 2.7724361419677734, + "learning_rate": 3.5516605166051667e-06, + "loss": 1.4819, + "step": 884 + }, + { + "epoch": 1.1014312383322962, + "grad_norm": 2.884434461593628, + "learning_rate": 3.5493542435424357e-06, + "loss": 1.433, + "step": 885 + }, + { + "epoch": 1.102675793403858, + "grad_norm": 2.887873411178589, + "learning_rate": 3.547047970479705e-06, + "loss": 1.4677, + "step": 886 + }, + { + "epoch": 1.10392034847542, + "grad_norm": 2.946845054626465, + "learning_rate": 3.5447416974169745e-06, + "loss": 1.4983, + "step": 887 + }, + { + "epoch": 1.105164903546982, + "grad_norm": 2.752552032470703, + "learning_rate": 3.5424354243542435e-06, + "loss": 1.4766, + "step": 888 + }, + { + "epoch": 1.1064094586185438, + "grad_norm": 2.788634777069092, + "learning_rate": 3.5401291512915133e-06, + "loss": 1.4747, + "step": 889 + }, + { + "epoch": 1.1076540136901059, + "grad_norm": 2.782792568206787, + "learning_rate": 3.5378228782287828e-06, + "loss": 1.5503, + "step": 890 + }, + { + "epoch": 1.1076540136901059, + "eval_loss": 1.6698333024978638, + "eval_runtime": 49.1373, + "eval_samples_per_second": 20.351, + "eval_steps_per_second": 0.855, + "step": 890 + }, + { + "epoch": 1.1088985687616677, + "grad_norm": 2.8351166248321533, + "learning_rate": 3.535516605166052e-06, + "loss": 1.4589, + "step": 891 + }, + { + "epoch": 1.1101431238332296, + "grad_norm": 2.798872709274292, + "learning_rate": 3.533210332103321e-06, + "loss": 1.4496, + "step": 892 + }, + { + "epoch": 1.1113876789047916, + "grad_norm": 3.0863239765167236, + "learning_rate": 3.5309040590405906e-06, + "loss": 1.4698, + "step": 893 + }, + { + "epoch": 1.1126322339763535, + "grad_norm": 2.853862762451172, + "learning_rate": 3.5285977859778605e-06, + "loss": 1.4429, + "step": 894 + }, + { + "epoch": 1.1138767890479153, + "grad_norm": 2.8946170806884766, + "learning_rate": 3.5262915129151295e-06, + "loss": 1.4697, + "step": 895 + }, + { + "epoch": 1.1151213441194774, + "grad_norm": 3.043823003768921, + "learning_rate": 3.523985239852399e-06, + "loss": 1.4666, + "step": 896 + }, + { + "epoch": 1.1163658991910392, + "grad_norm": 2.7822980880737305, + "learning_rate": 3.5216789667896683e-06, + "loss": 1.4523, + "step": 897 + }, + { + "epoch": 1.117610454262601, + "grad_norm": 2.9454171657562256, + "learning_rate": 3.5193726937269373e-06, + "loss": 1.5009, + "step": 898 + }, + { + "epoch": 1.1188550093341632, + "grad_norm": 2.8533740043640137, + "learning_rate": 3.5170664206642067e-06, + "loss": 1.4265, + "step": 899 + }, + { + "epoch": 1.120099564405725, + "grad_norm": 2.894937038421631, + "learning_rate": 3.5147601476014766e-06, + "loss": 1.4895, + "step": 900 + }, + { + "epoch": 1.120099564405725, + "eval_loss": 1.6701407432556152, + "eval_runtime": 45.477, + "eval_samples_per_second": 21.989, + "eval_steps_per_second": 0.924, + "step": 900 + }, + { + "epoch": 1.1213441194772868, + "grad_norm": 2.933821201324463, + "learning_rate": 3.5124538745387456e-06, + "loss": 1.468, + "step": 901 + }, + { + "epoch": 1.1225886745488487, + "grad_norm": 2.834265947341919, + "learning_rate": 3.510147601476015e-06, + "loss": 1.4621, + "step": 902 + }, + { + "epoch": 1.1238332296204108, + "grad_norm": 2.9064722061157227, + "learning_rate": 3.5078413284132844e-06, + "loss": 1.504, + "step": 903 + }, + { + "epoch": 1.1250777846919726, + "grad_norm": 2.8753886222839355, + "learning_rate": 3.505535055350554e-06, + "loss": 1.4643, + "step": 904 + }, + { + "epoch": 1.1263223397635345, + "grad_norm": 2.894659996032715, + "learning_rate": 3.503228782287823e-06, + "loss": 1.4909, + "step": 905 + }, + { + "epoch": 1.1275668948350965, + "grad_norm": 3.085977792739868, + "learning_rate": 3.5009225092250927e-06, + "loss": 1.459, + "step": 906 + }, + { + "epoch": 1.1288114499066584, + "grad_norm": 2.9958064556121826, + "learning_rate": 3.498616236162362e-06, + "loss": 1.4152, + "step": 907 + }, + { + "epoch": 1.1300560049782202, + "grad_norm": 2.9136762619018555, + "learning_rate": 3.496309963099631e-06, + "loss": 1.473, + "step": 908 + }, + { + "epoch": 1.1313005600497823, + "grad_norm": 2.956437826156616, + "learning_rate": 3.4940036900369005e-06, + "loss": 1.5167, + "step": 909 + }, + { + "epoch": 1.1325451151213441, + "grad_norm": 2.7354865074157715, + "learning_rate": 3.4916974169741704e-06, + "loss": 1.4413, + "step": 910 + }, + { + "epoch": 1.1325451151213441, + "eval_loss": 1.6629912853240967, + "eval_runtime": 41.5478, + "eval_samples_per_second": 24.069, + "eval_steps_per_second": 1.011, + "step": 910 + }, + { + "epoch": 1.133789670192906, + "grad_norm": 2.7443008422851562, + "learning_rate": 3.4893911439114394e-06, + "loss": 1.4475, + "step": 911 + }, + { + "epoch": 1.135034225264468, + "grad_norm": 2.7811238765716553, + "learning_rate": 3.487084870848709e-06, + "loss": 1.4257, + "step": 912 + }, + { + "epoch": 1.13627878033603, + "grad_norm": 2.7916479110717773, + "learning_rate": 3.4847785977859782e-06, + "loss": 1.4048, + "step": 913 + }, + { + "epoch": 1.1375233354075918, + "grad_norm": 2.7835633754730225, + "learning_rate": 3.4824723247232472e-06, + "loss": 1.3996, + "step": 914 + }, + { + "epoch": 1.1387678904791536, + "grad_norm": 2.8188705444335938, + "learning_rate": 3.4801660516605166e-06, + "loss": 1.4541, + "step": 915 + }, + { + "epoch": 1.1400124455507157, + "grad_norm": 3.0134100914001465, + "learning_rate": 3.4778597785977865e-06, + "loss": 1.4353, + "step": 916 + }, + { + "epoch": 1.1412570006222775, + "grad_norm": 2.6965606212615967, + "learning_rate": 3.475553505535056e-06, + "loss": 1.4308, + "step": 917 + }, + { + "epoch": 1.1425015556938394, + "grad_norm": 2.8179919719696045, + "learning_rate": 3.473247232472325e-06, + "loss": 1.5204, + "step": 918 + }, + { + "epoch": 1.1437461107654014, + "grad_norm": 2.7213902473449707, + "learning_rate": 3.4709409594095943e-06, + "loss": 1.4344, + "step": 919 + }, + { + "epoch": 1.1449906658369633, + "grad_norm": 2.850193738937378, + "learning_rate": 3.4686346863468638e-06, + "loss": 1.473, + "step": 920 + }, + { + "epoch": 1.1449906658369633, + "eval_loss": 1.663345217704773, + "eval_runtime": 43.8397, + "eval_samples_per_second": 22.81, + "eval_steps_per_second": 0.958, + "step": 920 + }, + { + "epoch": 1.1462352209085251, + "grad_norm": 2.780198574066162, + "learning_rate": 3.4663284132841328e-06, + "loss": 1.4472, + "step": 921 + }, + { + "epoch": 1.1474797759800872, + "grad_norm": 2.9373831748962402, + "learning_rate": 3.4640221402214026e-06, + "loss": 1.426, + "step": 922 + }, + { + "epoch": 1.148724331051649, + "grad_norm": 2.6986873149871826, + "learning_rate": 3.461715867158672e-06, + "loss": 1.3953, + "step": 923 + }, + { + "epoch": 1.149968886123211, + "grad_norm": 2.927441120147705, + "learning_rate": 3.459409594095941e-06, + "loss": 1.4578, + "step": 924 + }, + { + "epoch": 1.151213441194773, + "grad_norm": 2.8469648361206055, + "learning_rate": 3.4571033210332105e-06, + "loss": 1.4313, + "step": 925 + }, + { + "epoch": 1.1524579962663348, + "grad_norm": 2.8114013671875, + "learning_rate": 3.4547970479704803e-06, + "loss": 1.4267, + "step": 926 + }, + { + "epoch": 1.1537025513378967, + "grad_norm": 2.977605104446411, + "learning_rate": 3.4524907749077493e-06, + "loss": 1.4747, + "step": 927 + }, + { + "epoch": 1.1549471064094585, + "grad_norm": 2.9327852725982666, + "learning_rate": 3.4501845018450187e-06, + "loss": 1.4219, + "step": 928 + }, + { + "epoch": 1.1561916614810206, + "grad_norm": 2.844038963317871, + "learning_rate": 3.447878228782288e-06, + "loss": 1.5146, + "step": 929 + }, + { + "epoch": 1.1574362165525824, + "grad_norm": 2.9402201175689697, + "learning_rate": 3.4455719557195576e-06, + "loss": 1.45, + "step": 930 + }, + { + "epoch": 1.1574362165525824, + "eval_loss": 1.6620122194290161, + "eval_runtime": 46.1748, + "eval_samples_per_second": 21.657, + "eval_steps_per_second": 0.91, + "step": 930 + }, + { + "epoch": 1.1586807716241443, + "grad_norm": 2.842651128768921, + "learning_rate": 3.4432656826568266e-06, + "loss": 1.4415, + "step": 931 + }, + { + "epoch": 1.1599253266957064, + "grad_norm": 2.8909685611724854, + "learning_rate": 3.4409594095940964e-06, + "loss": 1.4529, + "step": 932 + }, + { + "epoch": 1.1611698817672682, + "grad_norm": 2.8025808334350586, + "learning_rate": 3.438653136531366e-06, + "loss": 1.4552, + "step": 933 + }, + { + "epoch": 1.16241443683883, + "grad_norm": 2.6995794773101807, + "learning_rate": 3.436346863468635e-06, + "loss": 1.4435, + "step": 934 + }, + { + "epoch": 1.1636589919103921, + "grad_norm": 2.7307121753692627, + "learning_rate": 3.4340405904059043e-06, + "loss": 1.4447, + "step": 935 + }, + { + "epoch": 1.164903546981954, + "grad_norm": 2.9532501697540283, + "learning_rate": 3.4317343173431737e-06, + "loss": 1.4747, + "step": 936 + }, + { + "epoch": 1.1661481020535158, + "grad_norm": 3.0135788917541504, + "learning_rate": 3.4294280442804427e-06, + "loss": 1.525, + "step": 937 + }, + { + "epoch": 1.167392657125078, + "grad_norm": 2.796355962753296, + "learning_rate": 3.4271217712177125e-06, + "loss": 1.4539, + "step": 938 + }, + { + "epoch": 1.1686372121966397, + "grad_norm": 2.9422309398651123, + "learning_rate": 3.424815498154982e-06, + "loss": 1.4664, + "step": 939 + }, + { + "epoch": 1.1698817672682016, + "grad_norm": 2.7680039405822754, + "learning_rate": 3.4225092250922514e-06, + "loss": 1.4837, + "step": 940 + }, + { + "epoch": 1.1698817672682016, + "eval_loss": 1.660908818244934, + "eval_runtime": 45.6998, + "eval_samples_per_second": 21.882, + "eval_steps_per_second": 0.919, + "step": 940 + }, + { + "epoch": 1.1711263223397634, + "grad_norm": 2.8888845443725586, + "learning_rate": 3.4202029520295204e-06, + "loss": 1.5315, + "step": 941 + }, + { + "epoch": 1.1723708774113255, + "grad_norm": 2.7124130725860596, + "learning_rate": 3.41789667896679e-06, + "loss": 1.4188, + "step": 942 + }, + { + "epoch": 1.1736154324828874, + "grad_norm": 2.75203013420105, + "learning_rate": 3.4155904059040596e-06, + "loss": 1.4533, + "step": 943 + }, + { + "epoch": 1.1748599875544492, + "grad_norm": 2.865480422973633, + "learning_rate": 3.4132841328413286e-06, + "loss": 1.4532, + "step": 944 + }, + { + "epoch": 1.1761045426260113, + "grad_norm": 2.8618321418762207, + "learning_rate": 3.410977859778598e-06, + "loss": 1.4386, + "step": 945 + }, + { + "epoch": 1.1773490976975731, + "grad_norm": 2.7656145095825195, + "learning_rate": 3.4086715867158675e-06, + "loss": 1.3996, + "step": 946 + }, + { + "epoch": 1.178593652769135, + "grad_norm": 2.76213002204895, + "learning_rate": 3.4063653136531365e-06, + "loss": 1.4979, + "step": 947 + }, + { + "epoch": 1.179838207840697, + "grad_norm": 2.8581957817077637, + "learning_rate": 3.4040590405904063e-06, + "loss": 1.4673, + "step": 948 + }, + { + "epoch": 1.1810827629122589, + "grad_norm": 2.7825586795806885, + "learning_rate": 3.4017527675276758e-06, + "loss": 1.4453, + "step": 949 + }, + { + "epoch": 1.1823273179838207, + "grad_norm": 2.9226131439208984, + "learning_rate": 3.3994464944649448e-06, + "loss": 1.4879, + "step": 950 + }, + { + "epoch": 1.1823273179838207, + "eval_loss": 1.6621263027191162, + "eval_runtime": 45.1222, + "eval_samples_per_second": 22.162, + "eval_steps_per_second": 0.931, + "step": 950 + }, + { + "epoch": 1.1835718730553828, + "grad_norm": 2.9997975826263428, + "learning_rate": 3.397140221402214e-06, + "loss": 1.505, + "step": 951 + }, + { + "epoch": 1.1848164281269447, + "grad_norm": 2.7225453853607178, + "learning_rate": 3.3948339483394836e-06, + "loss": 1.4695, + "step": 952 + }, + { + "epoch": 1.1860609831985065, + "grad_norm": 2.916473150253296, + "learning_rate": 3.3925276752767534e-06, + "loss": 1.4901, + "step": 953 + }, + { + "epoch": 1.1873055382700684, + "grad_norm": 2.8371644020080566, + "learning_rate": 3.3902214022140224e-06, + "loss": 1.4944, + "step": 954 + }, + { + "epoch": 1.1885500933416304, + "grad_norm": 2.813662528991699, + "learning_rate": 3.387915129151292e-06, + "loss": 1.4603, + "step": 955 + }, + { + "epoch": 1.1897946484131923, + "grad_norm": 2.991931676864624, + "learning_rate": 3.3856088560885613e-06, + "loss": 1.5496, + "step": 956 + }, + { + "epoch": 1.1910392034847541, + "grad_norm": 2.6252501010894775, + "learning_rate": 3.3833025830258303e-06, + "loss": 1.4405, + "step": 957 + }, + { + "epoch": 1.1922837585563162, + "grad_norm": 2.747725009918213, + "learning_rate": 3.3809963099630997e-06, + "loss": 1.4689, + "step": 958 + }, + { + "epoch": 1.193528313627878, + "grad_norm": 2.887763261795044, + "learning_rate": 3.3786900369003696e-06, + "loss": 1.4501, + "step": 959 + }, + { + "epoch": 1.1947728686994399, + "grad_norm": 2.9954354763031006, + "learning_rate": 3.3763837638376386e-06, + "loss": 1.4978, + "step": 960 + }, + { + "epoch": 1.1947728686994399, + "eval_loss": 1.664442539215088, + "eval_runtime": 49.1266, + "eval_samples_per_second": 20.356, + "eval_steps_per_second": 0.855, + "step": 960 + }, + { + "epoch": 1.196017423771002, + "grad_norm": 3.0174152851104736, + "learning_rate": 3.374077490774908e-06, + "loss": 1.4353, + "step": 961 + }, + { + "epoch": 1.1972619788425638, + "grad_norm": 2.9082882404327393, + "learning_rate": 3.3717712177121774e-06, + "loss": 1.4878, + "step": 962 + }, + { + "epoch": 1.1985065339141256, + "grad_norm": 2.897993326187134, + "learning_rate": 3.3694649446494464e-06, + "loss": 1.4975, + "step": 963 + }, + { + "epoch": 1.1997510889856877, + "grad_norm": 2.739470958709717, + "learning_rate": 3.3671586715867163e-06, + "loss": 1.4202, + "step": 964 + }, + { + "epoch": 1.2009956440572496, + "grad_norm": 2.817786693572998, + "learning_rate": 3.3648523985239857e-06, + "loss": 1.5195, + "step": 965 + }, + { + "epoch": 1.2022401991288114, + "grad_norm": 2.8476386070251465, + "learning_rate": 3.362546125461255e-06, + "loss": 1.444, + "step": 966 + }, + { + "epoch": 1.2034847542003733, + "grad_norm": 2.804047107696533, + "learning_rate": 3.360239852398524e-06, + "loss": 1.4442, + "step": 967 + }, + { + "epoch": 1.2047293092719353, + "grad_norm": 2.9285430908203125, + "learning_rate": 3.3579335793357935e-06, + "loss": 1.4877, + "step": 968 + }, + { + "epoch": 1.2059738643434972, + "grad_norm": 2.8814878463745117, + "learning_rate": 3.3556273062730634e-06, + "loss": 1.5078, + "step": 969 + }, + { + "epoch": 1.207218419415059, + "grad_norm": 2.970728635787964, + "learning_rate": 3.3533210332103324e-06, + "loss": 1.4166, + "step": 970 + }, + { + "epoch": 1.207218419415059, + "eval_loss": 1.6635315418243408, + "eval_runtime": 47.2321, + "eval_samples_per_second": 21.172, + "eval_steps_per_second": 0.889, + "step": 970 + }, + { + "epoch": 1.208462974486621, + "grad_norm": 2.9836292266845703, + "learning_rate": 3.3510147601476018e-06, + "loss": 1.4777, + "step": 971 + }, + { + "epoch": 1.209707529558183, + "grad_norm": 2.8495216369628906, + "learning_rate": 3.348708487084871e-06, + "loss": 1.4829, + "step": 972 + }, + { + "epoch": 1.2109520846297448, + "grad_norm": 2.7709312438964844, + "learning_rate": 3.34640221402214e-06, + "loss": 1.4242, + "step": 973 + }, + { + "epoch": 1.2121966397013069, + "grad_norm": 2.999462366104126, + "learning_rate": 3.3440959409594096e-06, + "loss": 1.4148, + "step": 974 + }, + { + "epoch": 1.2134411947728687, + "grad_norm": 2.8719074726104736, + "learning_rate": 3.3417896678966795e-06, + "loss": 1.4196, + "step": 975 + }, + { + "epoch": 1.2146857498444306, + "grad_norm": 2.8469338417053223, + "learning_rate": 3.3394833948339485e-06, + "loss": 1.4941, + "step": 976 + }, + { + "epoch": 1.2159303049159926, + "grad_norm": 2.9780938625335693, + "learning_rate": 3.337177121771218e-06, + "loss": 1.513, + "step": 977 + }, + { + "epoch": 1.2171748599875545, + "grad_norm": 2.814760684967041, + "learning_rate": 3.3348708487084873e-06, + "loss": 1.4204, + "step": 978 + }, + { + "epoch": 1.2184194150591163, + "grad_norm": 3.016261339187622, + "learning_rate": 3.332564575645757e-06, + "loss": 1.4389, + "step": 979 + }, + { + "epoch": 1.2196639701306782, + "grad_norm": 2.9385855197906494, + "learning_rate": 3.330258302583026e-06, + "loss": 1.4885, + "step": 980 + }, + { + "epoch": 1.2196639701306782, + "eval_loss": 1.6688873767852783, + "eval_runtime": 46.2942, + "eval_samples_per_second": 21.601, + "eval_steps_per_second": 0.907, + "step": 980 + }, + { + "epoch": 1.2209085252022402, + "grad_norm": 3.1434757709503174, + "learning_rate": 3.3279520295202956e-06, + "loss": 1.5157, + "step": 981 + }, + { + "epoch": 1.222153080273802, + "grad_norm": 3.0104827880859375, + "learning_rate": 3.325645756457565e-06, + "loss": 1.49, + "step": 982 + }, + { + "epoch": 1.223397635345364, + "grad_norm": 3.1913645267486572, + "learning_rate": 3.323339483394834e-06, + "loss": 1.4937, + "step": 983 + }, + { + "epoch": 1.224642190416926, + "grad_norm": 3.19496750831604, + "learning_rate": 3.3210332103321034e-06, + "loss": 1.5197, + "step": 984 + }, + { + "epoch": 1.2258867454884879, + "grad_norm": 3.046232223510742, + "learning_rate": 3.3187269372693733e-06, + "loss": 1.4497, + "step": 985 + }, + { + "epoch": 1.2271313005600497, + "grad_norm": 2.850675344467163, + "learning_rate": 3.3164206642066423e-06, + "loss": 1.4595, + "step": 986 + }, + { + "epoch": 1.2283758556316118, + "grad_norm": 2.7744576930999756, + "learning_rate": 3.3141143911439117e-06, + "loss": 1.4726, + "step": 987 + }, + { + "epoch": 1.2296204107031736, + "grad_norm": 3.017153024673462, + "learning_rate": 3.311808118081181e-06, + "loss": 1.493, + "step": 988 + }, + { + "epoch": 1.2308649657747355, + "grad_norm": 2.8821120262145996, + "learning_rate": 3.3095018450184506e-06, + "loss": 1.4734, + "step": 989 + }, + { + "epoch": 1.2321095208462975, + "grad_norm": 2.8805840015411377, + "learning_rate": 3.3071955719557196e-06, + "loss": 1.4562, + "step": 990 + }, + { + "epoch": 1.2321095208462975, + "eval_loss": 1.669345736503601, + "eval_runtime": 47.0893, + "eval_samples_per_second": 21.236, + "eval_steps_per_second": 0.892, + "step": 990 + }, + { + "epoch": 1.2333540759178594, + "grad_norm": 2.7599008083343506, + "learning_rate": 3.3048892988929894e-06, + "loss": 1.4218, + "step": 991 + }, + { + "epoch": 1.2345986309894212, + "grad_norm": 2.9835240840911865, + "learning_rate": 3.302583025830259e-06, + "loss": 1.4596, + "step": 992 + }, + { + "epoch": 1.235843186060983, + "grad_norm": 2.843358039855957, + "learning_rate": 3.300276752767528e-06, + "loss": 1.4561, + "step": 993 + }, + { + "epoch": 1.2370877411325452, + "grad_norm": 2.7386093139648438, + "learning_rate": 3.2979704797047972e-06, + "loss": 1.4475, + "step": 994 + }, + { + "epoch": 1.238332296204107, + "grad_norm": 2.668161392211914, + "learning_rate": 3.2956642066420667e-06, + "loss": 1.4661, + "step": 995 + }, + { + "epoch": 1.2395768512756689, + "grad_norm": 2.947235107421875, + "learning_rate": 3.2933579335793357e-06, + "loss": 1.4358, + "step": 996 + }, + { + "epoch": 1.240821406347231, + "grad_norm": 2.8159878253936768, + "learning_rate": 3.2910516605166055e-06, + "loss": 1.4432, + "step": 997 + }, + { + "epoch": 1.2420659614187928, + "grad_norm": 2.896597385406494, + "learning_rate": 3.288745387453875e-06, + "loss": 1.5585, + "step": 998 + }, + { + "epoch": 1.2433105164903546, + "grad_norm": 2.80241322517395, + "learning_rate": 3.286439114391144e-06, + "loss": 1.5076, + "step": 999 + }, + { + "epoch": 1.2445550715619167, + "grad_norm": 3.008939027786255, + "learning_rate": 3.2841328413284134e-06, + "loss": 1.4234, + "step": 1000 + }, + { + "epoch": 1.2445550715619167, + "eval_loss": 1.6621588468551636, + "eval_runtime": 50.4926, + "eval_samples_per_second": 19.805, + "eval_steps_per_second": 0.832, + "step": 1000 + }, + { + "epoch": 1.2457996266334785, + "grad_norm": 2.9872381687164307, + "learning_rate": 3.281826568265683e-06, + "loss": 1.453, + "step": 1001 + }, + { + "epoch": 1.2470441817050404, + "grad_norm": 2.704676628112793, + "learning_rate": 3.2795202952029526e-06, + "loss": 1.4711, + "step": 1002 + }, + { + "epoch": 1.2482887367766025, + "grad_norm": 2.8100757598876953, + "learning_rate": 3.2772140221402216e-06, + "loss": 1.3884, + "step": 1003 + }, + { + "epoch": 1.2495332918481643, + "grad_norm": 2.902953624725342, + "learning_rate": 3.274907749077491e-06, + "loss": 1.4986, + "step": 1004 + }, + { + "epoch": 1.2507778469197262, + "grad_norm": 2.980163097381592, + "learning_rate": 3.2726014760147605e-06, + "loss": 1.5067, + "step": 1005 + }, + { + "epoch": 1.252022401991288, + "grad_norm": 2.8820345401763916, + "learning_rate": 3.2702952029520295e-06, + "loss": 1.4995, + "step": 1006 + }, + { + "epoch": 1.25326695706285, + "grad_norm": 2.8219635486602783, + "learning_rate": 3.2679889298892993e-06, + "loss": 1.4881, + "step": 1007 + }, + { + "epoch": 1.254511512134412, + "grad_norm": 2.8593199253082275, + "learning_rate": 3.2656826568265687e-06, + "loss": 1.4469, + "step": 1008 + }, + { + "epoch": 1.255756067205974, + "grad_norm": 2.888357639312744, + "learning_rate": 3.2633763837638377e-06, + "loss": 1.4869, + "step": 1009 + }, + { + "epoch": 1.2570006222775358, + "grad_norm": 2.9985013008117676, + "learning_rate": 3.261070110701107e-06, + "loss": 1.4724, + "step": 1010 + }, + { + "epoch": 1.2570006222775358, + "eval_loss": 1.6492141485214233, + "eval_runtime": 45.2197, + "eval_samples_per_second": 22.114, + "eval_steps_per_second": 0.929, + "step": 1010 + }, + { + "epoch": 1.2582451773490977, + "grad_norm": 2.860780715942383, + "learning_rate": 3.2587638376383766e-06, + "loss": 1.4618, + "step": 1011 + }, + { + "epoch": 1.2594897324206595, + "grad_norm": 2.766479730606079, + "learning_rate": 3.2564575645756456e-06, + "loss": 1.4642, + "step": 1012 + }, + { + "epoch": 1.2607342874922216, + "grad_norm": 2.711759328842163, + "learning_rate": 3.2541512915129154e-06, + "loss": 1.4283, + "step": 1013 + }, + { + "epoch": 1.2619788425637835, + "grad_norm": 2.8365769386291504, + "learning_rate": 3.251845018450185e-06, + "loss": 1.4248, + "step": 1014 + }, + { + "epoch": 1.2632233976353453, + "grad_norm": 2.957620143890381, + "learning_rate": 3.2495387453874543e-06, + "loss": 1.4662, + "step": 1015 + }, + { + "epoch": 1.2644679527069074, + "grad_norm": 2.8066208362579346, + "learning_rate": 3.2472324723247233e-06, + "loss": 1.4447, + "step": 1016 + }, + { + "epoch": 1.2657125077784692, + "grad_norm": 2.902040958404541, + "learning_rate": 3.244926199261993e-06, + "loss": 1.4743, + "step": 1017 + }, + { + "epoch": 1.266957062850031, + "grad_norm": 2.7636687755584717, + "learning_rate": 3.2426199261992625e-06, + "loss": 1.4465, + "step": 1018 + }, + { + "epoch": 1.268201617921593, + "grad_norm": 2.95261549949646, + "learning_rate": 3.2403136531365315e-06, + "loss": 1.4769, + "step": 1019 + }, + { + "epoch": 1.269446172993155, + "grad_norm": 2.911128520965576, + "learning_rate": 3.238007380073801e-06, + "loss": 1.3843, + "step": 1020 + }, + { + "epoch": 1.269446172993155, + "eval_loss": 1.6526964902877808, + "eval_runtime": 48.2126, + "eval_samples_per_second": 20.741, + "eval_steps_per_second": 0.871, + "step": 1020 + }, + { + "epoch": 1.2706907280647168, + "grad_norm": 2.769951820373535, + "learning_rate": 3.2357011070110704e-06, + "loss": 1.4847, + "step": 1021 + }, + { + "epoch": 1.271935283136279, + "grad_norm": 3.0451674461364746, + "learning_rate": 3.2333948339483394e-06, + "loss": 1.4453, + "step": 1022 + }, + { + "epoch": 1.2731798382078408, + "grad_norm": 2.9543521404266357, + "learning_rate": 3.2310885608856092e-06, + "loss": 1.4789, + "step": 1023 + }, + { + "epoch": 1.2744243932794026, + "grad_norm": 2.9174394607543945, + "learning_rate": 3.2287822878228787e-06, + "loss": 1.4807, + "step": 1024 + }, + { + "epoch": 1.2756689483509644, + "grad_norm": 2.888258934020996, + "learning_rate": 3.2264760147601477e-06, + "loss": 1.4367, + "step": 1025 + }, + { + "epoch": 1.2769135034225265, + "grad_norm": 2.8346047401428223, + "learning_rate": 3.224169741697417e-06, + "loss": 1.4313, + "step": 1026 + }, + { + "epoch": 1.2781580584940884, + "grad_norm": 2.7531332969665527, + "learning_rate": 3.2218634686346865e-06, + "loss": 1.4018, + "step": 1027 + }, + { + "epoch": 1.2794026135656502, + "grad_norm": 3.136946439743042, + "learning_rate": 3.2195571955719564e-06, + "loss": 1.4417, + "step": 1028 + }, + { + "epoch": 1.2806471686372123, + "grad_norm": 2.9618165493011475, + "learning_rate": 3.2172509225092254e-06, + "loss": 1.5117, + "step": 1029 + }, + { + "epoch": 1.2818917237087741, + "grad_norm": 3.0740747451782227, + "learning_rate": 3.2149446494464948e-06, + "loss": 1.4641, + "step": 1030 + }, + { + "epoch": 1.2818917237087741, + "eval_loss": 1.6572761535644531, + "eval_runtime": 51.6696, + "eval_samples_per_second": 19.354, + "eval_steps_per_second": 0.813, + "step": 1030 + }, + { + "epoch": 1.283136278780336, + "grad_norm": 2.9225106239318848, + "learning_rate": 3.212638376383764e-06, + "loss": 1.4421, + "step": 1031 + }, + { + "epoch": 1.2843808338518978, + "grad_norm": 2.9530649185180664, + "learning_rate": 3.210332103321033e-06, + "loss": 1.5138, + "step": 1032 + }, + { + "epoch": 1.28562538892346, + "grad_norm": 2.8513238430023193, + "learning_rate": 3.208025830258303e-06, + "loss": 1.411, + "step": 1033 + }, + { + "epoch": 1.2868699439950217, + "grad_norm": 2.7440171241760254, + "learning_rate": 3.2057195571955725e-06, + "loss": 1.4182, + "step": 1034 + }, + { + "epoch": 1.2881144990665838, + "grad_norm": 2.8126630783081055, + "learning_rate": 3.2034132841328415e-06, + "loss": 1.4333, + "step": 1035 + }, + { + "epoch": 1.2893590541381457, + "grad_norm": 2.8598744869232178, + "learning_rate": 3.201107011070111e-06, + "loss": 1.439, + "step": 1036 + }, + { + "epoch": 1.2906036092097075, + "grad_norm": 2.901622772216797, + "learning_rate": 3.1988007380073803e-06, + "loss": 1.4461, + "step": 1037 + }, + { + "epoch": 1.2918481642812694, + "grad_norm": 2.7670180797576904, + "learning_rate": 3.1964944649446493e-06, + "loss": 1.4772, + "step": 1038 + }, + { + "epoch": 1.2930927193528314, + "grad_norm": 2.7860350608825684, + "learning_rate": 3.194188191881919e-06, + "loss": 1.4414, + "step": 1039 + }, + { + "epoch": 1.2943372744243933, + "grad_norm": 2.822052478790283, + "learning_rate": 3.1918819188191886e-06, + "loss": 1.4192, + "step": 1040 + }, + { + "epoch": 1.2943372744243933, + "eval_loss": 1.663284420967102, + "eval_runtime": 47.2822, + "eval_samples_per_second": 21.15, + "eval_steps_per_second": 0.888, + "step": 1040 + }, + { + "epoch": 1.2955818294959551, + "grad_norm": 2.909109115600586, + "learning_rate": 3.189575645756458e-06, + "loss": 1.4795, + "step": 1041 + }, + { + "epoch": 1.2968263845675172, + "grad_norm": 2.933673143386841, + "learning_rate": 3.187269372693727e-06, + "loss": 1.5038, + "step": 1042 + }, + { + "epoch": 1.298070939639079, + "grad_norm": 3.032518148422241, + "learning_rate": 3.1849630996309964e-06, + "loss": 1.5133, + "step": 1043 + }, + { + "epoch": 1.299315494710641, + "grad_norm": 2.826040506362915, + "learning_rate": 3.1826568265682663e-06, + "loss": 1.4395, + "step": 1044 + }, + { + "epoch": 1.3005600497822027, + "grad_norm": 2.8152658939361572, + "learning_rate": 3.1803505535055353e-06, + "loss": 1.4787, + "step": 1045 + }, + { + "epoch": 1.3018046048537648, + "grad_norm": 2.9410431385040283, + "learning_rate": 3.1780442804428047e-06, + "loss": 1.4375, + "step": 1046 + }, + { + "epoch": 1.3030491599253267, + "grad_norm": 2.948636531829834, + "learning_rate": 3.175738007380074e-06, + "loss": 1.4847, + "step": 1047 + }, + { + "epoch": 1.3042937149968887, + "grad_norm": 2.6287615299224854, + "learning_rate": 3.173431734317343e-06, + "loss": 1.4186, + "step": 1048 + }, + { + "epoch": 1.3055382700684506, + "grad_norm": 2.836029052734375, + "learning_rate": 3.1711254612546125e-06, + "loss": 1.4251, + "step": 1049 + }, + { + "epoch": 1.3067828251400124, + "grad_norm": 2.982081651687622, + "learning_rate": 3.1688191881918824e-06, + "loss": 1.4951, + "step": 1050 + }, + { + "epoch": 1.3067828251400124, + "eval_loss": 1.6578067541122437, + "eval_runtime": 47.5847, + "eval_samples_per_second": 21.015, + "eval_steps_per_second": 0.883, + "step": 1050 + }, + { + "epoch": 1.3080273802115743, + "grad_norm": 2.9297118186950684, + "learning_rate": 3.166512915129152e-06, + "loss": 1.4699, + "step": 1051 + }, + { + "epoch": 1.3092719352831363, + "grad_norm": 2.906435251235962, + "learning_rate": 3.164206642066421e-06, + "loss": 1.4562, + "step": 1052 + }, + { + "epoch": 1.3105164903546982, + "grad_norm": 2.8655896186828613, + "learning_rate": 3.1619003690036902e-06, + "loss": 1.4351, + "step": 1053 + }, + { + "epoch": 1.31176104542626, + "grad_norm": 2.779244899749756, + "learning_rate": 3.15959409594096e-06, + "loss": 1.4479, + "step": 1054 + }, + { + "epoch": 1.3130056004978221, + "grad_norm": 2.9863739013671875, + "learning_rate": 3.157287822878229e-06, + "loss": 1.4964, + "step": 1055 + }, + { + "epoch": 1.314250155569384, + "grad_norm": 2.821871042251587, + "learning_rate": 3.1549815498154985e-06, + "loss": 1.4173, + "step": 1056 + }, + { + "epoch": 1.3154947106409458, + "grad_norm": 2.8573882579803467, + "learning_rate": 3.152675276752768e-06, + "loss": 1.4877, + "step": 1057 + }, + { + "epoch": 1.3167392657125077, + "grad_norm": 2.9824137687683105, + "learning_rate": 3.150369003690037e-06, + "loss": 1.5124, + "step": 1058 + }, + { + "epoch": 1.3179838207840697, + "grad_norm": 2.881098747253418, + "learning_rate": 3.1480627306273063e-06, + "loss": 1.4346, + "step": 1059 + }, + { + "epoch": 1.3192283758556316, + "grad_norm": 2.918782949447632, + "learning_rate": 3.145756457564576e-06, + "loss": 1.4797, + "step": 1060 + }, + { + "epoch": 1.3192283758556316, + "eval_loss": 1.6510179042816162, + "eval_runtime": 53.6356, + "eval_samples_per_second": 18.644, + "eval_steps_per_second": 0.783, + "step": 1060 + }, + { + "epoch": 1.3204729309271936, + "grad_norm": 3.045776128768921, + "learning_rate": 3.143450184501845e-06, + "loss": 1.5039, + "step": 1061 + }, + { + "epoch": 1.3217174859987555, + "grad_norm": 3.041501760482788, + "learning_rate": 3.1411439114391146e-06, + "loss": 1.4878, + "step": 1062 + }, + { + "epoch": 1.3229620410703173, + "grad_norm": 2.917837142944336, + "learning_rate": 3.138837638376384e-06, + "loss": 1.4202, + "step": 1063 + }, + { + "epoch": 1.3242065961418792, + "grad_norm": 2.708070755004883, + "learning_rate": 3.1365313653136535e-06, + "loss": 1.4071, + "step": 1064 + }, + { + "epoch": 1.3254511512134413, + "grad_norm": 2.9146156311035156, + "learning_rate": 3.1342250922509225e-06, + "loss": 1.4428, + "step": 1065 + }, + { + "epoch": 1.326695706285003, + "grad_norm": 3.220919132232666, + "learning_rate": 3.1319188191881923e-06, + "loss": 1.4733, + "step": 1066 + }, + { + "epoch": 1.327940261356565, + "grad_norm": 2.9127554893493652, + "learning_rate": 3.1296125461254617e-06, + "loss": 1.4554, + "step": 1067 + }, + { + "epoch": 1.329184816428127, + "grad_norm": 3.053191661834717, + "learning_rate": 3.1273062730627307e-06, + "loss": 1.4044, + "step": 1068 + }, + { + "epoch": 1.3304293714996889, + "grad_norm": 3.2937021255493164, + "learning_rate": 3.125e-06, + "loss": 1.4949, + "step": 1069 + }, + { + "epoch": 1.3316739265712507, + "grad_norm": 2.976717948913574, + "learning_rate": 3.12269372693727e-06, + "loss": 1.446, + "step": 1070 + }, + { + "epoch": 1.3316739265712507, + "eval_loss": 1.6503105163574219, + "eval_runtime": 50.7579, + "eval_samples_per_second": 19.701, + "eval_steps_per_second": 0.827, + "step": 1070 + }, + { + "epoch": 1.3329184816428126, + "grad_norm": 3.0860936641693115, + "learning_rate": 3.120387453874539e-06, + "loss": 1.4055, + "step": 1071 + }, + { + "epoch": 1.3341630367143746, + "grad_norm": 3.0401217937469482, + "learning_rate": 3.1180811808118084e-06, + "loss": 1.4688, + "step": 1072 + }, + { + "epoch": 1.3354075917859365, + "grad_norm": 2.953253984451294, + "learning_rate": 3.115774907749078e-06, + "loss": 1.5001, + "step": 1073 + }, + { + "epoch": 1.3366521468574986, + "grad_norm": 3.0665059089660645, + "learning_rate": 3.113468634686347e-06, + "loss": 1.4217, + "step": 1074 + }, + { + "epoch": 1.3378967019290604, + "grad_norm": 3.121600389480591, + "learning_rate": 3.1111623616236163e-06, + "loss": 1.4508, + "step": 1075 + }, + { + "epoch": 1.3391412570006223, + "grad_norm": 2.8057339191436768, + "learning_rate": 3.108856088560886e-06, + "loss": 1.4241, + "step": 1076 + }, + { + "epoch": 1.340385812072184, + "grad_norm": 3.0147411823272705, + "learning_rate": 3.1065498154981555e-06, + "loss": 1.479, + "step": 1077 + }, + { + "epoch": 1.3416303671437462, + "grad_norm": 3.030010223388672, + "learning_rate": 3.1042435424354245e-06, + "loss": 1.4861, + "step": 1078 + }, + { + "epoch": 1.342874922215308, + "grad_norm": 3.027998208999634, + "learning_rate": 3.101937269372694e-06, + "loss": 1.4916, + "step": 1079 + }, + { + "epoch": 1.3441194772868699, + "grad_norm": 2.9252536296844482, + "learning_rate": 3.0996309963099634e-06, + "loss": 1.418, + "step": 1080 + }, + { + "epoch": 1.3441194772868699, + "eval_loss": 1.6490333080291748, + "eval_runtime": 47.4026, + "eval_samples_per_second": 21.096, + "eval_steps_per_second": 0.886, + "step": 1080 + }, + { + "epoch": 1.345364032358432, + "grad_norm": 3.110847234725952, + "learning_rate": 3.0973247232472324e-06, + "loss": 1.4127, + "step": 1081 + }, + { + "epoch": 1.3466085874299938, + "grad_norm": 2.846491575241089, + "learning_rate": 3.0950184501845022e-06, + "loss": 1.444, + "step": 1082 + }, + { + "epoch": 1.3478531425015556, + "grad_norm": 2.950533151626587, + "learning_rate": 3.0927121771217716e-06, + "loss": 1.4456, + "step": 1083 + }, + { + "epoch": 1.3490976975731175, + "grad_norm": 2.875964403152466, + "learning_rate": 3.0904059040590406e-06, + "loss": 1.4485, + "step": 1084 + }, + { + "epoch": 1.3503422526446796, + "grad_norm": 2.9612326622009277, + "learning_rate": 3.08809963099631e-06, + "loss": 1.4446, + "step": 1085 + }, + { + "epoch": 1.3515868077162414, + "grad_norm": 2.925297975540161, + "learning_rate": 3.0857933579335795e-06, + "loss": 1.4847, + "step": 1086 + }, + { + "epoch": 1.3528313627878035, + "grad_norm": 2.991616725921631, + "learning_rate": 3.0834870848708485e-06, + "loss": 1.5151, + "step": 1087 + }, + { + "epoch": 1.3540759178593653, + "grad_norm": 2.96329927444458, + "learning_rate": 3.0811808118081183e-06, + "loss": 1.4574, + "step": 1088 + }, + { + "epoch": 1.3553204729309272, + "grad_norm": 2.948702573776245, + "learning_rate": 3.0788745387453878e-06, + "loss": 1.4585, + "step": 1089 + }, + { + "epoch": 1.356565028002489, + "grad_norm": 3.218463659286499, + "learning_rate": 3.076568265682657e-06, + "loss": 1.4439, + "step": 1090 + }, + { + "epoch": 1.356565028002489, + "eval_loss": 1.645838975906372, + "eval_runtime": 47.048, + "eval_samples_per_second": 21.255, + "eval_steps_per_second": 0.893, + "step": 1090 + }, + { + "epoch": 1.357809583074051, + "grad_norm": 2.9799578189849854, + "learning_rate": 3.074261992619926e-06, + "loss": 1.4883, + "step": 1091 + }, + { + "epoch": 1.359054138145613, + "grad_norm": 2.7660505771636963, + "learning_rate": 3.071955719557196e-06, + "loss": 1.4349, + "step": 1092 + }, + { + "epoch": 1.3602986932171748, + "grad_norm": 3.0201916694641113, + "learning_rate": 3.0696494464944655e-06, + "loss": 1.418, + "step": 1093 + }, + { + "epoch": 1.3615432482887369, + "grad_norm": 2.844207763671875, + "learning_rate": 3.0673431734317345e-06, + "loss": 1.4769, + "step": 1094 + }, + { + "epoch": 1.3627878033602987, + "grad_norm": 2.8306753635406494, + "learning_rate": 3.065036900369004e-06, + "loss": 1.4653, + "step": 1095 + }, + { + "epoch": 1.3640323584318605, + "grad_norm": 2.805023431777954, + "learning_rate": 3.0627306273062733e-06, + "loss": 1.4771, + "step": 1096 + }, + { + "epoch": 1.3652769135034224, + "grad_norm": 3.020883083343506, + "learning_rate": 3.0604243542435423e-06, + "loss": 1.4237, + "step": 1097 + }, + { + "epoch": 1.3665214685749845, + "grad_norm": 2.8395731449127197, + "learning_rate": 3.058118081180812e-06, + "loss": 1.4647, + "step": 1098 + }, + { + "epoch": 1.3677660236465463, + "grad_norm": 2.871156692504883, + "learning_rate": 3.0558118081180816e-06, + "loss": 1.4196, + "step": 1099 + }, + { + "epoch": 1.3690105787181084, + "grad_norm": 2.974905014038086, + "learning_rate": 3.053505535055351e-06, + "loss": 1.4779, + "step": 1100 + }, + { + "epoch": 1.3690105787181084, + "eval_loss": 1.651503562927246, + "eval_runtime": 54.2713, + "eval_samples_per_second": 18.426, + "eval_steps_per_second": 0.774, + "step": 1100 + }, + { + "epoch": 1.3702551337896702, + "grad_norm": 2.7591328620910645, + "learning_rate": 3.05119926199262e-06, + "loss": 1.3899, + "step": 1101 + }, + { + "epoch": 1.371499688861232, + "grad_norm": 2.8575668334960938, + "learning_rate": 3.0488929889298894e-06, + "loss": 1.4482, + "step": 1102 + }, + { + "epoch": 1.372744243932794, + "grad_norm": 2.9049339294433594, + "learning_rate": 3.0465867158671593e-06, + "loss": 1.4785, + "step": 1103 + }, + { + "epoch": 1.373988799004356, + "grad_norm": 2.816453695297241, + "learning_rate": 3.0442804428044283e-06, + "loss": 1.4601, + "step": 1104 + }, + { + "epoch": 1.3752333540759178, + "grad_norm": 2.8844189643859863, + "learning_rate": 3.0419741697416977e-06, + "loss": 1.496, + "step": 1105 + }, + { + "epoch": 1.3764779091474797, + "grad_norm": 3.014744281768799, + "learning_rate": 3.039667896678967e-06, + "loss": 1.4098, + "step": 1106 + }, + { + "epoch": 1.3777224642190418, + "grad_norm": 2.8243560791015625, + "learning_rate": 3.037361623616236e-06, + "loss": 1.4086, + "step": 1107 + }, + { + "epoch": 1.3789670192906036, + "grad_norm": 2.8333253860473633, + "learning_rate": 3.035055350553506e-06, + "loss": 1.3516, + "step": 1108 + }, + { + "epoch": 1.3802115743621655, + "grad_norm": 2.926455020904541, + "learning_rate": 3.0327490774907754e-06, + "loss": 1.3976, + "step": 1109 + }, + { + "epoch": 1.3814561294337273, + "grad_norm": 2.900937080383301, + "learning_rate": 3.0304428044280444e-06, + "loss": 1.4336, + "step": 1110 + }, + { + "epoch": 1.3814561294337273, + "eval_loss": 1.647048830986023, + "eval_runtime": 42.3955, + "eval_samples_per_second": 23.587, + "eval_steps_per_second": 0.991, + "step": 1110 + }, + { + "epoch": 1.3827006845052894, + "grad_norm": 3.0756969451904297, + "learning_rate": 3.028136531365314e-06, + "loss": 1.4674, + "step": 1111 + }, + { + "epoch": 1.3839452395768512, + "grad_norm": 2.7754578590393066, + "learning_rate": 3.0258302583025832e-06, + "loss": 1.4435, + "step": 1112 + }, + { + "epoch": 1.3851897946484133, + "grad_norm": 2.8011393547058105, + "learning_rate": 3.023523985239853e-06, + "loss": 1.4212, + "step": 1113 + }, + { + "epoch": 1.3864343497199751, + "grad_norm": 2.7063710689544678, + "learning_rate": 3.021217712177122e-06, + "loss": 1.3769, + "step": 1114 + }, + { + "epoch": 1.387678904791537, + "grad_norm": 2.830411434173584, + "learning_rate": 3.0189114391143915e-06, + "loss": 1.4832, + "step": 1115 + }, + { + "epoch": 1.3889234598630988, + "grad_norm": 2.9075911045074463, + "learning_rate": 3.016605166051661e-06, + "loss": 1.486, + "step": 1116 + }, + { + "epoch": 1.390168014934661, + "grad_norm": 2.847822666168213, + "learning_rate": 3.01429889298893e-06, + "loss": 1.424, + "step": 1117 + }, + { + "epoch": 1.3914125700062228, + "grad_norm": 2.8822033405303955, + "learning_rate": 3.0119926199261993e-06, + "loss": 1.4218, + "step": 1118 + }, + { + "epoch": 1.3926571250777848, + "grad_norm": 2.8944740295410156, + "learning_rate": 3.009686346863469e-06, + "loss": 1.4283, + "step": 1119 + }, + { + "epoch": 1.3939016801493467, + "grad_norm": 2.765190839767456, + "learning_rate": 3.007380073800738e-06, + "loss": 1.4622, + "step": 1120 + }, + { + "epoch": 1.3939016801493467, + "eval_loss": 1.6451724767684937, + "eval_runtime": 43.7449, + "eval_samples_per_second": 22.86, + "eval_steps_per_second": 0.96, + "step": 1120 + }, + { + "epoch": 1.3951462352209085, + "grad_norm": 2.877002716064453, + "learning_rate": 3.0050738007380076e-06, + "loss": 1.4145, + "step": 1121 + }, + { + "epoch": 1.3963907902924704, + "grad_norm": 2.870389699935913, + "learning_rate": 3.002767527675277e-06, + "loss": 1.4401, + "step": 1122 + }, + { + "epoch": 1.3976353453640322, + "grad_norm": 2.7788920402526855, + "learning_rate": 3.000461254612546e-06, + "loss": 1.4514, + "step": 1123 + }, + { + "epoch": 1.3988799004355943, + "grad_norm": 2.9269332885742188, + "learning_rate": 2.998154981549816e-06, + "loss": 1.3961, + "step": 1124 + }, + { + "epoch": 1.4001244555071561, + "grad_norm": 3.013291835784912, + "learning_rate": 2.9958487084870853e-06, + "loss": 1.4597, + "step": 1125 + }, + { + "epoch": 1.4013690105787182, + "grad_norm": 2.9063880443573, + "learning_rate": 2.9935424354243547e-06, + "loss": 1.4668, + "step": 1126 + }, + { + "epoch": 1.40261356565028, + "grad_norm": 2.8926970958709717, + "learning_rate": 2.9912361623616237e-06, + "loss": 1.4332, + "step": 1127 + }, + { + "epoch": 1.403858120721842, + "grad_norm": 2.8563647270202637, + "learning_rate": 2.988929889298893e-06, + "loss": 1.407, + "step": 1128 + }, + { + "epoch": 1.4051026757934038, + "grad_norm": 2.8725359439849854, + "learning_rate": 2.986623616236163e-06, + "loss": 1.4434, + "step": 1129 + }, + { + "epoch": 1.4063472308649658, + "grad_norm": 2.906898260116577, + "learning_rate": 2.984317343173432e-06, + "loss": 1.4745, + "step": 1130 + }, + { + "epoch": 1.4063472308649658, + "eval_loss": 1.647884726524353, + "eval_runtime": 44.9221, + "eval_samples_per_second": 22.261, + "eval_steps_per_second": 0.935, + "step": 1130 + }, + { + "epoch": 1.4075917859365277, + "grad_norm": 2.8297719955444336, + "learning_rate": 2.9820110701107014e-06, + "loss": 1.4542, + "step": 1131 + }, + { + "epoch": 1.4088363410080897, + "grad_norm": 2.841031551361084, + "learning_rate": 2.979704797047971e-06, + "loss": 1.4105, + "step": 1132 + }, + { + "epoch": 1.4100808960796516, + "grad_norm": 2.867791175842285, + "learning_rate": 2.97739852398524e-06, + "loss": 1.4409, + "step": 1133 + }, + { + "epoch": 1.4113254511512134, + "grad_norm": 2.8156039714813232, + "learning_rate": 2.9750922509225093e-06, + "loss": 1.4284, + "step": 1134 + }, + { + "epoch": 1.4125700062227753, + "grad_norm": 2.825985908508301, + "learning_rate": 2.972785977859779e-06, + "loss": 1.4594, + "step": 1135 + }, + { + "epoch": 1.4138145612943374, + "grad_norm": 3.0134711265563965, + "learning_rate": 2.970479704797048e-06, + "loss": 1.4743, + "step": 1136 + }, + { + "epoch": 1.4150591163658992, + "grad_norm": 2.9829142093658447, + "learning_rate": 2.9681734317343175e-06, + "loss": 1.4935, + "step": 1137 + }, + { + "epoch": 1.416303671437461, + "grad_norm": 3.067620277404785, + "learning_rate": 2.965867158671587e-06, + "loss": 1.4266, + "step": 1138 + }, + { + "epoch": 1.4175482265090231, + "grad_norm": 2.8969919681549072, + "learning_rate": 2.9635608856088564e-06, + "loss": 1.451, + "step": 1139 + }, + { + "epoch": 1.418792781580585, + "grad_norm": 2.8706412315368652, + "learning_rate": 2.9612546125461254e-06, + "loss": 1.4834, + "step": 1140 + }, + { + "epoch": 1.418792781580585, + "eval_loss": 1.643480658531189, + "eval_runtime": 44.743, + "eval_samples_per_second": 22.35, + "eval_steps_per_second": 0.939, + "step": 1140 + }, + { + "epoch": 1.4200373366521468, + "grad_norm": 2.8801848888397217, + "learning_rate": 2.958948339483395e-06, + "loss": 1.4258, + "step": 1141 + }, + { + "epoch": 1.4212818917237087, + "grad_norm": 2.774635076522827, + "learning_rate": 2.9566420664206646e-06, + "loss": 1.416, + "step": 1142 + }, + { + "epoch": 1.4225264467952707, + "grad_norm": 2.8460140228271484, + "learning_rate": 2.9543357933579336e-06, + "loss": 1.4213, + "step": 1143 + }, + { + "epoch": 1.4237710018668326, + "grad_norm": 2.907888889312744, + "learning_rate": 2.952029520295203e-06, + "loss": 1.4507, + "step": 1144 + }, + { + "epoch": 1.4250155569383947, + "grad_norm": 2.789041757583618, + "learning_rate": 2.949723247232473e-06, + "loss": 1.4257, + "step": 1145 + }, + { + "epoch": 1.4262601120099565, + "grad_norm": 2.78859806060791, + "learning_rate": 2.947416974169742e-06, + "loss": 1.4377, + "step": 1146 + }, + { + "epoch": 1.4275046670815184, + "grad_norm": 2.840303421020508, + "learning_rate": 2.9451107011070113e-06, + "loss": 1.4684, + "step": 1147 + }, + { + "epoch": 1.4287492221530802, + "grad_norm": 2.8800530433654785, + "learning_rate": 2.9428044280442807e-06, + "loss": 1.4912, + "step": 1148 + }, + { + "epoch": 1.4299937772246423, + "grad_norm": 2.841092824935913, + "learning_rate": 2.9404981549815497e-06, + "loss": 1.4234, + "step": 1149 + }, + { + "epoch": 1.4312383322962041, + "grad_norm": 2.8226537704467773, + "learning_rate": 2.938191881918819e-06, + "loss": 1.4578, + "step": 1150 + }, + { + "epoch": 1.4312383322962041, + "eval_loss": 1.643249273300171, + "eval_runtime": 43.5586, + "eval_samples_per_second": 22.958, + "eval_steps_per_second": 0.964, + "step": 1150 + }, + { + "epoch": 1.432482887367766, + "grad_norm": 2.869736909866333, + "learning_rate": 2.935885608856089e-06, + "loss": 1.4701, + "step": 1151 + }, + { + "epoch": 1.433727442439328, + "grad_norm": 2.689211130142212, + "learning_rate": 2.9335793357933584e-06, + "loss": 1.4483, + "step": 1152 + }, + { + "epoch": 1.4349719975108899, + "grad_norm": 2.9267077445983887, + "learning_rate": 2.9312730627306274e-06, + "loss": 1.4446, + "step": 1153 + }, + { + "epoch": 1.4362165525824517, + "grad_norm": 2.906862497329712, + "learning_rate": 2.928966789667897e-06, + "loss": 1.4524, + "step": 1154 + }, + { + "epoch": 1.4374611076540136, + "grad_norm": 2.8249800205230713, + "learning_rate": 2.9266605166051663e-06, + "loss": 1.4208, + "step": 1155 + }, + { + "epoch": 1.4387056627255757, + "grad_norm": 2.9836032390594482, + "learning_rate": 2.9243542435424353e-06, + "loss": 1.4581, + "step": 1156 + }, + { + "epoch": 1.4399502177971375, + "grad_norm": 2.9595284461975098, + "learning_rate": 2.922047970479705e-06, + "loss": 1.4141, + "step": 1157 + }, + { + "epoch": 1.4411947728686996, + "grad_norm": 3.0363423824310303, + "learning_rate": 2.9197416974169746e-06, + "loss": 1.4859, + "step": 1158 + }, + { + "epoch": 1.4424393279402614, + "grad_norm": 2.911201000213623, + "learning_rate": 2.9174354243542436e-06, + "loss": 1.4346, + "step": 1159 + }, + { + "epoch": 1.4436838830118233, + "grad_norm": 2.8617193698883057, + "learning_rate": 2.915129151291513e-06, + "loss": 1.433, + "step": 1160 + }, + { + "epoch": 1.4436838830118233, + "eval_loss": 1.6437232494354248, + "eval_runtime": 43.961, + "eval_samples_per_second": 22.747, + "eval_steps_per_second": 0.955, + "step": 1160 + }, + { + "epoch": 1.4449284380833851, + "grad_norm": 2.9584295749664307, + "learning_rate": 2.912822878228783e-06, + "loss": 1.4847, + "step": 1161 + }, + { + "epoch": 1.4461729931549472, + "grad_norm": 2.8209400177001953, + "learning_rate": 2.9105166051660522e-06, + "loss": 1.421, + "step": 1162 + }, + { + "epoch": 1.447417548226509, + "grad_norm": 2.847637414932251, + "learning_rate": 2.9082103321033212e-06, + "loss": 1.4326, + "step": 1163 + }, + { + "epoch": 1.4486621032980709, + "grad_norm": 2.836228132247925, + "learning_rate": 2.9059040590405907e-06, + "loss": 1.4459, + "step": 1164 + }, + { + "epoch": 1.449906658369633, + "grad_norm": 2.8327207565307617, + "learning_rate": 2.90359778597786e-06, + "loss": 1.4189, + "step": 1165 + }, + { + "epoch": 1.4511512134411948, + "grad_norm": 2.880643606185913, + "learning_rate": 2.901291512915129e-06, + "loss": 1.4214, + "step": 1166 + }, + { + "epoch": 1.4523957685127566, + "grad_norm": 2.9637348651885986, + "learning_rate": 2.898985239852399e-06, + "loss": 1.4471, + "step": 1167 + }, + { + "epoch": 1.4536403235843185, + "grad_norm": 3.006145477294922, + "learning_rate": 2.8966789667896684e-06, + "loss": 1.4228, + "step": 1168 + }, + { + "epoch": 1.4548848786558806, + "grad_norm": 2.8547627925872803, + "learning_rate": 2.8943726937269374e-06, + "loss": 1.4707, + "step": 1169 + }, + { + "epoch": 1.4561294337274424, + "grad_norm": 2.973092794418335, + "learning_rate": 2.8920664206642068e-06, + "loss": 1.4804, + "step": 1170 + }, + { + "epoch": 1.4561294337274424, + "eval_loss": 1.6340677738189697, + "eval_runtime": 44.6094, + "eval_samples_per_second": 22.417, + "eval_steps_per_second": 0.942, + "step": 1170 + }, + { + "epoch": 1.4573739887990045, + "grad_norm": 2.9433603286743164, + "learning_rate": 2.889760147601476e-06, + "loss": 1.4354, + "step": 1171 + }, + { + "epoch": 1.4586185438705663, + "grad_norm": 2.872236728668213, + "learning_rate": 2.887453874538745e-06, + "loss": 1.418, + "step": 1172 + }, + { + "epoch": 1.4598630989421282, + "grad_norm": 2.847557783126831, + "learning_rate": 2.885147601476015e-06, + "loss": 1.4306, + "step": 1173 + }, + { + "epoch": 1.46110765401369, + "grad_norm": 2.7554969787597656, + "learning_rate": 2.8828413284132845e-06, + "loss": 1.4295, + "step": 1174 + }, + { + "epoch": 1.462352209085252, + "grad_norm": 2.8754842281341553, + "learning_rate": 2.880535055350554e-06, + "loss": 1.4482, + "step": 1175 + }, + { + "epoch": 1.463596764156814, + "grad_norm": 3.0115292072296143, + "learning_rate": 2.878228782287823e-06, + "loss": 1.4428, + "step": 1176 + }, + { + "epoch": 1.4648413192283758, + "grad_norm": 2.8976168632507324, + "learning_rate": 2.8759225092250923e-06, + "loss": 1.4707, + "step": 1177 + }, + { + "epoch": 1.4660858742999379, + "grad_norm": 2.7981812953948975, + "learning_rate": 2.873616236162362e-06, + "loss": 1.4693, + "step": 1178 + }, + { + "epoch": 1.4673304293714997, + "grad_norm": 2.8562371730804443, + "learning_rate": 2.871309963099631e-06, + "loss": 1.4242, + "step": 1179 + }, + { + "epoch": 1.4685749844430616, + "grad_norm": 2.8705570697784424, + "learning_rate": 2.8690036900369006e-06, + "loss": 1.4725, + "step": 1180 + }, + { + "epoch": 1.4685749844430616, + "eval_loss": 1.6338286399841309, + "eval_runtime": 44.7171, + "eval_samples_per_second": 22.363, + "eval_steps_per_second": 0.939, + "step": 1180 + }, + { + "epoch": 1.4698195395146234, + "grad_norm": 2.8025710582733154, + "learning_rate": 2.86669741697417e-06, + "loss": 1.427, + "step": 1181 + }, + { + "epoch": 1.4710640945861855, + "grad_norm": 2.845151901245117, + "learning_rate": 2.864391143911439e-06, + "loss": 1.4402, + "step": 1182 + }, + { + "epoch": 1.4723086496577473, + "grad_norm": 2.943373680114746, + "learning_rate": 2.862084870848709e-06, + "loss": 1.5194, + "step": 1183 + }, + { + "epoch": 1.4735532047293094, + "grad_norm": 2.896955966949463, + "learning_rate": 2.8597785977859783e-06, + "loss": 1.4242, + "step": 1184 + }, + { + "epoch": 1.4747977598008712, + "grad_norm": 3.131305456161499, + "learning_rate": 2.8574723247232473e-06, + "loss": 1.5008, + "step": 1185 + }, + { + "epoch": 1.476042314872433, + "grad_norm": 2.8605735301971436, + "learning_rate": 2.8551660516605167e-06, + "loss": 1.3533, + "step": 1186 + }, + { + "epoch": 1.477286869943995, + "grad_norm": 2.9158811569213867, + "learning_rate": 2.852859778597786e-06, + "loss": 1.4404, + "step": 1187 + }, + { + "epoch": 1.478531425015557, + "grad_norm": 2.7795321941375732, + "learning_rate": 2.850553505535056e-06, + "loss": 1.3977, + "step": 1188 + }, + { + "epoch": 1.4797759800871189, + "grad_norm": 2.8194212913513184, + "learning_rate": 2.848247232472325e-06, + "loss": 1.4111, + "step": 1189 + }, + { + "epoch": 1.4810205351586807, + "grad_norm": 2.983750104904175, + "learning_rate": 2.8459409594095944e-06, + "loss": 1.4061, + "step": 1190 + }, + { + "epoch": 1.4810205351586807, + "eval_loss": 1.6375409364700317, + "eval_runtime": 45.4026, + "eval_samples_per_second": 22.025, + "eval_steps_per_second": 0.925, + "step": 1190 + }, + { + "epoch": 1.4822650902302428, + "grad_norm": 3.0246665477752686, + "learning_rate": 2.843634686346864e-06, + "loss": 1.4532, + "step": 1191 + }, + { + "epoch": 1.4835096453018046, + "grad_norm": 2.8856449127197266, + "learning_rate": 2.841328413284133e-06, + "loss": 1.4528, + "step": 1192 + }, + { + "epoch": 1.4847542003733665, + "grad_norm": 2.848987102508545, + "learning_rate": 2.8390221402214022e-06, + "loss": 1.4369, + "step": 1193 + }, + { + "epoch": 1.4859987554449283, + "grad_norm": 2.970419406890869, + "learning_rate": 2.836715867158672e-06, + "loss": 1.4605, + "step": 1194 + }, + { + "epoch": 1.4872433105164904, + "grad_norm": 3.156613826751709, + "learning_rate": 2.834409594095941e-06, + "loss": 1.4489, + "step": 1195 + }, + { + "epoch": 1.4884878655880522, + "grad_norm": 3.095349073410034, + "learning_rate": 2.8321033210332105e-06, + "loss": 1.4604, + "step": 1196 + }, + { + "epoch": 1.4897324206596143, + "grad_norm": 2.870742082595825, + "learning_rate": 2.82979704797048e-06, + "loss": 1.4284, + "step": 1197 + }, + { + "epoch": 1.4909769757311762, + "grad_norm": 2.9415056705474854, + "learning_rate": 2.827490774907749e-06, + "loss": 1.4715, + "step": 1198 + }, + { + "epoch": 1.492221530802738, + "grad_norm": 2.8297767639160156, + "learning_rate": 2.8251845018450188e-06, + "loss": 1.4508, + "step": 1199 + }, + { + "epoch": 1.4934660858742999, + "grad_norm": 2.872086524963379, + "learning_rate": 2.822878228782288e-06, + "loss": 1.5098, + "step": 1200 + }, + { + "epoch": 1.4934660858742999, + "eval_loss": 1.6357526779174805, + "eval_runtime": 51.422, + "eval_samples_per_second": 19.447, + "eval_steps_per_second": 0.817, + "step": 1200 + }, + { + "epoch": 1.494710640945862, + "grad_norm": 2.7152082920074463, + "learning_rate": 2.8205719557195576e-06, + "loss": 1.4329, + "step": 1201 + }, + { + "epoch": 1.4959551960174238, + "grad_norm": 2.866170883178711, + "learning_rate": 2.8182656826568266e-06, + "loss": 1.4455, + "step": 1202 + }, + { + "epoch": 1.4971997510889856, + "grad_norm": 2.9010684490203857, + "learning_rate": 2.815959409594096e-06, + "loss": 1.423, + "step": 1203 + }, + { + "epoch": 1.4984443061605477, + "grad_norm": 2.8588502407073975, + "learning_rate": 2.813653136531366e-06, + "loss": 1.4041, + "step": 1204 + }, + { + "epoch": 1.4996888612321095, + "grad_norm": 2.932544231414795, + "learning_rate": 2.811346863468635e-06, + "loss": 1.4559, + "step": 1205 + }, + { + "epoch": 1.5009334163036714, + "grad_norm": 2.7486014366149902, + "learning_rate": 2.8090405904059043e-06, + "loss": 1.4337, + "step": 1206 + }, + { + "epoch": 1.5021779713752332, + "grad_norm": 3.0866589546203613, + "learning_rate": 2.8067343173431737e-06, + "loss": 1.4186, + "step": 1207 + }, + { + "epoch": 1.5034225264467953, + "grad_norm": 2.9710276126861572, + "learning_rate": 2.8044280442804427e-06, + "loss": 1.419, + "step": 1208 + }, + { + "epoch": 1.5046670815183572, + "grad_norm": 2.8379950523376465, + "learning_rate": 2.802121771217712e-06, + "loss": 1.4922, + "step": 1209 + }, + { + "epoch": 1.5059116365899192, + "grad_norm": 2.939629554748535, + "learning_rate": 2.799815498154982e-06, + "loss": 1.4402, + "step": 1210 + }, + { + "epoch": 1.5059116365899192, + "eval_loss": 1.6419442892074585, + "eval_runtime": 43.1202, + "eval_samples_per_second": 23.191, + "eval_steps_per_second": 0.974, + "step": 1210 + }, + { + "epoch": 1.507156191661481, + "grad_norm": 3.041389226913452, + "learning_rate": 2.7975092250922514e-06, + "loss": 1.4873, + "step": 1211 + }, + { + "epoch": 1.508400746733043, + "grad_norm": 2.9776995182037354, + "learning_rate": 2.7952029520295204e-06, + "loss": 1.4422, + "step": 1212 + }, + { + "epoch": 1.5096453018046048, + "grad_norm": 2.8798792362213135, + "learning_rate": 2.79289667896679e-06, + "loss": 1.4535, + "step": 1213 + }, + { + "epoch": 1.5108898568761666, + "grad_norm": 2.9016385078430176, + "learning_rate": 2.7905904059040597e-06, + "loss": 1.4462, + "step": 1214 + }, + { + "epoch": 1.5121344119477287, + "grad_norm": 2.8987581729888916, + "learning_rate": 2.7882841328413287e-06, + "loss": 1.4477, + "step": 1215 + }, + { + "epoch": 1.5133789670192908, + "grad_norm": 2.960266351699829, + "learning_rate": 2.785977859778598e-06, + "loss": 1.4811, + "step": 1216 + }, + { + "epoch": 1.5146235220908526, + "grad_norm": 2.8965611457824707, + "learning_rate": 2.7836715867158675e-06, + "loss": 1.4366, + "step": 1217 + }, + { + "epoch": 1.5158680771624145, + "grad_norm": 3.0667364597320557, + "learning_rate": 2.7813653136531365e-06, + "loss": 1.3828, + "step": 1218 + }, + { + "epoch": 1.5171126322339763, + "grad_norm": 2.952362537384033, + "learning_rate": 2.779059040590406e-06, + "loss": 1.3903, + "step": 1219 + }, + { + "epoch": 1.5183571873055381, + "grad_norm": 2.9899795055389404, + "learning_rate": 2.776752767527676e-06, + "loss": 1.4312, + "step": 1220 + }, + { + "epoch": 1.5183571873055381, + "eval_loss": 1.640383005142212, + "eval_runtime": 45.4128, + "eval_samples_per_second": 22.02, + "eval_steps_per_second": 0.925, + "step": 1220 + }, + { + "epoch": 1.5196017423771002, + "grad_norm": 3.2202725410461426, + "learning_rate": 2.774446494464945e-06, + "loss": 1.4344, + "step": 1221 + }, + { + "epoch": 1.520846297448662, + "grad_norm": 3.2202725410461426, + "learning_rate": 2.774446494464945e-06, + "loss": 1.4386, + "step": 1222 + }, + { + "epoch": 1.5220908525202241, + "grad_norm": 2.883223295211792, + "learning_rate": 2.7721402214022142e-06, + "loss": 1.4401, + "step": 1223 + }, + { + "epoch": 1.523335407591786, + "grad_norm": 2.8346424102783203, + "learning_rate": 2.7698339483394837e-06, + "loss": 1.3926, + "step": 1224 + }, + { + "epoch": 1.5245799626633478, + "grad_norm": 2.8898870944976807, + "learning_rate": 2.767527675276753e-06, + "loss": 1.4128, + "step": 1225 + }, + { + "epoch": 1.5258245177349097, + "grad_norm": 2.828594923019409, + "learning_rate": 2.765221402214022e-06, + "loss": 1.4103, + "step": 1226 + }, + { + "epoch": 1.5270690728064715, + "grad_norm": 2.882558822631836, + "learning_rate": 2.762915129151292e-06, + "loss": 1.4186, + "step": 1227 + }, + { + "epoch": 1.5283136278780336, + "grad_norm": 2.955760955810547, + "learning_rate": 2.7606088560885613e-06, + "loss": 1.3705, + "step": 1228 + }, + { + "epoch": 1.5295581829495957, + "grad_norm": 2.862934112548828, + "learning_rate": 2.7583025830258303e-06, + "loss": 1.5012, + "step": 1229 + }, + { + "epoch": 1.5308027380211575, + "grad_norm": 2.926501989364624, + "learning_rate": 2.7559963099630998e-06, + "loss": 1.3875, + "step": 1230 + }, + { + "epoch": 1.5308027380211575, + "eval_loss": 1.6376254558563232, + "eval_runtime": 47.9535, + "eval_samples_per_second": 20.854, + "eval_steps_per_second": 0.876, + "step": 1230 + }, + { + "epoch": 1.5320472930927194, + "grad_norm": 2.7648823261260986, + "learning_rate": 2.753690036900369e-06, + "loss": 1.3781, + "step": 1231 + }, + { + "epoch": 1.5332918481642812, + "grad_norm": 2.927103281021118, + "learning_rate": 2.751383763837638e-06, + "loss": 1.4091, + "step": 1232 + }, + { + "epoch": 1.534536403235843, + "grad_norm": 2.964823007583618, + "learning_rate": 2.749077490774908e-06, + "loss": 1.5075, + "step": 1233 + }, + { + "epoch": 1.5357809583074051, + "grad_norm": 2.7944016456604004, + "learning_rate": 2.7467712177121775e-06, + "loss": 1.3888, + "step": 1234 + }, + { + "epoch": 1.537025513378967, + "grad_norm": 2.891592025756836, + "learning_rate": 2.7444649446494465e-06, + "loss": 1.4825, + "step": 1235 + }, + { + "epoch": 1.538270068450529, + "grad_norm": 2.8120031356811523, + "learning_rate": 2.742158671586716e-06, + "loss": 1.4246, + "step": 1236 + }, + { + "epoch": 1.539514623522091, + "grad_norm": 2.9384255409240723, + "learning_rate": 2.7398523985239857e-06, + "loss": 1.4657, + "step": 1237 + }, + { + "epoch": 1.5407591785936527, + "grad_norm": 2.73111891746521, + "learning_rate": 2.737546125461255e-06, + "loss": 1.4689, + "step": 1238 + }, + { + "epoch": 1.5420037336652146, + "grad_norm": 2.858963966369629, + "learning_rate": 2.735239852398524e-06, + "loss": 1.4626, + "step": 1239 + }, + { + "epoch": 1.5432482887367764, + "grad_norm": 2.9148507118225098, + "learning_rate": 2.7329335793357936e-06, + "loss": 1.4787, + "step": 1240 + }, + { + "epoch": 1.5432482887367764, + "eval_loss": 1.6436412334442139, + "eval_runtime": 47.3139, + "eval_samples_per_second": 21.135, + "eval_steps_per_second": 0.888, + "step": 1240 + }, + { + "epoch": 1.5444928438083385, + "grad_norm": 2.8731327056884766, + "learning_rate": 2.730627306273063e-06, + "loss": 1.4748, + "step": 1241 + }, + { + "epoch": 1.5457373988799006, + "grad_norm": 2.990265369415283, + "learning_rate": 2.728321033210332e-06, + "loss": 1.4375, + "step": 1242 + }, + { + "epoch": 1.5469819539514624, + "grad_norm": 2.874321460723877, + "learning_rate": 2.726014760147602e-06, + "loss": 1.4154, + "step": 1243 + }, + { + "epoch": 1.5482265090230243, + "grad_norm": 2.9316225051879883, + "learning_rate": 2.7237084870848713e-06, + "loss": 1.4285, + "step": 1244 + }, + { + "epoch": 1.5494710640945861, + "grad_norm": 2.783966302871704, + "learning_rate": 2.7214022140221403e-06, + "loss": 1.409, + "step": 1245 + }, + { + "epoch": 1.550715619166148, + "grad_norm": 2.9578707218170166, + "learning_rate": 2.7190959409594097e-06, + "loss": 1.4325, + "step": 1246 + }, + { + "epoch": 1.55196017423771, + "grad_norm": 2.869893789291382, + "learning_rate": 2.716789667896679e-06, + "loss": 1.3989, + "step": 1247 + }, + { + "epoch": 1.553204729309272, + "grad_norm": 2.962944269180298, + "learning_rate": 2.714483394833948e-06, + "loss": 1.4302, + "step": 1248 + }, + { + "epoch": 1.554449284380834, + "grad_norm": 2.974494695663452, + "learning_rate": 2.712177121771218e-06, + "loss": 1.46, + "step": 1249 + }, + { + "epoch": 1.5556938394523958, + "grad_norm": 2.9154603481292725, + "learning_rate": 2.7098708487084874e-06, + "loss": 1.4388, + "step": 1250 + }, + { + "epoch": 1.5556938394523958, + "eval_loss": 1.6357773542404175, + "eval_runtime": 47.6085, + "eval_samples_per_second": 21.005, + "eval_steps_per_second": 0.882, + "step": 1250 + }, + { + "epoch": 1.5569383945239577, + "grad_norm": 3.0333547592163086, + "learning_rate": 2.707564575645757e-06, + "loss": 1.4846, + "step": 1251 + }, + { + "epoch": 1.5581829495955195, + "grad_norm": 3.06097674369812, + "learning_rate": 2.705258302583026e-06, + "loss": 1.4355, + "step": 1252 + }, + { + "epoch": 1.5594275046670814, + "grad_norm": 2.872680425643921, + "learning_rate": 2.7029520295202956e-06, + "loss": 1.4219, + "step": 1253 + }, + { + "epoch": 1.5606720597386434, + "grad_norm": 2.9066731929779053, + "learning_rate": 2.700645756457565e-06, + "loss": 1.384, + "step": 1254 + }, + { + "epoch": 1.5619166148102055, + "grad_norm": 3.0570194721221924, + "learning_rate": 2.698339483394834e-06, + "loss": 1.4687, + "step": 1255 + }, + { + "epoch": 1.5631611698817673, + "grad_norm": 3.1955857276916504, + "learning_rate": 2.6960332103321035e-06, + "loss": 1.4736, + "step": 1256 + }, + { + "epoch": 1.5644057249533292, + "grad_norm": 2.9686198234558105, + "learning_rate": 2.693726937269373e-06, + "loss": 1.4176, + "step": 1257 + }, + { + "epoch": 1.565650280024891, + "grad_norm": 3.1161272525787354, + "learning_rate": 2.691420664206642e-06, + "loss": 1.443, + "step": 1258 + }, + { + "epoch": 1.5668948350964529, + "grad_norm": 2.9031193256378174, + "learning_rate": 2.6891143911439118e-06, + "loss": 1.3871, + "step": 1259 + }, + { + "epoch": 1.568139390168015, + "grad_norm": 2.9120476245880127, + "learning_rate": 2.686808118081181e-06, + "loss": 1.4406, + "step": 1260 + }, + { + "epoch": 1.568139390168015, + "eval_loss": 1.6291446685791016, + "eval_runtime": 49.7006, + "eval_samples_per_second": 20.12, + "eval_steps_per_second": 0.845, + "step": 1260 + }, + { + "epoch": 1.5693839452395768, + "grad_norm": 3.3181824684143066, + "learning_rate": 2.6845018450184506e-06, + "loss": 1.5278, + "step": 1261 + }, + { + "epoch": 1.5706285003111389, + "grad_norm": 2.9628148078918457, + "learning_rate": 2.6821955719557196e-06, + "loss": 1.4206, + "step": 1262 + }, + { + "epoch": 1.5718730553827007, + "grad_norm": 2.9828474521636963, + "learning_rate": 2.679889298892989e-06, + "loss": 1.5478, + "step": 1263 + }, + { + "epoch": 1.5731176104542626, + "grad_norm": 3.018711805343628, + "learning_rate": 2.677583025830259e-06, + "loss": 1.4442, + "step": 1264 + }, + { + "epoch": 1.5743621655258244, + "grad_norm": 2.8596792221069336, + "learning_rate": 2.675276752767528e-06, + "loss": 1.4325, + "step": 1265 + }, + { + "epoch": 1.5756067205973865, + "grad_norm": 2.9592769145965576, + "learning_rate": 2.6729704797047973e-06, + "loss": 1.4059, + "step": 1266 + }, + { + "epoch": 1.5768512756689483, + "grad_norm": 2.773589611053467, + "learning_rate": 2.6706642066420667e-06, + "loss": 1.3608, + "step": 1267 + }, + { + "epoch": 1.5780958307405104, + "grad_norm": 3.313255548477173, + "learning_rate": 2.6683579335793357e-06, + "loss": 1.4552, + "step": 1268 + }, + { + "epoch": 1.5793403858120723, + "grad_norm": 2.977649211883545, + "learning_rate": 2.6660516605166056e-06, + "loss": 1.4221, + "step": 1269 + }, + { + "epoch": 1.580584940883634, + "grad_norm": 3.0672051906585693, + "learning_rate": 2.663745387453875e-06, + "loss": 1.4568, + "step": 1270 + }, + { + "epoch": 1.580584940883634, + "eval_loss": 1.6317907571792603, + "eval_runtime": 47.2354, + "eval_samples_per_second": 21.171, + "eval_steps_per_second": 0.889, + "step": 1270 + }, + { + "epoch": 1.581829495955196, + "grad_norm": 3.029320240020752, + "learning_rate": 2.661439114391144e-06, + "loss": 1.4432, + "step": 1271 + }, + { + "epoch": 1.5830740510267578, + "grad_norm": 3.0183846950531006, + "learning_rate": 2.6591328413284134e-06, + "loss": 1.4332, + "step": 1272 + }, + { + "epoch": 1.5843186060983199, + "grad_norm": 2.856551170349121, + "learning_rate": 2.656826568265683e-06, + "loss": 1.4291, + "step": 1273 + }, + { + "epoch": 1.5855631611698817, + "grad_norm": 3.124971389770508, + "learning_rate": 2.6545202952029527e-06, + "loss": 1.4363, + "step": 1274 + }, + { + "epoch": 1.5868077162414438, + "grad_norm": 2.9470791816711426, + "learning_rate": 2.6522140221402217e-06, + "loss": 1.4502, + "step": 1275 + }, + { + "epoch": 1.5880522713130056, + "grad_norm": 2.9116358757019043, + "learning_rate": 2.649907749077491e-06, + "loss": 1.4314, + "step": 1276 + }, + { + "epoch": 1.5892968263845675, + "grad_norm": 3.2575323581695557, + "learning_rate": 2.6476014760147605e-06, + "loss": 1.4749, + "step": 1277 + }, + { + "epoch": 1.5905413814561293, + "grad_norm": 2.935518503189087, + "learning_rate": 2.6452952029520295e-06, + "loss": 1.4265, + "step": 1278 + }, + { + "epoch": 1.5917859365276914, + "grad_norm": 2.9750072956085205, + "learning_rate": 2.642988929889299e-06, + "loss": 1.4501, + "step": 1279 + }, + { + "epoch": 1.5930304915992533, + "grad_norm": 2.8086845874786377, + "learning_rate": 2.640682656826569e-06, + "loss": 1.4557, + "step": 1280 + }, + { + "epoch": 1.5930304915992533, + "eval_loss": 1.6302106380462646, + "eval_runtime": 52.3962, + "eval_samples_per_second": 19.085, + "eval_steps_per_second": 0.802, + "step": 1280 + }, + { + "epoch": 1.5942750466708153, + "grad_norm": 2.741849422454834, + "learning_rate": 2.638376383763838e-06, + "loss": 1.3918, + "step": 1281 + }, + { + "epoch": 1.5955196017423772, + "grad_norm": 2.9414806365966797, + "learning_rate": 2.6360701107011072e-06, + "loss": 1.3882, + "step": 1282 + }, + { + "epoch": 1.596764156813939, + "grad_norm": 2.901514768600464, + "learning_rate": 2.6337638376383766e-06, + "loss": 1.4331, + "step": 1283 + }, + { + "epoch": 1.5980087118855009, + "grad_norm": 2.8159122467041016, + "learning_rate": 2.6314575645756456e-06, + "loss": 1.4064, + "step": 1284 + }, + { + "epoch": 1.5992532669570627, + "grad_norm": 2.900561571121216, + "learning_rate": 2.629151291512915e-06, + "loss": 1.3731, + "step": 1285 + }, + { + "epoch": 1.6004978220286248, + "grad_norm": 3.019540548324585, + "learning_rate": 2.626845018450185e-06, + "loss": 1.4506, + "step": 1286 + }, + { + "epoch": 1.6017423771001866, + "grad_norm": 2.9263620376586914, + "learning_rate": 2.6245387453874543e-06, + "loss": 1.4754, + "step": 1287 + }, + { + "epoch": 1.6029869321717487, + "grad_norm": 2.8681881427764893, + "learning_rate": 2.6222324723247233e-06, + "loss": 1.4124, + "step": 1288 + }, + { + "epoch": 1.6042314872433105, + "grad_norm": 2.922945261001587, + "learning_rate": 2.6199261992619928e-06, + "loss": 1.5101, + "step": 1289 + }, + { + "epoch": 1.6054760423148724, + "grad_norm": 2.9548606872558594, + "learning_rate": 2.6176199261992626e-06, + "loss": 1.41, + "step": 1290 + }, + { + "epoch": 1.6054760423148724, + "eval_loss": 1.6357502937316895, + "eval_runtime": 46.9547, + "eval_samples_per_second": 21.297, + "eval_steps_per_second": 0.894, + "step": 1290 + }, + { + "epoch": 1.6067205973864342, + "grad_norm": 2.837184190750122, + "learning_rate": 2.6153136531365316e-06, + "loss": 1.4354, + "step": 1291 + }, + { + "epoch": 1.6079651524579963, + "grad_norm": 2.818990468978882, + "learning_rate": 2.613007380073801e-06, + "loss": 1.4474, + "step": 1292 + }, + { + "epoch": 1.6092097075295582, + "grad_norm": 2.7880146503448486, + "learning_rate": 2.6107011070110704e-06, + "loss": 1.4156, + "step": 1293 + }, + { + "epoch": 1.6104542626011202, + "grad_norm": 2.790971040725708, + "learning_rate": 2.6083948339483394e-06, + "loss": 1.4671, + "step": 1294 + }, + { + "epoch": 1.611698817672682, + "grad_norm": 2.943129777908325, + "learning_rate": 2.606088560885609e-06, + "loss": 1.4801, + "step": 1295 + }, + { + "epoch": 1.612943372744244, + "grad_norm": 2.8589110374450684, + "learning_rate": 2.6037822878228787e-06, + "loss": 1.4894, + "step": 1296 + }, + { + "epoch": 1.6141879278158058, + "grad_norm": 2.8929262161254883, + "learning_rate": 2.6014760147601477e-06, + "loss": 1.4022, + "step": 1297 + }, + { + "epoch": 1.6154324828873676, + "grad_norm": 2.970099925994873, + "learning_rate": 2.599169741697417e-06, + "loss": 1.5136, + "step": 1298 + }, + { + "epoch": 1.6166770379589297, + "grad_norm": 2.928466796875, + "learning_rate": 2.5968634686346866e-06, + "loss": 1.4743, + "step": 1299 + }, + { + "epoch": 1.6179215930304915, + "grad_norm": 2.941580057144165, + "learning_rate": 2.594557195571956e-06, + "loss": 1.4397, + "step": 1300 + }, + { + "epoch": 1.6179215930304915, + "eval_loss": 1.6348339319229126, + "eval_runtime": 50.433, + "eval_samples_per_second": 19.828, + "eval_steps_per_second": 0.833, + "step": 1300 + }, + { + "epoch": 1.6191661481020536, + "grad_norm": 2.903212308883667, + "learning_rate": 2.592250922509225e-06, + "loss": 1.4221, + "step": 1301 + }, + { + "epoch": 1.6204107031736155, + "grad_norm": 2.948651075363159, + "learning_rate": 2.589944649446495e-06, + "loss": 1.4363, + "step": 1302 + }, + { + "epoch": 1.6216552582451773, + "grad_norm": 2.8940505981445312, + "learning_rate": 2.5876383763837643e-06, + "loss": 1.398, + "step": 1303 + }, + { + "epoch": 1.6228998133167392, + "grad_norm": 2.950056552886963, + "learning_rate": 2.5853321033210333e-06, + "loss": 1.4425, + "step": 1304 + }, + { + "epoch": 1.6241443683883012, + "grad_norm": 2.9674088954925537, + "learning_rate": 2.5830258302583027e-06, + "loss": 1.4473, + "step": 1305 + }, + { + "epoch": 1.625388923459863, + "grad_norm": 2.7541685104370117, + "learning_rate": 2.5807195571955725e-06, + "loss": 1.4483, + "step": 1306 + }, + { + "epoch": 1.6266334785314251, + "grad_norm": 2.8897035121917725, + "learning_rate": 2.5784132841328415e-06, + "loss": 1.4572, + "step": 1307 + }, + { + "epoch": 1.627878033602987, + "grad_norm": 3.021559238433838, + "learning_rate": 2.576107011070111e-06, + "loss": 1.5164, + "step": 1308 + }, + { + "epoch": 1.6291225886745488, + "grad_norm": 2.884009838104248, + "learning_rate": 2.5738007380073804e-06, + "loss": 1.4196, + "step": 1309 + }, + { + "epoch": 1.6303671437461107, + "grad_norm": 2.749497890472412, + "learning_rate": 2.5714944649446494e-06, + "loss": 1.4324, + "step": 1310 + }, + { + "epoch": 1.6303671437461107, + "eval_loss": 1.6314265727996826, + "eval_runtime": 42.4647, + "eval_samples_per_second": 23.549, + "eval_steps_per_second": 0.989, + "step": 1310 + }, + { + "epoch": 1.6316116988176725, + "grad_norm": 2.8908045291900635, + "learning_rate": 2.5691881918819188e-06, + "loss": 1.4388, + "step": 1311 + }, + { + "epoch": 1.6328562538892346, + "grad_norm": 2.832808256149292, + "learning_rate": 2.5668819188191886e-06, + "loss": 1.4368, + "step": 1312 + }, + { + "epoch": 1.6341008089607967, + "grad_norm": 2.9047906398773193, + "learning_rate": 2.564575645756458e-06, + "loss": 1.4186, + "step": 1313 + }, + { + "epoch": 1.6353453640323585, + "grad_norm": 2.9271419048309326, + "learning_rate": 2.562269372693727e-06, + "loss": 1.3776, + "step": 1314 + }, + { + "epoch": 1.6365899191039204, + "grad_norm": 2.760314702987671, + "learning_rate": 2.5599630996309965e-06, + "loss": 1.386, + "step": 1315 + }, + { + "epoch": 1.6378344741754822, + "grad_norm": 2.9694864749908447, + "learning_rate": 2.557656826568266e-06, + "loss": 1.4542, + "step": 1316 + }, + { + "epoch": 1.639079029247044, + "grad_norm": 2.9950435161590576, + "learning_rate": 2.555350553505535e-06, + "loss": 1.4242, + "step": 1317 + }, + { + "epoch": 1.6403235843186061, + "grad_norm": 2.9291114807128906, + "learning_rate": 2.5530442804428047e-06, + "loss": 1.4289, + "step": 1318 + }, + { + "epoch": 1.641568139390168, + "grad_norm": 2.981961488723755, + "learning_rate": 2.550738007380074e-06, + "loss": 1.4283, + "step": 1319 + }, + { + "epoch": 1.64281269446173, + "grad_norm": 2.734123706817627, + "learning_rate": 2.548431734317343e-06, + "loss": 1.4144, + "step": 1320 + }, + { + "epoch": 1.64281269446173, + "eval_loss": 1.6249239444732666, + "eval_runtime": 48.8116, + "eval_samples_per_second": 20.487, + "eval_steps_per_second": 0.86, + "step": 1320 + }, + { + "epoch": 1.644057249533292, + "grad_norm": 2.9752919673919678, + "learning_rate": 2.5461254612546126e-06, + "loss": 1.3641, + "step": 1321 + }, + { + "epoch": 1.6453018046048538, + "grad_norm": 2.972463607788086, + "learning_rate": 2.543819188191882e-06, + "loss": 1.3797, + "step": 1322 + }, + { + "epoch": 1.6465463596764156, + "grad_norm": 2.905499219894409, + "learning_rate": 2.541512915129152e-06, + "loss": 1.4221, + "step": 1323 + }, + { + "epoch": 1.6477909147479775, + "grad_norm": 2.7921793460845947, + "learning_rate": 2.539206642066421e-06, + "loss": 1.4118, + "step": 1324 + }, + { + "epoch": 1.6490354698195395, + "grad_norm": 3.064558267593384, + "learning_rate": 2.5369003690036903e-06, + "loss": 1.4611, + "step": 1325 + }, + { + "epoch": 1.6502800248911016, + "grad_norm": 2.9690096378326416, + "learning_rate": 2.5345940959409597e-06, + "loss": 1.461, + "step": 1326 + }, + { + "epoch": 1.6515245799626634, + "grad_norm": 3.0379040241241455, + "learning_rate": 2.5322878228782287e-06, + "loss": 1.451, + "step": 1327 + }, + { + "epoch": 1.6527691350342253, + "grad_norm": 2.9426493644714355, + "learning_rate": 2.5299815498154986e-06, + "loss": 1.4671, + "step": 1328 + }, + { + "epoch": 1.6540136901057871, + "grad_norm": 2.915076732635498, + "learning_rate": 2.527675276752768e-06, + "loss": 1.4159, + "step": 1329 + }, + { + "epoch": 1.655258245177349, + "grad_norm": 2.8588879108428955, + "learning_rate": 2.525369003690037e-06, + "loss": 1.4978, + "step": 1330 + }, + { + "epoch": 1.655258245177349, + "eval_loss": 1.624009132385254, + "eval_runtime": 47.0461, + "eval_samples_per_second": 21.256, + "eval_steps_per_second": 0.893, + "step": 1330 + }, + { + "epoch": 1.656502800248911, + "grad_norm": 2.9942209720611572, + "learning_rate": 2.5230627306273064e-06, + "loss": 1.3872, + "step": 1331 + }, + { + "epoch": 1.657747355320473, + "grad_norm": 2.8424410820007324, + "learning_rate": 2.520756457564576e-06, + "loss": 1.4181, + "step": 1332 + }, + { + "epoch": 1.658991910392035, + "grad_norm": 2.9234843254089355, + "learning_rate": 2.518450184501845e-06, + "loss": 1.4516, + "step": 1333 + }, + { + "epoch": 1.6602364654635968, + "grad_norm": 2.8367815017700195, + "learning_rate": 2.5161439114391147e-06, + "loss": 1.4214, + "step": 1334 + }, + { + "epoch": 1.6614810205351587, + "grad_norm": 2.819149971008301, + "learning_rate": 2.513837638376384e-06, + "loss": 1.3883, + "step": 1335 + }, + { + "epoch": 1.6627255756067205, + "grad_norm": 3.024693250656128, + "learning_rate": 2.5115313653136535e-06, + "loss": 1.3902, + "step": 1336 + }, + { + "epoch": 1.6639701306782824, + "grad_norm": 2.903599262237549, + "learning_rate": 2.5092250922509225e-06, + "loss": 1.4238, + "step": 1337 + }, + { + "epoch": 1.6652146857498444, + "grad_norm": 2.8296589851379395, + "learning_rate": 2.506918819188192e-06, + "loss": 1.3993, + "step": 1338 + }, + { + "epoch": 1.6664592408214065, + "grad_norm": 2.873272657394409, + "learning_rate": 2.5046125461254618e-06, + "loss": 1.4186, + "step": 1339 + }, + { + "epoch": 1.6677037958929684, + "grad_norm": 2.9268789291381836, + "learning_rate": 2.5023062730627308e-06, + "loss": 1.4242, + "step": 1340 + }, + { + "epoch": 1.6677037958929684, + "eval_loss": 1.6260672807693481, + "eval_runtime": 47.5895, + "eval_samples_per_second": 21.013, + "eval_steps_per_second": 0.883, + "step": 1340 + }, + { + "epoch": 1.6689483509645302, + "grad_norm": 3.057659387588501, + "learning_rate": 2.5e-06, + "loss": 1.4077, + "step": 1341 + }, + { + "epoch": 1.670192906036092, + "grad_norm": 2.875095844268799, + "learning_rate": 2.4976937269372696e-06, + "loss": 1.394, + "step": 1342 + }, + { + "epoch": 1.671437461107654, + "grad_norm": 2.7579872608184814, + "learning_rate": 2.495387453874539e-06, + "loss": 1.4029, + "step": 1343 + }, + { + "epoch": 1.672682016179216, + "grad_norm": 2.8981001377105713, + "learning_rate": 2.4930811808118085e-06, + "loss": 1.4494, + "step": 1344 + }, + { + "epoch": 1.6739265712507778, + "grad_norm": 2.9696900844573975, + "learning_rate": 2.490774907749078e-06, + "loss": 1.4692, + "step": 1345 + }, + { + "epoch": 1.6751711263223399, + "grad_norm": 2.8094892501831055, + "learning_rate": 2.488468634686347e-06, + "loss": 1.441, + "step": 1346 + }, + { + "epoch": 1.6764156813939017, + "grad_norm": 2.8619563579559326, + "learning_rate": 2.4861623616236163e-06, + "loss": 1.4885, + "step": 1347 + }, + { + "epoch": 1.6776602364654636, + "grad_norm": 2.854151487350464, + "learning_rate": 2.4838560885608857e-06, + "loss": 1.4338, + "step": 1348 + }, + { + "epoch": 1.6789047915370254, + "grad_norm": 2.9707868099212646, + "learning_rate": 2.481549815498155e-06, + "loss": 1.4529, + "step": 1349 + }, + { + "epoch": 1.6801493466085873, + "grad_norm": 3.0289969444274902, + "learning_rate": 2.4792435424354246e-06, + "loss": 1.4855, + "step": 1350 + }, + { + "epoch": 1.6801493466085873, + "eval_loss": 1.6256393194198608, + "eval_runtime": 45.9524, + "eval_samples_per_second": 21.762, + "eval_steps_per_second": 0.914, + "step": 1350 + }, + { + "epoch": 1.6813939016801493, + "grad_norm": 2.8407845497131348, + "learning_rate": 2.476937269372694e-06, + "loss": 1.3876, + "step": 1351 + }, + { + "epoch": 1.6826384567517114, + "grad_norm": 2.9876599311828613, + "learning_rate": 2.4746309963099634e-06, + "loss": 1.3931, + "step": 1352 + }, + { + "epoch": 1.6838830118232733, + "grad_norm": 3.008683919906616, + "learning_rate": 2.472324723247233e-06, + "loss": 1.4482, + "step": 1353 + }, + { + "epoch": 1.6851275668948351, + "grad_norm": 2.8110079765319824, + "learning_rate": 2.470018450184502e-06, + "loss": 1.3851, + "step": 1354 + }, + { + "epoch": 1.686372121966397, + "grad_norm": 2.9402263164520264, + "learning_rate": 2.4677121771217713e-06, + "loss": 1.4009, + "step": 1355 + }, + { + "epoch": 1.6876166770379588, + "grad_norm": 2.9846489429473877, + "learning_rate": 2.4654059040590407e-06, + "loss": 1.3975, + "step": 1356 + }, + { + "epoch": 1.6888612321095209, + "grad_norm": 2.835801362991333, + "learning_rate": 2.46309963099631e-06, + "loss": 1.3421, + "step": 1357 + }, + { + "epoch": 1.6901057871810827, + "grad_norm": 2.9438889026641846, + "learning_rate": 2.4607933579335795e-06, + "loss": 1.4422, + "step": 1358 + }, + { + "epoch": 1.6913503422526448, + "grad_norm": 3.168829917907715, + "learning_rate": 2.458487084870849e-06, + "loss": 1.4596, + "step": 1359 + }, + { + "epoch": 1.6925948973242066, + "grad_norm": 2.8412039279937744, + "learning_rate": 2.4561808118081184e-06, + "loss": 1.4263, + "step": 1360 + }, + { + "epoch": 1.6925948973242066, + "eval_loss": 1.6265885829925537, + "eval_runtime": 50.3402, + "eval_samples_per_second": 19.865, + "eval_steps_per_second": 0.834, + "step": 1360 + }, + { + "epoch": 1.6938394523957685, + "grad_norm": 3.0540060997009277, + "learning_rate": 2.453874538745388e-06, + "loss": 1.4123, + "step": 1361 + }, + { + "epoch": 1.6950840074673303, + "grad_norm": 3.0544989109039307, + "learning_rate": 2.451568265682657e-06, + "loss": 1.412, + "step": 1362 + }, + { + "epoch": 1.6963285625388922, + "grad_norm": 3.114126205444336, + "learning_rate": 2.4492619926199267e-06, + "loss": 1.4952, + "step": 1363 + }, + { + "epoch": 1.6975731176104543, + "grad_norm": 2.9366016387939453, + "learning_rate": 2.4469557195571957e-06, + "loss": 1.5155, + "step": 1364 + }, + { + "epoch": 1.6988176726820163, + "grad_norm": 2.910677671432495, + "learning_rate": 2.444649446494465e-06, + "loss": 1.4393, + "step": 1365 + }, + { + "epoch": 1.7000622277535782, + "grad_norm": 2.8373727798461914, + "learning_rate": 2.4423431734317345e-06, + "loss": 1.4223, + "step": 1366 + }, + { + "epoch": 1.70130678282514, + "grad_norm": 2.865161895751953, + "learning_rate": 2.440036900369004e-06, + "loss": 1.4605, + "step": 1367 + }, + { + "epoch": 1.7025513378967019, + "grad_norm": 2.7929177284240723, + "learning_rate": 2.437730627306273e-06, + "loss": 1.4695, + "step": 1368 + }, + { + "epoch": 1.7037958929682637, + "grad_norm": 2.867953300476074, + "learning_rate": 2.4354243542435428e-06, + "loss": 1.3524, + "step": 1369 + }, + { + "epoch": 1.7050404480398258, + "grad_norm": 2.8980207443237305, + "learning_rate": 2.4331180811808118e-06, + "loss": 1.4568, + "step": 1370 + }, + { + "epoch": 1.7050404480398258, + "eval_loss": 1.6297556161880493, + "eval_runtime": 51.447, + "eval_samples_per_second": 19.437, + "eval_steps_per_second": 0.816, + "step": 1370 + }, + { + "epoch": 1.7062850031113876, + "grad_norm": 2.954972505569458, + "learning_rate": 2.4308118081180816e-06, + "loss": 1.4, + "step": 1371 + }, + { + "epoch": 1.7075295581829497, + "grad_norm": 2.973191738128662, + "learning_rate": 2.4285055350553506e-06, + "loss": 1.4062, + "step": 1372 + }, + { + "epoch": 1.7087741132545116, + "grad_norm": 2.951444387435913, + "learning_rate": 2.42619926199262e-06, + "loss": 1.4523, + "step": 1373 + }, + { + "epoch": 1.7100186683260734, + "grad_norm": 2.850445508956909, + "learning_rate": 2.4238929889298895e-06, + "loss": 1.4282, + "step": 1374 + }, + { + "epoch": 1.7112632233976353, + "grad_norm": 2.7643561363220215, + "learning_rate": 2.421586715867159e-06, + "loss": 1.3807, + "step": 1375 + }, + { + "epoch": 1.712507778469197, + "grad_norm": 2.8872134685516357, + "learning_rate": 2.4192804428044283e-06, + "loss": 1.4801, + "step": 1376 + }, + { + "epoch": 1.7137523335407592, + "grad_norm": 2.8748483657836914, + "learning_rate": 2.4169741697416977e-06, + "loss": 1.381, + "step": 1377 + }, + { + "epoch": 1.7149968886123212, + "grad_norm": 2.7833831310272217, + "learning_rate": 2.4146678966789667e-06, + "loss": 1.389, + "step": 1378 + }, + { + "epoch": 1.716241443683883, + "grad_norm": 2.994715690612793, + "learning_rate": 2.4123616236162366e-06, + "loss": 1.3917, + "step": 1379 + }, + { + "epoch": 1.717485998755445, + "grad_norm": 2.887026786804199, + "learning_rate": 2.4100553505535056e-06, + "loss": 1.3725, + "step": 1380 + }, + { + "epoch": 1.717485998755445, + "eval_loss": 1.6283434629440308, + "eval_runtime": 50.596, + "eval_samples_per_second": 19.764, + "eval_steps_per_second": 0.83, + "step": 1380 + }, + { + "epoch": 1.7187305538270068, + "grad_norm": 3.2046546936035156, + "learning_rate": 2.407749077490775e-06, + "loss": 1.4422, + "step": 1381 + }, + { + "epoch": 1.7199751088985686, + "grad_norm": 3.0105698108673096, + "learning_rate": 2.4054428044280444e-06, + "loss": 1.4494, + "step": 1382 + }, + { + "epoch": 1.7212196639701307, + "grad_norm": 2.9917800426483154, + "learning_rate": 2.403136531365314e-06, + "loss": 1.4407, + "step": 1383 + }, + { + "epoch": 1.7224642190416926, + "grad_norm": 2.844325065612793, + "learning_rate": 2.4008302583025833e-06, + "loss": 1.4067, + "step": 1384 + }, + { + "epoch": 1.7237087741132546, + "grad_norm": 2.919576406478882, + "learning_rate": 2.3985239852398527e-06, + "loss": 1.4352, + "step": 1385 + }, + { + "epoch": 1.7249533291848165, + "grad_norm": 2.8357744216918945, + "learning_rate": 2.3962177121771217e-06, + "loss": 1.3978, + "step": 1386 + }, + { + "epoch": 1.7261978842563783, + "grad_norm": 2.82425856590271, + "learning_rate": 2.3939114391143915e-06, + "loss": 1.4439, + "step": 1387 + }, + { + "epoch": 1.7274424393279402, + "grad_norm": 2.9155638217926025, + "learning_rate": 2.3916051660516605e-06, + "loss": 1.4326, + "step": 1388 + }, + { + "epoch": 1.728686994399502, + "grad_norm": 2.862804651260376, + "learning_rate": 2.3892988929889304e-06, + "loss": 1.3766, + "step": 1389 + }, + { + "epoch": 1.729931549471064, + "grad_norm": 3.012230634689331, + "learning_rate": 2.3869926199261994e-06, + "loss": 1.4569, + "step": 1390 + }, + { + "epoch": 1.729931549471064, + "eval_loss": 1.6268510818481445, + "eval_runtime": 52.5438, + "eval_samples_per_second": 19.032, + "eval_steps_per_second": 0.799, + "step": 1390 + }, + { + "epoch": 1.7311761045426262, + "grad_norm": 2.9922425746917725, + "learning_rate": 2.384686346863469e-06, + "loss": 1.4066, + "step": 1391 + }, + { + "epoch": 1.732420659614188, + "grad_norm": 2.960145950317383, + "learning_rate": 2.3823800738007382e-06, + "loss": 1.4514, + "step": 1392 + }, + { + "epoch": 1.7336652146857499, + "grad_norm": 2.834998846054077, + "learning_rate": 2.3800738007380077e-06, + "loss": 1.4714, + "step": 1393 + }, + { + "epoch": 1.7349097697573117, + "grad_norm": 2.9646077156066895, + "learning_rate": 2.377767527675277e-06, + "loss": 1.4368, + "step": 1394 + }, + { + "epoch": 1.7361543248288736, + "grad_norm": 2.8233699798583984, + "learning_rate": 2.3754612546125465e-06, + "loss": 1.4598, + "step": 1395 + }, + { + "epoch": 1.7373988799004356, + "grad_norm": 2.895472764968872, + "learning_rate": 2.3731549815498155e-06, + "loss": 1.4551, + "step": 1396 + }, + { + "epoch": 1.7386434349719975, + "grad_norm": 2.7917752265930176, + "learning_rate": 2.3708487084870853e-06, + "loss": 1.4759, + "step": 1397 + }, + { + "epoch": 1.7398879900435595, + "grad_norm": 2.820672035217285, + "learning_rate": 2.3685424354243543e-06, + "loss": 1.3711, + "step": 1398 + }, + { + "epoch": 1.7411325451151214, + "grad_norm": 2.804952621459961, + "learning_rate": 2.3662361623616238e-06, + "loss": 1.4227, + "step": 1399 + }, + { + "epoch": 1.7423771001866832, + "grad_norm": 2.981553316116333, + "learning_rate": 2.363929889298893e-06, + "loss": 1.4639, + "step": 1400 + }, + { + "epoch": 1.7423771001866832, + "eval_loss": 1.6229957342147827, + "eval_runtime": 49.1507, + "eval_samples_per_second": 20.346, + "eval_steps_per_second": 0.855, + "step": 1400 + }, + { + "epoch": 1.743621655258245, + "grad_norm": 2.8649392127990723, + "learning_rate": 2.3616236162361626e-06, + "loss": 1.4283, + "step": 1401 + }, + { + "epoch": 1.744866210329807, + "grad_norm": 2.9438977241516113, + "learning_rate": 2.359317343173432e-06, + "loss": 1.444, + "step": 1402 + }, + { + "epoch": 1.746110765401369, + "grad_norm": 2.9270176887512207, + "learning_rate": 2.3570110701107015e-06, + "loss": 1.4044, + "step": 1403 + }, + { + "epoch": 1.747355320472931, + "grad_norm": 2.791527271270752, + "learning_rate": 2.3547047970479705e-06, + "loss": 1.4195, + "step": 1404 + }, + { + "epoch": 1.748599875544493, + "grad_norm": 2.8519608974456787, + "learning_rate": 2.3523985239852403e-06, + "loss": 1.4031, + "step": 1405 + }, + { + "epoch": 1.7498444306160548, + "grad_norm": 2.90872859954834, + "learning_rate": 2.3500922509225093e-06, + "loss": 1.4616, + "step": 1406 + }, + { + "epoch": 1.7510889856876166, + "grad_norm": 2.9190874099731445, + "learning_rate": 2.3477859778597787e-06, + "loss": 1.3917, + "step": 1407 + }, + { + "epoch": 1.7523335407591785, + "grad_norm": 2.868323802947998, + "learning_rate": 2.345479704797048e-06, + "loss": 1.4053, + "step": 1408 + }, + { + "epoch": 1.7535780958307405, + "grad_norm": 2.8638579845428467, + "learning_rate": 2.3431734317343176e-06, + "loss": 1.4042, + "step": 1409 + }, + { + "epoch": 1.7548226509023024, + "grad_norm": 3.1759817600250244, + "learning_rate": 2.340867158671587e-06, + "loss": 1.4937, + "step": 1410 + }, + { + "epoch": 1.7548226509023024, + "eval_loss": 1.6252617835998535, + "eval_runtime": 44.3041, + "eval_samples_per_second": 22.571, + "eval_steps_per_second": 0.948, + "step": 1410 + }, + { + "epoch": 1.7560672059738645, + "grad_norm": 2.8265459537506104, + "learning_rate": 2.3385608856088564e-06, + "loss": 1.406, + "step": 1411 + }, + { + "epoch": 1.7573117610454263, + "grad_norm": 2.856229305267334, + "learning_rate": 2.336254612546126e-06, + "loss": 1.4395, + "step": 1412 + }, + { + "epoch": 1.7585563161169881, + "grad_norm": 2.9112775325775146, + "learning_rate": 2.333948339483395e-06, + "loss": 1.427, + "step": 1413 + }, + { + "epoch": 1.75980087118855, + "grad_norm": 3.098846435546875, + "learning_rate": 2.3316420664206643e-06, + "loss": 1.4222, + "step": 1414 + }, + { + "epoch": 1.7610454262601118, + "grad_norm": 2.758100986480713, + "learning_rate": 2.3293357933579337e-06, + "loss": 1.3844, + "step": 1415 + }, + { + "epoch": 1.762289981331674, + "grad_norm": 2.8422274589538574, + "learning_rate": 2.327029520295203e-06, + "loss": 1.4033, + "step": 1416 + }, + { + "epoch": 1.763534536403236, + "grad_norm": 2.9257912635803223, + "learning_rate": 2.3247232472324725e-06, + "loss": 1.3961, + "step": 1417 + }, + { + "epoch": 1.7647790914747978, + "grad_norm": 2.9431657791137695, + "learning_rate": 2.322416974169742e-06, + "loss": 1.4451, + "step": 1418 + }, + { + "epoch": 1.7660236465463597, + "grad_norm": 2.8598814010620117, + "learning_rate": 2.3201107011070114e-06, + "loss": 1.3816, + "step": 1419 + }, + { + "epoch": 1.7672682016179215, + "grad_norm": 2.9868006706237793, + "learning_rate": 2.317804428044281e-06, + "loss": 1.4037, + "step": 1420 + }, + { + "epoch": 1.7672682016179215, + "eval_loss": 1.6231273412704468, + "eval_runtime": 48.8067, + "eval_samples_per_second": 20.489, + "eval_steps_per_second": 0.861, + "step": 1420 + }, + { + "epoch": 1.7685127566894834, + "grad_norm": 2.888206720352173, + "learning_rate": 2.31549815498155e-06, + "loss": 1.4087, + "step": 1421 + }, + { + "epoch": 1.7697573117610454, + "grad_norm": 2.9240078926086426, + "learning_rate": 2.3131918819188192e-06, + "loss": 1.3909, + "step": 1422 + }, + { + "epoch": 1.7710018668326073, + "grad_norm": 3.0401012897491455, + "learning_rate": 2.3108856088560886e-06, + "loss": 1.423, + "step": 1423 + }, + { + "epoch": 1.7722464219041694, + "grad_norm": 2.9484200477600098, + "learning_rate": 2.308579335793358e-06, + "loss": 1.449, + "step": 1424 + }, + { + "epoch": 1.7734909769757312, + "grad_norm": 2.863267660140991, + "learning_rate": 2.3062730627306275e-06, + "loss": 1.4215, + "step": 1425 + }, + { + "epoch": 1.774735532047293, + "grad_norm": 2.9615612030029297, + "learning_rate": 2.303966789667897e-06, + "loss": 1.4391, + "step": 1426 + }, + { + "epoch": 1.775980087118855, + "grad_norm": 2.7958569526672363, + "learning_rate": 2.3016605166051663e-06, + "loss": 1.3662, + "step": 1427 + }, + { + "epoch": 1.777224642190417, + "grad_norm": 2.898787021636963, + "learning_rate": 2.2993542435424358e-06, + "loss": 1.362, + "step": 1428 + }, + { + "epoch": 1.7784691972619788, + "grad_norm": 2.883049726486206, + "learning_rate": 2.2970479704797048e-06, + "loss": 1.4339, + "step": 1429 + }, + { + "epoch": 1.779713752333541, + "grad_norm": 3.0204827785491943, + "learning_rate": 2.294741697416974e-06, + "loss": 1.5174, + "step": 1430 + }, + { + "epoch": 1.779713752333541, + "eval_loss": 1.6246228218078613, + "eval_runtime": 47.5611, + "eval_samples_per_second": 21.026, + "eval_steps_per_second": 0.883, + "step": 1430 + }, + { + "epoch": 1.7809583074051027, + "grad_norm": 2.9354310035705566, + "learning_rate": 2.2924354243542436e-06, + "loss": 1.4115, + "step": 1431 + }, + { + "epoch": 1.7822028624766646, + "grad_norm": 2.9498608112335205, + "learning_rate": 2.290129151291513e-06, + "loss": 1.4058, + "step": 1432 + }, + { + "epoch": 1.7834474175482264, + "grad_norm": 2.9706921577453613, + "learning_rate": 2.2878228782287825e-06, + "loss": 1.426, + "step": 1433 + }, + { + "epoch": 1.7846919726197883, + "grad_norm": 2.8945024013519287, + "learning_rate": 2.285516605166052e-06, + "loss": 1.4011, + "step": 1434 + }, + { + "epoch": 1.7859365276913504, + "grad_norm": 2.8987081050872803, + "learning_rate": 2.2832103321033213e-06, + "loss": 1.4163, + "step": 1435 + }, + { + "epoch": 1.7871810827629122, + "grad_norm": 2.9797592163085938, + "learning_rate": 2.2809040590405907e-06, + "loss": 1.4367, + "step": 1436 + }, + { + "epoch": 1.7884256378344743, + "grad_norm": 2.8872063159942627, + "learning_rate": 2.2785977859778597e-06, + "loss": 1.4226, + "step": 1437 + }, + { + "epoch": 1.7896701929060361, + "grad_norm": 2.942655563354492, + "learning_rate": 2.2762915129151296e-06, + "loss": 1.4132, + "step": 1438 + }, + { + "epoch": 1.790914747977598, + "grad_norm": 3.027392625808716, + "learning_rate": 2.2739852398523986e-06, + "loss": 1.4148, + "step": 1439 + }, + { + "epoch": 1.7921593030491598, + "grad_norm": 2.9475181102752686, + "learning_rate": 2.271678966789668e-06, + "loss": 1.4585, + "step": 1440 + }, + { + "epoch": 1.7921593030491598, + "eval_loss": 1.6234897375106812, + "eval_runtime": 53.3737, + "eval_samples_per_second": 18.736, + "eval_steps_per_second": 0.787, + "step": 1440 + }, + { + "epoch": 1.793403858120722, + "grad_norm": 2.846304416656494, + "learning_rate": 2.2693726937269374e-06, + "loss": 1.3667, + "step": 1441 + }, + { + "epoch": 1.7946484131922837, + "grad_norm": 2.8925700187683105, + "learning_rate": 2.267066420664207e-06, + "loss": 1.4318, + "step": 1442 + }, + { + "epoch": 1.7958929682638458, + "grad_norm": 2.884025812149048, + "learning_rate": 2.2647601476014763e-06, + "loss": 1.4395, + "step": 1443 + }, + { + "epoch": 1.7971375233354077, + "grad_norm": 2.9763753414154053, + "learning_rate": 2.2624538745387457e-06, + "loss": 1.3742, + "step": 1444 + }, + { + "epoch": 1.7983820784069695, + "grad_norm": 3.0834898948669434, + "learning_rate": 2.2601476014760147e-06, + "loss": 1.4651, + "step": 1445 + }, + { + "epoch": 1.7996266334785314, + "grad_norm": 3.0673978328704834, + "learning_rate": 2.2578413284132845e-06, + "loss": 1.475, + "step": 1446 + }, + { + "epoch": 1.8008711885500932, + "grad_norm": 2.931521415710449, + "learning_rate": 2.2555350553505535e-06, + "loss": 1.4506, + "step": 1447 + }, + { + "epoch": 1.8021157436216553, + "grad_norm": 2.953301429748535, + "learning_rate": 2.253228782287823e-06, + "loss": 1.4224, + "step": 1448 + }, + { + "epoch": 1.8033602986932171, + "grad_norm": 3.0210883617401123, + "learning_rate": 2.2509225092250924e-06, + "loss": 1.4387, + "step": 1449 + }, + { + "epoch": 1.8046048537647792, + "grad_norm": 2.975741386413574, + "learning_rate": 2.248616236162362e-06, + "loss": 1.4318, + "step": 1450 + }, + { + "epoch": 1.8046048537647792, + "eval_loss": 1.618930697441101, + "eval_runtime": 49.6692, + "eval_samples_per_second": 20.133, + "eval_steps_per_second": 0.846, + "step": 1450 + }, + { + "epoch": 1.805849408836341, + "grad_norm": 2.9083876609802246, + "learning_rate": 2.2463099630996312e-06, + "loss": 1.3837, + "step": 1451 + }, + { + "epoch": 1.807093963907903, + "grad_norm": 3.364248275756836, + "learning_rate": 2.2440036900369006e-06, + "loss": 1.4589, + "step": 1452 + }, + { + "epoch": 1.8083385189794647, + "grad_norm": 2.935478925704956, + "learning_rate": 2.2416974169741696e-06, + "loss": 1.4119, + "step": 1453 + }, + { + "epoch": 1.8095830740510268, + "grad_norm": 3.1742911338806152, + "learning_rate": 2.2393911439114395e-06, + "loss": 1.4203, + "step": 1454 + }, + { + "epoch": 1.8108276291225887, + "grad_norm": 2.96966290473938, + "learning_rate": 2.2370848708487085e-06, + "loss": 1.4054, + "step": 1455 + }, + { + "epoch": 1.8120721841941507, + "grad_norm": 3.0919055938720703, + "learning_rate": 2.2347785977859783e-06, + "loss": 1.4289, + "step": 1456 + }, + { + "epoch": 1.8133167392657126, + "grad_norm": 2.9061055183410645, + "learning_rate": 2.2324723247232473e-06, + "loss": 1.4958, + "step": 1457 + }, + { + "epoch": 1.8145612943372744, + "grad_norm": 2.968348503112793, + "learning_rate": 2.2301660516605168e-06, + "loss": 1.4033, + "step": 1458 + }, + { + "epoch": 1.8158058494088363, + "grad_norm": 3.0435378551483154, + "learning_rate": 2.227859778597786e-06, + "loss": 1.414, + "step": 1459 + }, + { + "epoch": 1.8170504044803981, + "grad_norm": 2.9067916870117188, + "learning_rate": 2.2255535055350556e-06, + "loss": 1.3954, + "step": 1460 + }, + { + "epoch": 1.8170504044803981, + "eval_loss": 1.6240814924240112, + "eval_runtime": 51.7164, + "eval_samples_per_second": 19.336, + "eval_steps_per_second": 0.812, + "step": 1460 + }, + { + "epoch": 1.8182949595519602, + "grad_norm": 3.156384229660034, + "learning_rate": 2.2232472324723246e-06, + "loss": 1.4658, + "step": 1461 + }, + { + "epoch": 1.819539514623522, + "grad_norm": 2.9885313510894775, + "learning_rate": 2.2209409594095944e-06, + "loss": 1.4123, + "step": 1462 + }, + { + "epoch": 1.820784069695084, + "grad_norm": 3.227780818939209, + "learning_rate": 2.2186346863468634e-06, + "loss": 1.3587, + "step": 1463 + }, + { + "epoch": 1.822028624766646, + "grad_norm": 3.0375192165374756, + "learning_rate": 2.2163284132841333e-06, + "loss": 1.4015, + "step": 1464 + }, + { + "epoch": 1.8232731798382078, + "grad_norm": 3.0200846195220947, + "learning_rate": 2.2140221402214023e-06, + "loss": 1.419, + "step": 1465 + }, + { + "epoch": 1.8245177349097697, + "grad_norm": 2.9456615447998047, + "learning_rate": 2.2117158671586717e-06, + "loss": 1.4075, + "step": 1466 + }, + { + "epoch": 1.8257622899813317, + "grad_norm": 3.3129165172576904, + "learning_rate": 2.209409594095941e-06, + "loss": 1.4241, + "step": 1467 + }, + { + "epoch": 1.8270068450528936, + "grad_norm": 3.013141393661499, + "learning_rate": 2.2071033210332106e-06, + "loss": 1.4066, + "step": 1468 + }, + { + "epoch": 1.8282514001244556, + "grad_norm": 2.8414366245269775, + "learning_rate": 2.20479704797048e-06, + "loss": 1.4009, + "step": 1469 + }, + { + "epoch": 1.8294959551960175, + "grad_norm": 2.8920438289642334, + "learning_rate": 2.2024907749077494e-06, + "loss": 1.3916, + "step": 1470 + }, + { + "epoch": 1.8294959551960175, + "eval_loss": 1.6220412254333496, + "eval_runtime": 52.6199, + "eval_samples_per_second": 19.004, + "eval_steps_per_second": 0.798, + "step": 1470 + }, + { + "epoch": 1.8307405102675793, + "grad_norm": 3.032353639602661, + "learning_rate": 2.2001845018450184e-06, + "loss": 1.402, + "step": 1471 + }, + { + "epoch": 1.8319850653391412, + "grad_norm": 2.9512081146240234, + "learning_rate": 2.1978782287822883e-06, + "loss": 1.429, + "step": 1472 + }, + { + "epoch": 1.833229620410703, + "grad_norm": 2.846081018447876, + "learning_rate": 2.1955719557195573e-06, + "loss": 1.4203, + "step": 1473 + }, + { + "epoch": 1.834474175482265, + "grad_norm": 3.093378782272339, + "learning_rate": 2.1932656826568267e-06, + "loss": 1.4307, + "step": 1474 + }, + { + "epoch": 1.8357187305538272, + "grad_norm": 2.808326005935669, + "learning_rate": 2.190959409594096e-06, + "loss": 1.3753, + "step": 1475 + }, + { + "epoch": 1.836963285625389, + "grad_norm": 3.0328383445739746, + "learning_rate": 2.1886531365313655e-06, + "loss": 1.4534, + "step": 1476 + }, + { + "epoch": 1.8382078406969509, + "grad_norm": 3.045897960662842, + "learning_rate": 2.186346863468635e-06, + "loss": 1.3259, + "step": 1477 + }, + { + "epoch": 1.8394523957685127, + "grad_norm": 3.171846628189087, + "learning_rate": 2.1840405904059044e-06, + "loss": 1.397, + "step": 1478 + }, + { + "epoch": 1.8406969508400746, + "grad_norm": 2.893061637878418, + "learning_rate": 2.1817343173431734e-06, + "loss": 1.4201, + "step": 1479 + }, + { + "epoch": 1.8419415059116366, + "grad_norm": 3.0151467323303223, + "learning_rate": 2.1794280442804432e-06, + "loss": 1.4266, + "step": 1480 + }, + { + "epoch": 1.8419415059116366, + "eval_loss": 1.6166906356811523, + "eval_runtime": 51.8266, + "eval_samples_per_second": 19.295, + "eval_steps_per_second": 0.81, + "step": 1480 + }, + { + "epoch": 1.8431860609831985, + "grad_norm": 2.942431688308716, + "learning_rate": 2.177121771217712e-06, + "loss": 1.3744, + "step": 1481 + }, + { + "epoch": 1.8444306160547606, + "grad_norm": 2.8482232093811035, + "learning_rate": 2.1748154981549816e-06, + "loss": 1.3914, + "step": 1482 + }, + { + "epoch": 1.8456751711263224, + "grad_norm": 2.9315359592437744, + "learning_rate": 2.172509225092251e-06, + "loss": 1.419, + "step": 1483 + }, + { + "epoch": 1.8469197261978842, + "grad_norm": 3.191800594329834, + "learning_rate": 2.1702029520295205e-06, + "loss": 1.3524, + "step": 1484 + }, + { + "epoch": 1.848164281269446, + "grad_norm": 2.9308040142059326, + "learning_rate": 2.16789667896679e-06, + "loss": 1.3742, + "step": 1485 + }, + { + "epoch": 1.849408836341008, + "grad_norm": 2.7678964138031006, + "learning_rate": 2.1655904059040593e-06, + "loss": 1.3786, + "step": 1486 + }, + { + "epoch": 1.85065339141257, + "grad_norm": 3.0300636291503906, + "learning_rate": 2.1632841328413287e-06, + "loss": 1.4469, + "step": 1487 + }, + { + "epoch": 1.851897946484132, + "grad_norm": 3.142781972885132, + "learning_rate": 2.160977859778598e-06, + "loss": 1.4565, + "step": 1488 + }, + { + "epoch": 1.853142501555694, + "grad_norm": 2.954916000366211, + "learning_rate": 2.158671586715867e-06, + "loss": 1.4356, + "step": 1489 + }, + { + "epoch": 1.8543870566272558, + "grad_norm": 2.864875316619873, + "learning_rate": 2.1563653136531366e-06, + "loss": 1.4521, + "step": 1490 + }, + { + "epoch": 1.8543870566272558, + "eval_loss": 1.6207302808761597, + "eval_runtime": 49.2215, + "eval_samples_per_second": 20.316, + "eval_steps_per_second": 0.853, + "step": 1490 + }, + { + "epoch": 1.8556316116988176, + "grad_norm": 2.8908467292785645, + "learning_rate": 2.154059040590406e-06, + "loss": 1.4304, + "step": 1491 + }, + { + "epoch": 1.8568761667703795, + "grad_norm": 2.9439549446105957, + "learning_rate": 2.1517527675276754e-06, + "loss": 1.398, + "step": 1492 + }, + { + "epoch": 1.8581207218419415, + "grad_norm": 2.967728853225708, + "learning_rate": 2.149446494464945e-06, + "loss": 1.5038, + "step": 1493 + }, + { + "epoch": 1.8593652769135034, + "grad_norm": 2.938123941421509, + "learning_rate": 2.1471402214022143e-06, + "loss": 1.4395, + "step": 1494 + }, + { + "epoch": 1.8606098319850655, + "grad_norm": 2.93609881401062, + "learning_rate": 2.1448339483394837e-06, + "loss": 1.3939, + "step": 1495 + }, + { + "epoch": 1.8618543870566273, + "grad_norm": 2.9093775749206543, + "learning_rate": 2.142527675276753e-06, + "loss": 1.3942, + "step": 1496 + }, + { + "epoch": 1.8630989421281892, + "grad_norm": 2.773960590362549, + "learning_rate": 2.140221402214022e-06, + "loss": 1.3886, + "step": 1497 + }, + { + "epoch": 1.864343497199751, + "grad_norm": 2.9858205318450928, + "learning_rate": 2.1379151291512916e-06, + "loss": 1.3814, + "step": 1498 + }, + { + "epoch": 1.8655880522713129, + "grad_norm": 3.1233417987823486, + "learning_rate": 2.135608856088561e-06, + "loss": 1.4313, + "step": 1499 + }, + { + "epoch": 1.866832607342875, + "grad_norm": 2.9569480419158936, + "learning_rate": 2.1333025830258304e-06, + "loss": 1.4302, + "step": 1500 + }, + { + "epoch": 1.866832607342875, + "eval_loss": 1.6130231618881226, + "eval_runtime": 46.2834, + "eval_samples_per_second": 21.606, + "eval_steps_per_second": 0.907, + "step": 1500 + }, + { + "epoch": 1.868077162414437, + "grad_norm": 2.926203727722168, + "learning_rate": 2.1309963099631e-06, + "loss": 1.4236, + "step": 1501 + }, + { + "epoch": 1.8693217174859988, + "grad_norm": 2.9496357440948486, + "learning_rate": 2.1286900369003692e-06, + "loss": 1.4014, + "step": 1502 + }, + { + "epoch": 1.8705662725575607, + "grad_norm": 2.9314279556274414, + "learning_rate": 2.1263837638376387e-06, + "loss": 1.4414, + "step": 1503 + }, + { + "epoch": 1.8718108276291225, + "grad_norm": 2.889460802078247, + "learning_rate": 2.124077490774908e-06, + "loss": 1.3883, + "step": 1504 + }, + { + "epoch": 1.8730553827006844, + "grad_norm": 2.9859671592712402, + "learning_rate": 2.1217712177121775e-06, + "loss": 1.4551, + "step": 1505 + }, + { + "epoch": 1.8742999377722465, + "grad_norm": 2.9330570697784424, + "learning_rate": 2.1194649446494465e-06, + "loss": 1.3895, + "step": 1506 + }, + { + "epoch": 1.8755444928438083, + "grad_norm": 2.9062387943267822, + "learning_rate": 2.117158671586716e-06, + "loss": 1.4492, + "step": 1507 + }, + { + "epoch": 1.8767890479153704, + "grad_norm": 3.0337791442871094, + "learning_rate": 2.1148523985239854e-06, + "loss": 1.4682, + "step": 1508 + }, + { + "epoch": 1.8780336029869322, + "grad_norm": 3.0448431968688965, + "learning_rate": 2.1125461254612548e-06, + "loss": 1.4836, + "step": 1509 + }, + { + "epoch": 1.879278158058494, + "grad_norm": 2.912764310836792, + "learning_rate": 2.110239852398524e-06, + "loss": 1.4644, + "step": 1510 + }, + { + "epoch": 1.879278158058494, + "eval_loss": 1.6124104261398315, + "eval_runtime": 41.9713, + "eval_samples_per_second": 23.826, + "eval_steps_per_second": 1.001, + "step": 1510 + }, + { + "epoch": 1.880522713130056, + "grad_norm": 3.0399203300476074, + "learning_rate": 2.1079335793357936e-06, + "loss": 1.4391, + "step": 1511 + }, + { + "epoch": 1.8817672682016178, + "grad_norm": 2.9852235317230225, + "learning_rate": 2.1056273062730626e-06, + "loss": 1.346, + "step": 1512 + }, + { + "epoch": 1.8830118232731798, + "grad_norm": 2.904437780380249, + "learning_rate": 2.1033210332103325e-06, + "loss": 1.4523, + "step": 1513 + }, + { + "epoch": 1.884256378344742, + "grad_norm": 2.8988049030303955, + "learning_rate": 2.1010147601476015e-06, + "loss": 1.4001, + "step": 1514 + }, + { + "epoch": 1.8855009334163038, + "grad_norm": 2.9890260696411133, + "learning_rate": 2.098708487084871e-06, + "loss": 1.4074, + "step": 1515 + }, + { + "epoch": 1.8867454884878656, + "grad_norm": 3.0486819744110107, + "learning_rate": 2.0964022140221403e-06, + "loss": 1.461, + "step": 1516 + }, + { + "epoch": 1.8879900435594275, + "grad_norm": 2.8746016025543213, + "learning_rate": 2.0940959409594097e-06, + "loss": 1.4013, + "step": 1517 + }, + { + "epoch": 1.8892345986309893, + "grad_norm": 2.979480266571045, + "learning_rate": 2.091789667896679e-06, + "loss": 1.4567, + "step": 1518 + }, + { + "epoch": 1.8904791537025514, + "grad_norm": 2.8351497650146484, + "learning_rate": 2.0894833948339486e-06, + "loss": 1.3655, + "step": 1519 + }, + { + "epoch": 1.8917237087741132, + "grad_norm": 2.9061646461486816, + "learning_rate": 2.0871771217712176e-06, + "loss": 1.4165, + "step": 1520 + }, + { + "epoch": 1.8917237087741132, + "eval_loss": 1.6172568798065186, + "eval_runtime": 42.998, + "eval_samples_per_second": 23.257, + "eval_steps_per_second": 0.977, + "step": 1520 + }, + { + "epoch": 1.8929682638456753, + "grad_norm": 2.8751838207244873, + "learning_rate": 2.0848708487084874e-06, + "loss": 1.4133, + "step": 1521 + }, + { + "epoch": 1.8942128189172371, + "grad_norm": 2.7513587474823, + "learning_rate": 2.0825645756457564e-06, + "loss": 1.3734, + "step": 1522 + }, + { + "epoch": 1.895457373988799, + "grad_norm": 2.9244418144226074, + "learning_rate": 2.0802583025830263e-06, + "loss": 1.4181, + "step": 1523 + }, + { + "epoch": 1.8967019290603608, + "grad_norm": 2.877495765686035, + "learning_rate": 2.0779520295202953e-06, + "loss": 1.4009, + "step": 1524 + }, + { + "epoch": 1.8979464841319227, + "grad_norm": 2.9316184520721436, + "learning_rate": 2.0756457564575647e-06, + "loss": 1.3796, + "step": 1525 + }, + { + "epoch": 1.8991910392034848, + "grad_norm": 3.0403976440429688, + "learning_rate": 2.073339483394834e-06, + "loss": 1.4101, + "step": 1526 + }, + { + "epoch": 1.9004355942750468, + "grad_norm": 3.0020930767059326, + "learning_rate": 2.0710332103321035e-06, + "loss": 1.4166, + "step": 1527 + }, + { + "epoch": 1.9016801493466087, + "grad_norm": 3.0937304496765137, + "learning_rate": 2.0687269372693725e-06, + "loss": 1.4599, + "step": 1528 + }, + { + "epoch": 1.9029247044181705, + "grad_norm": 2.901540517807007, + "learning_rate": 2.0664206642066424e-06, + "loss": 1.3642, + "step": 1529 + }, + { + "epoch": 1.9041692594897324, + "grad_norm": 3.0517938137054443, + "learning_rate": 2.0641143911439114e-06, + "loss": 1.451, + "step": 1530 + }, + { + "epoch": 1.9041692594897324, + "eval_loss": 1.6178728342056274, + "eval_runtime": 43.9067, + "eval_samples_per_second": 22.776, + "eval_steps_per_second": 0.957, + "step": 1530 + }, + { + "epoch": 1.9054138145612942, + "grad_norm": 3.12484073638916, + "learning_rate": 2.0618081180811812e-06, + "loss": 1.4522, + "step": 1531 + }, + { + "epoch": 1.9066583696328563, + "grad_norm": 2.9447414875030518, + "learning_rate": 2.0595018450184502e-06, + "loss": 1.4154, + "step": 1532 + }, + { + "epoch": 1.9079029247044181, + "grad_norm": 2.909738779067993, + "learning_rate": 2.0571955719557197e-06, + "loss": 1.4133, + "step": 1533 + }, + { + "epoch": 1.9091474797759802, + "grad_norm": 3.1522421836853027, + "learning_rate": 2.054889298892989e-06, + "loss": 1.4777, + "step": 1534 + }, + { + "epoch": 1.910392034847542, + "grad_norm": 3.041447639465332, + "learning_rate": 2.0525830258302585e-06, + "loss": 1.4104, + "step": 1535 + }, + { + "epoch": 1.911636589919104, + "grad_norm": 3.01129412651062, + "learning_rate": 2.050276752767528e-06, + "loss": 1.3852, + "step": 1536 + }, + { + "epoch": 1.9128811449906657, + "grad_norm": 3.0697543621063232, + "learning_rate": 2.0479704797047974e-06, + "loss": 1.4389, + "step": 1537 + }, + { + "epoch": 1.9141257000622276, + "grad_norm": 2.9391236305236816, + "learning_rate": 2.0456642066420664e-06, + "loss": 1.4239, + "step": 1538 + }, + { + "epoch": 1.9153702551337897, + "grad_norm": 2.978236675262451, + "learning_rate": 2.043357933579336e-06, + "loss": 1.462, + "step": 1539 + }, + { + "epoch": 1.9166148102053517, + "grad_norm": 2.8124167919158936, + "learning_rate": 2.041051660516605e-06, + "loss": 1.3186, + "step": 1540 + }, + { + "epoch": 1.9166148102053517, + "eval_loss": 1.6120468378067017, + "eval_runtime": 43.3787, + "eval_samples_per_second": 23.053, + "eval_steps_per_second": 0.968, + "step": 1540 + }, + { + "epoch": 1.9178593652769136, + "grad_norm": 3.029939889907837, + "learning_rate": 2.0387453874538746e-06, + "loss": 1.4481, + "step": 1541 + }, + { + "epoch": 1.9191039203484754, + "grad_norm": 2.8749356269836426, + "learning_rate": 2.036439114391144e-06, + "loss": 1.3735, + "step": 1542 + }, + { + "epoch": 1.9203484754200373, + "grad_norm": 2.9181101322174072, + "learning_rate": 2.0341328413284135e-06, + "loss": 1.4085, + "step": 1543 + }, + { + "epoch": 1.9215930304915991, + "grad_norm": 2.936319589614868, + "learning_rate": 2.031826568265683e-06, + "loss": 1.3827, + "step": 1544 + }, + { + "epoch": 1.9228375855631612, + "grad_norm": 2.9286844730377197, + "learning_rate": 2.0295202952029523e-06, + "loss": 1.4563, + "step": 1545 + }, + { + "epoch": 1.924082140634723, + "grad_norm": 2.790752649307251, + "learning_rate": 2.0272140221402213e-06, + "loss": 1.3935, + "step": 1546 + }, + { + "epoch": 1.9253266957062851, + "grad_norm": 2.878164529800415, + "learning_rate": 2.024907749077491e-06, + "loss": 1.4159, + "step": 1547 + }, + { + "epoch": 1.926571250777847, + "grad_norm": 3.0583767890930176, + "learning_rate": 2.02260147601476e-06, + "loss": 1.4036, + "step": 1548 + }, + { + "epoch": 1.9278158058494088, + "grad_norm": 2.943686008453369, + "learning_rate": 2.02029520295203e-06, + "loss": 1.4178, + "step": 1549 + }, + { + "epoch": 1.9290603609209707, + "grad_norm": 2.983553886413574, + "learning_rate": 2.017988929889299e-06, + "loss": 1.3959, + "step": 1550 + }, + { + "epoch": 1.9290603609209707, + "eval_loss": 1.610864520072937, + "eval_runtime": 44.5262, + "eval_samples_per_second": 22.459, + "eval_steps_per_second": 0.943, + "step": 1550 + }, + { + "epoch": 1.9303049159925325, + "grad_norm": 3.0323703289031982, + "learning_rate": 2.0156826568265684e-06, + "loss": 1.4362, + "step": 1551 + }, + { + "epoch": 1.9315494710640946, + "grad_norm": 3.036627769470215, + "learning_rate": 2.013376383763838e-06, + "loss": 1.4132, + "step": 1552 + }, + { + "epoch": 1.9327940261356567, + "grad_norm": 2.901071310043335, + "learning_rate": 2.0110701107011073e-06, + "loss": 1.3846, + "step": 1553 + }, + { + "epoch": 1.9340385812072185, + "grad_norm": 3.002803325653076, + "learning_rate": 2.0087638376383767e-06, + "loss": 1.4402, + "step": 1554 + }, + { + "epoch": 1.9352831362787803, + "grad_norm": 2.969228982925415, + "learning_rate": 2.006457564575646e-06, + "loss": 1.4387, + "step": 1555 + }, + { + "epoch": 1.9365276913503422, + "grad_norm": 2.9773998260498047, + "learning_rate": 2.004151291512915e-06, + "loss": 1.4149, + "step": 1556 + }, + { + "epoch": 1.937772246421904, + "grad_norm": 2.831655502319336, + "learning_rate": 2.0018450184501845e-06, + "loss": 1.3753, + "step": 1557 + }, + { + "epoch": 1.9390168014934661, + "grad_norm": 2.835994005203247, + "learning_rate": 1.999538745387454e-06, + "loss": 1.345, + "step": 1558 + }, + { + "epoch": 1.940261356565028, + "grad_norm": 3.0328211784362793, + "learning_rate": 1.9972324723247234e-06, + "loss": 1.3473, + "step": 1559 + }, + { + "epoch": 1.94150591163659, + "grad_norm": 2.9441378116607666, + "learning_rate": 1.994926199261993e-06, + "loss": 1.3913, + "step": 1560 + }, + { + "epoch": 1.94150591163659, + "eval_loss": 1.6193854808807373, + "eval_runtime": 44.6194, + "eval_samples_per_second": 22.412, + "eval_steps_per_second": 0.941, + "step": 1560 + }, + { + "epoch": 1.9427504667081519, + "grad_norm": 2.905486822128296, + "learning_rate": 1.9926199261992622e-06, + "loss": 1.4043, + "step": 1561 + }, + { + "epoch": 1.9439950217797137, + "grad_norm": 2.7758026123046875, + "learning_rate": 1.9903136531365317e-06, + "loss": 1.3974, + "step": 1562 + }, + { + "epoch": 1.9452395768512756, + "grad_norm": 2.7883355617523193, + "learning_rate": 1.988007380073801e-06, + "loss": 1.4246, + "step": 1563 + }, + { + "epoch": 1.9464841319228374, + "grad_norm": 2.8322808742523193, + "learning_rate": 1.98570110701107e-06, + "loss": 1.4002, + "step": 1564 + }, + { + "epoch": 1.9477286869943995, + "grad_norm": 2.948850631713867, + "learning_rate": 1.9833948339483395e-06, + "loss": 1.3866, + "step": 1565 + }, + { + "epoch": 1.9489732420659616, + "grad_norm": 2.9772586822509766, + "learning_rate": 1.981088560885609e-06, + "loss": 1.4473, + "step": 1566 + }, + { + "epoch": 1.9502177971375234, + "grad_norm": 2.9183740615844727, + "learning_rate": 1.9787822878228783e-06, + "loss": 1.3943, + "step": 1567 + }, + { + "epoch": 1.9514623522090853, + "grad_norm": 3.10422682762146, + "learning_rate": 1.9764760147601478e-06, + "loss": 1.465, + "step": 1568 + }, + { + "epoch": 1.952706907280647, + "grad_norm": 3.001533031463623, + "learning_rate": 1.974169741697417e-06, + "loss": 1.4133, + "step": 1569 + }, + { + "epoch": 1.953951462352209, + "grad_norm": 2.9521870613098145, + "learning_rate": 1.9718634686346866e-06, + "loss": 1.4652, + "step": 1570 + }, + { + "epoch": 1.953951462352209, + "eval_loss": 1.617977499961853, + "eval_runtime": 45.7189, + "eval_samples_per_second": 21.873, + "eval_steps_per_second": 0.919, + "step": 1570 + }, + { + "epoch": 1.955196017423771, + "grad_norm": 2.9504826068878174, + "learning_rate": 1.969557195571956e-06, + "loss": 1.4117, + "step": 1571 + }, + { + "epoch": 1.9564405724953329, + "grad_norm": 2.9383559226989746, + "learning_rate": 1.9672509225092255e-06, + "loss": 1.3439, + "step": 1572 + }, + { + "epoch": 1.957685127566895, + "grad_norm": 2.850644588470459, + "learning_rate": 1.9649446494464945e-06, + "loss": 1.3521, + "step": 1573 + }, + { + "epoch": 1.9589296826384568, + "grad_norm": 2.8787410259246826, + "learning_rate": 1.962638376383764e-06, + "loss": 1.4268, + "step": 1574 + }, + { + "epoch": 1.9601742377100186, + "grad_norm": 2.966080665588379, + "learning_rate": 1.9603321033210333e-06, + "loss": 1.3367, + "step": 1575 + }, + { + "epoch": 1.9614187927815805, + "grad_norm": 2.858222007751465, + "learning_rate": 1.9580258302583027e-06, + "loss": 1.3842, + "step": 1576 + }, + { + "epoch": 1.9626633478531423, + "grad_norm": 2.924525737762451, + "learning_rate": 1.955719557195572e-06, + "loss": 1.4561, + "step": 1577 + }, + { + "epoch": 1.9639079029247044, + "grad_norm": 2.9281182289123535, + "learning_rate": 1.9534132841328416e-06, + "loss": 1.4179, + "step": 1578 + }, + { + "epoch": 1.9651524579962665, + "grad_norm": 2.982023000717163, + "learning_rate": 1.951107011070111e-06, + "loss": 1.4413, + "step": 1579 + }, + { + "epoch": 1.9663970130678283, + "grad_norm": 2.7912652492523193, + "learning_rate": 1.9488007380073804e-06, + "loss": 1.3577, + "step": 1580 + }, + { + "epoch": 1.9663970130678283, + "eval_loss": 1.6062211990356445, + "eval_runtime": 46.7931, + "eval_samples_per_second": 21.371, + "eval_steps_per_second": 0.898, + "step": 1580 + }, + { + "epoch": 1.9676415681393902, + "grad_norm": 2.9585583209991455, + "learning_rate": 1.9464944649446494e-06, + "loss": 1.4084, + "step": 1581 + }, + { + "epoch": 1.968886123210952, + "grad_norm": 2.879201889038086, + "learning_rate": 1.944188191881919e-06, + "loss": 1.4164, + "step": 1582 + }, + { + "epoch": 1.9701306782825139, + "grad_norm": 2.9510116577148438, + "learning_rate": 1.9418819188191883e-06, + "loss": 1.4211, + "step": 1583 + }, + { + "epoch": 1.971375233354076, + "grad_norm": 2.973844528198242, + "learning_rate": 1.9395756457564577e-06, + "loss": 1.4358, + "step": 1584 + }, + { + "epoch": 1.9726197884256378, + "grad_norm": 2.939131736755371, + "learning_rate": 1.937269372693727e-06, + "loss": 1.407, + "step": 1585 + }, + { + "epoch": 1.9738643434971999, + "grad_norm": 2.956646680831909, + "learning_rate": 1.9349630996309965e-06, + "loss": 1.3872, + "step": 1586 + }, + { + "epoch": 1.9751088985687617, + "grad_norm": 3.070354461669922, + "learning_rate": 1.932656826568266e-06, + "loss": 1.412, + "step": 1587 + }, + { + "epoch": 1.9763534536403236, + "grad_norm": 2.8143749237060547, + "learning_rate": 1.9303505535055354e-06, + "loss": 1.3522, + "step": 1588 + }, + { + "epoch": 1.9775980087118854, + "grad_norm": 2.8610002994537354, + "learning_rate": 1.9280442804428044e-06, + "loss": 1.3988, + "step": 1589 + }, + { + "epoch": 1.9788425637834475, + "grad_norm": 2.951023817062378, + "learning_rate": 1.925738007380074e-06, + "loss": 1.3855, + "step": 1590 + }, + { + "epoch": 1.9788425637834475, + "eval_loss": 1.613527536392212, + "eval_runtime": 46.5242, + "eval_samples_per_second": 21.494, + "eval_steps_per_second": 0.903, + "step": 1590 + }, + { + "epoch": 1.9800871188550093, + "grad_norm": 2.9333510398864746, + "learning_rate": 1.9234317343173432e-06, + "loss": 1.4518, + "step": 1591 + }, + { + "epoch": 1.9813316739265714, + "grad_norm": 2.847015619277954, + "learning_rate": 1.9211254612546126e-06, + "loss": 1.3999, + "step": 1592 + }, + { + "epoch": 1.9825762289981332, + "grad_norm": 2.969320058822632, + "learning_rate": 1.918819188191882e-06, + "loss": 1.4241, + "step": 1593 + }, + { + "epoch": 1.983820784069695, + "grad_norm": 2.862797498703003, + "learning_rate": 1.9165129151291515e-06, + "loss": 1.3929, + "step": 1594 + }, + { + "epoch": 1.985065339141257, + "grad_norm": 2.90384840965271, + "learning_rate": 1.914206642066421e-06, + "loss": 1.4127, + "step": 1595 + }, + { + "epoch": 1.9863098942128188, + "grad_norm": 2.8059256076812744, + "learning_rate": 1.9119003690036903e-06, + "loss": 1.4155, + "step": 1596 + }, + { + "epoch": 1.9875544492843809, + "grad_norm": 2.9068455696105957, + "learning_rate": 1.9095940959409593e-06, + "loss": 1.3924, + "step": 1597 + }, + { + "epoch": 1.9887990043559427, + "grad_norm": 2.903542995452881, + "learning_rate": 1.907287822878229e-06, + "loss": 1.4039, + "step": 1598 + }, + { + "epoch": 1.9900435594275048, + "grad_norm": 2.8566174507141113, + "learning_rate": 1.9049815498154984e-06, + "loss": 1.4276, + "step": 1599 + }, + { + "epoch": 1.9912881144990666, + "grad_norm": 2.8891515731811523, + "learning_rate": 1.9026752767527676e-06, + "loss": 1.4555, + "step": 1600 + }, + { + "epoch": 1.9912881144990666, + "eval_loss": 1.6086686849594116, + "eval_runtime": 46.2785, + "eval_samples_per_second": 21.608, + "eval_steps_per_second": 0.908, + "step": 1600 + }, + { + "epoch": 1.9925326695706285, + "grad_norm": 2.933851957321167, + "learning_rate": 1.900369003690037e-06, + "loss": 1.3881, + "step": 1601 + }, + { + "epoch": 1.9937772246421903, + "grad_norm": 2.8393521308898926, + "learning_rate": 1.8980627306273065e-06, + "loss": 1.3959, + "step": 1602 + }, + { + "epoch": 1.9950217797137524, + "grad_norm": 2.839740753173828, + "learning_rate": 1.8957564575645759e-06, + "loss": 1.384, + "step": 1603 + }, + { + "epoch": 1.9962663347853142, + "grad_norm": 2.7918097972869873, + "learning_rate": 1.8934501845018453e-06, + "loss": 1.4173, + "step": 1604 + }, + { + "epoch": 1.9975108898568763, + "grad_norm": 2.920839548110962, + "learning_rate": 1.8911439114391145e-06, + "loss": 1.3693, + "step": 1605 + }, + { + "epoch": 1.9987554449284382, + "grad_norm": 2.9545562267303467, + "learning_rate": 1.888837638376384e-06, + "loss": 1.4062, + "step": 1606 + }, + { + "epoch": 2.0, + "grad_norm": 2.912815570831299, + "learning_rate": 1.8865313653136534e-06, + "loss": 1.4146, + "step": 1607 + }, + { + "epoch": 2.001244555071562, + "grad_norm": 3.1125903129577637, + "learning_rate": 1.8842250922509226e-06, + "loss": 1.3294, + "step": 1608 + }, + { + "epoch": 2.0024891101431237, + "grad_norm": 3.123905658721924, + "learning_rate": 1.881918819188192e-06, + "loss": 1.4175, + "step": 1609 + }, + { + "epoch": 2.0037336652146855, + "grad_norm": 3.055100440979004, + "learning_rate": 1.8796125461254614e-06, + "loss": 1.3628, + "step": 1610 + }, + { + "epoch": 2.0037336652146855, + "eval_loss": 1.6198724508285522, + "eval_runtime": 42.5155, + "eval_samples_per_second": 23.521, + "eval_steps_per_second": 0.988, + "step": 1610 + }, + { + "epoch": 2.004978220286248, + "grad_norm": 3.087313413619995, + "learning_rate": 1.8773062730627308e-06, + "loss": 1.292, + "step": 1611 + }, + { + "epoch": 2.0062227753578097, + "grad_norm": 3.1849849224090576, + "learning_rate": 1.8750000000000003e-06, + "loss": 1.3393, + "step": 1612 + }, + { + "epoch": 2.0074673304293715, + "grad_norm": 3.243750810623169, + "learning_rate": 1.8726937269372695e-06, + "loss": 1.3367, + "step": 1613 + }, + { + "epoch": 2.0087118855009334, + "grad_norm": 3.3086085319519043, + "learning_rate": 1.8703874538745389e-06, + "loss": 1.3099, + "step": 1614 + }, + { + "epoch": 2.0099564405724952, + "grad_norm": 3.388633966445923, + "learning_rate": 1.8680811808118083e-06, + "loss": 1.3816, + "step": 1615 + }, + { + "epoch": 2.011200995644057, + "grad_norm": 3.422671318054199, + "learning_rate": 1.8657749077490777e-06, + "loss": 1.3799, + "step": 1616 + }, + { + "epoch": 2.0124455507156194, + "grad_norm": 3.1104466915130615, + "learning_rate": 1.863468634686347e-06, + "loss": 1.3752, + "step": 1617 + }, + { + "epoch": 2.013690105787181, + "grad_norm": 3.2728488445281982, + "learning_rate": 1.8611623616236164e-06, + "loss": 1.322, + "step": 1618 + }, + { + "epoch": 2.014934660858743, + "grad_norm": 3.3357644081115723, + "learning_rate": 1.8588560885608858e-06, + "loss": 1.3193, + "step": 1619 + }, + { + "epoch": 2.016179215930305, + "grad_norm": 3.1026456356048584, + "learning_rate": 1.856549815498155e-06, + "loss": 1.3394, + "step": 1620 + }, + { + "epoch": 2.016179215930305, + "eval_loss": 1.6217585802078247, + "eval_runtime": 45.9794, + "eval_samples_per_second": 21.749, + "eval_steps_per_second": 0.913, + "step": 1620 + }, + { + "epoch": 2.0174237710018668, + "grad_norm": 3.314291000366211, + "learning_rate": 1.8542435424354244e-06, + "loss": 1.3778, + "step": 1621 + }, + { + "epoch": 2.0186683260734286, + "grad_norm": 3.3624770641326904, + "learning_rate": 1.8519372693726939e-06, + "loss": 1.3948, + "step": 1622 + }, + { + "epoch": 2.0199128811449905, + "grad_norm": 3.4041574001312256, + "learning_rate": 1.8496309963099633e-06, + "loss": 1.3109, + "step": 1623 + }, + { + "epoch": 2.0211574362165527, + "grad_norm": 3.5128726959228516, + "learning_rate": 1.8473247232472327e-06, + "loss": 1.4223, + "step": 1624 + }, + { + "epoch": 2.0224019912881146, + "grad_norm": 3.444768190383911, + "learning_rate": 1.845018450184502e-06, + "loss": 1.3672, + "step": 1625 + }, + { + "epoch": 2.0236465463596764, + "grad_norm": 3.478060722351074, + "learning_rate": 1.8427121771217713e-06, + "loss": 1.4384, + "step": 1626 + }, + { + "epoch": 2.0248911014312383, + "grad_norm": 3.211465835571289, + "learning_rate": 1.8404059040590408e-06, + "loss": 1.3891, + "step": 1627 + }, + { + "epoch": 2.0261356565028, + "grad_norm": 3.1283977031707764, + "learning_rate": 1.83809963099631e-06, + "loss": 1.3752, + "step": 1628 + }, + { + "epoch": 2.027380211574362, + "grad_norm": 3.3699934482574463, + "learning_rate": 1.8357933579335796e-06, + "loss": 1.3751, + "step": 1629 + }, + { + "epoch": 2.0286247666459243, + "grad_norm": 3.0729823112487793, + "learning_rate": 1.8334870848708488e-06, + "loss": 1.3452, + "step": 1630 + }, + { + "epoch": 2.0286247666459243, + "eval_loss": 1.6185853481292725, + "eval_runtime": 45.2873, + "eval_samples_per_second": 22.081, + "eval_steps_per_second": 0.927, + "step": 1630 + }, + { + "epoch": 2.029869321717486, + "grad_norm": 3.274139642715454, + "learning_rate": 1.8311808118081182e-06, + "loss": 1.3242, + "step": 1631 + }, + { + "epoch": 2.031113876789048, + "grad_norm": 3.2869648933410645, + "learning_rate": 1.8288745387453877e-06, + "loss": 1.3435, + "step": 1632 + }, + { + "epoch": 2.03235843186061, + "grad_norm": 3.3614306449890137, + "learning_rate": 1.8265682656826569e-06, + "loss": 1.3705, + "step": 1633 + }, + { + "epoch": 2.0336029869321717, + "grad_norm": 3.411914587020874, + "learning_rate": 1.8242619926199265e-06, + "loss": 1.3942, + "step": 1634 + }, + { + "epoch": 2.0348475420037335, + "grad_norm": 3.274519920349121, + "learning_rate": 1.8219557195571957e-06, + "loss": 1.3393, + "step": 1635 + }, + { + "epoch": 2.0360920970752954, + "grad_norm": 3.993478775024414, + "learning_rate": 1.819649446494465e-06, + "loss": 1.3713, + "step": 1636 + }, + { + "epoch": 2.0373366521468577, + "grad_norm": 3.273571491241455, + "learning_rate": 1.8173431734317346e-06, + "loss": 1.3337, + "step": 1637 + }, + { + "epoch": 2.0385812072184195, + "grad_norm": 3.1542446613311768, + "learning_rate": 1.8150369003690038e-06, + "loss": 1.3793, + "step": 1638 + }, + { + "epoch": 2.0398257622899814, + "grad_norm": 3.263773202896118, + "learning_rate": 1.812730627306273e-06, + "loss": 1.3844, + "step": 1639 + }, + { + "epoch": 2.041070317361543, + "grad_norm": 3.268772840499878, + "learning_rate": 1.8104243542435426e-06, + "loss": 1.3947, + "step": 1640 + }, + { + "epoch": 2.041070317361543, + "eval_loss": 1.6149734258651733, + "eval_runtime": 48.5557, + "eval_samples_per_second": 20.595, + "eval_steps_per_second": 0.865, + "step": 1640 + }, + { + "epoch": 2.042314872433105, + "grad_norm": 3.190168619155884, + "learning_rate": 1.8081180811808118e-06, + "loss": 1.3276, + "step": 1641 + }, + { + "epoch": 2.043559427504667, + "grad_norm": 3.1573212146759033, + "learning_rate": 1.8058118081180815e-06, + "loss": 1.3307, + "step": 1642 + }, + { + "epoch": 2.044803982576229, + "grad_norm": 3.286513328552246, + "learning_rate": 1.8035055350553507e-06, + "loss": 1.3359, + "step": 1643 + }, + { + "epoch": 2.046048537647791, + "grad_norm": 3.342024803161621, + "learning_rate": 1.8011992619926199e-06, + "loss": 1.3472, + "step": 1644 + }, + { + "epoch": 2.047293092719353, + "grad_norm": 3.3237056732177734, + "learning_rate": 1.7988929889298895e-06, + "loss": 1.3898, + "step": 1645 + }, + { + "epoch": 2.0485376477909147, + "grad_norm": 3.370035409927368, + "learning_rate": 1.7965867158671587e-06, + "loss": 1.3537, + "step": 1646 + }, + { + "epoch": 2.0497822028624766, + "grad_norm": 3.1636598110198975, + "learning_rate": 1.7942804428044284e-06, + "loss": 1.3455, + "step": 1647 + }, + { + "epoch": 2.0510267579340384, + "grad_norm": 3.239396810531616, + "learning_rate": 1.7919741697416976e-06, + "loss": 1.3212, + "step": 1648 + }, + { + "epoch": 2.0522713130056003, + "grad_norm": 3.2765328884124756, + "learning_rate": 1.7896678966789668e-06, + "loss": 1.396, + "step": 1649 + }, + { + "epoch": 2.0535158680771626, + "grad_norm": 3.2259459495544434, + "learning_rate": 1.7873616236162364e-06, + "loss": 1.3614, + "step": 1650 + }, + { + "epoch": 2.0535158680771626, + "eval_loss": 1.614806056022644, + "eval_runtime": 49.8704, + "eval_samples_per_second": 20.052, + "eval_steps_per_second": 0.842, + "step": 1650 + }, + { + "epoch": 2.0547604231487244, + "grad_norm": 3.257683277130127, + "learning_rate": 1.7850553505535056e-06, + "loss": 1.3663, + "step": 1651 + }, + { + "epoch": 2.0560049782202863, + "grad_norm": 3.292070150375366, + "learning_rate": 1.7827490774907748e-06, + "loss": 1.3548, + "step": 1652 + }, + { + "epoch": 2.057249533291848, + "grad_norm": 3.3517003059387207, + "learning_rate": 1.7804428044280445e-06, + "loss": 1.4015, + "step": 1653 + }, + { + "epoch": 2.05849408836341, + "grad_norm": 3.5100908279418945, + "learning_rate": 1.7781365313653137e-06, + "loss": 1.3666, + "step": 1654 + }, + { + "epoch": 2.059738643434972, + "grad_norm": 3.3054633140563965, + "learning_rate": 1.7758302583025833e-06, + "loss": 1.3691, + "step": 1655 + }, + { + "epoch": 2.060983198506534, + "grad_norm": 3.1036577224731445, + "learning_rate": 1.7735239852398525e-06, + "loss": 1.2721, + "step": 1656 + }, + { + "epoch": 2.062227753578096, + "grad_norm": 3.604623317718506, + "learning_rate": 1.7712177121771217e-06, + "loss": 1.376, + "step": 1657 + }, + { + "epoch": 2.063472308649658, + "grad_norm": 3.5246458053588867, + "learning_rate": 1.7689114391143914e-06, + "loss": 1.3891, + "step": 1658 + }, + { + "epoch": 2.0647168637212197, + "grad_norm": 3.286940336227417, + "learning_rate": 1.7666051660516606e-06, + "loss": 1.3978, + "step": 1659 + }, + { + "epoch": 2.0659614187927815, + "grad_norm": 3.1022047996520996, + "learning_rate": 1.7642988929889302e-06, + "loss": 1.3373, + "step": 1660 + }, + { + "epoch": 2.0659614187927815, + "eval_loss": 1.6144907474517822, + "eval_runtime": 48.4382, + "eval_samples_per_second": 20.645, + "eval_steps_per_second": 0.867, + "step": 1660 + }, + { + "epoch": 2.0672059738643433, + "grad_norm": 3.4111568927764893, + "learning_rate": 1.7619926199261994e-06, + "loss": 1.3816, + "step": 1661 + }, + { + "epoch": 2.068450528935905, + "grad_norm": 3.478947162628174, + "learning_rate": 1.7596863468634687e-06, + "loss": 1.3601, + "step": 1662 + }, + { + "epoch": 2.0696950840074675, + "grad_norm": 3.3320810794830322, + "learning_rate": 1.7573800738007383e-06, + "loss": 1.3454, + "step": 1663 + }, + { + "epoch": 2.0709396390790293, + "grad_norm": 3.475529909133911, + "learning_rate": 1.7550738007380075e-06, + "loss": 1.3666, + "step": 1664 + }, + { + "epoch": 2.072184194150591, + "grad_norm": 3.302342653274536, + "learning_rate": 1.752767527675277e-06, + "loss": 1.3824, + "step": 1665 + }, + { + "epoch": 2.073428749222153, + "grad_norm": 3.198641777038574, + "learning_rate": 1.7504612546125463e-06, + "loss": 1.3398, + "step": 1666 + }, + { + "epoch": 2.074673304293715, + "grad_norm": 3.421834945678711, + "learning_rate": 1.7481549815498156e-06, + "loss": 1.363, + "step": 1667 + }, + { + "epoch": 2.0759178593652767, + "grad_norm": 3.654343843460083, + "learning_rate": 1.7458487084870852e-06, + "loss": 1.3852, + "step": 1668 + }, + { + "epoch": 2.077162414436839, + "grad_norm": 3.3543035984039307, + "learning_rate": 1.7435424354243544e-06, + "loss": 1.3554, + "step": 1669 + }, + { + "epoch": 2.078406969508401, + "grad_norm": 3.2144811153411865, + "learning_rate": 1.7412361623616236e-06, + "loss": 1.3137, + "step": 1670 + }, + { + "epoch": 2.078406969508401, + "eval_loss": 1.6155465841293335, + "eval_runtime": 52.4111, + "eval_samples_per_second": 19.08, + "eval_steps_per_second": 0.801, + "step": 1670 + }, + { + "epoch": 2.0796515245799627, + "grad_norm": 3.2240991592407227, + "learning_rate": 1.7389298892988932e-06, + "loss": 1.3578, + "step": 1671 + }, + { + "epoch": 2.0808960796515246, + "grad_norm": 3.1450018882751465, + "learning_rate": 1.7366236162361625e-06, + "loss": 1.2999, + "step": 1672 + }, + { + "epoch": 2.0821406347230864, + "grad_norm": 3.4396843910217285, + "learning_rate": 1.7343173431734319e-06, + "loss": 1.3522, + "step": 1673 + }, + { + "epoch": 2.0833851897946483, + "grad_norm": 3.1719677448272705, + "learning_rate": 1.7320110701107013e-06, + "loss": 1.3552, + "step": 1674 + }, + { + "epoch": 2.08462974486621, + "grad_norm": 3.4420199394226074, + "learning_rate": 1.7297047970479705e-06, + "loss": 1.4033, + "step": 1675 + }, + { + "epoch": 2.0858742999377724, + "grad_norm": 3.3366570472717285, + "learning_rate": 1.7273985239852401e-06, + "loss": 1.3407, + "step": 1676 + }, + { + "epoch": 2.0871188550093343, + "grad_norm": 3.3416335582733154, + "learning_rate": 1.7250922509225094e-06, + "loss": 1.3882, + "step": 1677 + }, + { + "epoch": 2.088363410080896, + "grad_norm": 3.4189534187316895, + "learning_rate": 1.7227859778597788e-06, + "loss": 1.3996, + "step": 1678 + }, + { + "epoch": 2.089607965152458, + "grad_norm": 3.309095859527588, + "learning_rate": 1.7204797047970482e-06, + "loss": 1.3799, + "step": 1679 + }, + { + "epoch": 2.09085252022402, + "grad_norm": 3.369511365890503, + "learning_rate": 1.7181734317343174e-06, + "loss": 1.4159, + "step": 1680 + }, + { + "epoch": 2.09085252022402, + "eval_loss": 1.6145448684692383, + "eval_runtime": 49.0228, + "eval_samples_per_second": 20.399, + "eval_steps_per_second": 0.857, + "step": 1680 + }, + { + "epoch": 2.0920970752955816, + "grad_norm": 3.323286771774292, + "learning_rate": 1.7158671586715868e-06, + "loss": 1.4035, + "step": 1681 + }, + { + "epoch": 2.093341630367144, + "grad_norm": 3.3053157329559326, + "learning_rate": 1.7135608856088563e-06, + "loss": 1.3498, + "step": 1682 + }, + { + "epoch": 2.094586185438706, + "grad_norm": 3.141890048980713, + "learning_rate": 1.7112546125461257e-06, + "loss": 1.3438, + "step": 1683 + }, + { + "epoch": 2.0958307405102676, + "grad_norm": 3.135058879852295, + "learning_rate": 1.708948339483395e-06, + "loss": 1.3726, + "step": 1684 + }, + { + "epoch": 2.0970752955818295, + "grad_norm": 3.2389063835144043, + "learning_rate": 1.7066420664206643e-06, + "loss": 1.3661, + "step": 1685 + }, + { + "epoch": 2.0983198506533913, + "grad_norm": 3.1741721630096436, + "learning_rate": 1.7043357933579337e-06, + "loss": 1.3438, + "step": 1686 + }, + { + "epoch": 2.099564405724953, + "grad_norm": 3.3524587154388428, + "learning_rate": 1.7020295202952032e-06, + "loss": 1.3388, + "step": 1687 + }, + { + "epoch": 2.100808960796515, + "grad_norm": 3.3574740886688232, + "learning_rate": 1.6997232472324724e-06, + "loss": 1.3702, + "step": 1688 + }, + { + "epoch": 2.1020535158680773, + "grad_norm": 3.458645820617676, + "learning_rate": 1.6974169741697418e-06, + "loss": 1.3785, + "step": 1689 + }, + { + "epoch": 2.103298070939639, + "grad_norm": 3.346677303314209, + "learning_rate": 1.6951107011070112e-06, + "loss": 1.3295, + "step": 1690 + }, + { + "epoch": 2.103298070939639, + "eval_loss": 1.6184555292129517, + "eval_runtime": 46.6544, + "eval_samples_per_second": 21.434, + "eval_steps_per_second": 0.9, + "step": 1690 + }, + { + "epoch": 2.104542626011201, + "grad_norm": 3.3403804302215576, + "learning_rate": 1.6928044280442806e-06, + "loss": 1.3912, + "step": 1691 + }, + { + "epoch": 2.105787181082763, + "grad_norm": 3.2638344764709473, + "learning_rate": 1.6904981549815499e-06, + "loss": 1.3684, + "step": 1692 + }, + { + "epoch": 2.1070317361543247, + "grad_norm": 3.192701578140259, + "learning_rate": 1.6881918819188193e-06, + "loss": 1.2997, + "step": 1693 + }, + { + "epoch": 2.1082762912258866, + "grad_norm": 3.482121706008911, + "learning_rate": 1.6858856088560887e-06, + "loss": 1.3515, + "step": 1694 + }, + { + "epoch": 2.109520846297449, + "grad_norm": 3.292353630065918, + "learning_rate": 1.6835793357933581e-06, + "loss": 1.3603, + "step": 1695 + }, + { + "epoch": 2.1107654013690107, + "grad_norm": 3.34220290184021, + "learning_rate": 1.6812730627306275e-06, + "loss": 1.3666, + "step": 1696 + }, + { + "epoch": 2.1120099564405725, + "grad_norm": 3.190058469772339, + "learning_rate": 1.6789667896678968e-06, + "loss": 1.3549, + "step": 1697 + }, + { + "epoch": 2.1132545115121344, + "grad_norm": 3.3759775161743164, + "learning_rate": 1.6766605166051662e-06, + "loss": 1.3628, + "step": 1698 + }, + { + "epoch": 2.1144990665836962, + "grad_norm": 3.419440507888794, + "learning_rate": 1.6743542435424356e-06, + "loss": 1.3618, + "step": 1699 + }, + { + "epoch": 2.115743621655258, + "grad_norm": 3.5021324157714844, + "learning_rate": 1.6720479704797048e-06, + "loss": 1.4023, + "step": 1700 + }, + { + "epoch": 2.115743621655258, + "eval_loss": 1.6218053102493286, + "eval_runtime": 48.3483, + "eval_samples_per_second": 20.683, + "eval_steps_per_second": 0.869, + "step": 1700 + } + ], + "logging_steps": 1, + "max_steps": 2409, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 100, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 5.299425995915264e+17, + "train_batch_size": 3, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/training_args.bin b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..6cdf93817e8f6d23b60d79888b226cc66ed2e88b --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a50e4918f3c49370326e14a7ed2f73b48b05e521f4b3399b53438fac6abee926 +size 6011 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/zero_to_fp32.py b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..24cc342e78d1a006c782b3a4cd68d9ce786d8fd8 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1700/zero_to_fp32.py @@ -0,0 +1,604 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: python zero_to_fp32.py . pytorch_model.bin + +import argparse +import torch +import glob +import math +import os +import re +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + + total_files = len(files) + state_dicts = [] + for f in files: + state_dict = torch.load(f, map_location=device) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + if zero_stage <= 2: + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + elif zero_stage == 3: + # if there is more than one param group, there will be multiple flattened tensors - one + # flattened tensor per group - for simplicity merge them into a single tensor + # + # XXX: could make the script more memory efficient for when there are multiple groups - it + # will require matching the sub-lists of param_shapes for each param group flattened tensor + + fp32_flat_groups = [ + torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts)) + ] + + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = fp32_flat_groups[0].numel() * world_size + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + for name, shape in param_shapes.items(): + + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # XXX: memory usage doubles here + state_dict[name] = torch.cat( + tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)), + 0).narrow(0, 0, unpartitioned_numel).view(shape) + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + + Returns: + - pytorch ``state_dict`` + + Note: this approach may not work if your application doesn't have sufficient free CPU memory and + you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters) + print(f"Saving fp32 state dict to {output_file}") + torch.save(state_dict, output_file) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info(f"Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info(f"Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument( + "output_file", + type=str, + help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_file, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fb0f9a53a507ffcdb60410deb71d60ef801bf350 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/config.json @@ -0,0 +1,36 @@ +{ + "_name_or_path": "meta-llama/Llama-3.2-3B", + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "eos_token_id": 128001, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 3072, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 24, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "torch_dtype": "float16", + "transformers_version": "4.45.1", + "use_cache": true, + "vocab_size": 128256 +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/generation_config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9c389b841a9a29ddd904e02d1eb0e08dcce82ad9 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": 128001, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.45.1" +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/latest b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/latest new file mode 100644 index 0000000000000000000000000000000000000000..e5bdf58d4f29d34e909da25905fad376f73e7c29 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/latest @@ -0,0 +1 @@ +global_step400 \ No newline at end of file diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/model.safetensors.index.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..ed64de846d720b9a7859dc20575fea8e8ca51940 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/model.safetensors.index.json @@ -0,0 +1,262 @@ +{ + "metadata": { + "total_size": 7213504512 + }, + "weight_map": { + "lm_head.weight": "model-00002-of-00002.safetensors", + "model.embed_tokens.weight": "model-00001-of-00002.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.norm.weight": "model-00002-of-00002.safetensors" + } +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/rng_state_0.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..d5b3eaa75e01e0564b8641c8f89a308c359a8a6c --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf3509a33c2e615b42fa452a51f3b448cd738037d76a4830d79b89c785555ed0 +size 16567 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/rng_state_1.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..f31c91e6fc0dcc1eb5c101a4c40e35f5b315d26f --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c32df687c2bf9ce5fb1beafdcec2d9cfb49bc02e3a06d089279c235d40a82ca8 +size 16567 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/rng_state_2.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..c1492af50e3afb56d1aecd7362d9e876ea1c01da --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fef8236d6b1951c0a5c3e8cf66c0be4ae93551b780648507e1981a0117386378 +size 16567 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/scheduler.pt b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..207bc81dcbfdea224b4e9a71bb746a20689aab5b --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5340761043b7ad61c7ba658f6fc46681f4d609c5ac3728de2c2013a9e0ce4e79 +size 627 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/special_tokens_map.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..6949c5975ee0e961ef61cf31010dce04df0a03f8 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "[PAD]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/tokenizer.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f28ecaeab53ae07feed29ccf8624d2b0a8344df9 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19fb2e1e3cdd6f7433d89fd6d62c82042599dd4984f342efe7fec6e159e6a8f6 +size 17210734 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/tokenizer_config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8991b49e9c2a43fc527dab9e09ad8171f0cc5943 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/tokenizer_config.json @@ -0,0 +1,2086 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "[PAD]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128257": { + "content": "🅂", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "128258": { + "content": "🄿", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "[PAD]", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/trainer_state.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..1eccaf06ef38e304ae8445a8f76140723b9499b5 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/trainer_state.json @@ -0,0 +1,3153 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.49782202862476665, + "eval_steps": 10, + "global_step": 400, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0012445550715619166, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8368, + "step": 1 + }, + { + "epoch": 0.002489110143123833, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.86, + "step": 2 + }, + { + "epoch": 0.00373366521468575, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8265, + "step": 3 + }, + { + "epoch": 0.004978220286247666, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8271, + "step": 4 + }, + { + "epoch": 0.006222775357809583, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8768, + "step": 5 + }, + { + "epoch": 0.0074673304293715, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8402, + "step": 6 + }, + { + "epoch": 0.008711885500933417, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.785, + "step": 7 + }, + { + "epoch": 0.009956440572495333, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8466, + "step": 8 + }, + { + "epoch": 0.01120099564405725, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.7673, + "step": 9 + }, + { + "epoch": 0.012445550715619166, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8074, + "step": 10 + }, + { + "epoch": 0.012445550715619166, + "eval_loss": 2.8535053730010986, + "eval_runtime": 42.446, + "eval_samples_per_second": 23.559, + "eval_steps_per_second": 0.989, + "step": 10 + }, + { + "epoch": 0.013690105787181083, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8592, + "step": 11 + }, + { + "epoch": 0.014934660858743, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.804, + "step": 12 + }, + { + "epoch": 0.016179215930304917, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8076, + "step": 13 + }, + { + "epoch": 0.017423771001866834, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8219, + "step": 14 + }, + { + "epoch": 0.018668326073428748, + "grad_norm": 5.105710506439209, + "learning_rate": 2.0746887966804982e-08, + "loss": 2.7995, + "step": 15 + }, + { + "epoch": 0.019912881144990666, + "grad_norm": 5.156588077545166, + "learning_rate": 4.1493775933609963e-08, + "loss": 2.8577, + "step": 16 + }, + { + "epoch": 0.021157436216552583, + "grad_norm": 5.709373950958252, + "learning_rate": 6.224066390041494e-08, + "loss": 2.8433, + "step": 17 + }, + { + "epoch": 0.0224019912881145, + "grad_norm": 5.64658784866333, + "learning_rate": 8.298755186721993e-08, + "loss": 2.8428, + "step": 18 + }, + { + "epoch": 0.023646546359676415, + "grad_norm": 5.541937351226807, + "learning_rate": 1.037344398340249e-07, + "loss": 2.7785, + "step": 19 + }, + { + "epoch": 0.024891101431238332, + "grad_norm": 5.233374118804932, + "learning_rate": 1.2448132780082988e-07, + "loss": 2.8599, + "step": 20 + }, + { + "epoch": 0.024891101431238332, + "eval_loss": 2.8521182537078857, + "eval_runtime": 43.5664, + "eval_samples_per_second": 22.953, + "eval_steps_per_second": 0.964, + "step": 20 + }, + { + "epoch": 0.02613565650280025, + "grad_norm": 5.407598972320557, + "learning_rate": 1.4522821576763488e-07, + "loss": 2.8081, + "step": 21 + }, + { + "epoch": 0.027380211574362167, + "grad_norm": 5.0483317375183105, + "learning_rate": 1.6597510373443985e-07, + "loss": 2.8489, + "step": 22 + }, + { + "epoch": 0.02862476664592408, + "grad_norm": 4.985588550567627, + "learning_rate": 1.8672199170124483e-07, + "loss": 2.8375, + "step": 23 + }, + { + "epoch": 0.029869321717486, + "grad_norm": 5.201700687408447, + "learning_rate": 2.074688796680498e-07, + "loss": 2.8355, + "step": 24 + }, + { + "epoch": 0.031113876789047916, + "grad_norm": 5.009235382080078, + "learning_rate": 2.2821576763485478e-07, + "loss": 2.7774, + "step": 25 + }, + { + "epoch": 0.03235843186060983, + "grad_norm": 4.509210109710693, + "learning_rate": 2.4896265560165975e-07, + "loss": 2.7992, + "step": 26 + }, + { + "epoch": 0.03360298693217175, + "grad_norm": 4.529262065887451, + "learning_rate": 2.6970954356846476e-07, + "loss": 2.836, + "step": 27 + }, + { + "epoch": 0.03484754200373367, + "grad_norm": 4.526787281036377, + "learning_rate": 2.9045643153526976e-07, + "loss": 2.8128, + "step": 28 + }, + { + "epoch": 0.03609209707529558, + "grad_norm": 4.016609191894531, + "learning_rate": 3.112033195020747e-07, + "loss": 2.7947, + "step": 29 + }, + { + "epoch": 0.037336652146857496, + "grad_norm": 3.4784562587738037, + "learning_rate": 3.319502074688797e-07, + "loss": 2.7854, + "step": 30 + }, + { + "epoch": 0.037336652146857496, + "eval_loss": 2.8020100593566895, + "eval_runtime": 44.8464, + "eval_samples_per_second": 22.298, + "eval_steps_per_second": 0.937, + "step": 30 + }, + { + "epoch": 0.038581207218419414, + "grad_norm": 3.4887945652008057, + "learning_rate": 3.5269709543568466e-07, + "loss": 2.7908, + "step": 31 + }, + { + "epoch": 0.03982576228998133, + "grad_norm": 3.4533374309539795, + "learning_rate": 3.7344398340248966e-07, + "loss": 2.7358, + "step": 32 + }, + { + "epoch": 0.04107031736154325, + "grad_norm": 3.270954132080078, + "learning_rate": 3.941908713692946e-07, + "loss": 2.8013, + "step": 33 + }, + { + "epoch": 0.042314872433105166, + "grad_norm": 3.61322283744812, + "learning_rate": 4.149377593360996e-07, + "loss": 2.7521, + "step": 34 + }, + { + "epoch": 0.043559427504667084, + "grad_norm": 3.158719062805176, + "learning_rate": 4.3568464730290456e-07, + "loss": 2.7573, + "step": 35 + }, + { + "epoch": 0.044803982576229, + "grad_norm": 2.888746500015259, + "learning_rate": 4.5643153526970956e-07, + "loss": 2.7795, + "step": 36 + }, + { + "epoch": 0.04604853764779091, + "grad_norm": 3.022629499435425, + "learning_rate": 4.771784232365145e-07, + "loss": 2.7891, + "step": 37 + }, + { + "epoch": 0.04729309271935283, + "grad_norm": 3.038097381591797, + "learning_rate": 4.979253112033195e-07, + "loss": 2.7376, + "step": 38 + }, + { + "epoch": 0.04853764779091475, + "grad_norm": 2.9992239475250244, + "learning_rate": 5.186721991701245e-07, + "loss": 2.7279, + "step": 39 + }, + { + "epoch": 0.049782202862476664, + "grad_norm": 2.908842086791992, + "learning_rate": 5.394190871369295e-07, + "loss": 2.6582, + "step": 40 + }, + { + "epoch": 0.049782202862476664, + "eval_loss": 2.725968837738037, + "eval_runtime": 44.9498, + "eval_samples_per_second": 22.247, + "eval_steps_per_second": 0.934, + "step": 40 + }, + { + "epoch": 0.05102675793403858, + "grad_norm": 2.8025174140930176, + "learning_rate": 5.601659751037345e-07, + "loss": 2.6809, + "step": 41 + }, + { + "epoch": 0.0522713130056005, + "grad_norm": 2.7555699348449707, + "learning_rate": 5.809128630705395e-07, + "loss": 2.6954, + "step": 42 + }, + { + "epoch": 0.053515868077162417, + "grad_norm": 2.4732933044433594, + "learning_rate": 6.016597510373444e-07, + "loss": 2.7307, + "step": 43 + }, + { + "epoch": 0.054760423148724334, + "grad_norm": 2.451366424560547, + "learning_rate": 6.224066390041494e-07, + "loss": 2.7064, + "step": 44 + }, + { + "epoch": 0.056004978220286245, + "grad_norm": 2.5768678188323975, + "learning_rate": 6.431535269709543e-07, + "loss": 2.6466, + "step": 45 + }, + { + "epoch": 0.05724953329184816, + "grad_norm": 2.579332113265991, + "learning_rate": 6.639004149377594e-07, + "loss": 2.6127, + "step": 46 + }, + { + "epoch": 0.05849408836341008, + "grad_norm": 2.231207847595215, + "learning_rate": 6.846473029045644e-07, + "loss": 2.6429, + "step": 47 + }, + { + "epoch": 0.059738643434972, + "grad_norm": 2.134375810623169, + "learning_rate": 7.053941908713693e-07, + "loss": 2.6423, + "step": 48 + }, + { + "epoch": 0.060983198506533914, + "grad_norm": 2.0485494136810303, + "learning_rate": 7.261410788381744e-07, + "loss": 2.581, + "step": 49 + }, + { + "epoch": 0.06222775357809583, + "grad_norm": 2.02909517288208, + "learning_rate": 7.468879668049793e-07, + "loss": 2.6357, + "step": 50 + }, + { + "epoch": 0.06222775357809583, + "eval_loss": 2.632822036743164, + "eval_runtime": 47.0104, + "eval_samples_per_second": 21.272, + "eval_steps_per_second": 0.893, + "step": 50 + }, + { + "epoch": 0.06347230864965775, + "grad_norm": 1.892500877380371, + "learning_rate": 7.676348547717843e-07, + "loss": 2.5945, + "step": 51 + }, + { + "epoch": 0.06471686372121967, + "grad_norm": 2.107921600341797, + "learning_rate": 7.883817427385892e-07, + "loss": 2.5867, + "step": 52 + }, + { + "epoch": 0.06596141879278158, + "grad_norm": 1.8287527561187744, + "learning_rate": 8.091286307053943e-07, + "loss": 2.5862, + "step": 53 + }, + { + "epoch": 0.0672059738643435, + "grad_norm": 1.7342065572738647, + "learning_rate": 8.298755186721992e-07, + "loss": 2.5735, + "step": 54 + }, + { + "epoch": 0.06845052893590542, + "grad_norm": 1.7905038595199585, + "learning_rate": 8.506224066390042e-07, + "loss": 2.572, + "step": 55 + }, + { + "epoch": 0.06969508400746734, + "grad_norm": 1.9715144634246826, + "learning_rate": 8.713692946058091e-07, + "loss": 2.5797, + "step": 56 + }, + { + "epoch": 0.07093963907902924, + "grad_norm": 1.788966417312622, + "learning_rate": 8.921161825726142e-07, + "loss": 2.5567, + "step": 57 + }, + { + "epoch": 0.07218419415059116, + "grad_norm": 1.8978915214538574, + "learning_rate": 9.128630705394191e-07, + "loss": 2.5776, + "step": 58 + }, + { + "epoch": 0.07342874922215308, + "grad_norm": 1.6401689052581787, + "learning_rate": 9.336099585062241e-07, + "loss": 2.5662, + "step": 59 + }, + { + "epoch": 0.07467330429371499, + "grad_norm": 1.8200798034667969, + "learning_rate": 9.54356846473029e-07, + "loss": 2.5001, + "step": 60 + }, + { + "epoch": 0.07467330429371499, + "eval_loss": 2.569110870361328, + "eval_runtime": 47.3292, + "eval_samples_per_second": 21.129, + "eval_steps_per_second": 0.887, + "step": 60 + }, + { + "epoch": 0.07591785936527691, + "grad_norm": 1.5468674898147583, + "learning_rate": 9.751037344398341e-07, + "loss": 2.5308, + "step": 61 + }, + { + "epoch": 0.07716241443683883, + "grad_norm": 1.723684310913086, + "learning_rate": 9.95850622406639e-07, + "loss": 2.4907, + "step": 62 + }, + { + "epoch": 0.07840696950840075, + "grad_norm": 1.6346293687820435, + "learning_rate": 1.0165975103734441e-06, + "loss": 2.514, + "step": 63 + }, + { + "epoch": 0.07965152457996266, + "grad_norm": 1.8902088403701782, + "learning_rate": 1.037344398340249e-06, + "loss": 2.4859, + "step": 64 + }, + { + "epoch": 0.08089607965152458, + "grad_norm": 2.092611312866211, + "learning_rate": 1.058091286307054e-06, + "loss": 2.4888, + "step": 65 + }, + { + "epoch": 0.0821406347230865, + "grad_norm": 1.758482575416565, + "learning_rate": 1.078838174273859e-06, + "loss": 2.4537, + "step": 66 + }, + { + "epoch": 0.08338518979464841, + "grad_norm": 1.8941714763641357, + "learning_rate": 1.099585062240664e-06, + "loss": 2.5416, + "step": 67 + }, + { + "epoch": 0.08462974486621033, + "grad_norm": 1.7030234336853027, + "learning_rate": 1.120331950207469e-06, + "loss": 2.4791, + "step": 68 + }, + { + "epoch": 0.08587429993777225, + "grad_norm": 1.7602269649505615, + "learning_rate": 1.141078838174274e-06, + "loss": 2.4316, + "step": 69 + }, + { + "epoch": 0.08711885500933417, + "grad_norm": 1.7635431289672852, + "learning_rate": 1.161825726141079e-06, + "loss": 2.4674, + "step": 70 + }, + { + "epoch": 0.08711885500933417, + "eval_loss": 2.4922080039978027, + "eval_runtime": 45.3028, + "eval_samples_per_second": 22.074, + "eval_steps_per_second": 0.927, + "step": 70 + }, + { + "epoch": 0.08836341008089608, + "grad_norm": 2.0913803577423096, + "learning_rate": 1.182572614107884e-06, + "loss": 2.4341, + "step": 71 + }, + { + "epoch": 0.089607965152458, + "grad_norm": 1.8292183876037598, + "learning_rate": 1.2033195020746888e-06, + "loss": 2.4291, + "step": 72 + }, + { + "epoch": 0.09085252022401992, + "grad_norm": 2.206770181655884, + "learning_rate": 1.224066390041494e-06, + "loss": 2.3764, + "step": 73 + }, + { + "epoch": 0.09209707529558182, + "grad_norm": 1.8263559341430664, + "learning_rate": 1.2448132780082988e-06, + "loss": 2.4082, + "step": 74 + }, + { + "epoch": 0.09334163036714374, + "grad_norm": 1.9405455589294434, + "learning_rate": 1.2655601659751037e-06, + "loss": 2.4673, + "step": 75 + }, + { + "epoch": 0.09458618543870566, + "grad_norm": 2.138108015060425, + "learning_rate": 1.2863070539419086e-06, + "loss": 2.3879, + "step": 76 + }, + { + "epoch": 0.09583074051026758, + "grad_norm": 1.9024745225906372, + "learning_rate": 1.307053941908714e-06, + "loss": 2.4344, + "step": 77 + }, + { + "epoch": 0.0970752955818295, + "grad_norm": 1.9136689901351929, + "learning_rate": 1.3278008298755188e-06, + "loss": 2.474, + "step": 78 + }, + { + "epoch": 0.09831985065339141, + "grad_norm": 2.4752795696258545, + "learning_rate": 1.3485477178423237e-06, + "loss": 2.3412, + "step": 79 + }, + { + "epoch": 0.09956440572495333, + "grad_norm": 1.8720006942749023, + "learning_rate": 1.3692946058091288e-06, + "loss": 2.3438, + "step": 80 + }, + { + "epoch": 0.09956440572495333, + "eval_loss": 2.3980512619018555, + "eval_runtime": 48.2788, + "eval_samples_per_second": 20.713, + "eval_steps_per_second": 0.87, + "step": 80 + }, + { + "epoch": 0.10080896079651525, + "grad_norm": 2.671691656112671, + "learning_rate": 1.3900414937759337e-06, + "loss": 2.3336, + "step": 81 + }, + { + "epoch": 0.10205351586807716, + "grad_norm": 2.2953391075134277, + "learning_rate": 1.4107883817427386e-06, + "loss": 2.377, + "step": 82 + }, + { + "epoch": 0.10329807093963908, + "grad_norm": 3.009018898010254, + "learning_rate": 1.4315352697095435e-06, + "loss": 2.2977, + "step": 83 + }, + { + "epoch": 0.104542626011201, + "grad_norm": 2.664454936981201, + "learning_rate": 1.4522821576763488e-06, + "loss": 2.3271, + "step": 84 + }, + { + "epoch": 0.10578718108276292, + "grad_norm": 3.017303705215454, + "learning_rate": 1.4730290456431537e-06, + "loss": 2.3251, + "step": 85 + }, + { + "epoch": 0.10703173615432483, + "grad_norm": 2.634716510772705, + "learning_rate": 1.4937759336099586e-06, + "loss": 2.332, + "step": 86 + }, + { + "epoch": 0.10827629122588675, + "grad_norm": 3.059644937515259, + "learning_rate": 1.5145228215767635e-06, + "loss": 2.3478, + "step": 87 + }, + { + "epoch": 0.10952084629744867, + "grad_norm": 2.6962637901306152, + "learning_rate": 1.5352697095435686e-06, + "loss": 2.2792, + "step": 88 + }, + { + "epoch": 0.11076540136901059, + "grad_norm": 3.419729709625244, + "learning_rate": 1.5560165975103735e-06, + "loss": 2.2571, + "step": 89 + }, + { + "epoch": 0.11200995644057249, + "grad_norm": 2.740781545639038, + "learning_rate": 1.5767634854771784e-06, + "loss": 2.2875, + "step": 90 + }, + { + "epoch": 0.11200995644057249, + "eval_loss": 2.30843186378479, + "eval_runtime": 49.9264, + "eval_samples_per_second": 20.029, + "eval_steps_per_second": 0.841, + "step": 90 + }, + { + "epoch": 0.1132545115121344, + "grad_norm": 2.5608789920806885, + "learning_rate": 1.5975103734439833e-06, + "loss": 2.2592, + "step": 91 + }, + { + "epoch": 0.11449906658369632, + "grad_norm": 2.687999963760376, + "learning_rate": 1.6182572614107886e-06, + "loss": 2.2546, + "step": 92 + }, + { + "epoch": 0.11574362165525824, + "grad_norm": 2.695909023284912, + "learning_rate": 1.6390041493775935e-06, + "loss": 2.2525, + "step": 93 + }, + { + "epoch": 0.11698817672682016, + "grad_norm": 2.818357467651367, + "learning_rate": 1.6597510373443984e-06, + "loss": 2.216, + "step": 94 + }, + { + "epoch": 0.11823273179838208, + "grad_norm": 2.884119987487793, + "learning_rate": 1.6804979253112035e-06, + "loss": 2.2321, + "step": 95 + }, + { + "epoch": 0.119477286869944, + "grad_norm": 2.52104115486145, + "learning_rate": 1.7012448132780084e-06, + "loss": 2.199, + "step": 96 + }, + { + "epoch": 0.12072184194150591, + "grad_norm": 2.420313596725464, + "learning_rate": 1.7219917012448133e-06, + "loss": 2.1862, + "step": 97 + }, + { + "epoch": 0.12196639701306783, + "grad_norm": 2.8047542572021484, + "learning_rate": 1.7427385892116182e-06, + "loss": 2.1793, + "step": 98 + }, + { + "epoch": 0.12321095208462975, + "grad_norm": 2.836482286453247, + "learning_rate": 1.7634854771784235e-06, + "loss": 2.2271, + "step": 99 + }, + { + "epoch": 0.12445550715619166, + "grad_norm": 2.5282301902770996, + "learning_rate": 1.7842323651452284e-06, + "loss": 2.1768, + "step": 100 + }, + { + "epoch": 0.12445550715619166, + "eval_loss": 2.2312686443328857, + "eval_runtime": 49.2382, + "eval_samples_per_second": 20.309, + "eval_steps_per_second": 0.853, + "step": 100 + }, + { + "epoch": 0.12570006222775357, + "grad_norm": 3.3407280445098877, + "learning_rate": 1.8049792531120333e-06, + "loss": 2.1666, + "step": 101 + }, + { + "epoch": 0.1269446172993155, + "grad_norm": 2.4754133224487305, + "learning_rate": 1.8257261410788382e-06, + "loss": 2.1768, + "step": 102 + }, + { + "epoch": 0.1281891723708774, + "grad_norm": 3.430889129638672, + "learning_rate": 1.8464730290456433e-06, + "loss": 2.1953, + "step": 103 + }, + { + "epoch": 0.12943372744243933, + "grad_norm": 2.835294246673584, + "learning_rate": 1.8672199170124482e-06, + "loss": 2.146, + "step": 104 + }, + { + "epoch": 0.13067828251400124, + "grad_norm": 3.1532323360443115, + "learning_rate": 1.8879668049792531e-06, + "loss": 2.1729, + "step": 105 + }, + { + "epoch": 0.13192283758556317, + "grad_norm": 3.2278342247009277, + "learning_rate": 1.908713692946058e-06, + "loss": 2.1336, + "step": 106 + }, + { + "epoch": 0.13316739265712507, + "grad_norm": 2.7892515659332275, + "learning_rate": 1.929460580912863e-06, + "loss": 2.141, + "step": 107 + }, + { + "epoch": 0.134411947728687, + "grad_norm": 3.0179977416992188, + "learning_rate": 1.9502074688796682e-06, + "loss": 2.1255, + "step": 108 + }, + { + "epoch": 0.1356565028002489, + "grad_norm": 2.977935552597046, + "learning_rate": 1.970954356846473e-06, + "loss": 2.0986, + "step": 109 + }, + { + "epoch": 0.13690105787181084, + "grad_norm": 3.3042492866516113, + "learning_rate": 1.991701244813278e-06, + "loss": 2.0992, + "step": 110 + }, + { + "epoch": 0.13690105787181084, + "eval_loss": 2.171299695968628, + "eval_runtime": 42.9511, + "eval_samples_per_second": 23.282, + "eval_steps_per_second": 0.978, + "step": 110 + }, + { + "epoch": 0.13814561294337274, + "grad_norm": 2.9750890731811523, + "learning_rate": 2.012448132780083e-06, + "loss": 2.1116, + "step": 111 + }, + { + "epoch": 0.13939016801493467, + "grad_norm": 2.8199286460876465, + "learning_rate": 2.0331950207468883e-06, + "loss": 2.0822, + "step": 112 + }, + { + "epoch": 0.14063472308649658, + "grad_norm": 2.930532217025757, + "learning_rate": 2.053941908713693e-06, + "loss": 2.161, + "step": 113 + }, + { + "epoch": 0.14187927815805848, + "grad_norm": 3.3400321006774902, + "learning_rate": 2.074688796680498e-06, + "loss": 2.0953, + "step": 114 + }, + { + "epoch": 0.1431238332296204, + "grad_norm": 2.716564655303955, + "learning_rate": 2.095435684647303e-06, + "loss": 2.0757, + "step": 115 + }, + { + "epoch": 0.14436838830118232, + "grad_norm": 2.7586236000061035, + "learning_rate": 2.116182572614108e-06, + "loss": 2.0979, + "step": 116 + }, + { + "epoch": 0.14561294337274425, + "grad_norm": 2.622126340866089, + "learning_rate": 2.136929460580913e-06, + "loss": 2.0748, + "step": 117 + }, + { + "epoch": 0.14685749844430615, + "grad_norm": 2.646477460861206, + "learning_rate": 2.157676348547718e-06, + "loss": 2.0605, + "step": 118 + }, + { + "epoch": 0.14810205351586808, + "grad_norm": 2.8882410526275635, + "learning_rate": 2.178423236514523e-06, + "loss": 2.0501, + "step": 119 + }, + { + "epoch": 0.14934660858742999, + "grad_norm": 2.8302247524261475, + "learning_rate": 2.199170124481328e-06, + "loss": 2.0592, + "step": 120 + }, + { + "epoch": 0.14934660858742999, + "eval_loss": 2.1291966438293457, + "eval_runtime": 47.5399, + "eval_samples_per_second": 21.035, + "eval_steps_per_second": 0.883, + "step": 120 + }, + { + "epoch": 0.15059116365899192, + "grad_norm": 3.26979660987854, + "learning_rate": 2.219917012448133e-06, + "loss": 2.0422, + "step": 121 + }, + { + "epoch": 0.15183571873055382, + "grad_norm": 2.7483913898468018, + "learning_rate": 2.240663900414938e-06, + "loss": 2.0872, + "step": 122 + }, + { + "epoch": 0.15308027380211575, + "grad_norm": 3.1252024173736572, + "learning_rate": 2.2614107883817427e-06, + "loss": 2.0684, + "step": 123 + }, + { + "epoch": 0.15432482887367766, + "grad_norm": 2.9175283908843994, + "learning_rate": 2.282157676348548e-06, + "loss": 2.0522, + "step": 124 + }, + { + "epoch": 0.1555693839452396, + "grad_norm": 2.7950754165649414, + "learning_rate": 2.302904564315353e-06, + "loss": 2.065, + "step": 125 + }, + { + "epoch": 0.1568139390168015, + "grad_norm": 2.84049654006958, + "learning_rate": 2.323651452282158e-06, + "loss": 2.0781, + "step": 126 + }, + { + "epoch": 0.15805849408836342, + "grad_norm": 2.7396647930145264, + "learning_rate": 2.3443983402489627e-06, + "loss": 2.1758, + "step": 127 + }, + { + "epoch": 0.15930304915992533, + "grad_norm": 3.0765926837921143, + "learning_rate": 2.365145228215768e-06, + "loss": 2.0172, + "step": 128 + }, + { + "epoch": 0.16054760423148726, + "grad_norm": 3.217189073562622, + "learning_rate": 2.385892116182573e-06, + "loss": 2.0286, + "step": 129 + }, + { + "epoch": 0.16179215930304916, + "grad_norm": 3.141545295715332, + "learning_rate": 2.4066390041493776e-06, + "loss": 2.0583, + "step": 130 + }, + { + "epoch": 0.16179215930304916, + "eval_loss": 2.0946149826049805, + "eval_runtime": 54.92, + "eval_samples_per_second": 18.208, + "eval_steps_per_second": 0.765, + "step": 130 + }, + { + "epoch": 0.16303671437461106, + "grad_norm": 2.850052833557129, + "learning_rate": 2.4273858921161828e-06, + "loss": 2.0746, + "step": 131 + }, + { + "epoch": 0.164281269446173, + "grad_norm": 3.28913950920105, + "learning_rate": 2.448132780082988e-06, + "loss": 2.0539, + "step": 132 + }, + { + "epoch": 0.1655258245177349, + "grad_norm": 2.8819124698638916, + "learning_rate": 2.468879668049793e-06, + "loss": 1.9843, + "step": 133 + }, + { + "epoch": 0.16677037958929683, + "grad_norm": 3.6254632472991943, + "learning_rate": 2.4896265560165977e-06, + "loss": 2.0233, + "step": 134 + }, + { + "epoch": 0.16801493466085873, + "grad_norm": 2.7385146617889404, + "learning_rate": 2.5103734439834028e-06, + "loss": 1.9667, + "step": 135 + }, + { + "epoch": 0.16925948973242066, + "grad_norm": 2.9722647666931152, + "learning_rate": 2.5311203319502074e-06, + "loss": 1.9527, + "step": 136 + }, + { + "epoch": 0.17050404480398257, + "grad_norm": 3.139526605606079, + "learning_rate": 2.5518672199170125e-06, + "loss": 1.9967, + "step": 137 + }, + { + "epoch": 0.1717485998755445, + "grad_norm": 3.399920701980591, + "learning_rate": 2.5726141078838172e-06, + "loss": 1.9798, + "step": 138 + }, + { + "epoch": 0.1729931549471064, + "grad_norm": 2.8017327785491943, + "learning_rate": 2.5933609958506228e-06, + "loss": 1.983, + "step": 139 + }, + { + "epoch": 0.17423771001866833, + "grad_norm": 3.9967198371887207, + "learning_rate": 2.614107883817428e-06, + "loss": 1.977, + "step": 140 + }, + { + "epoch": 0.17423771001866833, + "eval_loss": 2.059037685394287, + "eval_runtime": 52.6232, + "eval_samples_per_second": 19.003, + "eval_steps_per_second": 0.798, + "step": 140 + }, + { + "epoch": 0.17548226509023024, + "grad_norm": 2.893092393875122, + "learning_rate": 2.6348547717842326e-06, + "loss": 1.9888, + "step": 141 + }, + { + "epoch": 0.17672682016179217, + "grad_norm": 3.634352207183838, + "learning_rate": 2.6556016597510377e-06, + "loss": 2.0726, + "step": 142 + }, + { + "epoch": 0.17797137523335407, + "grad_norm": 3.3651444911956787, + "learning_rate": 2.6763485477178423e-06, + "loss": 1.973, + "step": 143 + }, + { + "epoch": 0.179215930304916, + "grad_norm": 3.968986988067627, + "learning_rate": 2.6970954356846475e-06, + "loss": 1.9948, + "step": 144 + }, + { + "epoch": 0.1804604853764779, + "grad_norm": 3.20105242729187, + "learning_rate": 2.717842323651452e-06, + "loss": 1.9238, + "step": 145 + }, + { + "epoch": 0.18170504044803984, + "grad_norm": 3.648339033126831, + "learning_rate": 2.7385892116182577e-06, + "loss": 1.9923, + "step": 146 + }, + { + "epoch": 0.18294959551960174, + "grad_norm": 3.0856316089630127, + "learning_rate": 2.7593360995850628e-06, + "loss": 1.9749, + "step": 147 + }, + { + "epoch": 0.18419415059116365, + "grad_norm": 3.2681071758270264, + "learning_rate": 2.7800829875518675e-06, + "loss": 1.9304, + "step": 148 + }, + { + "epoch": 0.18543870566272558, + "grad_norm": 2.634958267211914, + "learning_rate": 2.8008298755186726e-06, + "loss": 1.9237, + "step": 149 + }, + { + "epoch": 0.18668326073428748, + "grad_norm": 2.769491672515869, + "learning_rate": 2.8215767634854773e-06, + "loss": 1.8963, + "step": 150 + }, + { + "epoch": 0.18668326073428748, + "eval_loss": 2.0394654273986816, + "eval_runtime": 55.2136, + "eval_samples_per_second": 18.111, + "eval_steps_per_second": 0.761, + "step": 150 + }, + { + "epoch": 0.1879278158058494, + "grad_norm": 3.4345781803131104, + "learning_rate": 2.8423236514522824e-06, + "loss": 1.9836, + "step": 151 + }, + { + "epoch": 0.18917237087741132, + "grad_norm": 3.115727424621582, + "learning_rate": 2.863070539419087e-06, + "loss": 1.9352, + "step": 152 + }, + { + "epoch": 0.19041692594897325, + "grad_norm": 3.050652503967285, + "learning_rate": 2.883817427385892e-06, + "loss": 1.9031, + "step": 153 + }, + { + "epoch": 0.19166148102053515, + "grad_norm": 2.9404428005218506, + "learning_rate": 2.9045643153526977e-06, + "loss": 1.9196, + "step": 154 + }, + { + "epoch": 0.19290603609209708, + "grad_norm": 3.080810785293579, + "learning_rate": 2.9253112033195024e-06, + "loss": 1.9405, + "step": 155 + }, + { + "epoch": 0.194150591163659, + "grad_norm": 3.058558702468872, + "learning_rate": 2.9460580912863075e-06, + "loss": 1.9052, + "step": 156 + }, + { + "epoch": 0.19539514623522092, + "grad_norm": 3.307955026626587, + "learning_rate": 2.966804979253112e-06, + "loss": 1.9245, + "step": 157 + }, + { + "epoch": 0.19663970130678282, + "grad_norm": 2.845506191253662, + "learning_rate": 2.9875518672199173e-06, + "loss": 1.9461, + "step": 158 + }, + { + "epoch": 0.19788425637834475, + "grad_norm": 2.758654832839966, + "learning_rate": 3.008298755186722e-06, + "loss": 1.8885, + "step": 159 + }, + { + "epoch": 0.19912881144990666, + "grad_norm": 3.161252737045288, + "learning_rate": 3.029045643153527e-06, + "loss": 1.8535, + "step": 160 + }, + { + "epoch": 0.19912881144990666, + "eval_loss": 2.018101453781128, + "eval_runtime": 51.0744, + "eval_samples_per_second": 19.579, + "eval_steps_per_second": 0.822, + "step": 160 + }, + { + "epoch": 0.2003733665214686, + "grad_norm": 3.196847677230835, + "learning_rate": 3.0497925311203326e-06, + "loss": 1.8853, + "step": 161 + }, + { + "epoch": 0.2016179215930305, + "grad_norm": 2.8364663124084473, + "learning_rate": 3.0705394190871373e-06, + "loss": 1.9463, + "step": 162 + }, + { + "epoch": 0.2028624766645924, + "grad_norm": 3.0874054431915283, + "learning_rate": 3.0912863070539424e-06, + "loss": 1.9254, + "step": 163 + }, + { + "epoch": 0.20410703173615433, + "grad_norm": 2.7914493083953857, + "learning_rate": 3.112033195020747e-06, + "loss": 1.9213, + "step": 164 + }, + { + "epoch": 0.20535158680771623, + "grad_norm": 3.3871428966522217, + "learning_rate": 3.132780082987552e-06, + "loss": 1.8993, + "step": 165 + }, + { + "epoch": 0.20659614187927816, + "grad_norm": 3.096653461456299, + "learning_rate": 3.153526970954357e-06, + "loss": 1.8827, + "step": 166 + }, + { + "epoch": 0.20784069695084006, + "grad_norm": 2.8276076316833496, + "learning_rate": 3.174273858921162e-06, + "loss": 1.912, + "step": 167 + }, + { + "epoch": 0.209085252022402, + "grad_norm": 3.3058435916900635, + "learning_rate": 3.1950207468879666e-06, + "loss": 1.8381, + "step": 168 + }, + { + "epoch": 0.2103298070939639, + "grad_norm": 3.5017333030700684, + "learning_rate": 3.215767634854772e-06, + "loss": 1.9544, + "step": 169 + }, + { + "epoch": 0.21157436216552583, + "grad_norm": 3.457296133041382, + "learning_rate": 3.2365145228215773e-06, + "loss": 1.922, + "step": 170 + }, + { + "epoch": 0.21157436216552583, + "eval_loss": 1.988455891609192, + "eval_runtime": 56.2099, + "eval_samples_per_second": 17.79, + "eval_steps_per_second": 0.747, + "step": 170 + }, + { + "epoch": 0.21281891723708773, + "grad_norm": 3.1964566707611084, + "learning_rate": 3.257261410788382e-06, + "loss": 1.8925, + "step": 171 + }, + { + "epoch": 0.21406347230864967, + "grad_norm": 3.234652042388916, + "learning_rate": 3.278008298755187e-06, + "loss": 1.884, + "step": 172 + }, + { + "epoch": 0.21530802738021157, + "grad_norm": 3.14414119720459, + "learning_rate": 3.2987551867219918e-06, + "loss": 1.8905, + "step": 173 + }, + { + "epoch": 0.2165525824517735, + "grad_norm": 3.606379508972168, + "learning_rate": 3.319502074688797e-06, + "loss": 1.946, + "step": 174 + }, + { + "epoch": 0.2177971375233354, + "grad_norm": 2.986646890640259, + "learning_rate": 3.3402489626556016e-06, + "loss": 1.8691, + "step": 175 + }, + { + "epoch": 0.21904169259489734, + "grad_norm": 3.7298269271850586, + "learning_rate": 3.360995850622407e-06, + "loss": 1.8717, + "step": 176 + }, + { + "epoch": 0.22028624766645924, + "grad_norm": 3.422295093536377, + "learning_rate": 3.381742738589212e-06, + "loss": 1.8568, + "step": 177 + }, + { + "epoch": 0.22153080273802117, + "grad_norm": 3.2293782234191895, + "learning_rate": 3.402489626556017e-06, + "loss": 1.8471, + "step": 178 + }, + { + "epoch": 0.22277535780958307, + "grad_norm": 3.2293782234191895, + "learning_rate": 3.402489626556017e-06, + "loss": 1.9303, + "step": 179 + }, + { + "epoch": 0.22401991288114498, + "grad_norm": 3.506223440170288, + "learning_rate": 3.423236514522822e-06, + "loss": 1.8237, + "step": 180 + }, + { + "epoch": 0.22401991288114498, + "eval_loss": 1.9734643697738647, + "eval_runtime": 52.2815, + "eval_samples_per_second": 19.127, + "eval_steps_per_second": 0.803, + "step": 180 + }, + { + "epoch": 0.2252644679527069, + "grad_norm": 3.0156619548797607, + "learning_rate": 3.4439834024896267e-06, + "loss": 1.908, + "step": 181 + }, + { + "epoch": 0.2265090230242688, + "grad_norm": 3.55517315864563, + "learning_rate": 3.4647302904564318e-06, + "loss": 1.9104, + "step": 182 + }, + { + "epoch": 0.22775357809583074, + "grad_norm": 3.144984006881714, + "learning_rate": 3.4854771784232365e-06, + "loss": 1.8198, + "step": 183 + }, + { + "epoch": 0.22899813316739265, + "grad_norm": 4.302074432373047, + "learning_rate": 3.5062240663900416e-06, + "loss": 1.8237, + "step": 184 + }, + { + "epoch": 0.23024268823895458, + "grad_norm": 3.0522446632385254, + "learning_rate": 3.526970954356847e-06, + "loss": 1.8409, + "step": 185 + }, + { + "epoch": 0.23148724331051648, + "grad_norm": 3.8607394695281982, + "learning_rate": 3.5477178423236518e-06, + "loss": 1.8807, + "step": 186 + }, + { + "epoch": 0.23273179838207841, + "grad_norm": 2.9236302375793457, + "learning_rate": 3.568464730290457e-06, + "loss": 1.7873, + "step": 187 + }, + { + "epoch": 0.23397635345364032, + "grad_norm": 4.013780117034912, + "learning_rate": 3.5892116182572616e-06, + "loss": 1.7909, + "step": 188 + }, + { + "epoch": 0.23522090852520225, + "grad_norm": 3.0933122634887695, + "learning_rate": 3.6099585062240667e-06, + "loss": 1.8469, + "step": 189 + }, + { + "epoch": 0.23646546359676415, + "grad_norm": 3.487816095352173, + "learning_rate": 3.6307053941908714e-06, + "loss": 1.8469, + "step": 190 + }, + { + "epoch": 0.23646546359676415, + "eval_loss": 1.9604240655899048, + "eval_runtime": 53.4604, + "eval_samples_per_second": 18.705, + "eval_steps_per_second": 0.786, + "step": 190 + }, + { + "epoch": 0.23771001866832608, + "grad_norm": 2.829219341278076, + "learning_rate": 3.6514522821576765e-06, + "loss": 1.8639, + "step": 191 + }, + { + "epoch": 0.238954573739888, + "grad_norm": 3.595534324645996, + "learning_rate": 3.672199170124482e-06, + "loss": 1.8988, + "step": 192 + }, + { + "epoch": 0.24019912881144992, + "grad_norm": 2.9022483825683594, + "learning_rate": 3.6929460580912867e-06, + "loss": 1.8972, + "step": 193 + }, + { + "epoch": 0.24144368388301182, + "grad_norm": 3.0667994022369385, + "learning_rate": 3.713692946058092e-06, + "loss": 1.8499, + "step": 194 + }, + { + "epoch": 0.24268823895457373, + "grad_norm": 3.0498485565185547, + "learning_rate": 3.7344398340248965e-06, + "loss": 1.8341, + "step": 195 + }, + { + "epoch": 0.24393279402613566, + "grad_norm": 4.058084964752197, + "learning_rate": 3.7551867219917016e-06, + "loss": 1.8641, + "step": 196 + }, + { + "epoch": 0.24517734909769756, + "grad_norm": 2.9995028972625732, + "learning_rate": 3.7759336099585063e-06, + "loss": 1.7885, + "step": 197 + }, + { + "epoch": 0.2464219041692595, + "grad_norm": 3.254704475402832, + "learning_rate": 3.7966804979253114e-06, + "loss": 1.7544, + "step": 198 + }, + { + "epoch": 0.2476664592408214, + "grad_norm": 2.9126744270324707, + "learning_rate": 3.817427385892116e-06, + "loss": 1.8461, + "step": 199 + }, + { + "epoch": 0.24891101431238333, + "grad_norm": 3.475247621536255, + "learning_rate": 3.838174273858922e-06, + "loss": 1.8095, + "step": 200 + }, + { + "epoch": 0.24891101431238333, + "eval_loss": 1.9373760223388672, + "eval_runtime": 51.8733, + "eval_samples_per_second": 19.278, + "eval_steps_per_second": 0.81, + "step": 200 + }, + { + "epoch": 0.25015556938394523, + "grad_norm": 3.2663731575012207, + "learning_rate": 3.858921161825726e-06, + "loss": 1.8076, + "step": 201 + }, + { + "epoch": 0.25140012445550713, + "grad_norm": 3.1151726245880127, + "learning_rate": 3.879668049792531e-06, + "loss": 1.8143, + "step": 202 + }, + { + "epoch": 0.2526446795270691, + "grad_norm": 3.102038860321045, + "learning_rate": 3.9004149377593365e-06, + "loss": 1.8279, + "step": 203 + }, + { + "epoch": 0.253889234598631, + "grad_norm": 3.370642900466919, + "learning_rate": 3.921161825726142e-06, + "loss": 1.8522, + "step": 204 + }, + { + "epoch": 0.2551337896701929, + "grad_norm": 3.116128921508789, + "learning_rate": 3.941908713692946e-06, + "loss": 1.8104, + "step": 205 + }, + { + "epoch": 0.2563783447417548, + "grad_norm": 3.12028431892395, + "learning_rate": 3.962655601659751e-06, + "loss": 1.7723, + "step": 206 + }, + { + "epoch": 0.25762289981331676, + "grad_norm": 3.1251418590545654, + "learning_rate": 3.983402489626556e-06, + "loss": 1.863, + "step": 207 + }, + { + "epoch": 0.25886745488487867, + "grad_norm": 3.073702812194824, + "learning_rate": 4.004149377593361e-06, + "loss": 1.805, + "step": 208 + }, + { + "epoch": 0.26011200995644057, + "grad_norm": 2.9772469997406006, + "learning_rate": 4.024896265560166e-06, + "loss": 1.8251, + "step": 209 + }, + { + "epoch": 0.2613565650280025, + "grad_norm": 2.9970037937164307, + "learning_rate": 4.045643153526971e-06, + "loss": 1.8386, + "step": 210 + }, + { + "epoch": 0.2613565650280025, + "eval_loss": 1.9167065620422363, + "eval_runtime": 42.3328, + "eval_samples_per_second": 23.622, + "eval_steps_per_second": 0.992, + "step": 210 + }, + { + "epoch": 0.26260112009956443, + "grad_norm": 3.1391561031341553, + "learning_rate": 4.0663900414937765e-06, + "loss": 1.8109, + "step": 211 + }, + { + "epoch": 0.26384567517112634, + "grad_norm": 3.161538600921631, + "learning_rate": 4.087136929460581e-06, + "loss": 1.8441, + "step": 212 + }, + { + "epoch": 0.26509023024268824, + "grad_norm": 3.12412166595459, + "learning_rate": 4.107883817427386e-06, + "loss": 1.7736, + "step": 213 + }, + { + "epoch": 0.26633478531425014, + "grad_norm": 3.3241145610809326, + "learning_rate": 4.128630705394191e-06, + "loss": 1.8067, + "step": 214 + }, + { + "epoch": 0.26757934038581205, + "grad_norm": 3.503307819366455, + "learning_rate": 4.149377593360996e-06, + "loss": 1.7858, + "step": 215 + }, + { + "epoch": 0.268823895457374, + "grad_norm": 3.1009578704833984, + "learning_rate": 4.170124481327801e-06, + "loss": 1.7542, + "step": 216 + }, + { + "epoch": 0.2700684505289359, + "grad_norm": 3.2236011028289795, + "learning_rate": 4.190871369294606e-06, + "loss": 1.8006, + "step": 217 + }, + { + "epoch": 0.2713130056004978, + "grad_norm": 3.048935651779175, + "learning_rate": 4.211618257261411e-06, + "loss": 1.7706, + "step": 218 + }, + { + "epoch": 0.2725575606720597, + "grad_norm": 3.3416147232055664, + "learning_rate": 4.232365145228216e-06, + "loss": 1.8108, + "step": 219 + }, + { + "epoch": 0.2738021157436217, + "grad_norm": 3.2010738849639893, + "learning_rate": 4.253112033195021e-06, + "loss": 1.8561, + "step": 220 + }, + { + "epoch": 0.2738021157436217, + "eval_loss": 1.9111930131912231, + "eval_runtime": 50.4077, + "eval_samples_per_second": 19.838, + "eval_steps_per_second": 0.833, + "step": 220 + }, + { + "epoch": 0.2750466708151836, + "grad_norm": 3.4033374786376953, + "learning_rate": 4.273858921161826e-06, + "loss": 1.8218, + "step": 221 + }, + { + "epoch": 0.2762912258867455, + "grad_norm": 3.2086002826690674, + "learning_rate": 4.294605809128631e-06, + "loss": 1.8213, + "step": 222 + }, + { + "epoch": 0.2775357809583074, + "grad_norm": 4.090956211090088, + "learning_rate": 4.315352697095436e-06, + "loss": 1.8455, + "step": 223 + }, + { + "epoch": 0.27878033602986935, + "grad_norm": 3.1142985820770264, + "learning_rate": 4.336099585062241e-06, + "loss": 1.7737, + "step": 224 + }, + { + "epoch": 0.28002489110143125, + "grad_norm": 3.369669198989868, + "learning_rate": 4.356846473029046e-06, + "loss": 1.824, + "step": 225 + }, + { + "epoch": 0.28126944617299315, + "grad_norm": 3.165672779083252, + "learning_rate": 4.3775933609958506e-06, + "loss": 1.7768, + "step": 226 + }, + { + "epoch": 0.28251400124455506, + "grad_norm": 3.3343470096588135, + "learning_rate": 4.398340248962656e-06, + "loss": 1.7278, + "step": 227 + }, + { + "epoch": 0.28375855631611696, + "grad_norm": 3.2574994564056396, + "learning_rate": 4.419087136929461e-06, + "loss": 1.8439, + "step": 228 + }, + { + "epoch": 0.2850031113876789, + "grad_norm": 3.043928623199463, + "learning_rate": 4.439834024896266e-06, + "loss": 1.818, + "step": 229 + }, + { + "epoch": 0.2862476664592408, + "grad_norm": 2.9703125953674316, + "learning_rate": 4.460580912863071e-06, + "loss": 1.773, + "step": 230 + }, + { + "epoch": 0.2862476664592408, + "eval_loss": 1.898109793663025, + "eval_runtime": 45.6023, + "eval_samples_per_second": 21.929, + "eval_steps_per_second": 0.921, + "step": 230 + }, + { + "epoch": 0.2874922215308027, + "grad_norm": 2.9997220039367676, + "learning_rate": 4.481327800829876e-06, + "loss": 1.7727, + "step": 231 + }, + { + "epoch": 0.28873677660236463, + "grad_norm": 3.1358482837677, + "learning_rate": 4.502074688796681e-06, + "loss": 1.7764, + "step": 232 + }, + { + "epoch": 0.2899813316739266, + "grad_norm": 3.395747661590576, + "learning_rate": 4.5228215767634855e-06, + "loss": 1.7826, + "step": 233 + }, + { + "epoch": 0.2912258867454885, + "grad_norm": 3.7494754791259766, + "learning_rate": 4.543568464730291e-06, + "loss": 1.7181, + "step": 234 + }, + { + "epoch": 0.2924704418170504, + "grad_norm": 3.42293381690979, + "learning_rate": 4.564315352697096e-06, + "loss": 1.7773, + "step": 235 + }, + { + "epoch": 0.2937149968886123, + "grad_norm": 3.2524514198303223, + "learning_rate": 4.585062240663901e-06, + "loss": 1.7294, + "step": 236 + }, + { + "epoch": 0.29495955196017426, + "grad_norm": 3.657869577407837, + "learning_rate": 4.605809128630706e-06, + "loss": 1.762, + "step": 237 + }, + { + "epoch": 0.29620410703173616, + "grad_norm": 3.127372980117798, + "learning_rate": 4.626556016597511e-06, + "loss": 1.7623, + "step": 238 + }, + { + "epoch": 0.29744866210329807, + "grad_norm": 4.166962146759033, + "learning_rate": 4.647302904564316e-06, + "loss": 1.6995, + "step": 239 + }, + { + "epoch": 0.29869321717485997, + "grad_norm": 3.094264030456543, + "learning_rate": 4.66804979253112e-06, + "loss": 1.7469, + "step": 240 + }, + { + "epoch": 0.29869321717485997, + "eval_loss": 1.8908178806304932, + "eval_runtime": 44.6332, + "eval_samples_per_second": 22.405, + "eval_steps_per_second": 0.941, + "step": 240 + }, + { + "epoch": 0.29993777224642193, + "grad_norm": 3.8644745349884033, + "learning_rate": 4.6887966804979255e-06, + "loss": 1.7644, + "step": 241 + }, + { + "epoch": 0.30118232731798383, + "grad_norm": 3.1488852500915527, + "learning_rate": 4.709543568464731e-06, + "loss": 1.772, + "step": 242 + }, + { + "epoch": 0.30242688238954574, + "grad_norm": 3.3179638385772705, + "learning_rate": 4.730290456431536e-06, + "loss": 1.7588, + "step": 243 + }, + { + "epoch": 0.30367143746110764, + "grad_norm": 3.328355550765991, + "learning_rate": 4.751037344398341e-06, + "loss": 1.7417, + "step": 244 + }, + { + "epoch": 0.30491599253266954, + "grad_norm": 3.2690482139587402, + "learning_rate": 4.771784232365146e-06, + "loss": 1.7562, + "step": 245 + }, + { + "epoch": 0.3061605476042315, + "grad_norm": 3.2759454250335693, + "learning_rate": 4.792531120331951e-06, + "loss": 1.7536, + "step": 246 + }, + { + "epoch": 0.3074051026757934, + "grad_norm": 2.937964916229248, + "learning_rate": 4.813278008298755e-06, + "loss": 1.7518, + "step": 247 + }, + { + "epoch": 0.3086496577473553, + "grad_norm": 3.0995302200317383, + "learning_rate": 4.83402489626556e-06, + "loss": 1.7593, + "step": 248 + }, + { + "epoch": 0.3098942128189172, + "grad_norm": 3.230459213256836, + "learning_rate": 4.8547717842323655e-06, + "loss": 1.6962, + "step": 249 + }, + { + "epoch": 0.3111387678904792, + "grad_norm": 3.242576837539673, + "learning_rate": 4.875518672199171e-06, + "loss": 1.7307, + "step": 250 + }, + { + "epoch": 0.3111387678904792, + "eval_loss": 1.8805371522903442, + "eval_runtime": 45.9886, + "eval_samples_per_second": 21.745, + "eval_steps_per_second": 0.913, + "step": 250 + }, + { + "epoch": 0.3123833229620411, + "grad_norm": 3.097045421600342, + "learning_rate": 4.896265560165976e-06, + "loss": 1.7438, + "step": 251 + }, + { + "epoch": 0.313627878033603, + "grad_norm": 3.2428948879241943, + "learning_rate": 4.91701244813278e-06, + "loss": 1.7179, + "step": 252 + }, + { + "epoch": 0.3148724331051649, + "grad_norm": 3.196274518966675, + "learning_rate": 4.937759336099586e-06, + "loss": 1.7855, + "step": 253 + }, + { + "epoch": 0.31611698817672684, + "grad_norm": 2.978203535079956, + "learning_rate": 4.95850622406639e-06, + "loss": 1.7144, + "step": 254 + }, + { + "epoch": 0.31736154324828875, + "grad_norm": 3.2641701698303223, + "learning_rate": 4.979253112033195e-06, + "loss": 1.7427, + "step": 255 + }, + { + "epoch": 0.31860609831985065, + "grad_norm": 2.7441232204437256, + "learning_rate": 5e-06, + "loss": 1.7619, + "step": 256 + }, + { + "epoch": 0.31985065339141255, + "grad_norm": 3.0723495483398438, + "learning_rate": 4.99769372693727e-06, + "loss": 1.8017, + "step": 257 + }, + { + "epoch": 0.3210952084629745, + "grad_norm": 3.104752540588379, + "learning_rate": 4.995387453874539e-06, + "loss": 1.7761, + "step": 258 + }, + { + "epoch": 0.3223397635345364, + "grad_norm": 3.138627052307129, + "learning_rate": 4.993081180811809e-06, + "loss": 1.7879, + "step": 259 + }, + { + "epoch": 0.3235843186060983, + "grad_norm": 2.8567333221435547, + "learning_rate": 4.990774907749078e-06, + "loss": 1.7561, + "step": 260 + }, + { + "epoch": 0.3235843186060983, + "eval_loss": 1.8606494665145874, + "eval_runtime": 46.453, + "eval_samples_per_second": 21.527, + "eval_steps_per_second": 0.904, + "step": 260 + }, + { + "epoch": 0.3248288736776602, + "grad_norm": 3.2696540355682373, + "learning_rate": 4.988468634686347e-06, + "loss": 1.7201, + "step": 261 + }, + { + "epoch": 0.3260734287492221, + "grad_norm": 2.6692731380462646, + "learning_rate": 4.986162361623617e-06, + "loss": 1.7264, + "step": 262 + }, + { + "epoch": 0.3273179838207841, + "grad_norm": 3.047549247741699, + "learning_rate": 4.983856088560886e-06, + "loss": 1.7362, + "step": 263 + }, + { + "epoch": 0.328562538892346, + "grad_norm": 3.0667457580566406, + "learning_rate": 4.981549815498156e-06, + "loss": 1.7321, + "step": 264 + }, + { + "epoch": 0.3298070939639079, + "grad_norm": 2.560047149658203, + "learning_rate": 4.979243542435424e-06, + "loss": 1.7508, + "step": 265 + }, + { + "epoch": 0.3310516490354698, + "grad_norm": 3.26595401763916, + "learning_rate": 4.976937269372694e-06, + "loss": 1.7248, + "step": 266 + }, + { + "epoch": 0.33229620410703176, + "grad_norm": 2.929210662841797, + "learning_rate": 4.974630996309964e-06, + "loss": 1.697, + "step": 267 + }, + { + "epoch": 0.33354075917859366, + "grad_norm": 3.1526286602020264, + "learning_rate": 4.972324723247233e-06, + "loss": 1.7385, + "step": 268 + }, + { + "epoch": 0.33478531425015556, + "grad_norm": 3.001619577407837, + "learning_rate": 4.970018450184502e-06, + "loss": 1.7072, + "step": 269 + }, + { + "epoch": 0.33602986932171747, + "grad_norm": 3.2464189529418945, + "learning_rate": 4.9677121771217715e-06, + "loss": 1.7361, + "step": 270 + }, + { + "epoch": 0.33602986932171747, + "eval_loss": 1.860226035118103, + "eval_runtime": 46.6881, + "eval_samples_per_second": 21.419, + "eval_steps_per_second": 0.9, + "step": 270 + }, + { + "epoch": 0.3372744243932794, + "grad_norm": 3.2104530334472656, + "learning_rate": 4.965405904059041e-06, + "loss": 1.7171, + "step": 271 + }, + { + "epoch": 0.33851897946484133, + "grad_norm": 3.146847724914551, + "learning_rate": 4.96309963099631e-06, + "loss": 1.7311, + "step": 272 + }, + { + "epoch": 0.33976353453640323, + "grad_norm": 3.1431286334991455, + "learning_rate": 4.96079335793358e-06, + "loss": 1.7345, + "step": 273 + }, + { + "epoch": 0.34100808960796514, + "grad_norm": 2.847163677215576, + "learning_rate": 4.958487084870849e-06, + "loss": 1.7262, + "step": 274 + }, + { + "epoch": 0.3422526446795271, + "grad_norm": 3.1987810134887695, + "learning_rate": 4.956180811808119e-06, + "loss": 1.7945, + "step": 275 + }, + { + "epoch": 0.343497199751089, + "grad_norm": 2.7475385665893555, + "learning_rate": 4.953874538745388e-06, + "loss": 1.76, + "step": 276 + }, + { + "epoch": 0.3447417548226509, + "grad_norm": 3.0427663326263428, + "learning_rate": 4.9515682656826574e-06, + "loss": 1.7324, + "step": 277 + }, + { + "epoch": 0.3459863098942128, + "grad_norm": 3.605212688446045, + "learning_rate": 4.949261992619927e-06, + "loss": 1.7712, + "step": 278 + }, + { + "epoch": 0.3472308649657747, + "grad_norm": 3.0564935207366943, + "learning_rate": 4.946955719557196e-06, + "loss": 1.6792, + "step": 279 + }, + { + "epoch": 0.34847542003733667, + "grad_norm": 3.1069741249084473, + "learning_rate": 4.944649446494466e-06, + "loss": 1.7145, + "step": 280 + }, + { + "epoch": 0.34847542003733667, + "eval_loss": 1.8537051677703857, + "eval_runtime": 47.4813, + "eval_samples_per_second": 21.061, + "eval_steps_per_second": 0.885, + "step": 280 + }, + { + "epoch": 0.3497199751088986, + "grad_norm": 2.8021512031555176, + "learning_rate": 4.942343173431734e-06, + "loss": 1.7145, + "step": 281 + }, + { + "epoch": 0.3509645301804605, + "grad_norm": 3.5549023151397705, + "learning_rate": 4.940036900369004e-06, + "loss": 1.7642, + "step": 282 + }, + { + "epoch": 0.3522090852520224, + "grad_norm": 2.7648985385894775, + "learning_rate": 4.937730627306274e-06, + "loss": 1.7255, + "step": 283 + }, + { + "epoch": 0.35345364032358434, + "grad_norm": 3.0815863609313965, + "learning_rate": 4.9354243542435426e-06, + "loss": 1.7055, + "step": 284 + }, + { + "epoch": 0.35469819539514624, + "grad_norm": 2.9009227752685547, + "learning_rate": 4.933118081180812e-06, + "loss": 1.7019, + "step": 285 + }, + { + "epoch": 0.35594275046670815, + "grad_norm": 2.618429660797119, + "learning_rate": 4.930811808118081e-06, + "loss": 1.6318, + "step": 286 + }, + { + "epoch": 0.35718730553827005, + "grad_norm": 3.368230104446411, + "learning_rate": 4.928505535055351e-06, + "loss": 1.7304, + "step": 287 + }, + { + "epoch": 0.358431860609832, + "grad_norm": 3.256889820098877, + "learning_rate": 4.92619926199262e-06, + "loss": 1.7019, + "step": 288 + }, + { + "epoch": 0.3596764156813939, + "grad_norm": 2.9366304874420166, + "learning_rate": 4.92389298892989e-06, + "loss": 1.734, + "step": 289 + }, + { + "epoch": 0.3609209707529558, + "grad_norm": 3.0393142700195312, + "learning_rate": 4.921586715867159e-06, + "loss": 1.7504, + "step": 290 + }, + { + "epoch": 0.3609209707529558, + "eval_loss": 1.8359886407852173, + "eval_runtime": 49.7622, + "eval_samples_per_second": 20.096, + "eval_steps_per_second": 0.844, + "step": 290 + }, + { + "epoch": 0.3621655258245177, + "grad_norm": 2.7288589477539062, + "learning_rate": 4.9192804428044285e-06, + "loss": 1.7004, + "step": 291 + }, + { + "epoch": 0.3634100808960797, + "grad_norm": 3.2793378829956055, + "learning_rate": 4.916974169741698e-06, + "loss": 1.7028, + "step": 292 + }, + { + "epoch": 0.3646546359676416, + "grad_norm": 2.960880756378174, + "learning_rate": 4.914667896678967e-06, + "loss": 1.6759, + "step": 293 + }, + { + "epoch": 0.3658991910392035, + "grad_norm": 2.836421012878418, + "learning_rate": 4.912361623616237e-06, + "loss": 1.6475, + "step": 294 + }, + { + "epoch": 0.3671437461107654, + "grad_norm": 2.9495465755462646, + "learning_rate": 4.910055350553506e-06, + "loss": 1.7076, + "step": 295 + }, + { + "epoch": 0.3683883011823273, + "grad_norm": 2.954730272293091, + "learning_rate": 4.907749077490776e-06, + "loss": 1.7261, + "step": 296 + }, + { + "epoch": 0.36963285625388925, + "grad_norm": 2.8167543411254883, + "learning_rate": 4.905442804428044e-06, + "loss": 1.6939, + "step": 297 + }, + { + "epoch": 0.37087741132545116, + "grad_norm": 2.857316017150879, + "learning_rate": 4.903136531365314e-06, + "loss": 1.7362, + "step": 298 + }, + { + "epoch": 0.37212196639701306, + "grad_norm": 2.8612918853759766, + "learning_rate": 4.900830258302584e-06, + "loss": 1.6489, + "step": 299 + }, + { + "epoch": 0.37336652146857496, + "grad_norm": 2.9618914127349854, + "learning_rate": 4.898523985239853e-06, + "loss": 1.718, + "step": 300 + }, + { + "epoch": 0.37336652146857496, + "eval_loss": 1.827571988105774, + "eval_runtime": 46.4552, + "eval_samples_per_second": 21.526, + "eval_steps_per_second": 0.904, + "step": 300 + }, + { + "epoch": 0.3746110765401369, + "grad_norm": 2.8706953525543213, + "learning_rate": 4.896217712177122e-06, + "loss": 1.6925, + "step": 301 + }, + { + "epoch": 0.3758556316116988, + "grad_norm": 3.2565090656280518, + "learning_rate": 4.893911439114391e-06, + "loss": 1.7369, + "step": 302 + }, + { + "epoch": 0.37710018668326073, + "grad_norm": 2.928858995437622, + "learning_rate": 4.891605166051661e-06, + "loss": 1.6882, + "step": 303 + }, + { + "epoch": 0.37834474175482263, + "grad_norm": 3.042314052581787, + "learning_rate": 4.88929889298893e-06, + "loss": 1.6466, + "step": 304 + }, + { + "epoch": 0.3795892968263846, + "grad_norm": 2.7509143352508545, + "learning_rate": 4.8869926199262e-06, + "loss": 1.685, + "step": 305 + }, + { + "epoch": 0.3808338518979465, + "grad_norm": 2.9601588249206543, + "learning_rate": 4.884686346863469e-06, + "loss": 1.6772, + "step": 306 + }, + { + "epoch": 0.3820784069695084, + "grad_norm": 2.618608236312866, + "learning_rate": 4.8823800738007384e-06, + "loss": 1.7127, + "step": 307 + }, + { + "epoch": 0.3833229620410703, + "grad_norm": 3.047581195831299, + "learning_rate": 4.880073800738008e-06, + "loss": 1.7174, + "step": 308 + }, + { + "epoch": 0.3845675171126322, + "grad_norm": 2.87735652923584, + "learning_rate": 4.877767527675277e-06, + "loss": 1.6677, + "step": 309 + }, + { + "epoch": 0.38581207218419417, + "grad_norm": 2.7169721126556396, + "learning_rate": 4.875461254612546e-06, + "loss": 1.7486, + "step": 310 + }, + { + "epoch": 0.38581207218419417, + "eval_loss": 1.8221794366836548, + "eval_runtime": 43.1681, + "eval_samples_per_second": 23.165, + "eval_steps_per_second": 0.973, + "step": 310 + }, + { + "epoch": 0.38705662725575607, + "grad_norm": 2.7094991207122803, + "learning_rate": 4.873154981549816e-06, + "loss": 1.7348, + "step": 311 + }, + { + "epoch": 0.388301182327318, + "grad_norm": 2.8989078998565674, + "learning_rate": 4.8708487084870856e-06, + "loss": 1.7352, + "step": 312 + }, + { + "epoch": 0.3895457373988799, + "grad_norm": 2.9565601348876953, + "learning_rate": 4.868542435424355e-06, + "loss": 1.7063, + "step": 313 + }, + { + "epoch": 0.39079029247044184, + "grad_norm": 2.8354082107543945, + "learning_rate": 4.8662361623616235e-06, + "loss": 1.6808, + "step": 314 + }, + { + "epoch": 0.39203484754200374, + "grad_norm": 2.8287479877471924, + "learning_rate": 4.863929889298894e-06, + "loss": 1.6731, + "step": 315 + }, + { + "epoch": 0.39327940261356564, + "grad_norm": 2.6537821292877197, + "learning_rate": 4.861623616236163e-06, + "loss": 1.695, + "step": 316 + }, + { + "epoch": 0.39452395768512755, + "grad_norm": 2.7135190963745117, + "learning_rate": 4.859317343173432e-06, + "loss": 1.6897, + "step": 317 + }, + { + "epoch": 0.3957685127566895, + "grad_norm": 3.1019129753112793, + "learning_rate": 4.857011070110701e-06, + "loss": 1.7219, + "step": 318 + }, + { + "epoch": 0.3970130678282514, + "grad_norm": 2.945737838745117, + "learning_rate": 4.854704797047971e-06, + "loss": 1.7316, + "step": 319 + }, + { + "epoch": 0.3982576228998133, + "grad_norm": 2.7668251991271973, + "learning_rate": 4.85239852398524e-06, + "loss": 1.6461, + "step": 320 + }, + { + "epoch": 0.3982576228998133, + "eval_loss": 1.8210570812225342, + "eval_runtime": 46.2552, + "eval_samples_per_second": 21.619, + "eval_steps_per_second": 0.908, + "step": 320 + }, + { + "epoch": 0.3995021779713752, + "grad_norm": 2.738109588623047, + "learning_rate": 4.8500922509225095e-06, + "loss": 1.6907, + "step": 321 + }, + { + "epoch": 0.4007467330429372, + "grad_norm": 3.0195062160491943, + "learning_rate": 4.847785977859779e-06, + "loss": 1.7099, + "step": 322 + }, + { + "epoch": 0.4019912881144991, + "grad_norm": 2.8672404289245605, + "learning_rate": 4.845479704797048e-06, + "loss": 1.6509, + "step": 323 + }, + { + "epoch": 0.403235843186061, + "grad_norm": 2.9041008949279785, + "learning_rate": 4.843173431734318e-06, + "loss": 1.5818, + "step": 324 + }, + { + "epoch": 0.4044803982576229, + "grad_norm": 2.9105985164642334, + "learning_rate": 4.840867158671587e-06, + "loss": 1.7002, + "step": 325 + }, + { + "epoch": 0.4057249533291848, + "grad_norm": 2.749009847640991, + "learning_rate": 4.838560885608857e-06, + "loss": 1.7021, + "step": 326 + }, + { + "epoch": 0.40696950840074675, + "grad_norm": 2.6998822689056396, + "learning_rate": 4.836254612546126e-06, + "loss": 1.6853, + "step": 327 + }, + { + "epoch": 0.40821406347230865, + "grad_norm": 2.678201675415039, + "learning_rate": 4.8339483394833955e-06, + "loss": 1.6934, + "step": 328 + }, + { + "epoch": 0.40945861854387056, + "grad_norm": 2.774845600128174, + "learning_rate": 4.831642066420665e-06, + "loss": 1.657, + "step": 329 + }, + { + "epoch": 0.41070317361543246, + "grad_norm": 2.6363000869750977, + "learning_rate": 4.8293357933579335e-06, + "loss": 1.6555, + "step": 330 + }, + { + "epoch": 0.41070317361543246, + "eval_loss": 1.8126581907272339, + "eval_runtime": 52.7717, + "eval_samples_per_second": 18.95, + "eval_steps_per_second": 0.796, + "step": 330 + }, + { + "epoch": 0.4119477286869944, + "grad_norm": 3.011659622192383, + "learning_rate": 4.827029520295204e-06, + "loss": 1.7202, + "step": 331 + }, + { + "epoch": 0.4131922837585563, + "grad_norm": 2.8322594165802, + "learning_rate": 4.824723247232473e-06, + "loss": 1.6736, + "step": 332 + }, + { + "epoch": 0.4144368388301182, + "grad_norm": 2.9360146522521973, + "learning_rate": 4.822416974169742e-06, + "loss": 1.7213, + "step": 333 + }, + { + "epoch": 0.41568139390168013, + "grad_norm": 2.810206413269043, + "learning_rate": 4.820110701107011e-06, + "loss": 1.6867, + "step": 334 + }, + { + "epoch": 0.4169259489732421, + "grad_norm": 2.6864097118377686, + "learning_rate": 4.817804428044281e-06, + "loss": 1.6854, + "step": 335 + }, + { + "epoch": 0.418170504044804, + "grad_norm": 2.7396061420440674, + "learning_rate": 4.81549815498155e-06, + "loss": 1.68, + "step": 336 + }, + { + "epoch": 0.4194150591163659, + "grad_norm": 2.602771520614624, + "learning_rate": 4.8131918819188194e-06, + "loss": 1.6269, + "step": 337 + }, + { + "epoch": 0.4206596141879278, + "grad_norm": 2.8174757957458496, + "learning_rate": 4.810885608856089e-06, + "loss": 1.6981, + "step": 338 + }, + { + "epoch": 0.42190416925948976, + "grad_norm": 3.2398130893707275, + "learning_rate": 4.808579335793358e-06, + "loss": 1.7061, + "step": 339 + }, + { + "epoch": 0.42314872433105166, + "grad_norm": 2.7119736671447754, + "learning_rate": 4.806273062730628e-06, + "loss": 1.6875, + "step": 340 + }, + { + "epoch": 0.42314872433105166, + "eval_loss": 1.7973068952560425, + "eval_runtime": 53.5791, + "eval_samples_per_second": 18.664, + "eval_steps_per_second": 0.784, + "step": 340 + }, + { + "epoch": 0.42439327940261357, + "grad_norm": 2.868533134460449, + "learning_rate": 4.803966789667897e-06, + "loss": 1.6967, + "step": 341 + }, + { + "epoch": 0.42563783447417547, + "grad_norm": 2.528083562850952, + "learning_rate": 4.8016605166051665e-06, + "loss": 1.6195, + "step": 342 + }, + { + "epoch": 0.4268823895457374, + "grad_norm": 2.7885115146636963, + "learning_rate": 4.799354243542436e-06, + "loss": 1.6197, + "step": 343 + }, + { + "epoch": 0.42812694461729933, + "grad_norm": 2.768247604370117, + "learning_rate": 4.797047970479705e-06, + "loss": 1.6271, + "step": 344 + }, + { + "epoch": 0.42937149968886124, + "grad_norm": 2.7710275650024414, + "learning_rate": 4.794741697416975e-06, + "loss": 1.6768, + "step": 345 + }, + { + "epoch": 0.43061605476042314, + "grad_norm": 2.968337297439575, + "learning_rate": 4.792435424354243e-06, + "loss": 1.6433, + "step": 346 + }, + { + "epoch": 0.43186060983198504, + "grad_norm": 2.5739808082580566, + "learning_rate": 4.790129151291514e-06, + "loss": 1.6327, + "step": 347 + }, + { + "epoch": 0.433105164903547, + "grad_norm": 2.7676408290863037, + "learning_rate": 4.787822878228783e-06, + "loss": 1.6178, + "step": 348 + }, + { + "epoch": 0.4343497199751089, + "grad_norm": 2.959059476852417, + "learning_rate": 4.7855166051660525e-06, + "loss": 1.6809, + "step": 349 + }, + { + "epoch": 0.4355942750466708, + "grad_norm": 2.559967279434204, + "learning_rate": 4.783210332103321e-06, + "loss": 1.6704, + "step": 350 + }, + { + "epoch": 0.4355942750466708, + "eval_loss": 1.8013949394226074, + "eval_runtime": 50.4649, + "eval_samples_per_second": 19.816, + "eval_steps_per_second": 0.832, + "step": 350 + }, + { + "epoch": 0.4368388301182327, + "grad_norm": 2.817901611328125, + "learning_rate": 4.7809040590405905e-06, + "loss": 1.6296, + "step": 351 + }, + { + "epoch": 0.43808338518979467, + "grad_norm": 2.7116055488586426, + "learning_rate": 4.778597785977861e-06, + "loss": 1.6652, + "step": 352 + }, + { + "epoch": 0.4393279402613566, + "grad_norm": 2.544750452041626, + "learning_rate": 4.776291512915129e-06, + "loss": 1.5668, + "step": 353 + }, + { + "epoch": 0.4405724953329185, + "grad_norm": 2.720534086227417, + "learning_rate": 4.773985239852399e-06, + "loss": 1.6152, + "step": 354 + }, + { + "epoch": 0.4418170504044804, + "grad_norm": 2.7270126342773438, + "learning_rate": 4.771678966789668e-06, + "loss": 1.5624, + "step": 355 + }, + { + "epoch": 0.44306160547604234, + "grad_norm": 2.7986624240875244, + "learning_rate": 4.769372693726938e-06, + "loss": 1.6621, + "step": 356 + }, + { + "epoch": 0.44430616054760425, + "grad_norm": 2.943107843399048, + "learning_rate": 4.767066420664207e-06, + "loss": 1.6713, + "step": 357 + }, + { + "epoch": 0.44555071561916615, + "grad_norm": 2.739898681640625, + "learning_rate": 4.7647601476014765e-06, + "loss": 1.6363, + "step": 358 + }, + { + "epoch": 0.44679527069072805, + "grad_norm": 2.8729329109191895, + "learning_rate": 4.762453874538746e-06, + "loss": 1.6605, + "step": 359 + }, + { + "epoch": 0.44803982576228996, + "grad_norm": 2.909029483795166, + "learning_rate": 4.760147601476015e-06, + "loss": 1.6277, + "step": 360 + }, + { + "epoch": 0.44803982576228996, + "eval_loss": 1.8110179901123047, + "eval_runtime": 47.924, + "eval_samples_per_second": 20.866, + "eval_steps_per_second": 0.876, + "step": 360 + }, + { + "epoch": 0.4492843808338519, + "grad_norm": 3.3826682567596436, + "learning_rate": 4.757841328413285e-06, + "loss": 1.7155, + "step": 361 + }, + { + "epoch": 0.4505289359054138, + "grad_norm": 3.0586729049682617, + "learning_rate": 4.755535055350554e-06, + "loss": 1.5991, + "step": 362 + }, + { + "epoch": 0.4517734909769757, + "grad_norm": 2.645576238632202, + "learning_rate": 4.753228782287823e-06, + "loss": 1.6504, + "step": 363 + }, + { + "epoch": 0.4530180460485376, + "grad_norm": 2.705967664718628, + "learning_rate": 4.750922509225093e-06, + "loss": 1.6761, + "step": 364 + }, + { + "epoch": 0.4542626011200996, + "grad_norm": 2.7651922702789307, + "learning_rate": 4.748616236162362e-06, + "loss": 1.61, + "step": 365 + }, + { + "epoch": 0.4555071561916615, + "grad_norm": 2.5654966831207275, + "learning_rate": 4.746309963099631e-06, + "loss": 1.6335, + "step": 366 + }, + { + "epoch": 0.4567517112632234, + "grad_norm": 2.6606173515319824, + "learning_rate": 4.7440036900369e-06, + "loss": 1.6207, + "step": 367 + }, + { + "epoch": 0.4579962663347853, + "grad_norm": 2.724755048751831, + "learning_rate": 4.741697416974171e-06, + "loss": 1.6329, + "step": 368 + }, + { + "epoch": 0.45924082140634725, + "grad_norm": 2.7019615173339844, + "learning_rate": 4.739391143911439e-06, + "loss": 1.6763, + "step": 369 + }, + { + "epoch": 0.46048537647790916, + "grad_norm": 2.6594254970550537, + "learning_rate": 4.737084870848709e-06, + "loss": 1.6174, + "step": 370 + }, + { + "epoch": 0.46048537647790916, + "eval_loss": 1.7924141883850098, + "eval_runtime": 53.0349, + "eval_samples_per_second": 18.855, + "eval_steps_per_second": 0.792, + "step": 370 + }, + { + "epoch": 0.46172993154947106, + "grad_norm": 2.668421745300293, + "learning_rate": 4.734778597785978e-06, + "loss": 1.6421, + "step": 371 + }, + { + "epoch": 0.46297448662103297, + "grad_norm": 2.7148995399475098, + "learning_rate": 4.7324723247232475e-06, + "loss": 1.6635, + "step": 372 + }, + { + "epoch": 0.4642190416925949, + "grad_norm": 3.156646490097046, + "learning_rate": 4.730166051660517e-06, + "loss": 1.6679, + "step": 373 + }, + { + "epoch": 0.46546359676415683, + "grad_norm": 2.8170645236968994, + "learning_rate": 4.727859778597786e-06, + "loss": 1.6492, + "step": 374 + }, + { + "epoch": 0.46670815183571873, + "grad_norm": 2.9796457290649414, + "learning_rate": 4.725553505535056e-06, + "loss": 1.6025, + "step": 375 + }, + { + "epoch": 0.46795270690728064, + "grad_norm": 2.6493406295776367, + "learning_rate": 4.723247232472325e-06, + "loss": 1.6398, + "step": 376 + }, + { + "epoch": 0.46919726197884254, + "grad_norm": 3.2643542289733887, + "learning_rate": 4.720940959409595e-06, + "loss": 1.6413, + "step": 377 + }, + { + "epoch": 0.4704418170504045, + "grad_norm": 2.7613179683685303, + "learning_rate": 4.718634686346864e-06, + "loss": 1.6251, + "step": 378 + }, + { + "epoch": 0.4716863721219664, + "grad_norm": 3.1216518878936768, + "learning_rate": 4.716328413284133e-06, + "loss": 1.6268, + "step": 379 + }, + { + "epoch": 0.4729309271935283, + "grad_norm": 2.571647882461548, + "learning_rate": 4.714022140221403e-06, + "loss": 1.6245, + "step": 380 + }, + { + "epoch": 0.4729309271935283, + "eval_loss": 1.7875893115997314, + "eval_runtime": 49.3058, + "eval_samples_per_second": 20.282, + "eval_steps_per_second": 0.852, + "step": 380 + }, + { + "epoch": 0.4741754822650902, + "grad_norm": 3.443906545639038, + "learning_rate": 4.711715867158672e-06, + "loss": 1.6799, + "step": 381 + }, + { + "epoch": 0.47542003733665217, + "grad_norm": 2.7887957096099854, + "learning_rate": 4.709409594095941e-06, + "loss": 1.6589, + "step": 382 + }, + { + "epoch": 0.47666459240821407, + "grad_norm": 3.2833151817321777, + "learning_rate": 4.70710332103321e-06, + "loss": 1.6064, + "step": 383 + }, + { + "epoch": 0.477909147479776, + "grad_norm": 2.6975057125091553, + "learning_rate": 4.704797047970481e-06, + "loss": 1.6029, + "step": 384 + }, + { + "epoch": 0.4791537025513379, + "grad_norm": 2.745246171951294, + "learning_rate": 4.702490774907749e-06, + "loss": 1.7127, + "step": 385 + }, + { + "epoch": 0.48039825762289984, + "grad_norm": 3.0522775650024414, + "learning_rate": 4.700184501845019e-06, + "loss": 1.6508, + "step": 386 + }, + { + "epoch": 0.48164281269446174, + "grad_norm": 2.689218044281006, + "learning_rate": 4.697878228782288e-06, + "loss": 1.6971, + "step": 387 + }, + { + "epoch": 0.48288736776602365, + "grad_norm": 3.060361862182617, + "learning_rate": 4.6955719557195575e-06, + "loss": 1.6464, + "step": 388 + }, + { + "epoch": 0.48413192283758555, + "grad_norm": 2.872262477874756, + "learning_rate": 4.693265682656827e-06, + "loss": 1.6293, + "step": 389 + }, + { + "epoch": 0.48537647790914745, + "grad_norm": 2.980050563812256, + "learning_rate": 4.690959409594096e-06, + "loss": 1.6435, + "step": 390 + }, + { + "epoch": 0.48537647790914745, + "eval_loss": 1.7791496515274048, + "eval_runtime": 50.0299, + "eval_samples_per_second": 19.988, + "eval_steps_per_second": 0.839, + "step": 390 + }, + { + "epoch": 0.4866210329807094, + "grad_norm": 2.764827251434326, + "learning_rate": 4.688653136531366e-06, + "loss": 1.6279, + "step": 391 + }, + { + "epoch": 0.4878655880522713, + "grad_norm": 2.6912333965301514, + "learning_rate": 4.686346863468635e-06, + "loss": 1.7276, + "step": 392 + }, + { + "epoch": 0.4891101431238332, + "grad_norm": 2.8358330726623535, + "learning_rate": 4.6840405904059046e-06, + "loss": 1.6482, + "step": 393 + }, + { + "epoch": 0.4903546981953951, + "grad_norm": 2.6581575870513916, + "learning_rate": 4.681734317343174e-06, + "loss": 1.6853, + "step": 394 + }, + { + "epoch": 0.4915992532669571, + "grad_norm": 2.941425323486328, + "learning_rate": 4.6794280442804426e-06, + "loss": 1.6075, + "step": 395 + }, + { + "epoch": 0.492843808338519, + "grad_norm": 3.2334814071655273, + "learning_rate": 4.677121771217713e-06, + "loss": 1.6157, + "step": 396 + }, + { + "epoch": 0.4940883634100809, + "grad_norm": 2.984410524368286, + "learning_rate": 4.674815498154982e-06, + "loss": 1.6257, + "step": 397 + }, + { + "epoch": 0.4953329184816428, + "grad_norm": 3.0650789737701416, + "learning_rate": 4.672509225092252e-06, + "loss": 1.6573, + "step": 398 + }, + { + "epoch": 0.49657747355320475, + "grad_norm": 2.798664093017578, + "learning_rate": 4.67020295202952e-06, + "loss": 1.638, + "step": 399 + }, + { + "epoch": 0.49782202862476665, + "grad_norm": 2.8266706466674805, + "learning_rate": 4.66789667896679e-06, + "loss": 1.6484, + "step": 400 + }, + { + "epoch": 0.49782202862476665, + "eval_loss": 1.7714862823486328, + "eval_runtime": 48.0657, + "eval_samples_per_second": 20.805, + "eval_steps_per_second": 0.874, + "step": 400 + } + ], + "logging_steps": 1, + "max_steps": 2409, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 100, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.246923763744768e+17, + "train_batch_size": 3, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/training_args.bin b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..6cdf93817e8f6d23b60d79888b226cc66ed2e88b --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a50e4918f3c49370326e14a7ed2f73b48b05e521f4b3399b53438fac6abee926 +size 6011 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/zero_to_fp32.py b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..24cc342e78d1a006c782b3a4cd68d9ce786d8fd8 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/zero_to_fp32.py @@ -0,0 +1,604 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: python zero_to_fp32.py . pytorch_model.bin + +import argparse +import torch +import glob +import math +import os +import re +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + + total_files = len(files) + state_dicts = [] + for f in files: + state_dict = torch.load(f, map_location=device) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + if zero_stage <= 2: + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + elif zero_stage == 3: + # if there is more than one param group, there will be multiple flattened tensors - one + # flattened tensor per group - for simplicity merge them into a single tensor + # + # XXX: could make the script more memory efficient for when there are multiple groups - it + # will require matching the sub-lists of param_shapes for each param group flattened tensor + + fp32_flat_groups = [ + torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts)) + ] + + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = fp32_flat_groups[0].numel() * world_size + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + for name, shape in param_shapes.items(): + + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # XXX: memory usage doubles here + state_dict[name] = torch.cat( + tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)), + 0).narrow(0, 0, unpartitioned_numel).view(shape) + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + + Returns: + - pytorch ``state_dict`` + + Note: this approach may not work if your application doesn't have sufficient free CPU memory and + you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters) + print(f"Saving fp32 state dict to {output_file}") + torch.save(state_dict, output_file) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info(f"Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info(f"Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument( + "output_file", + type=str, + help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_file, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fb0f9a53a507ffcdb60410deb71d60ef801bf350 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/config.json @@ -0,0 +1,36 @@ +{ + "_name_or_path": "meta-llama/Llama-3.2-3B", + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "eos_token_id": 128001, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 3072, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 24, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "torch_dtype": "float16", + "transformers_version": "4.45.1", + "use_cache": true, + "vocab_size": 128256 +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/generation_config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9c389b841a9a29ddd904e02d1eb0e08dcce82ad9 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": 128001, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.45.1" +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/latest b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/latest new file mode 100644 index 0000000000000000000000000000000000000000..12cae1adf3af8546b4141c6f62261c8e99839a54 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/latest @@ -0,0 +1 @@ +global_step600 \ No newline at end of file diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/model.safetensors.index.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..ed64de846d720b9a7859dc20575fea8e8ca51940 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/model.safetensors.index.json @@ -0,0 +1,262 @@ +{ + "metadata": { + "total_size": 7213504512 + }, + "weight_map": { + "lm_head.weight": "model-00002-of-00002.safetensors", + "model.embed_tokens.weight": "model-00001-of-00002.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.norm.weight": "model-00002-of-00002.safetensors" + } +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/rng_state_0.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..d5f37008b65b2b6b774ba8ca3e08ea7f2886beeb --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:898e9b2b9cf458e184c0b3debd303a4fd7c10dad3b82242d58682e89cace8b1e +size 16567 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/rng_state_1.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..43698f33ac8bdb91266f12645eda8b36787da828 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:050132a910188198ac7405dcc27a0ec731bdadfd7fd9c07e7468ac838c90fae0 +size 16567 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/rng_state_2.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..90d0b3f139f4cb1368a680fad6f12df4c66f078f --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21ab24991874894fe74f6d6882102cbe2b931a423ab076862a8f74e33fa92f52 +size 16567 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/scheduler.pt b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..aeba18b3f74be376a645e91e43000677785c02d2 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:030ba726537291b026d46325eab79e8e9916823e9059bf1870c5c7ab595e53e3 +size 627 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/special_tokens_map.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..6949c5975ee0e961ef61cf31010dce04df0a03f8 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "[PAD]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/tokenizer.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f28ecaeab53ae07feed29ccf8624d2b0a8344df9 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19fb2e1e3cdd6f7433d89fd6d62c82042599dd4984f342efe7fec6e159e6a8f6 +size 17210734 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/tokenizer_config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8991b49e9c2a43fc527dab9e09ad8171f0cc5943 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/tokenizer_config.json @@ -0,0 +1,2086 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "[PAD]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128257": { + "content": "🅂", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "128258": { + "content": "🄿", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "[PAD]", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/trainer_state.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..9b1c01a3a030fd74d714935fe40c0e252f8ee96f --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/trainer_state.json @@ -0,0 +1,4713 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.7467330429371499, + "eval_steps": 10, + "global_step": 600, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0012445550715619166, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8368, + "step": 1 + }, + { + "epoch": 0.002489110143123833, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.86, + "step": 2 + }, + { + "epoch": 0.00373366521468575, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8265, + "step": 3 + }, + { + "epoch": 0.004978220286247666, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8271, + "step": 4 + }, + { + "epoch": 0.006222775357809583, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8768, + "step": 5 + }, + { + "epoch": 0.0074673304293715, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8402, + "step": 6 + }, + { + "epoch": 0.008711885500933417, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.785, + "step": 7 + }, + { + "epoch": 0.009956440572495333, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8466, + "step": 8 + }, + { + "epoch": 0.01120099564405725, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.7673, + "step": 9 + }, + { + "epoch": 0.012445550715619166, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8074, + "step": 10 + }, + { + "epoch": 0.012445550715619166, + "eval_loss": 2.8535053730010986, + "eval_runtime": 42.446, + "eval_samples_per_second": 23.559, + "eval_steps_per_second": 0.989, + "step": 10 + }, + { + "epoch": 0.013690105787181083, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8592, + "step": 11 + }, + { + "epoch": 0.014934660858743, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.804, + "step": 12 + }, + { + "epoch": 0.016179215930304917, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8076, + "step": 13 + }, + { + "epoch": 0.017423771001866834, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 2.8219, + "step": 14 + }, + { + "epoch": 0.018668326073428748, + "grad_norm": 5.105710506439209, + "learning_rate": 2.0746887966804982e-08, + "loss": 2.7995, + "step": 15 + }, + { + "epoch": 0.019912881144990666, + "grad_norm": 5.156588077545166, + "learning_rate": 4.1493775933609963e-08, + "loss": 2.8577, + "step": 16 + }, + { + "epoch": 0.021157436216552583, + "grad_norm": 5.709373950958252, + "learning_rate": 6.224066390041494e-08, + "loss": 2.8433, + "step": 17 + }, + { + "epoch": 0.0224019912881145, + "grad_norm": 5.64658784866333, + "learning_rate": 8.298755186721993e-08, + "loss": 2.8428, + "step": 18 + }, + { + "epoch": 0.023646546359676415, + "grad_norm": 5.541937351226807, + "learning_rate": 1.037344398340249e-07, + "loss": 2.7785, + "step": 19 + }, + { + "epoch": 0.024891101431238332, + "grad_norm": 5.233374118804932, + "learning_rate": 1.2448132780082988e-07, + "loss": 2.8599, + "step": 20 + }, + { + "epoch": 0.024891101431238332, + "eval_loss": 2.8521182537078857, + "eval_runtime": 43.5664, + "eval_samples_per_second": 22.953, + "eval_steps_per_second": 0.964, + "step": 20 + }, + { + "epoch": 0.02613565650280025, + "grad_norm": 5.407598972320557, + "learning_rate": 1.4522821576763488e-07, + "loss": 2.8081, + "step": 21 + }, + { + "epoch": 0.027380211574362167, + "grad_norm": 5.0483317375183105, + "learning_rate": 1.6597510373443985e-07, + "loss": 2.8489, + "step": 22 + }, + { + "epoch": 0.02862476664592408, + "grad_norm": 4.985588550567627, + "learning_rate": 1.8672199170124483e-07, + "loss": 2.8375, + "step": 23 + }, + { + "epoch": 0.029869321717486, + "grad_norm": 5.201700687408447, + "learning_rate": 2.074688796680498e-07, + "loss": 2.8355, + "step": 24 + }, + { + "epoch": 0.031113876789047916, + "grad_norm": 5.009235382080078, + "learning_rate": 2.2821576763485478e-07, + "loss": 2.7774, + "step": 25 + }, + { + "epoch": 0.03235843186060983, + "grad_norm": 4.509210109710693, + "learning_rate": 2.4896265560165975e-07, + "loss": 2.7992, + "step": 26 + }, + { + "epoch": 0.03360298693217175, + "grad_norm": 4.529262065887451, + "learning_rate": 2.6970954356846476e-07, + "loss": 2.836, + "step": 27 + }, + { + "epoch": 0.03484754200373367, + "grad_norm": 4.526787281036377, + "learning_rate": 2.9045643153526976e-07, + "loss": 2.8128, + "step": 28 + }, + { + "epoch": 0.03609209707529558, + "grad_norm": 4.016609191894531, + "learning_rate": 3.112033195020747e-07, + "loss": 2.7947, + "step": 29 + }, + { + "epoch": 0.037336652146857496, + "grad_norm": 3.4784562587738037, + "learning_rate": 3.319502074688797e-07, + "loss": 2.7854, + "step": 30 + }, + { + "epoch": 0.037336652146857496, + "eval_loss": 2.8020100593566895, + "eval_runtime": 44.8464, + "eval_samples_per_second": 22.298, + "eval_steps_per_second": 0.937, + "step": 30 + }, + { + "epoch": 0.038581207218419414, + "grad_norm": 3.4887945652008057, + "learning_rate": 3.5269709543568466e-07, + "loss": 2.7908, + "step": 31 + }, + { + "epoch": 0.03982576228998133, + "grad_norm": 3.4533374309539795, + "learning_rate": 3.7344398340248966e-07, + "loss": 2.7358, + "step": 32 + }, + { + "epoch": 0.04107031736154325, + "grad_norm": 3.270954132080078, + "learning_rate": 3.941908713692946e-07, + "loss": 2.8013, + "step": 33 + }, + { + "epoch": 0.042314872433105166, + "grad_norm": 3.61322283744812, + "learning_rate": 4.149377593360996e-07, + "loss": 2.7521, + "step": 34 + }, + { + "epoch": 0.043559427504667084, + "grad_norm": 3.158719062805176, + "learning_rate": 4.3568464730290456e-07, + "loss": 2.7573, + "step": 35 + }, + { + "epoch": 0.044803982576229, + "grad_norm": 2.888746500015259, + "learning_rate": 4.5643153526970956e-07, + "loss": 2.7795, + "step": 36 + }, + { + "epoch": 0.04604853764779091, + "grad_norm": 3.022629499435425, + "learning_rate": 4.771784232365145e-07, + "loss": 2.7891, + "step": 37 + }, + { + "epoch": 0.04729309271935283, + "grad_norm": 3.038097381591797, + "learning_rate": 4.979253112033195e-07, + "loss": 2.7376, + "step": 38 + }, + { + "epoch": 0.04853764779091475, + "grad_norm": 2.9992239475250244, + "learning_rate": 5.186721991701245e-07, + "loss": 2.7279, + "step": 39 + }, + { + "epoch": 0.049782202862476664, + "grad_norm": 2.908842086791992, + "learning_rate": 5.394190871369295e-07, + "loss": 2.6582, + "step": 40 + }, + { + "epoch": 0.049782202862476664, + "eval_loss": 2.725968837738037, + "eval_runtime": 44.9498, + "eval_samples_per_second": 22.247, + "eval_steps_per_second": 0.934, + "step": 40 + }, + { + "epoch": 0.05102675793403858, + "grad_norm": 2.8025174140930176, + "learning_rate": 5.601659751037345e-07, + "loss": 2.6809, + "step": 41 + }, + { + "epoch": 0.0522713130056005, + "grad_norm": 2.7555699348449707, + "learning_rate": 5.809128630705395e-07, + "loss": 2.6954, + "step": 42 + }, + { + "epoch": 0.053515868077162417, + "grad_norm": 2.4732933044433594, + "learning_rate": 6.016597510373444e-07, + "loss": 2.7307, + "step": 43 + }, + { + "epoch": 0.054760423148724334, + "grad_norm": 2.451366424560547, + "learning_rate": 6.224066390041494e-07, + "loss": 2.7064, + "step": 44 + }, + { + "epoch": 0.056004978220286245, + "grad_norm": 2.5768678188323975, + "learning_rate": 6.431535269709543e-07, + "loss": 2.6466, + "step": 45 + }, + { + "epoch": 0.05724953329184816, + "grad_norm": 2.579332113265991, + "learning_rate": 6.639004149377594e-07, + "loss": 2.6127, + "step": 46 + }, + { + "epoch": 0.05849408836341008, + "grad_norm": 2.231207847595215, + "learning_rate": 6.846473029045644e-07, + "loss": 2.6429, + "step": 47 + }, + { + "epoch": 0.059738643434972, + "grad_norm": 2.134375810623169, + "learning_rate": 7.053941908713693e-07, + "loss": 2.6423, + "step": 48 + }, + { + "epoch": 0.060983198506533914, + "grad_norm": 2.0485494136810303, + "learning_rate": 7.261410788381744e-07, + "loss": 2.581, + "step": 49 + }, + { + "epoch": 0.06222775357809583, + "grad_norm": 2.02909517288208, + "learning_rate": 7.468879668049793e-07, + "loss": 2.6357, + "step": 50 + }, + { + "epoch": 0.06222775357809583, + "eval_loss": 2.632822036743164, + "eval_runtime": 47.0104, + "eval_samples_per_second": 21.272, + "eval_steps_per_second": 0.893, + "step": 50 + }, + { + "epoch": 0.06347230864965775, + "grad_norm": 1.892500877380371, + "learning_rate": 7.676348547717843e-07, + "loss": 2.5945, + "step": 51 + }, + { + "epoch": 0.06471686372121967, + "grad_norm": 2.107921600341797, + "learning_rate": 7.883817427385892e-07, + "loss": 2.5867, + "step": 52 + }, + { + "epoch": 0.06596141879278158, + "grad_norm": 1.8287527561187744, + "learning_rate": 8.091286307053943e-07, + "loss": 2.5862, + "step": 53 + }, + { + "epoch": 0.0672059738643435, + "grad_norm": 1.7342065572738647, + "learning_rate": 8.298755186721992e-07, + "loss": 2.5735, + "step": 54 + }, + { + "epoch": 0.06845052893590542, + "grad_norm": 1.7905038595199585, + "learning_rate": 8.506224066390042e-07, + "loss": 2.572, + "step": 55 + }, + { + "epoch": 0.06969508400746734, + "grad_norm": 1.9715144634246826, + "learning_rate": 8.713692946058091e-07, + "loss": 2.5797, + "step": 56 + }, + { + "epoch": 0.07093963907902924, + "grad_norm": 1.788966417312622, + "learning_rate": 8.921161825726142e-07, + "loss": 2.5567, + "step": 57 + }, + { + "epoch": 0.07218419415059116, + "grad_norm": 1.8978915214538574, + "learning_rate": 9.128630705394191e-07, + "loss": 2.5776, + "step": 58 + }, + { + "epoch": 0.07342874922215308, + "grad_norm": 1.6401689052581787, + "learning_rate": 9.336099585062241e-07, + "loss": 2.5662, + "step": 59 + }, + { + "epoch": 0.07467330429371499, + "grad_norm": 1.8200798034667969, + "learning_rate": 9.54356846473029e-07, + "loss": 2.5001, + "step": 60 + }, + { + "epoch": 0.07467330429371499, + "eval_loss": 2.569110870361328, + "eval_runtime": 47.3292, + "eval_samples_per_second": 21.129, + "eval_steps_per_second": 0.887, + "step": 60 + }, + { + "epoch": 0.07591785936527691, + "grad_norm": 1.5468674898147583, + "learning_rate": 9.751037344398341e-07, + "loss": 2.5308, + "step": 61 + }, + { + "epoch": 0.07716241443683883, + "grad_norm": 1.723684310913086, + "learning_rate": 9.95850622406639e-07, + "loss": 2.4907, + "step": 62 + }, + { + "epoch": 0.07840696950840075, + "grad_norm": 1.6346293687820435, + "learning_rate": 1.0165975103734441e-06, + "loss": 2.514, + "step": 63 + }, + { + "epoch": 0.07965152457996266, + "grad_norm": 1.8902088403701782, + "learning_rate": 1.037344398340249e-06, + "loss": 2.4859, + "step": 64 + }, + { + "epoch": 0.08089607965152458, + "grad_norm": 2.092611312866211, + "learning_rate": 1.058091286307054e-06, + "loss": 2.4888, + "step": 65 + }, + { + "epoch": 0.0821406347230865, + "grad_norm": 1.758482575416565, + "learning_rate": 1.078838174273859e-06, + "loss": 2.4537, + "step": 66 + }, + { + "epoch": 0.08338518979464841, + "grad_norm": 1.8941714763641357, + "learning_rate": 1.099585062240664e-06, + "loss": 2.5416, + "step": 67 + }, + { + "epoch": 0.08462974486621033, + "grad_norm": 1.7030234336853027, + "learning_rate": 1.120331950207469e-06, + "loss": 2.4791, + "step": 68 + }, + { + "epoch": 0.08587429993777225, + "grad_norm": 1.7602269649505615, + "learning_rate": 1.141078838174274e-06, + "loss": 2.4316, + "step": 69 + }, + { + "epoch": 0.08711885500933417, + "grad_norm": 1.7635431289672852, + "learning_rate": 1.161825726141079e-06, + "loss": 2.4674, + "step": 70 + }, + { + "epoch": 0.08711885500933417, + "eval_loss": 2.4922080039978027, + "eval_runtime": 45.3028, + "eval_samples_per_second": 22.074, + "eval_steps_per_second": 0.927, + "step": 70 + }, + { + "epoch": 0.08836341008089608, + "grad_norm": 2.0913803577423096, + "learning_rate": 1.182572614107884e-06, + "loss": 2.4341, + "step": 71 + }, + { + "epoch": 0.089607965152458, + "grad_norm": 1.8292183876037598, + "learning_rate": 1.2033195020746888e-06, + "loss": 2.4291, + "step": 72 + }, + { + "epoch": 0.09085252022401992, + "grad_norm": 2.206770181655884, + "learning_rate": 1.224066390041494e-06, + "loss": 2.3764, + "step": 73 + }, + { + "epoch": 0.09209707529558182, + "grad_norm": 1.8263559341430664, + "learning_rate": 1.2448132780082988e-06, + "loss": 2.4082, + "step": 74 + }, + { + "epoch": 0.09334163036714374, + "grad_norm": 1.9405455589294434, + "learning_rate": 1.2655601659751037e-06, + "loss": 2.4673, + "step": 75 + }, + { + "epoch": 0.09458618543870566, + "grad_norm": 2.138108015060425, + "learning_rate": 1.2863070539419086e-06, + "loss": 2.3879, + "step": 76 + }, + { + "epoch": 0.09583074051026758, + "grad_norm": 1.9024745225906372, + "learning_rate": 1.307053941908714e-06, + "loss": 2.4344, + "step": 77 + }, + { + "epoch": 0.0970752955818295, + "grad_norm": 1.9136689901351929, + "learning_rate": 1.3278008298755188e-06, + "loss": 2.474, + "step": 78 + }, + { + "epoch": 0.09831985065339141, + "grad_norm": 2.4752795696258545, + "learning_rate": 1.3485477178423237e-06, + "loss": 2.3412, + "step": 79 + }, + { + "epoch": 0.09956440572495333, + "grad_norm": 1.8720006942749023, + "learning_rate": 1.3692946058091288e-06, + "loss": 2.3438, + "step": 80 + }, + { + "epoch": 0.09956440572495333, + "eval_loss": 2.3980512619018555, + "eval_runtime": 48.2788, + "eval_samples_per_second": 20.713, + "eval_steps_per_second": 0.87, + "step": 80 + }, + { + "epoch": 0.10080896079651525, + "grad_norm": 2.671691656112671, + "learning_rate": 1.3900414937759337e-06, + "loss": 2.3336, + "step": 81 + }, + { + "epoch": 0.10205351586807716, + "grad_norm": 2.2953391075134277, + "learning_rate": 1.4107883817427386e-06, + "loss": 2.377, + "step": 82 + }, + { + "epoch": 0.10329807093963908, + "grad_norm": 3.009018898010254, + "learning_rate": 1.4315352697095435e-06, + "loss": 2.2977, + "step": 83 + }, + { + "epoch": 0.104542626011201, + "grad_norm": 2.664454936981201, + "learning_rate": 1.4522821576763488e-06, + "loss": 2.3271, + "step": 84 + }, + { + "epoch": 0.10578718108276292, + "grad_norm": 3.017303705215454, + "learning_rate": 1.4730290456431537e-06, + "loss": 2.3251, + "step": 85 + }, + { + "epoch": 0.10703173615432483, + "grad_norm": 2.634716510772705, + "learning_rate": 1.4937759336099586e-06, + "loss": 2.332, + "step": 86 + }, + { + "epoch": 0.10827629122588675, + "grad_norm": 3.059644937515259, + "learning_rate": 1.5145228215767635e-06, + "loss": 2.3478, + "step": 87 + }, + { + "epoch": 0.10952084629744867, + "grad_norm": 2.6962637901306152, + "learning_rate": 1.5352697095435686e-06, + "loss": 2.2792, + "step": 88 + }, + { + "epoch": 0.11076540136901059, + "grad_norm": 3.419729709625244, + "learning_rate": 1.5560165975103735e-06, + "loss": 2.2571, + "step": 89 + }, + { + "epoch": 0.11200995644057249, + "grad_norm": 2.740781545639038, + "learning_rate": 1.5767634854771784e-06, + "loss": 2.2875, + "step": 90 + }, + { + "epoch": 0.11200995644057249, + "eval_loss": 2.30843186378479, + "eval_runtime": 49.9264, + "eval_samples_per_second": 20.029, + "eval_steps_per_second": 0.841, + "step": 90 + }, + { + "epoch": 0.1132545115121344, + "grad_norm": 2.5608789920806885, + "learning_rate": 1.5975103734439833e-06, + "loss": 2.2592, + "step": 91 + }, + { + "epoch": 0.11449906658369632, + "grad_norm": 2.687999963760376, + "learning_rate": 1.6182572614107886e-06, + "loss": 2.2546, + "step": 92 + }, + { + "epoch": 0.11574362165525824, + "grad_norm": 2.695909023284912, + "learning_rate": 1.6390041493775935e-06, + "loss": 2.2525, + "step": 93 + }, + { + "epoch": 0.11698817672682016, + "grad_norm": 2.818357467651367, + "learning_rate": 1.6597510373443984e-06, + "loss": 2.216, + "step": 94 + }, + { + "epoch": 0.11823273179838208, + "grad_norm": 2.884119987487793, + "learning_rate": 1.6804979253112035e-06, + "loss": 2.2321, + "step": 95 + }, + { + "epoch": 0.119477286869944, + "grad_norm": 2.52104115486145, + "learning_rate": 1.7012448132780084e-06, + "loss": 2.199, + "step": 96 + }, + { + "epoch": 0.12072184194150591, + "grad_norm": 2.420313596725464, + "learning_rate": 1.7219917012448133e-06, + "loss": 2.1862, + "step": 97 + }, + { + "epoch": 0.12196639701306783, + "grad_norm": 2.8047542572021484, + "learning_rate": 1.7427385892116182e-06, + "loss": 2.1793, + "step": 98 + }, + { + "epoch": 0.12321095208462975, + "grad_norm": 2.836482286453247, + "learning_rate": 1.7634854771784235e-06, + "loss": 2.2271, + "step": 99 + }, + { + "epoch": 0.12445550715619166, + "grad_norm": 2.5282301902770996, + "learning_rate": 1.7842323651452284e-06, + "loss": 2.1768, + "step": 100 + }, + { + "epoch": 0.12445550715619166, + "eval_loss": 2.2312686443328857, + "eval_runtime": 49.2382, + "eval_samples_per_second": 20.309, + "eval_steps_per_second": 0.853, + "step": 100 + }, + { + "epoch": 0.12570006222775357, + "grad_norm": 3.3407280445098877, + "learning_rate": 1.8049792531120333e-06, + "loss": 2.1666, + "step": 101 + }, + { + "epoch": 0.1269446172993155, + "grad_norm": 2.4754133224487305, + "learning_rate": 1.8257261410788382e-06, + "loss": 2.1768, + "step": 102 + }, + { + "epoch": 0.1281891723708774, + "grad_norm": 3.430889129638672, + "learning_rate": 1.8464730290456433e-06, + "loss": 2.1953, + "step": 103 + }, + { + "epoch": 0.12943372744243933, + "grad_norm": 2.835294246673584, + "learning_rate": 1.8672199170124482e-06, + "loss": 2.146, + "step": 104 + }, + { + "epoch": 0.13067828251400124, + "grad_norm": 3.1532323360443115, + "learning_rate": 1.8879668049792531e-06, + "loss": 2.1729, + "step": 105 + }, + { + "epoch": 0.13192283758556317, + "grad_norm": 3.2278342247009277, + "learning_rate": 1.908713692946058e-06, + "loss": 2.1336, + "step": 106 + }, + { + "epoch": 0.13316739265712507, + "grad_norm": 2.7892515659332275, + "learning_rate": 1.929460580912863e-06, + "loss": 2.141, + "step": 107 + }, + { + "epoch": 0.134411947728687, + "grad_norm": 3.0179977416992188, + "learning_rate": 1.9502074688796682e-06, + "loss": 2.1255, + "step": 108 + }, + { + "epoch": 0.1356565028002489, + "grad_norm": 2.977935552597046, + "learning_rate": 1.970954356846473e-06, + "loss": 2.0986, + "step": 109 + }, + { + "epoch": 0.13690105787181084, + "grad_norm": 3.3042492866516113, + "learning_rate": 1.991701244813278e-06, + "loss": 2.0992, + "step": 110 + }, + { + "epoch": 0.13690105787181084, + "eval_loss": 2.171299695968628, + "eval_runtime": 42.9511, + "eval_samples_per_second": 23.282, + "eval_steps_per_second": 0.978, + "step": 110 + }, + { + "epoch": 0.13814561294337274, + "grad_norm": 2.9750890731811523, + "learning_rate": 2.012448132780083e-06, + "loss": 2.1116, + "step": 111 + }, + { + "epoch": 0.13939016801493467, + "grad_norm": 2.8199286460876465, + "learning_rate": 2.0331950207468883e-06, + "loss": 2.0822, + "step": 112 + }, + { + "epoch": 0.14063472308649658, + "grad_norm": 2.930532217025757, + "learning_rate": 2.053941908713693e-06, + "loss": 2.161, + "step": 113 + }, + { + "epoch": 0.14187927815805848, + "grad_norm": 3.3400321006774902, + "learning_rate": 2.074688796680498e-06, + "loss": 2.0953, + "step": 114 + }, + { + "epoch": 0.1431238332296204, + "grad_norm": 2.716564655303955, + "learning_rate": 2.095435684647303e-06, + "loss": 2.0757, + "step": 115 + }, + { + "epoch": 0.14436838830118232, + "grad_norm": 2.7586236000061035, + "learning_rate": 2.116182572614108e-06, + "loss": 2.0979, + "step": 116 + }, + { + "epoch": 0.14561294337274425, + "grad_norm": 2.622126340866089, + "learning_rate": 2.136929460580913e-06, + "loss": 2.0748, + "step": 117 + }, + { + "epoch": 0.14685749844430615, + "grad_norm": 2.646477460861206, + "learning_rate": 2.157676348547718e-06, + "loss": 2.0605, + "step": 118 + }, + { + "epoch": 0.14810205351586808, + "grad_norm": 2.8882410526275635, + "learning_rate": 2.178423236514523e-06, + "loss": 2.0501, + "step": 119 + }, + { + "epoch": 0.14934660858742999, + "grad_norm": 2.8302247524261475, + "learning_rate": 2.199170124481328e-06, + "loss": 2.0592, + "step": 120 + }, + { + "epoch": 0.14934660858742999, + "eval_loss": 2.1291966438293457, + "eval_runtime": 47.5399, + "eval_samples_per_second": 21.035, + "eval_steps_per_second": 0.883, + "step": 120 + }, + { + "epoch": 0.15059116365899192, + "grad_norm": 3.26979660987854, + "learning_rate": 2.219917012448133e-06, + "loss": 2.0422, + "step": 121 + }, + { + "epoch": 0.15183571873055382, + "grad_norm": 2.7483913898468018, + "learning_rate": 2.240663900414938e-06, + "loss": 2.0872, + "step": 122 + }, + { + "epoch": 0.15308027380211575, + "grad_norm": 3.1252024173736572, + "learning_rate": 2.2614107883817427e-06, + "loss": 2.0684, + "step": 123 + }, + { + "epoch": 0.15432482887367766, + "grad_norm": 2.9175283908843994, + "learning_rate": 2.282157676348548e-06, + "loss": 2.0522, + "step": 124 + }, + { + "epoch": 0.1555693839452396, + "grad_norm": 2.7950754165649414, + "learning_rate": 2.302904564315353e-06, + "loss": 2.065, + "step": 125 + }, + { + "epoch": 0.1568139390168015, + "grad_norm": 2.84049654006958, + "learning_rate": 2.323651452282158e-06, + "loss": 2.0781, + "step": 126 + }, + { + "epoch": 0.15805849408836342, + "grad_norm": 2.7396647930145264, + "learning_rate": 2.3443983402489627e-06, + "loss": 2.1758, + "step": 127 + }, + { + "epoch": 0.15930304915992533, + "grad_norm": 3.0765926837921143, + "learning_rate": 2.365145228215768e-06, + "loss": 2.0172, + "step": 128 + }, + { + "epoch": 0.16054760423148726, + "grad_norm": 3.217189073562622, + "learning_rate": 2.385892116182573e-06, + "loss": 2.0286, + "step": 129 + }, + { + "epoch": 0.16179215930304916, + "grad_norm": 3.141545295715332, + "learning_rate": 2.4066390041493776e-06, + "loss": 2.0583, + "step": 130 + }, + { + "epoch": 0.16179215930304916, + "eval_loss": 2.0946149826049805, + "eval_runtime": 54.92, + "eval_samples_per_second": 18.208, + "eval_steps_per_second": 0.765, + "step": 130 + }, + { + "epoch": 0.16303671437461106, + "grad_norm": 2.850052833557129, + "learning_rate": 2.4273858921161828e-06, + "loss": 2.0746, + "step": 131 + }, + { + "epoch": 0.164281269446173, + "grad_norm": 3.28913950920105, + "learning_rate": 2.448132780082988e-06, + "loss": 2.0539, + "step": 132 + }, + { + "epoch": 0.1655258245177349, + "grad_norm": 2.8819124698638916, + "learning_rate": 2.468879668049793e-06, + "loss": 1.9843, + "step": 133 + }, + { + "epoch": 0.16677037958929683, + "grad_norm": 3.6254632472991943, + "learning_rate": 2.4896265560165977e-06, + "loss": 2.0233, + "step": 134 + }, + { + "epoch": 0.16801493466085873, + "grad_norm": 2.7385146617889404, + "learning_rate": 2.5103734439834028e-06, + "loss": 1.9667, + "step": 135 + }, + { + "epoch": 0.16925948973242066, + "grad_norm": 2.9722647666931152, + "learning_rate": 2.5311203319502074e-06, + "loss": 1.9527, + "step": 136 + }, + { + "epoch": 0.17050404480398257, + "grad_norm": 3.139526605606079, + "learning_rate": 2.5518672199170125e-06, + "loss": 1.9967, + "step": 137 + }, + { + "epoch": 0.1717485998755445, + "grad_norm": 3.399920701980591, + "learning_rate": 2.5726141078838172e-06, + "loss": 1.9798, + "step": 138 + }, + { + "epoch": 0.1729931549471064, + "grad_norm": 2.8017327785491943, + "learning_rate": 2.5933609958506228e-06, + "loss": 1.983, + "step": 139 + }, + { + "epoch": 0.17423771001866833, + "grad_norm": 3.9967198371887207, + "learning_rate": 2.614107883817428e-06, + "loss": 1.977, + "step": 140 + }, + { + "epoch": 0.17423771001866833, + "eval_loss": 2.059037685394287, + "eval_runtime": 52.6232, + "eval_samples_per_second": 19.003, + "eval_steps_per_second": 0.798, + "step": 140 + }, + { + "epoch": 0.17548226509023024, + "grad_norm": 2.893092393875122, + "learning_rate": 2.6348547717842326e-06, + "loss": 1.9888, + "step": 141 + }, + { + "epoch": 0.17672682016179217, + "grad_norm": 3.634352207183838, + "learning_rate": 2.6556016597510377e-06, + "loss": 2.0726, + "step": 142 + }, + { + "epoch": 0.17797137523335407, + "grad_norm": 3.3651444911956787, + "learning_rate": 2.6763485477178423e-06, + "loss": 1.973, + "step": 143 + }, + { + "epoch": 0.179215930304916, + "grad_norm": 3.968986988067627, + "learning_rate": 2.6970954356846475e-06, + "loss": 1.9948, + "step": 144 + }, + { + "epoch": 0.1804604853764779, + "grad_norm": 3.20105242729187, + "learning_rate": 2.717842323651452e-06, + "loss": 1.9238, + "step": 145 + }, + { + "epoch": 0.18170504044803984, + "grad_norm": 3.648339033126831, + "learning_rate": 2.7385892116182577e-06, + "loss": 1.9923, + "step": 146 + }, + { + "epoch": 0.18294959551960174, + "grad_norm": 3.0856316089630127, + "learning_rate": 2.7593360995850628e-06, + "loss": 1.9749, + "step": 147 + }, + { + "epoch": 0.18419415059116365, + "grad_norm": 3.2681071758270264, + "learning_rate": 2.7800829875518675e-06, + "loss": 1.9304, + "step": 148 + }, + { + "epoch": 0.18543870566272558, + "grad_norm": 2.634958267211914, + "learning_rate": 2.8008298755186726e-06, + "loss": 1.9237, + "step": 149 + }, + { + "epoch": 0.18668326073428748, + "grad_norm": 2.769491672515869, + "learning_rate": 2.8215767634854773e-06, + "loss": 1.8963, + "step": 150 + }, + { + "epoch": 0.18668326073428748, + "eval_loss": 2.0394654273986816, + "eval_runtime": 55.2136, + "eval_samples_per_second": 18.111, + "eval_steps_per_second": 0.761, + "step": 150 + }, + { + "epoch": 0.1879278158058494, + "grad_norm": 3.4345781803131104, + "learning_rate": 2.8423236514522824e-06, + "loss": 1.9836, + "step": 151 + }, + { + "epoch": 0.18917237087741132, + "grad_norm": 3.115727424621582, + "learning_rate": 2.863070539419087e-06, + "loss": 1.9352, + "step": 152 + }, + { + "epoch": 0.19041692594897325, + "grad_norm": 3.050652503967285, + "learning_rate": 2.883817427385892e-06, + "loss": 1.9031, + "step": 153 + }, + { + "epoch": 0.19166148102053515, + "grad_norm": 2.9404428005218506, + "learning_rate": 2.9045643153526977e-06, + "loss": 1.9196, + "step": 154 + }, + { + "epoch": 0.19290603609209708, + "grad_norm": 3.080810785293579, + "learning_rate": 2.9253112033195024e-06, + "loss": 1.9405, + "step": 155 + }, + { + "epoch": 0.194150591163659, + "grad_norm": 3.058558702468872, + "learning_rate": 2.9460580912863075e-06, + "loss": 1.9052, + "step": 156 + }, + { + "epoch": 0.19539514623522092, + "grad_norm": 3.307955026626587, + "learning_rate": 2.966804979253112e-06, + "loss": 1.9245, + "step": 157 + }, + { + "epoch": 0.19663970130678282, + "grad_norm": 2.845506191253662, + "learning_rate": 2.9875518672199173e-06, + "loss": 1.9461, + "step": 158 + }, + { + "epoch": 0.19788425637834475, + "grad_norm": 2.758654832839966, + "learning_rate": 3.008298755186722e-06, + "loss": 1.8885, + "step": 159 + }, + { + "epoch": 0.19912881144990666, + "grad_norm": 3.161252737045288, + "learning_rate": 3.029045643153527e-06, + "loss": 1.8535, + "step": 160 + }, + { + "epoch": 0.19912881144990666, + "eval_loss": 2.018101453781128, + "eval_runtime": 51.0744, + "eval_samples_per_second": 19.579, + "eval_steps_per_second": 0.822, + "step": 160 + }, + { + "epoch": 0.2003733665214686, + "grad_norm": 3.196847677230835, + "learning_rate": 3.0497925311203326e-06, + "loss": 1.8853, + "step": 161 + }, + { + "epoch": 0.2016179215930305, + "grad_norm": 2.8364663124084473, + "learning_rate": 3.0705394190871373e-06, + "loss": 1.9463, + "step": 162 + }, + { + "epoch": 0.2028624766645924, + "grad_norm": 3.0874054431915283, + "learning_rate": 3.0912863070539424e-06, + "loss": 1.9254, + "step": 163 + }, + { + "epoch": 0.20410703173615433, + "grad_norm": 2.7914493083953857, + "learning_rate": 3.112033195020747e-06, + "loss": 1.9213, + "step": 164 + }, + { + "epoch": 0.20535158680771623, + "grad_norm": 3.3871428966522217, + "learning_rate": 3.132780082987552e-06, + "loss": 1.8993, + "step": 165 + }, + { + "epoch": 0.20659614187927816, + "grad_norm": 3.096653461456299, + "learning_rate": 3.153526970954357e-06, + "loss": 1.8827, + "step": 166 + }, + { + "epoch": 0.20784069695084006, + "grad_norm": 2.8276076316833496, + "learning_rate": 3.174273858921162e-06, + "loss": 1.912, + "step": 167 + }, + { + "epoch": 0.209085252022402, + "grad_norm": 3.3058435916900635, + "learning_rate": 3.1950207468879666e-06, + "loss": 1.8381, + "step": 168 + }, + { + "epoch": 0.2103298070939639, + "grad_norm": 3.5017333030700684, + "learning_rate": 3.215767634854772e-06, + "loss": 1.9544, + "step": 169 + }, + { + "epoch": 0.21157436216552583, + "grad_norm": 3.457296133041382, + "learning_rate": 3.2365145228215773e-06, + "loss": 1.922, + "step": 170 + }, + { + "epoch": 0.21157436216552583, + "eval_loss": 1.988455891609192, + "eval_runtime": 56.2099, + "eval_samples_per_second": 17.79, + "eval_steps_per_second": 0.747, + "step": 170 + }, + { + "epoch": 0.21281891723708773, + "grad_norm": 3.1964566707611084, + "learning_rate": 3.257261410788382e-06, + "loss": 1.8925, + "step": 171 + }, + { + "epoch": 0.21406347230864967, + "grad_norm": 3.234652042388916, + "learning_rate": 3.278008298755187e-06, + "loss": 1.884, + "step": 172 + }, + { + "epoch": 0.21530802738021157, + "grad_norm": 3.14414119720459, + "learning_rate": 3.2987551867219918e-06, + "loss": 1.8905, + "step": 173 + }, + { + "epoch": 0.2165525824517735, + "grad_norm": 3.606379508972168, + "learning_rate": 3.319502074688797e-06, + "loss": 1.946, + "step": 174 + }, + { + "epoch": 0.2177971375233354, + "grad_norm": 2.986646890640259, + "learning_rate": 3.3402489626556016e-06, + "loss": 1.8691, + "step": 175 + }, + { + "epoch": 0.21904169259489734, + "grad_norm": 3.7298269271850586, + "learning_rate": 3.360995850622407e-06, + "loss": 1.8717, + "step": 176 + }, + { + "epoch": 0.22028624766645924, + "grad_norm": 3.422295093536377, + "learning_rate": 3.381742738589212e-06, + "loss": 1.8568, + "step": 177 + }, + { + "epoch": 0.22153080273802117, + "grad_norm": 3.2293782234191895, + "learning_rate": 3.402489626556017e-06, + "loss": 1.8471, + "step": 178 + }, + { + "epoch": 0.22277535780958307, + "grad_norm": 3.2293782234191895, + "learning_rate": 3.402489626556017e-06, + "loss": 1.9303, + "step": 179 + }, + { + "epoch": 0.22401991288114498, + "grad_norm": 3.506223440170288, + "learning_rate": 3.423236514522822e-06, + "loss": 1.8237, + "step": 180 + }, + { + "epoch": 0.22401991288114498, + "eval_loss": 1.9734643697738647, + "eval_runtime": 52.2815, + "eval_samples_per_second": 19.127, + "eval_steps_per_second": 0.803, + "step": 180 + }, + { + "epoch": 0.2252644679527069, + "grad_norm": 3.0156619548797607, + "learning_rate": 3.4439834024896267e-06, + "loss": 1.908, + "step": 181 + }, + { + "epoch": 0.2265090230242688, + "grad_norm": 3.55517315864563, + "learning_rate": 3.4647302904564318e-06, + "loss": 1.9104, + "step": 182 + }, + { + "epoch": 0.22775357809583074, + "grad_norm": 3.144984006881714, + "learning_rate": 3.4854771784232365e-06, + "loss": 1.8198, + "step": 183 + }, + { + "epoch": 0.22899813316739265, + "grad_norm": 4.302074432373047, + "learning_rate": 3.5062240663900416e-06, + "loss": 1.8237, + "step": 184 + }, + { + "epoch": 0.23024268823895458, + "grad_norm": 3.0522446632385254, + "learning_rate": 3.526970954356847e-06, + "loss": 1.8409, + "step": 185 + }, + { + "epoch": 0.23148724331051648, + "grad_norm": 3.8607394695281982, + "learning_rate": 3.5477178423236518e-06, + "loss": 1.8807, + "step": 186 + }, + { + "epoch": 0.23273179838207841, + "grad_norm": 2.9236302375793457, + "learning_rate": 3.568464730290457e-06, + "loss": 1.7873, + "step": 187 + }, + { + "epoch": 0.23397635345364032, + "grad_norm": 4.013780117034912, + "learning_rate": 3.5892116182572616e-06, + "loss": 1.7909, + "step": 188 + }, + { + "epoch": 0.23522090852520225, + "grad_norm": 3.0933122634887695, + "learning_rate": 3.6099585062240667e-06, + "loss": 1.8469, + "step": 189 + }, + { + "epoch": 0.23646546359676415, + "grad_norm": 3.487816095352173, + "learning_rate": 3.6307053941908714e-06, + "loss": 1.8469, + "step": 190 + }, + { + "epoch": 0.23646546359676415, + "eval_loss": 1.9604240655899048, + "eval_runtime": 53.4604, + "eval_samples_per_second": 18.705, + "eval_steps_per_second": 0.786, + "step": 190 + }, + { + "epoch": 0.23771001866832608, + "grad_norm": 2.829219341278076, + "learning_rate": 3.6514522821576765e-06, + "loss": 1.8639, + "step": 191 + }, + { + "epoch": 0.238954573739888, + "grad_norm": 3.595534324645996, + "learning_rate": 3.672199170124482e-06, + "loss": 1.8988, + "step": 192 + }, + { + "epoch": 0.24019912881144992, + "grad_norm": 2.9022483825683594, + "learning_rate": 3.6929460580912867e-06, + "loss": 1.8972, + "step": 193 + }, + { + "epoch": 0.24144368388301182, + "grad_norm": 3.0667994022369385, + "learning_rate": 3.713692946058092e-06, + "loss": 1.8499, + "step": 194 + }, + { + "epoch": 0.24268823895457373, + "grad_norm": 3.0498485565185547, + "learning_rate": 3.7344398340248965e-06, + "loss": 1.8341, + "step": 195 + }, + { + "epoch": 0.24393279402613566, + "grad_norm": 4.058084964752197, + "learning_rate": 3.7551867219917016e-06, + "loss": 1.8641, + "step": 196 + }, + { + "epoch": 0.24517734909769756, + "grad_norm": 2.9995028972625732, + "learning_rate": 3.7759336099585063e-06, + "loss": 1.7885, + "step": 197 + }, + { + "epoch": 0.2464219041692595, + "grad_norm": 3.254704475402832, + "learning_rate": 3.7966804979253114e-06, + "loss": 1.7544, + "step": 198 + }, + { + "epoch": 0.2476664592408214, + "grad_norm": 2.9126744270324707, + "learning_rate": 3.817427385892116e-06, + "loss": 1.8461, + "step": 199 + }, + { + "epoch": 0.24891101431238333, + "grad_norm": 3.475247621536255, + "learning_rate": 3.838174273858922e-06, + "loss": 1.8095, + "step": 200 + }, + { + "epoch": 0.24891101431238333, + "eval_loss": 1.9373760223388672, + "eval_runtime": 51.8733, + "eval_samples_per_second": 19.278, + "eval_steps_per_second": 0.81, + "step": 200 + }, + { + "epoch": 0.25015556938394523, + "grad_norm": 3.2663731575012207, + "learning_rate": 3.858921161825726e-06, + "loss": 1.8076, + "step": 201 + }, + { + "epoch": 0.25140012445550713, + "grad_norm": 3.1151726245880127, + "learning_rate": 3.879668049792531e-06, + "loss": 1.8143, + "step": 202 + }, + { + "epoch": 0.2526446795270691, + "grad_norm": 3.102038860321045, + "learning_rate": 3.9004149377593365e-06, + "loss": 1.8279, + "step": 203 + }, + { + "epoch": 0.253889234598631, + "grad_norm": 3.370642900466919, + "learning_rate": 3.921161825726142e-06, + "loss": 1.8522, + "step": 204 + }, + { + "epoch": 0.2551337896701929, + "grad_norm": 3.116128921508789, + "learning_rate": 3.941908713692946e-06, + "loss": 1.8104, + "step": 205 + }, + { + "epoch": 0.2563783447417548, + "grad_norm": 3.12028431892395, + "learning_rate": 3.962655601659751e-06, + "loss": 1.7723, + "step": 206 + }, + { + "epoch": 0.25762289981331676, + "grad_norm": 3.1251418590545654, + "learning_rate": 3.983402489626556e-06, + "loss": 1.863, + "step": 207 + }, + { + "epoch": 0.25886745488487867, + "grad_norm": 3.073702812194824, + "learning_rate": 4.004149377593361e-06, + "loss": 1.805, + "step": 208 + }, + { + "epoch": 0.26011200995644057, + "grad_norm": 2.9772469997406006, + "learning_rate": 4.024896265560166e-06, + "loss": 1.8251, + "step": 209 + }, + { + "epoch": 0.2613565650280025, + "grad_norm": 2.9970037937164307, + "learning_rate": 4.045643153526971e-06, + "loss": 1.8386, + "step": 210 + }, + { + "epoch": 0.2613565650280025, + "eval_loss": 1.9167065620422363, + "eval_runtime": 42.3328, + "eval_samples_per_second": 23.622, + "eval_steps_per_second": 0.992, + "step": 210 + }, + { + "epoch": 0.26260112009956443, + "grad_norm": 3.1391561031341553, + "learning_rate": 4.0663900414937765e-06, + "loss": 1.8109, + "step": 211 + }, + { + "epoch": 0.26384567517112634, + "grad_norm": 3.161538600921631, + "learning_rate": 4.087136929460581e-06, + "loss": 1.8441, + "step": 212 + }, + { + "epoch": 0.26509023024268824, + "grad_norm": 3.12412166595459, + "learning_rate": 4.107883817427386e-06, + "loss": 1.7736, + "step": 213 + }, + { + "epoch": 0.26633478531425014, + "grad_norm": 3.3241145610809326, + "learning_rate": 4.128630705394191e-06, + "loss": 1.8067, + "step": 214 + }, + { + "epoch": 0.26757934038581205, + "grad_norm": 3.503307819366455, + "learning_rate": 4.149377593360996e-06, + "loss": 1.7858, + "step": 215 + }, + { + "epoch": 0.268823895457374, + "grad_norm": 3.1009578704833984, + "learning_rate": 4.170124481327801e-06, + "loss": 1.7542, + "step": 216 + }, + { + "epoch": 0.2700684505289359, + "grad_norm": 3.2236011028289795, + "learning_rate": 4.190871369294606e-06, + "loss": 1.8006, + "step": 217 + }, + { + "epoch": 0.2713130056004978, + "grad_norm": 3.048935651779175, + "learning_rate": 4.211618257261411e-06, + "loss": 1.7706, + "step": 218 + }, + { + "epoch": 0.2725575606720597, + "grad_norm": 3.3416147232055664, + "learning_rate": 4.232365145228216e-06, + "loss": 1.8108, + "step": 219 + }, + { + "epoch": 0.2738021157436217, + "grad_norm": 3.2010738849639893, + "learning_rate": 4.253112033195021e-06, + "loss": 1.8561, + "step": 220 + }, + { + "epoch": 0.2738021157436217, + "eval_loss": 1.9111930131912231, + "eval_runtime": 50.4077, + "eval_samples_per_second": 19.838, + "eval_steps_per_second": 0.833, + "step": 220 + }, + { + "epoch": 0.2750466708151836, + "grad_norm": 3.4033374786376953, + "learning_rate": 4.273858921161826e-06, + "loss": 1.8218, + "step": 221 + }, + { + "epoch": 0.2762912258867455, + "grad_norm": 3.2086002826690674, + "learning_rate": 4.294605809128631e-06, + "loss": 1.8213, + "step": 222 + }, + { + "epoch": 0.2775357809583074, + "grad_norm": 4.090956211090088, + "learning_rate": 4.315352697095436e-06, + "loss": 1.8455, + "step": 223 + }, + { + "epoch": 0.27878033602986935, + "grad_norm": 3.1142985820770264, + "learning_rate": 4.336099585062241e-06, + "loss": 1.7737, + "step": 224 + }, + { + "epoch": 0.28002489110143125, + "grad_norm": 3.369669198989868, + "learning_rate": 4.356846473029046e-06, + "loss": 1.824, + "step": 225 + }, + { + "epoch": 0.28126944617299315, + "grad_norm": 3.165672779083252, + "learning_rate": 4.3775933609958506e-06, + "loss": 1.7768, + "step": 226 + }, + { + "epoch": 0.28251400124455506, + "grad_norm": 3.3343470096588135, + "learning_rate": 4.398340248962656e-06, + "loss": 1.7278, + "step": 227 + }, + { + "epoch": 0.28375855631611696, + "grad_norm": 3.2574994564056396, + "learning_rate": 4.419087136929461e-06, + "loss": 1.8439, + "step": 228 + }, + { + "epoch": 0.2850031113876789, + "grad_norm": 3.043928623199463, + "learning_rate": 4.439834024896266e-06, + "loss": 1.818, + "step": 229 + }, + { + "epoch": 0.2862476664592408, + "grad_norm": 2.9703125953674316, + "learning_rate": 4.460580912863071e-06, + "loss": 1.773, + "step": 230 + }, + { + "epoch": 0.2862476664592408, + "eval_loss": 1.898109793663025, + "eval_runtime": 45.6023, + "eval_samples_per_second": 21.929, + "eval_steps_per_second": 0.921, + "step": 230 + }, + { + "epoch": 0.2874922215308027, + "grad_norm": 2.9997220039367676, + "learning_rate": 4.481327800829876e-06, + "loss": 1.7727, + "step": 231 + }, + { + "epoch": 0.28873677660236463, + "grad_norm": 3.1358482837677, + "learning_rate": 4.502074688796681e-06, + "loss": 1.7764, + "step": 232 + }, + { + "epoch": 0.2899813316739266, + "grad_norm": 3.395747661590576, + "learning_rate": 4.5228215767634855e-06, + "loss": 1.7826, + "step": 233 + }, + { + "epoch": 0.2912258867454885, + "grad_norm": 3.7494754791259766, + "learning_rate": 4.543568464730291e-06, + "loss": 1.7181, + "step": 234 + }, + { + "epoch": 0.2924704418170504, + "grad_norm": 3.42293381690979, + "learning_rate": 4.564315352697096e-06, + "loss": 1.7773, + "step": 235 + }, + { + "epoch": 0.2937149968886123, + "grad_norm": 3.2524514198303223, + "learning_rate": 4.585062240663901e-06, + "loss": 1.7294, + "step": 236 + }, + { + "epoch": 0.29495955196017426, + "grad_norm": 3.657869577407837, + "learning_rate": 4.605809128630706e-06, + "loss": 1.762, + "step": 237 + }, + { + "epoch": 0.29620410703173616, + "grad_norm": 3.127372980117798, + "learning_rate": 4.626556016597511e-06, + "loss": 1.7623, + "step": 238 + }, + { + "epoch": 0.29744866210329807, + "grad_norm": 4.166962146759033, + "learning_rate": 4.647302904564316e-06, + "loss": 1.6995, + "step": 239 + }, + { + "epoch": 0.29869321717485997, + "grad_norm": 3.094264030456543, + "learning_rate": 4.66804979253112e-06, + "loss": 1.7469, + "step": 240 + }, + { + "epoch": 0.29869321717485997, + "eval_loss": 1.8908178806304932, + "eval_runtime": 44.6332, + "eval_samples_per_second": 22.405, + "eval_steps_per_second": 0.941, + "step": 240 + }, + { + "epoch": 0.29993777224642193, + "grad_norm": 3.8644745349884033, + "learning_rate": 4.6887966804979255e-06, + "loss": 1.7644, + "step": 241 + }, + { + "epoch": 0.30118232731798383, + "grad_norm": 3.1488852500915527, + "learning_rate": 4.709543568464731e-06, + "loss": 1.772, + "step": 242 + }, + { + "epoch": 0.30242688238954574, + "grad_norm": 3.3179638385772705, + "learning_rate": 4.730290456431536e-06, + "loss": 1.7588, + "step": 243 + }, + { + "epoch": 0.30367143746110764, + "grad_norm": 3.328355550765991, + "learning_rate": 4.751037344398341e-06, + "loss": 1.7417, + "step": 244 + }, + { + "epoch": 0.30491599253266954, + "grad_norm": 3.2690482139587402, + "learning_rate": 4.771784232365146e-06, + "loss": 1.7562, + "step": 245 + }, + { + "epoch": 0.3061605476042315, + "grad_norm": 3.2759454250335693, + "learning_rate": 4.792531120331951e-06, + "loss": 1.7536, + "step": 246 + }, + { + "epoch": 0.3074051026757934, + "grad_norm": 2.937964916229248, + "learning_rate": 4.813278008298755e-06, + "loss": 1.7518, + "step": 247 + }, + { + "epoch": 0.3086496577473553, + "grad_norm": 3.0995302200317383, + "learning_rate": 4.83402489626556e-06, + "loss": 1.7593, + "step": 248 + }, + { + "epoch": 0.3098942128189172, + "grad_norm": 3.230459213256836, + "learning_rate": 4.8547717842323655e-06, + "loss": 1.6962, + "step": 249 + }, + { + "epoch": 0.3111387678904792, + "grad_norm": 3.242576837539673, + "learning_rate": 4.875518672199171e-06, + "loss": 1.7307, + "step": 250 + }, + { + "epoch": 0.3111387678904792, + "eval_loss": 1.8805371522903442, + "eval_runtime": 45.9886, + "eval_samples_per_second": 21.745, + "eval_steps_per_second": 0.913, + "step": 250 + }, + { + "epoch": 0.3123833229620411, + "grad_norm": 3.097045421600342, + "learning_rate": 4.896265560165976e-06, + "loss": 1.7438, + "step": 251 + }, + { + "epoch": 0.313627878033603, + "grad_norm": 3.2428948879241943, + "learning_rate": 4.91701244813278e-06, + "loss": 1.7179, + "step": 252 + }, + { + "epoch": 0.3148724331051649, + "grad_norm": 3.196274518966675, + "learning_rate": 4.937759336099586e-06, + "loss": 1.7855, + "step": 253 + }, + { + "epoch": 0.31611698817672684, + "grad_norm": 2.978203535079956, + "learning_rate": 4.95850622406639e-06, + "loss": 1.7144, + "step": 254 + }, + { + "epoch": 0.31736154324828875, + "grad_norm": 3.2641701698303223, + "learning_rate": 4.979253112033195e-06, + "loss": 1.7427, + "step": 255 + }, + { + "epoch": 0.31860609831985065, + "grad_norm": 2.7441232204437256, + "learning_rate": 5e-06, + "loss": 1.7619, + "step": 256 + }, + { + "epoch": 0.31985065339141255, + "grad_norm": 3.0723495483398438, + "learning_rate": 4.99769372693727e-06, + "loss": 1.8017, + "step": 257 + }, + { + "epoch": 0.3210952084629745, + "grad_norm": 3.104752540588379, + "learning_rate": 4.995387453874539e-06, + "loss": 1.7761, + "step": 258 + }, + { + "epoch": 0.3223397635345364, + "grad_norm": 3.138627052307129, + "learning_rate": 4.993081180811809e-06, + "loss": 1.7879, + "step": 259 + }, + { + "epoch": 0.3235843186060983, + "grad_norm": 2.8567333221435547, + "learning_rate": 4.990774907749078e-06, + "loss": 1.7561, + "step": 260 + }, + { + "epoch": 0.3235843186060983, + "eval_loss": 1.8606494665145874, + "eval_runtime": 46.453, + "eval_samples_per_second": 21.527, + "eval_steps_per_second": 0.904, + "step": 260 + }, + { + "epoch": 0.3248288736776602, + "grad_norm": 3.2696540355682373, + "learning_rate": 4.988468634686347e-06, + "loss": 1.7201, + "step": 261 + }, + { + "epoch": 0.3260734287492221, + "grad_norm": 2.6692731380462646, + "learning_rate": 4.986162361623617e-06, + "loss": 1.7264, + "step": 262 + }, + { + "epoch": 0.3273179838207841, + "grad_norm": 3.047549247741699, + "learning_rate": 4.983856088560886e-06, + "loss": 1.7362, + "step": 263 + }, + { + "epoch": 0.328562538892346, + "grad_norm": 3.0667457580566406, + "learning_rate": 4.981549815498156e-06, + "loss": 1.7321, + "step": 264 + }, + { + "epoch": 0.3298070939639079, + "grad_norm": 2.560047149658203, + "learning_rate": 4.979243542435424e-06, + "loss": 1.7508, + "step": 265 + }, + { + "epoch": 0.3310516490354698, + "grad_norm": 3.26595401763916, + "learning_rate": 4.976937269372694e-06, + "loss": 1.7248, + "step": 266 + }, + { + "epoch": 0.33229620410703176, + "grad_norm": 2.929210662841797, + "learning_rate": 4.974630996309964e-06, + "loss": 1.697, + "step": 267 + }, + { + "epoch": 0.33354075917859366, + "grad_norm": 3.1526286602020264, + "learning_rate": 4.972324723247233e-06, + "loss": 1.7385, + "step": 268 + }, + { + "epoch": 0.33478531425015556, + "grad_norm": 3.001619577407837, + "learning_rate": 4.970018450184502e-06, + "loss": 1.7072, + "step": 269 + }, + { + "epoch": 0.33602986932171747, + "grad_norm": 3.2464189529418945, + "learning_rate": 4.9677121771217715e-06, + "loss": 1.7361, + "step": 270 + }, + { + "epoch": 0.33602986932171747, + "eval_loss": 1.860226035118103, + "eval_runtime": 46.6881, + "eval_samples_per_second": 21.419, + "eval_steps_per_second": 0.9, + "step": 270 + }, + { + "epoch": 0.3372744243932794, + "grad_norm": 3.2104530334472656, + "learning_rate": 4.965405904059041e-06, + "loss": 1.7171, + "step": 271 + }, + { + "epoch": 0.33851897946484133, + "grad_norm": 3.146847724914551, + "learning_rate": 4.96309963099631e-06, + "loss": 1.7311, + "step": 272 + }, + { + "epoch": 0.33976353453640323, + "grad_norm": 3.1431286334991455, + "learning_rate": 4.96079335793358e-06, + "loss": 1.7345, + "step": 273 + }, + { + "epoch": 0.34100808960796514, + "grad_norm": 2.847163677215576, + "learning_rate": 4.958487084870849e-06, + "loss": 1.7262, + "step": 274 + }, + { + "epoch": 0.3422526446795271, + "grad_norm": 3.1987810134887695, + "learning_rate": 4.956180811808119e-06, + "loss": 1.7945, + "step": 275 + }, + { + "epoch": 0.343497199751089, + "grad_norm": 2.7475385665893555, + "learning_rate": 4.953874538745388e-06, + "loss": 1.76, + "step": 276 + }, + { + "epoch": 0.3447417548226509, + "grad_norm": 3.0427663326263428, + "learning_rate": 4.9515682656826574e-06, + "loss": 1.7324, + "step": 277 + }, + { + "epoch": 0.3459863098942128, + "grad_norm": 3.605212688446045, + "learning_rate": 4.949261992619927e-06, + "loss": 1.7712, + "step": 278 + }, + { + "epoch": 0.3472308649657747, + "grad_norm": 3.0564935207366943, + "learning_rate": 4.946955719557196e-06, + "loss": 1.6792, + "step": 279 + }, + { + "epoch": 0.34847542003733667, + "grad_norm": 3.1069741249084473, + "learning_rate": 4.944649446494466e-06, + "loss": 1.7145, + "step": 280 + }, + { + "epoch": 0.34847542003733667, + "eval_loss": 1.8537051677703857, + "eval_runtime": 47.4813, + "eval_samples_per_second": 21.061, + "eval_steps_per_second": 0.885, + "step": 280 + }, + { + "epoch": 0.3497199751088986, + "grad_norm": 2.8021512031555176, + "learning_rate": 4.942343173431734e-06, + "loss": 1.7145, + "step": 281 + }, + { + "epoch": 0.3509645301804605, + "grad_norm": 3.5549023151397705, + "learning_rate": 4.940036900369004e-06, + "loss": 1.7642, + "step": 282 + }, + { + "epoch": 0.3522090852520224, + "grad_norm": 2.7648985385894775, + "learning_rate": 4.937730627306274e-06, + "loss": 1.7255, + "step": 283 + }, + { + "epoch": 0.35345364032358434, + "grad_norm": 3.0815863609313965, + "learning_rate": 4.9354243542435426e-06, + "loss": 1.7055, + "step": 284 + }, + { + "epoch": 0.35469819539514624, + "grad_norm": 2.9009227752685547, + "learning_rate": 4.933118081180812e-06, + "loss": 1.7019, + "step": 285 + }, + { + "epoch": 0.35594275046670815, + "grad_norm": 2.618429660797119, + "learning_rate": 4.930811808118081e-06, + "loss": 1.6318, + "step": 286 + }, + { + "epoch": 0.35718730553827005, + "grad_norm": 3.368230104446411, + "learning_rate": 4.928505535055351e-06, + "loss": 1.7304, + "step": 287 + }, + { + "epoch": 0.358431860609832, + "grad_norm": 3.256889820098877, + "learning_rate": 4.92619926199262e-06, + "loss": 1.7019, + "step": 288 + }, + { + "epoch": 0.3596764156813939, + "grad_norm": 2.9366304874420166, + "learning_rate": 4.92389298892989e-06, + "loss": 1.734, + "step": 289 + }, + { + "epoch": 0.3609209707529558, + "grad_norm": 3.0393142700195312, + "learning_rate": 4.921586715867159e-06, + "loss": 1.7504, + "step": 290 + }, + { + "epoch": 0.3609209707529558, + "eval_loss": 1.8359886407852173, + "eval_runtime": 49.7622, + "eval_samples_per_second": 20.096, + "eval_steps_per_second": 0.844, + "step": 290 + }, + { + "epoch": 0.3621655258245177, + "grad_norm": 2.7288589477539062, + "learning_rate": 4.9192804428044285e-06, + "loss": 1.7004, + "step": 291 + }, + { + "epoch": 0.3634100808960797, + "grad_norm": 3.2793378829956055, + "learning_rate": 4.916974169741698e-06, + "loss": 1.7028, + "step": 292 + }, + { + "epoch": 0.3646546359676416, + "grad_norm": 2.960880756378174, + "learning_rate": 4.914667896678967e-06, + "loss": 1.6759, + "step": 293 + }, + { + "epoch": 0.3658991910392035, + "grad_norm": 2.836421012878418, + "learning_rate": 4.912361623616237e-06, + "loss": 1.6475, + "step": 294 + }, + { + "epoch": 0.3671437461107654, + "grad_norm": 2.9495465755462646, + "learning_rate": 4.910055350553506e-06, + "loss": 1.7076, + "step": 295 + }, + { + "epoch": 0.3683883011823273, + "grad_norm": 2.954730272293091, + "learning_rate": 4.907749077490776e-06, + "loss": 1.7261, + "step": 296 + }, + { + "epoch": 0.36963285625388925, + "grad_norm": 2.8167543411254883, + "learning_rate": 4.905442804428044e-06, + "loss": 1.6939, + "step": 297 + }, + { + "epoch": 0.37087741132545116, + "grad_norm": 2.857316017150879, + "learning_rate": 4.903136531365314e-06, + "loss": 1.7362, + "step": 298 + }, + { + "epoch": 0.37212196639701306, + "grad_norm": 2.8612918853759766, + "learning_rate": 4.900830258302584e-06, + "loss": 1.6489, + "step": 299 + }, + { + "epoch": 0.37336652146857496, + "grad_norm": 2.9618914127349854, + "learning_rate": 4.898523985239853e-06, + "loss": 1.718, + "step": 300 + }, + { + "epoch": 0.37336652146857496, + "eval_loss": 1.827571988105774, + "eval_runtime": 46.4552, + "eval_samples_per_second": 21.526, + "eval_steps_per_second": 0.904, + "step": 300 + }, + { + "epoch": 0.3746110765401369, + "grad_norm": 2.8706953525543213, + "learning_rate": 4.896217712177122e-06, + "loss": 1.6925, + "step": 301 + }, + { + "epoch": 0.3758556316116988, + "grad_norm": 3.2565090656280518, + "learning_rate": 4.893911439114391e-06, + "loss": 1.7369, + "step": 302 + }, + { + "epoch": 0.37710018668326073, + "grad_norm": 2.928858995437622, + "learning_rate": 4.891605166051661e-06, + "loss": 1.6882, + "step": 303 + }, + { + "epoch": 0.37834474175482263, + "grad_norm": 3.042314052581787, + "learning_rate": 4.88929889298893e-06, + "loss": 1.6466, + "step": 304 + }, + { + "epoch": 0.3795892968263846, + "grad_norm": 2.7509143352508545, + "learning_rate": 4.8869926199262e-06, + "loss": 1.685, + "step": 305 + }, + { + "epoch": 0.3808338518979465, + "grad_norm": 2.9601588249206543, + "learning_rate": 4.884686346863469e-06, + "loss": 1.6772, + "step": 306 + }, + { + "epoch": 0.3820784069695084, + "grad_norm": 2.618608236312866, + "learning_rate": 4.8823800738007384e-06, + "loss": 1.7127, + "step": 307 + }, + { + "epoch": 0.3833229620410703, + "grad_norm": 3.047581195831299, + "learning_rate": 4.880073800738008e-06, + "loss": 1.7174, + "step": 308 + }, + { + "epoch": 0.3845675171126322, + "grad_norm": 2.87735652923584, + "learning_rate": 4.877767527675277e-06, + "loss": 1.6677, + "step": 309 + }, + { + "epoch": 0.38581207218419417, + "grad_norm": 2.7169721126556396, + "learning_rate": 4.875461254612546e-06, + "loss": 1.7486, + "step": 310 + }, + { + "epoch": 0.38581207218419417, + "eval_loss": 1.8221794366836548, + "eval_runtime": 43.1681, + "eval_samples_per_second": 23.165, + "eval_steps_per_second": 0.973, + "step": 310 + }, + { + "epoch": 0.38705662725575607, + "grad_norm": 2.7094991207122803, + "learning_rate": 4.873154981549816e-06, + "loss": 1.7348, + "step": 311 + }, + { + "epoch": 0.388301182327318, + "grad_norm": 2.8989078998565674, + "learning_rate": 4.8708487084870856e-06, + "loss": 1.7352, + "step": 312 + }, + { + "epoch": 0.3895457373988799, + "grad_norm": 2.9565601348876953, + "learning_rate": 4.868542435424355e-06, + "loss": 1.7063, + "step": 313 + }, + { + "epoch": 0.39079029247044184, + "grad_norm": 2.8354082107543945, + "learning_rate": 4.8662361623616235e-06, + "loss": 1.6808, + "step": 314 + }, + { + "epoch": 0.39203484754200374, + "grad_norm": 2.8287479877471924, + "learning_rate": 4.863929889298894e-06, + "loss": 1.6731, + "step": 315 + }, + { + "epoch": 0.39327940261356564, + "grad_norm": 2.6537821292877197, + "learning_rate": 4.861623616236163e-06, + "loss": 1.695, + "step": 316 + }, + { + "epoch": 0.39452395768512755, + "grad_norm": 2.7135190963745117, + "learning_rate": 4.859317343173432e-06, + "loss": 1.6897, + "step": 317 + }, + { + "epoch": 0.3957685127566895, + "grad_norm": 3.1019129753112793, + "learning_rate": 4.857011070110701e-06, + "loss": 1.7219, + "step": 318 + }, + { + "epoch": 0.3970130678282514, + "grad_norm": 2.945737838745117, + "learning_rate": 4.854704797047971e-06, + "loss": 1.7316, + "step": 319 + }, + { + "epoch": 0.3982576228998133, + "grad_norm": 2.7668251991271973, + "learning_rate": 4.85239852398524e-06, + "loss": 1.6461, + "step": 320 + }, + { + "epoch": 0.3982576228998133, + "eval_loss": 1.8210570812225342, + "eval_runtime": 46.2552, + "eval_samples_per_second": 21.619, + "eval_steps_per_second": 0.908, + "step": 320 + }, + { + "epoch": 0.3995021779713752, + "grad_norm": 2.738109588623047, + "learning_rate": 4.8500922509225095e-06, + "loss": 1.6907, + "step": 321 + }, + { + "epoch": 0.4007467330429372, + "grad_norm": 3.0195062160491943, + "learning_rate": 4.847785977859779e-06, + "loss": 1.7099, + "step": 322 + }, + { + "epoch": 0.4019912881144991, + "grad_norm": 2.8672404289245605, + "learning_rate": 4.845479704797048e-06, + "loss": 1.6509, + "step": 323 + }, + { + "epoch": 0.403235843186061, + "grad_norm": 2.9041008949279785, + "learning_rate": 4.843173431734318e-06, + "loss": 1.5818, + "step": 324 + }, + { + "epoch": 0.4044803982576229, + "grad_norm": 2.9105985164642334, + "learning_rate": 4.840867158671587e-06, + "loss": 1.7002, + "step": 325 + }, + { + "epoch": 0.4057249533291848, + "grad_norm": 2.749009847640991, + "learning_rate": 4.838560885608857e-06, + "loss": 1.7021, + "step": 326 + }, + { + "epoch": 0.40696950840074675, + "grad_norm": 2.6998822689056396, + "learning_rate": 4.836254612546126e-06, + "loss": 1.6853, + "step": 327 + }, + { + "epoch": 0.40821406347230865, + "grad_norm": 2.678201675415039, + "learning_rate": 4.8339483394833955e-06, + "loss": 1.6934, + "step": 328 + }, + { + "epoch": 0.40945861854387056, + "grad_norm": 2.774845600128174, + "learning_rate": 4.831642066420665e-06, + "loss": 1.657, + "step": 329 + }, + { + "epoch": 0.41070317361543246, + "grad_norm": 2.6363000869750977, + "learning_rate": 4.8293357933579335e-06, + "loss": 1.6555, + "step": 330 + }, + { + "epoch": 0.41070317361543246, + "eval_loss": 1.8126581907272339, + "eval_runtime": 52.7717, + "eval_samples_per_second": 18.95, + "eval_steps_per_second": 0.796, + "step": 330 + }, + { + "epoch": 0.4119477286869944, + "grad_norm": 3.011659622192383, + "learning_rate": 4.827029520295204e-06, + "loss": 1.7202, + "step": 331 + }, + { + "epoch": 0.4131922837585563, + "grad_norm": 2.8322594165802, + "learning_rate": 4.824723247232473e-06, + "loss": 1.6736, + "step": 332 + }, + { + "epoch": 0.4144368388301182, + "grad_norm": 2.9360146522521973, + "learning_rate": 4.822416974169742e-06, + "loss": 1.7213, + "step": 333 + }, + { + "epoch": 0.41568139390168013, + "grad_norm": 2.810206413269043, + "learning_rate": 4.820110701107011e-06, + "loss": 1.6867, + "step": 334 + }, + { + "epoch": 0.4169259489732421, + "grad_norm": 2.6864097118377686, + "learning_rate": 4.817804428044281e-06, + "loss": 1.6854, + "step": 335 + }, + { + "epoch": 0.418170504044804, + "grad_norm": 2.7396061420440674, + "learning_rate": 4.81549815498155e-06, + "loss": 1.68, + "step": 336 + }, + { + "epoch": 0.4194150591163659, + "grad_norm": 2.602771520614624, + "learning_rate": 4.8131918819188194e-06, + "loss": 1.6269, + "step": 337 + }, + { + "epoch": 0.4206596141879278, + "grad_norm": 2.8174757957458496, + "learning_rate": 4.810885608856089e-06, + "loss": 1.6981, + "step": 338 + }, + { + "epoch": 0.42190416925948976, + "grad_norm": 3.2398130893707275, + "learning_rate": 4.808579335793358e-06, + "loss": 1.7061, + "step": 339 + }, + { + "epoch": 0.42314872433105166, + "grad_norm": 2.7119736671447754, + "learning_rate": 4.806273062730628e-06, + "loss": 1.6875, + "step": 340 + }, + { + "epoch": 0.42314872433105166, + "eval_loss": 1.7973068952560425, + "eval_runtime": 53.5791, + "eval_samples_per_second": 18.664, + "eval_steps_per_second": 0.784, + "step": 340 + }, + { + "epoch": 0.42439327940261357, + "grad_norm": 2.868533134460449, + "learning_rate": 4.803966789667897e-06, + "loss": 1.6967, + "step": 341 + }, + { + "epoch": 0.42563783447417547, + "grad_norm": 2.528083562850952, + "learning_rate": 4.8016605166051665e-06, + "loss": 1.6195, + "step": 342 + }, + { + "epoch": 0.4268823895457374, + "grad_norm": 2.7885115146636963, + "learning_rate": 4.799354243542436e-06, + "loss": 1.6197, + "step": 343 + }, + { + "epoch": 0.42812694461729933, + "grad_norm": 2.768247604370117, + "learning_rate": 4.797047970479705e-06, + "loss": 1.6271, + "step": 344 + }, + { + "epoch": 0.42937149968886124, + "grad_norm": 2.7710275650024414, + "learning_rate": 4.794741697416975e-06, + "loss": 1.6768, + "step": 345 + }, + { + "epoch": 0.43061605476042314, + "grad_norm": 2.968337297439575, + "learning_rate": 4.792435424354243e-06, + "loss": 1.6433, + "step": 346 + }, + { + "epoch": 0.43186060983198504, + "grad_norm": 2.5739808082580566, + "learning_rate": 4.790129151291514e-06, + "loss": 1.6327, + "step": 347 + }, + { + "epoch": 0.433105164903547, + "grad_norm": 2.7676408290863037, + "learning_rate": 4.787822878228783e-06, + "loss": 1.6178, + "step": 348 + }, + { + "epoch": 0.4343497199751089, + "grad_norm": 2.959059476852417, + "learning_rate": 4.7855166051660525e-06, + "loss": 1.6809, + "step": 349 + }, + { + "epoch": 0.4355942750466708, + "grad_norm": 2.559967279434204, + "learning_rate": 4.783210332103321e-06, + "loss": 1.6704, + "step": 350 + }, + { + "epoch": 0.4355942750466708, + "eval_loss": 1.8013949394226074, + "eval_runtime": 50.4649, + "eval_samples_per_second": 19.816, + "eval_steps_per_second": 0.832, + "step": 350 + }, + { + "epoch": 0.4368388301182327, + "grad_norm": 2.817901611328125, + "learning_rate": 4.7809040590405905e-06, + "loss": 1.6296, + "step": 351 + }, + { + "epoch": 0.43808338518979467, + "grad_norm": 2.7116055488586426, + "learning_rate": 4.778597785977861e-06, + "loss": 1.6652, + "step": 352 + }, + { + "epoch": 0.4393279402613566, + "grad_norm": 2.544750452041626, + "learning_rate": 4.776291512915129e-06, + "loss": 1.5668, + "step": 353 + }, + { + "epoch": 0.4405724953329185, + "grad_norm": 2.720534086227417, + "learning_rate": 4.773985239852399e-06, + "loss": 1.6152, + "step": 354 + }, + { + "epoch": 0.4418170504044804, + "grad_norm": 2.7270126342773438, + "learning_rate": 4.771678966789668e-06, + "loss": 1.5624, + "step": 355 + }, + { + "epoch": 0.44306160547604234, + "grad_norm": 2.7986624240875244, + "learning_rate": 4.769372693726938e-06, + "loss": 1.6621, + "step": 356 + }, + { + "epoch": 0.44430616054760425, + "grad_norm": 2.943107843399048, + "learning_rate": 4.767066420664207e-06, + "loss": 1.6713, + "step": 357 + }, + { + "epoch": 0.44555071561916615, + "grad_norm": 2.739898681640625, + "learning_rate": 4.7647601476014765e-06, + "loss": 1.6363, + "step": 358 + }, + { + "epoch": 0.44679527069072805, + "grad_norm": 2.8729329109191895, + "learning_rate": 4.762453874538746e-06, + "loss": 1.6605, + "step": 359 + }, + { + "epoch": 0.44803982576228996, + "grad_norm": 2.909029483795166, + "learning_rate": 4.760147601476015e-06, + "loss": 1.6277, + "step": 360 + }, + { + "epoch": 0.44803982576228996, + "eval_loss": 1.8110179901123047, + "eval_runtime": 47.924, + "eval_samples_per_second": 20.866, + "eval_steps_per_second": 0.876, + "step": 360 + }, + { + "epoch": 0.4492843808338519, + "grad_norm": 3.3826682567596436, + "learning_rate": 4.757841328413285e-06, + "loss": 1.7155, + "step": 361 + }, + { + "epoch": 0.4505289359054138, + "grad_norm": 3.0586729049682617, + "learning_rate": 4.755535055350554e-06, + "loss": 1.5991, + "step": 362 + }, + { + "epoch": 0.4517734909769757, + "grad_norm": 2.645576238632202, + "learning_rate": 4.753228782287823e-06, + "loss": 1.6504, + "step": 363 + }, + { + "epoch": 0.4530180460485376, + "grad_norm": 2.705967664718628, + "learning_rate": 4.750922509225093e-06, + "loss": 1.6761, + "step": 364 + }, + { + "epoch": 0.4542626011200996, + "grad_norm": 2.7651922702789307, + "learning_rate": 4.748616236162362e-06, + "loss": 1.61, + "step": 365 + }, + { + "epoch": 0.4555071561916615, + "grad_norm": 2.5654966831207275, + "learning_rate": 4.746309963099631e-06, + "loss": 1.6335, + "step": 366 + }, + { + "epoch": 0.4567517112632234, + "grad_norm": 2.6606173515319824, + "learning_rate": 4.7440036900369e-06, + "loss": 1.6207, + "step": 367 + }, + { + "epoch": 0.4579962663347853, + "grad_norm": 2.724755048751831, + "learning_rate": 4.741697416974171e-06, + "loss": 1.6329, + "step": 368 + }, + { + "epoch": 0.45924082140634725, + "grad_norm": 2.7019615173339844, + "learning_rate": 4.739391143911439e-06, + "loss": 1.6763, + "step": 369 + }, + { + "epoch": 0.46048537647790916, + "grad_norm": 2.6594254970550537, + "learning_rate": 4.737084870848709e-06, + "loss": 1.6174, + "step": 370 + }, + { + "epoch": 0.46048537647790916, + "eval_loss": 1.7924141883850098, + "eval_runtime": 53.0349, + "eval_samples_per_second": 18.855, + "eval_steps_per_second": 0.792, + "step": 370 + }, + { + "epoch": 0.46172993154947106, + "grad_norm": 2.668421745300293, + "learning_rate": 4.734778597785978e-06, + "loss": 1.6421, + "step": 371 + }, + { + "epoch": 0.46297448662103297, + "grad_norm": 2.7148995399475098, + "learning_rate": 4.7324723247232475e-06, + "loss": 1.6635, + "step": 372 + }, + { + "epoch": 0.4642190416925949, + "grad_norm": 3.156646490097046, + "learning_rate": 4.730166051660517e-06, + "loss": 1.6679, + "step": 373 + }, + { + "epoch": 0.46546359676415683, + "grad_norm": 2.8170645236968994, + "learning_rate": 4.727859778597786e-06, + "loss": 1.6492, + "step": 374 + }, + { + "epoch": 0.46670815183571873, + "grad_norm": 2.9796457290649414, + "learning_rate": 4.725553505535056e-06, + "loss": 1.6025, + "step": 375 + }, + { + "epoch": 0.46795270690728064, + "grad_norm": 2.6493406295776367, + "learning_rate": 4.723247232472325e-06, + "loss": 1.6398, + "step": 376 + }, + { + "epoch": 0.46919726197884254, + "grad_norm": 3.2643542289733887, + "learning_rate": 4.720940959409595e-06, + "loss": 1.6413, + "step": 377 + }, + { + "epoch": 0.4704418170504045, + "grad_norm": 2.7613179683685303, + "learning_rate": 4.718634686346864e-06, + "loss": 1.6251, + "step": 378 + }, + { + "epoch": 0.4716863721219664, + "grad_norm": 3.1216518878936768, + "learning_rate": 4.716328413284133e-06, + "loss": 1.6268, + "step": 379 + }, + { + "epoch": 0.4729309271935283, + "grad_norm": 2.571647882461548, + "learning_rate": 4.714022140221403e-06, + "loss": 1.6245, + "step": 380 + }, + { + "epoch": 0.4729309271935283, + "eval_loss": 1.7875893115997314, + "eval_runtime": 49.3058, + "eval_samples_per_second": 20.282, + "eval_steps_per_second": 0.852, + "step": 380 + }, + { + "epoch": 0.4741754822650902, + "grad_norm": 3.443906545639038, + "learning_rate": 4.711715867158672e-06, + "loss": 1.6799, + "step": 381 + }, + { + "epoch": 0.47542003733665217, + "grad_norm": 2.7887957096099854, + "learning_rate": 4.709409594095941e-06, + "loss": 1.6589, + "step": 382 + }, + { + "epoch": 0.47666459240821407, + "grad_norm": 3.2833151817321777, + "learning_rate": 4.70710332103321e-06, + "loss": 1.6064, + "step": 383 + }, + { + "epoch": 0.477909147479776, + "grad_norm": 2.6975057125091553, + "learning_rate": 4.704797047970481e-06, + "loss": 1.6029, + "step": 384 + }, + { + "epoch": 0.4791537025513379, + "grad_norm": 2.745246171951294, + "learning_rate": 4.702490774907749e-06, + "loss": 1.7127, + "step": 385 + }, + { + "epoch": 0.48039825762289984, + "grad_norm": 3.0522775650024414, + "learning_rate": 4.700184501845019e-06, + "loss": 1.6508, + "step": 386 + }, + { + "epoch": 0.48164281269446174, + "grad_norm": 2.689218044281006, + "learning_rate": 4.697878228782288e-06, + "loss": 1.6971, + "step": 387 + }, + { + "epoch": 0.48288736776602365, + "grad_norm": 3.060361862182617, + "learning_rate": 4.6955719557195575e-06, + "loss": 1.6464, + "step": 388 + }, + { + "epoch": 0.48413192283758555, + "grad_norm": 2.872262477874756, + "learning_rate": 4.693265682656827e-06, + "loss": 1.6293, + "step": 389 + }, + { + "epoch": 0.48537647790914745, + "grad_norm": 2.980050563812256, + "learning_rate": 4.690959409594096e-06, + "loss": 1.6435, + "step": 390 + }, + { + "epoch": 0.48537647790914745, + "eval_loss": 1.7791496515274048, + "eval_runtime": 50.0299, + "eval_samples_per_second": 19.988, + "eval_steps_per_second": 0.839, + "step": 390 + }, + { + "epoch": 0.4866210329807094, + "grad_norm": 2.764827251434326, + "learning_rate": 4.688653136531366e-06, + "loss": 1.6279, + "step": 391 + }, + { + "epoch": 0.4878655880522713, + "grad_norm": 2.6912333965301514, + "learning_rate": 4.686346863468635e-06, + "loss": 1.7276, + "step": 392 + }, + { + "epoch": 0.4891101431238332, + "grad_norm": 2.8358330726623535, + "learning_rate": 4.6840405904059046e-06, + "loss": 1.6482, + "step": 393 + }, + { + "epoch": 0.4903546981953951, + "grad_norm": 2.6581575870513916, + "learning_rate": 4.681734317343174e-06, + "loss": 1.6853, + "step": 394 + }, + { + "epoch": 0.4915992532669571, + "grad_norm": 2.941425323486328, + "learning_rate": 4.6794280442804426e-06, + "loss": 1.6075, + "step": 395 + }, + { + "epoch": 0.492843808338519, + "grad_norm": 3.2334814071655273, + "learning_rate": 4.677121771217713e-06, + "loss": 1.6157, + "step": 396 + }, + { + "epoch": 0.4940883634100809, + "grad_norm": 2.984410524368286, + "learning_rate": 4.674815498154982e-06, + "loss": 1.6257, + "step": 397 + }, + { + "epoch": 0.4953329184816428, + "grad_norm": 3.0650789737701416, + "learning_rate": 4.672509225092252e-06, + "loss": 1.6573, + "step": 398 + }, + { + "epoch": 0.49657747355320475, + "grad_norm": 2.798664093017578, + "learning_rate": 4.67020295202952e-06, + "loss": 1.638, + "step": 399 + }, + { + "epoch": 0.49782202862476665, + "grad_norm": 2.8266706466674805, + "learning_rate": 4.66789667896679e-06, + "loss": 1.6484, + "step": 400 + }, + { + "epoch": 0.49782202862476665, + "eval_loss": 1.7714862823486328, + "eval_runtime": 48.0657, + "eval_samples_per_second": 20.805, + "eval_steps_per_second": 0.874, + "step": 400 + }, + { + "epoch": 0.49906658369632856, + "grad_norm": 2.9928388595581055, + "learning_rate": 4.66559040590406e-06, + "loss": 1.6721, + "step": 401 + }, + { + "epoch": 0.5003111387678905, + "grad_norm": 2.573514461517334, + "learning_rate": 4.6632841328413285e-06, + "loss": 1.679, + "step": 402 + }, + { + "epoch": 0.5015556938394524, + "grad_norm": 3.1736996173858643, + "learning_rate": 4.660977859778598e-06, + "loss": 1.7124, + "step": 403 + }, + { + "epoch": 0.5028002489110143, + "grad_norm": 2.858849287033081, + "learning_rate": 4.658671586715867e-06, + "loss": 1.6622, + "step": 404 + }, + { + "epoch": 0.5040448039825762, + "grad_norm": 2.825698137283325, + "learning_rate": 4.656365313653137e-06, + "loss": 1.6464, + "step": 405 + }, + { + "epoch": 0.5052893590541382, + "grad_norm": 3.783891439437866, + "learning_rate": 4.654059040590406e-06, + "loss": 1.6665, + "step": 406 + }, + { + "epoch": 0.5065339141257, + "grad_norm": 2.789813756942749, + "learning_rate": 4.651752767527676e-06, + "loss": 1.6064, + "step": 407 + }, + { + "epoch": 0.507778469197262, + "grad_norm": 3.3106348514556885, + "learning_rate": 4.649446494464945e-06, + "loss": 1.6542, + "step": 408 + }, + { + "epoch": 0.5090230242688238, + "grad_norm": 2.8805112838745117, + "learning_rate": 4.6471402214022145e-06, + "loss": 1.6058, + "step": 409 + }, + { + "epoch": 0.5102675793403858, + "grad_norm": 3.30898118019104, + "learning_rate": 4.644833948339484e-06, + "loss": 1.5933, + "step": 410 + }, + { + "epoch": 0.5102675793403858, + "eval_loss": 1.7787818908691406, + "eval_runtime": 44.0876, + "eval_samples_per_second": 22.682, + "eval_steps_per_second": 0.953, + "step": 410 + }, + { + "epoch": 0.5115121344119478, + "grad_norm": 3.3563625812530518, + "learning_rate": 4.642527675276753e-06, + "loss": 1.6751, + "step": 411 + }, + { + "epoch": 0.5127566894835096, + "grad_norm": 2.968120574951172, + "learning_rate": 4.640221402214023e-06, + "loss": 1.663, + "step": 412 + }, + { + "epoch": 0.5140012445550716, + "grad_norm": 3.310248374938965, + "learning_rate": 4.637915129151292e-06, + "loss": 1.658, + "step": 413 + }, + { + "epoch": 0.5152457996266335, + "grad_norm": 2.7011170387268066, + "learning_rate": 4.635608856088562e-06, + "loss": 1.6894, + "step": 414 + }, + { + "epoch": 0.5164903546981954, + "grad_norm": 2.804901361465454, + "learning_rate": 4.63330258302583e-06, + "loss": 1.602, + "step": 415 + }, + { + "epoch": 0.5177349097697573, + "grad_norm": 3.2412750720977783, + "learning_rate": 4.6309963099631e-06, + "loss": 1.6158, + "step": 416 + }, + { + "epoch": 0.5189794648413192, + "grad_norm": 2.699909210205078, + "learning_rate": 4.62869003690037e-06, + "loss": 1.5842, + "step": 417 + }, + { + "epoch": 0.5202240199128811, + "grad_norm": 3.065929412841797, + "learning_rate": 4.6263837638376384e-06, + "loss": 1.6434, + "step": 418 + }, + { + "epoch": 0.5214685749844431, + "grad_norm": 2.895305633544922, + "learning_rate": 4.624077490774908e-06, + "loss": 1.5969, + "step": 419 + }, + { + "epoch": 0.522713130056005, + "grad_norm": 3.1590664386749268, + "learning_rate": 4.621771217712177e-06, + "loss": 1.6211, + "step": 420 + }, + { + "epoch": 0.522713130056005, + "eval_loss": 1.7674189805984497, + "eval_runtime": 53.328, + "eval_samples_per_second": 18.752, + "eval_steps_per_second": 0.788, + "step": 420 + }, + { + "epoch": 0.5239576851275669, + "grad_norm": 2.7126312255859375, + "learning_rate": 4.619464944649447e-06, + "loss": 1.6246, + "step": 421 + }, + { + "epoch": 0.5252022401991289, + "grad_norm": 2.918508768081665, + "learning_rate": 4.617158671586716e-06, + "loss": 1.6937, + "step": 422 + }, + { + "epoch": 0.5264467952706907, + "grad_norm": 3.008610725402832, + "learning_rate": 4.6148523985239856e-06, + "loss": 1.6864, + "step": 423 + }, + { + "epoch": 0.5276913503422527, + "grad_norm": 2.6612043380737305, + "learning_rate": 4.612546125461255e-06, + "loss": 1.5785, + "step": 424 + }, + { + "epoch": 0.5289359054138145, + "grad_norm": 2.850679874420166, + "learning_rate": 4.610239852398524e-06, + "loss": 1.5899, + "step": 425 + }, + { + "epoch": 0.5301804604853765, + "grad_norm": 2.9205031394958496, + "learning_rate": 4.607933579335794e-06, + "loss": 1.5867, + "step": 426 + }, + { + "epoch": 0.5314250155569384, + "grad_norm": 2.929532527923584, + "learning_rate": 4.605627306273063e-06, + "loss": 1.6182, + "step": 427 + }, + { + "epoch": 0.5326695706285003, + "grad_norm": 2.874936103820801, + "learning_rate": 4.603321033210333e-06, + "loss": 1.5789, + "step": 428 + }, + { + "epoch": 0.5339141257000622, + "grad_norm": 2.8703525066375732, + "learning_rate": 4.601014760147602e-06, + "loss": 1.5859, + "step": 429 + }, + { + "epoch": 0.5351586807716241, + "grad_norm": 2.7076902389526367, + "learning_rate": 4.5987084870848715e-06, + "loss": 1.644, + "step": 430 + }, + { + "epoch": 0.5351586807716241, + "eval_loss": 1.7592095136642456, + "eval_runtime": 47.9438, + "eval_samples_per_second": 20.858, + "eval_steps_per_second": 0.876, + "step": 430 + }, + { + "epoch": 0.536403235843186, + "grad_norm": 2.839160680770874, + "learning_rate": 4.59640221402214e-06, + "loss": 1.6357, + "step": 431 + }, + { + "epoch": 0.537647790914748, + "grad_norm": 2.674025535583496, + "learning_rate": 4.5940959409594095e-06, + "loss": 1.676, + "step": 432 + }, + { + "epoch": 0.5388923459863099, + "grad_norm": 3.0619537830352783, + "learning_rate": 4.59178966789668e-06, + "loss": 1.5617, + "step": 433 + }, + { + "epoch": 0.5401369010578718, + "grad_norm": 3.058418035507202, + "learning_rate": 4.589483394833948e-06, + "loss": 1.5951, + "step": 434 + }, + { + "epoch": 0.5413814561294338, + "grad_norm": 3.0046396255493164, + "learning_rate": 4.587177121771218e-06, + "loss": 1.6473, + "step": 435 + }, + { + "epoch": 0.5426260112009956, + "grad_norm": 2.7736752033233643, + "learning_rate": 4.584870848708487e-06, + "loss": 1.6262, + "step": 436 + }, + { + "epoch": 0.5438705662725576, + "grad_norm": 2.7804994583129883, + "learning_rate": 4.5825645756457575e-06, + "loss": 1.6886, + "step": 437 + }, + { + "epoch": 0.5451151213441194, + "grad_norm": 3.0717954635620117, + "learning_rate": 4.580258302583026e-06, + "loss": 1.6287, + "step": 438 + }, + { + "epoch": 0.5463596764156814, + "grad_norm": 2.800243854522705, + "learning_rate": 4.5779520295202955e-06, + "loss": 1.6169, + "step": 439 + }, + { + "epoch": 0.5476042314872434, + "grad_norm": 2.9581644535064697, + "learning_rate": 4.575645756457565e-06, + "loss": 1.6617, + "step": 440 + }, + { + "epoch": 0.5476042314872434, + "eval_loss": 1.7642868757247925, + "eval_runtime": 51.2646, + "eval_samples_per_second": 19.507, + "eval_steps_per_second": 0.819, + "step": 440 + }, + { + "epoch": 0.5488487865588052, + "grad_norm": 3.0067460536956787, + "learning_rate": 4.573339483394834e-06, + "loss": 1.6482, + "step": 441 + }, + { + "epoch": 0.5500933416303672, + "grad_norm": 2.9075019359588623, + "learning_rate": 4.571033210332104e-06, + "loss": 1.6595, + "step": 442 + }, + { + "epoch": 0.551337896701929, + "grad_norm": 2.7788755893707275, + "learning_rate": 4.568726937269373e-06, + "loss": 1.5733, + "step": 443 + }, + { + "epoch": 0.552582451773491, + "grad_norm": 2.7775425910949707, + "learning_rate": 4.566420664206643e-06, + "loss": 1.6925, + "step": 444 + }, + { + "epoch": 0.5538270068450529, + "grad_norm": 2.6949567794799805, + "learning_rate": 4.564114391143912e-06, + "loss": 1.5738, + "step": 445 + }, + { + "epoch": 0.5550715619166148, + "grad_norm": 2.7093007564544678, + "learning_rate": 4.5618081180811814e-06, + "loss": 1.5848, + "step": 446 + }, + { + "epoch": 0.5563161169881767, + "grad_norm": 2.6920173168182373, + "learning_rate": 4.559501845018451e-06, + "loss": 1.6092, + "step": 447 + }, + { + "epoch": 0.5575606720597387, + "grad_norm": 2.733731985092163, + "learning_rate": 4.5571955719557194e-06, + "loss": 1.6056, + "step": 448 + }, + { + "epoch": 0.5588052271313005, + "grad_norm": 2.9156274795532227, + "learning_rate": 4.55488929889299e-06, + "loss": 1.6436, + "step": 449 + }, + { + "epoch": 0.5600497822028625, + "grad_norm": 2.8802616596221924, + "learning_rate": 4.552583025830259e-06, + "loss": 1.6309, + "step": 450 + }, + { + "epoch": 0.5600497822028625, + "eval_loss": 1.7486340999603271, + "eval_runtime": 47.2219, + "eval_samples_per_second": 21.177, + "eval_steps_per_second": 0.889, + "step": 450 + }, + { + "epoch": 0.5612943372744243, + "grad_norm": 2.729743242263794, + "learning_rate": 4.550276752767528e-06, + "loss": 1.7171, + "step": 451 + }, + { + "epoch": 0.5625388923459863, + "grad_norm": 2.800049066543579, + "learning_rate": 4.547970479704797e-06, + "loss": 1.6098, + "step": 452 + }, + { + "epoch": 0.5637834474175483, + "grad_norm": 2.9659311771392822, + "learning_rate": 4.5456642066420666e-06, + "loss": 1.6646, + "step": 453 + }, + { + "epoch": 0.5650280024891101, + "grad_norm": 2.5417754650115967, + "learning_rate": 4.543357933579336e-06, + "loss": 1.5847, + "step": 454 + }, + { + "epoch": 0.5662725575606721, + "grad_norm": 2.8471338748931885, + "learning_rate": 4.541051660516605e-06, + "loss": 1.6235, + "step": 455 + }, + { + "epoch": 0.5675171126322339, + "grad_norm": 2.7039637565612793, + "learning_rate": 4.538745387453875e-06, + "loss": 1.5613, + "step": 456 + }, + { + "epoch": 0.5687616677037959, + "grad_norm": 3.0383341312408447, + "learning_rate": 4.536439114391144e-06, + "loss": 1.6485, + "step": 457 + }, + { + "epoch": 0.5700062227753578, + "grad_norm": 2.528388261795044, + "learning_rate": 4.534132841328414e-06, + "loss": 1.6117, + "step": 458 + }, + { + "epoch": 0.5712507778469197, + "grad_norm": 2.7623119354248047, + "learning_rate": 4.531826568265683e-06, + "loss": 1.6017, + "step": 459 + }, + { + "epoch": 0.5724953329184816, + "grad_norm": 2.9213945865631104, + "learning_rate": 4.5295202952029525e-06, + "loss": 1.643, + "step": 460 + }, + { + "epoch": 0.5724953329184816, + "eval_loss": 1.747575044631958, + "eval_runtime": 45.2844, + "eval_samples_per_second": 22.083, + "eval_steps_per_second": 0.927, + "step": 460 + }, + { + "epoch": 0.5737398879900436, + "grad_norm": 2.664275884628296, + "learning_rate": 4.527214022140222e-06, + "loss": 1.6154, + "step": 461 + }, + { + "epoch": 0.5749844430616055, + "grad_norm": 2.5891788005828857, + "learning_rate": 4.524907749077491e-06, + "loss": 1.5975, + "step": 462 + }, + { + "epoch": 0.5762289981331674, + "grad_norm": 2.8126487731933594, + "learning_rate": 4.522601476014761e-06, + "loss": 1.6306, + "step": 463 + }, + { + "epoch": 0.5774735532047293, + "grad_norm": 2.799391031265259, + "learning_rate": 4.520295202952029e-06, + "loss": 1.6139, + "step": 464 + }, + { + "epoch": 0.5787181082762912, + "grad_norm": 2.7323718070983887, + "learning_rate": 4.5179889298893e-06, + "loss": 1.632, + "step": 465 + }, + { + "epoch": 0.5799626633478532, + "grad_norm": 2.601386785507202, + "learning_rate": 4.515682656826569e-06, + "loss": 1.6139, + "step": 466 + }, + { + "epoch": 0.581207218419415, + "grad_norm": 2.562162399291992, + "learning_rate": 4.513376383763838e-06, + "loss": 1.5168, + "step": 467 + }, + { + "epoch": 0.582451773490977, + "grad_norm": 2.6912002563476562, + "learning_rate": 4.511070110701107e-06, + "loss": 1.6249, + "step": 468 + }, + { + "epoch": 0.583696328562539, + "grad_norm": 2.6971495151519775, + "learning_rate": 4.5087638376383765e-06, + "loss": 1.616, + "step": 469 + }, + { + "epoch": 0.5849408836341008, + "grad_norm": 2.6898910999298096, + "learning_rate": 4.506457564575646e-06, + "loss": 1.614, + "step": 470 + }, + { + "epoch": 0.5849408836341008, + "eval_loss": 1.7460769414901733, + "eval_runtime": 45.3055, + "eval_samples_per_second": 22.072, + "eval_steps_per_second": 0.927, + "step": 470 + }, + { + "epoch": 0.5861854387056628, + "grad_norm": 2.698180675506592, + "learning_rate": 4.504151291512915e-06, + "loss": 1.5993, + "step": 471 + }, + { + "epoch": 0.5874299937772246, + "grad_norm": 2.7125210762023926, + "learning_rate": 4.501845018450185e-06, + "loss": 1.6113, + "step": 472 + }, + { + "epoch": 0.5886745488487866, + "grad_norm": 2.754445791244507, + "learning_rate": 4.499538745387454e-06, + "loss": 1.6719, + "step": 473 + }, + { + "epoch": 0.5899191039203485, + "grad_norm": 2.6979362964630127, + "learning_rate": 4.497232472324724e-06, + "loss": 1.608, + "step": 474 + }, + { + "epoch": 0.5911636589919104, + "grad_norm": 2.7694528102874756, + "learning_rate": 4.494926199261993e-06, + "loss": 1.5695, + "step": 475 + }, + { + "epoch": 0.5924082140634723, + "grad_norm": 2.653353214263916, + "learning_rate": 4.4926199261992624e-06, + "loss": 1.5523, + "step": 476 + }, + { + "epoch": 0.5936527691350342, + "grad_norm": 2.6632070541381836, + "learning_rate": 4.490313653136532e-06, + "loss": 1.5675, + "step": 477 + }, + { + "epoch": 0.5948973242065961, + "grad_norm": 3.038543939590454, + "learning_rate": 4.488007380073801e-06, + "loss": 1.5692, + "step": 478 + }, + { + "epoch": 0.5961418792781581, + "grad_norm": 2.8123953342437744, + "learning_rate": 4.485701107011071e-06, + "loss": 1.6223, + "step": 479 + }, + { + "epoch": 0.5973864343497199, + "grad_norm": 2.5845773220062256, + "learning_rate": 4.483394833948339e-06, + "loss": 1.5773, + "step": 480 + }, + { + "epoch": 0.5973864343497199, + "eval_loss": 1.7419319152832031, + "eval_runtime": 44.6842, + "eval_samples_per_second": 22.379, + "eval_steps_per_second": 0.94, + "step": 480 + }, + { + "epoch": 0.5986309894212819, + "grad_norm": 3.069936752319336, + "learning_rate": 4.4810885608856096e-06, + "loss": 1.6323, + "step": 481 + }, + { + "epoch": 0.5998755444928439, + "grad_norm": 2.9085781574249268, + "learning_rate": 4.478782287822879e-06, + "loss": 1.5845, + "step": 482 + }, + { + "epoch": 0.6011200995644057, + "grad_norm": 2.8568010330200195, + "learning_rate": 4.4764760147601476e-06, + "loss": 1.5898, + "step": 483 + }, + { + "epoch": 0.6023646546359677, + "grad_norm": 3.089081048965454, + "learning_rate": 4.474169741697417e-06, + "loss": 1.6622, + "step": 484 + }, + { + "epoch": 0.6036092097075295, + "grad_norm": 2.8515470027923584, + "learning_rate": 4.471863468634686e-06, + "loss": 1.5859, + "step": 485 + }, + { + "epoch": 0.6048537647790915, + "grad_norm": 2.647765874862671, + "learning_rate": 4.469557195571957e-06, + "loss": 1.5926, + "step": 486 + }, + { + "epoch": 0.6060983198506534, + "grad_norm": 2.61676025390625, + "learning_rate": 4.467250922509225e-06, + "loss": 1.5684, + "step": 487 + }, + { + "epoch": 0.6073428749222153, + "grad_norm": 2.7127039432525635, + "learning_rate": 4.464944649446495e-06, + "loss": 1.5702, + "step": 488 + }, + { + "epoch": 0.6085874299937772, + "grad_norm": 2.9932055473327637, + "learning_rate": 4.462638376383764e-06, + "loss": 1.5976, + "step": 489 + }, + { + "epoch": 0.6098319850653391, + "grad_norm": 2.6508774757385254, + "learning_rate": 4.4603321033210335e-06, + "loss": 1.586, + "step": 490 + }, + { + "epoch": 0.6098319850653391, + "eval_loss": 1.7357326745986938, + "eval_runtime": 45.393, + "eval_samples_per_second": 22.03, + "eval_steps_per_second": 0.925, + "step": 490 + }, + { + "epoch": 0.611076540136901, + "grad_norm": 2.7626044750213623, + "learning_rate": 4.458025830258303e-06, + "loss": 1.5773, + "step": 491 + }, + { + "epoch": 0.612321095208463, + "grad_norm": 2.595003604888916, + "learning_rate": 4.455719557195572e-06, + "loss": 1.601, + "step": 492 + }, + { + "epoch": 0.6135656502800249, + "grad_norm": 2.595767021179199, + "learning_rate": 4.453413284132842e-06, + "loss": 1.6287, + "step": 493 + }, + { + "epoch": 0.6148102053515868, + "grad_norm": 2.755845308303833, + "learning_rate": 4.451107011070111e-06, + "loss": 1.593, + "step": 494 + }, + { + "epoch": 0.6160547604231488, + "grad_norm": 2.7302653789520264, + "learning_rate": 4.448800738007381e-06, + "loss": 1.5768, + "step": 495 + }, + { + "epoch": 0.6172993154947106, + "grad_norm": 2.6493024826049805, + "learning_rate": 4.446494464944649e-06, + "loss": 1.6059, + "step": 496 + }, + { + "epoch": 0.6185438705662726, + "grad_norm": 2.8633735179901123, + "learning_rate": 4.4441881918819195e-06, + "loss": 1.598, + "step": 497 + }, + { + "epoch": 0.6197884256378344, + "grad_norm": 2.65639066696167, + "learning_rate": 4.441881918819189e-06, + "loss": 1.6036, + "step": 498 + }, + { + "epoch": 0.6210329807093964, + "grad_norm": 2.6867759227752686, + "learning_rate": 4.439575645756458e-06, + "loss": 1.5676, + "step": 499 + }, + { + "epoch": 0.6222775357809583, + "grad_norm": 2.7986443042755127, + "learning_rate": 4.437269372693727e-06, + "loss": 1.524, + "step": 500 + }, + { + "epoch": 0.6222775357809583, + "eval_loss": 1.7449125051498413, + "eval_runtime": 43.9329, + "eval_samples_per_second": 22.762, + "eval_steps_per_second": 0.956, + "step": 500 + }, + { + "epoch": 0.6235220908525202, + "grad_norm": 2.665905714035034, + "learning_rate": 4.434963099630996e-06, + "loss": 1.5635, + "step": 501 + }, + { + "epoch": 0.6247666459240822, + "grad_norm": 2.902435541152954, + "learning_rate": 4.432656826568267e-06, + "loss": 1.6377, + "step": 502 + }, + { + "epoch": 0.6260112009956441, + "grad_norm": 2.6872262954711914, + "learning_rate": 4.430350553505535e-06, + "loss": 1.5553, + "step": 503 + }, + { + "epoch": 0.627255756067206, + "grad_norm": 2.676621913909912, + "learning_rate": 4.428044280442805e-06, + "loss": 1.6101, + "step": 504 + }, + { + "epoch": 0.6285003111387679, + "grad_norm": 2.756347179412842, + "learning_rate": 4.425738007380074e-06, + "loss": 1.5884, + "step": 505 + }, + { + "epoch": 0.6297448662103298, + "grad_norm": 2.856882333755493, + "learning_rate": 4.4234317343173434e-06, + "loss": 1.5635, + "step": 506 + }, + { + "epoch": 0.6309894212818917, + "grad_norm": 2.7708330154418945, + "learning_rate": 4.421125461254613e-06, + "loss": 1.6028, + "step": 507 + }, + { + "epoch": 0.6322339763534537, + "grad_norm": 2.8167600631713867, + "learning_rate": 4.418819188191882e-06, + "loss": 1.6026, + "step": 508 + }, + { + "epoch": 0.6334785314250155, + "grad_norm": 2.54194974899292, + "learning_rate": 4.416512915129152e-06, + "loss": 1.6001, + "step": 509 + }, + { + "epoch": 0.6347230864965775, + "grad_norm": 2.683037519454956, + "learning_rate": 4.414206642066421e-06, + "loss": 1.6084, + "step": 510 + }, + { + "epoch": 0.6347230864965775, + "eval_loss": 1.7376186847686768, + "eval_runtime": 42.3026, + "eval_samples_per_second": 23.639, + "eval_steps_per_second": 0.993, + "step": 510 + }, + { + "epoch": 0.6359676415681393, + "grad_norm": 2.82627010345459, + "learning_rate": 4.4119003690036905e-06, + "loss": 1.6258, + "step": 511 + }, + { + "epoch": 0.6372121966397013, + "grad_norm": 2.6139848232269287, + "learning_rate": 4.40959409594096e-06, + "loss": 1.6029, + "step": 512 + }, + { + "epoch": 0.6384567517112633, + "grad_norm": 2.646712303161621, + "learning_rate": 4.407287822878229e-06, + "loss": 1.5602, + "step": 513 + }, + { + "epoch": 0.6397013067828251, + "grad_norm": 2.910935163497925, + "learning_rate": 4.404981549815499e-06, + "loss": 1.5757, + "step": 514 + }, + { + "epoch": 0.6409458618543871, + "grad_norm": 2.6001148223876953, + "learning_rate": 4.402675276752768e-06, + "loss": 1.5857, + "step": 515 + }, + { + "epoch": 0.642190416925949, + "grad_norm": 2.8525235652923584, + "learning_rate": 4.400369003690037e-06, + "loss": 1.6092, + "step": 516 + }, + { + "epoch": 0.6434349719975109, + "grad_norm": 2.5335488319396973, + "learning_rate": 4.398062730627306e-06, + "loss": 1.5849, + "step": 517 + }, + { + "epoch": 0.6446795270690728, + "grad_norm": 2.5787103176116943, + "learning_rate": 4.3957564575645765e-06, + "loss": 1.634, + "step": 518 + }, + { + "epoch": 0.6459240821406347, + "grad_norm": 2.6188197135925293, + "learning_rate": 4.393450184501845e-06, + "loss": 1.5477, + "step": 519 + }, + { + "epoch": 0.6471686372121966, + "grad_norm": 2.6548666954040527, + "learning_rate": 4.3911439114391145e-06, + "loss": 1.6133, + "step": 520 + }, + { + "epoch": 0.6471686372121966, + "eval_loss": 1.7393039464950562, + "eval_runtime": 45.9234, + "eval_samples_per_second": 21.775, + "eval_steps_per_second": 0.915, + "step": 520 + }, + { + "epoch": 0.6484131922837586, + "grad_norm": 2.907928228378296, + "learning_rate": 4.388837638376384e-06, + "loss": 1.587, + "step": 521 + }, + { + "epoch": 0.6496577473553204, + "grad_norm": 2.7899692058563232, + "learning_rate": 4.386531365313653e-06, + "loss": 1.6126, + "step": 522 + }, + { + "epoch": 0.6509023024268824, + "grad_norm": 2.680147886276245, + "learning_rate": 4.384225092250923e-06, + "loss": 1.565, + "step": 523 + }, + { + "epoch": 0.6521468574984443, + "grad_norm": 2.6590754985809326, + "learning_rate": 4.381918819188192e-06, + "loss": 1.6127, + "step": 524 + }, + { + "epoch": 0.6533914125700062, + "grad_norm": 2.6638906002044678, + "learning_rate": 4.379612546125462e-06, + "loss": 1.5615, + "step": 525 + }, + { + "epoch": 0.6546359676415682, + "grad_norm": 2.6518194675445557, + "learning_rate": 4.377306273062731e-06, + "loss": 1.6202, + "step": 526 + }, + { + "epoch": 0.65588052271313, + "grad_norm": 2.5323619842529297, + "learning_rate": 4.3750000000000005e-06, + "loss": 1.5305, + "step": 527 + }, + { + "epoch": 0.657125077784692, + "grad_norm": 2.6871442794799805, + "learning_rate": 4.37269372693727e-06, + "loss": 1.6043, + "step": 528 + }, + { + "epoch": 0.6583696328562539, + "grad_norm": 2.760746717453003, + "learning_rate": 4.370387453874539e-06, + "loss": 1.6059, + "step": 529 + }, + { + "epoch": 0.6596141879278158, + "grad_norm": 2.780482769012451, + "learning_rate": 4.368081180811809e-06, + "loss": 1.6082, + "step": 530 + }, + { + "epoch": 0.6596141879278158, + "eval_loss": 1.7219713926315308, + "eval_runtime": 51.8478, + "eval_samples_per_second": 19.287, + "eval_steps_per_second": 0.81, + "step": 530 + }, + { + "epoch": 0.6608587429993777, + "grad_norm": 2.605890989303589, + "learning_rate": 4.365774907749078e-06, + "loss": 1.5554, + "step": 531 + }, + { + "epoch": 0.6621032980709396, + "grad_norm": 2.731555461883545, + "learning_rate": 4.363468634686347e-06, + "loss": 1.606, + "step": 532 + }, + { + "epoch": 0.6633478531425016, + "grad_norm": 2.8356943130493164, + "learning_rate": 4.361162361623616e-06, + "loss": 1.5601, + "step": 533 + }, + { + "epoch": 0.6645924082140635, + "grad_norm": 2.7196593284606934, + "learning_rate": 4.3588560885608864e-06, + "loss": 1.5722, + "step": 534 + }, + { + "epoch": 0.6658369632856254, + "grad_norm": 2.621371269226074, + "learning_rate": 4.356549815498156e-06, + "loss": 1.6194, + "step": 535 + }, + { + "epoch": 0.6670815183571873, + "grad_norm": 2.653916120529175, + "learning_rate": 4.354243542435424e-06, + "loss": 1.5592, + "step": 536 + }, + { + "epoch": 0.6683260734287492, + "grad_norm": 2.88431453704834, + "learning_rate": 4.351937269372694e-06, + "loss": 1.6296, + "step": 537 + }, + { + "epoch": 0.6695706285003111, + "grad_norm": 2.667130470275879, + "learning_rate": 4.349630996309963e-06, + "loss": 1.5624, + "step": 538 + }, + { + "epoch": 0.6708151835718731, + "grad_norm": 2.6453566551208496, + "learning_rate": 4.347324723247233e-06, + "loss": 1.6025, + "step": 539 + }, + { + "epoch": 0.6720597386434349, + "grad_norm": 3.032271146774292, + "learning_rate": 4.345018450184502e-06, + "loss": 1.6771, + "step": 540 + }, + { + "epoch": 0.6720597386434349, + "eval_loss": 1.721895456314087, + "eval_runtime": 50.62, + "eval_samples_per_second": 19.755, + "eval_steps_per_second": 0.83, + "step": 540 + }, + { + "epoch": 0.6733042937149969, + "grad_norm": 2.8476362228393555, + "learning_rate": 4.3427121771217715e-06, + "loss": 1.5663, + "step": 541 + }, + { + "epoch": 0.6745488487865589, + "grad_norm": 2.615602970123291, + "learning_rate": 4.340405904059041e-06, + "loss": 1.5815, + "step": 542 + }, + { + "epoch": 0.6757934038581207, + "grad_norm": 3.221571683883667, + "learning_rate": 4.33809963099631e-06, + "loss": 1.5955, + "step": 543 + }, + { + "epoch": 0.6770379589296827, + "grad_norm": 2.602994203567505, + "learning_rate": 4.33579335793358e-06, + "loss": 1.5667, + "step": 544 + }, + { + "epoch": 0.6782825140012445, + "grad_norm": 2.9380433559417725, + "learning_rate": 4.333487084870848e-06, + "loss": 1.5651, + "step": 545 + }, + { + "epoch": 0.6795270690728065, + "grad_norm": 3.0689425468444824, + "learning_rate": 4.331180811808119e-06, + "loss": 1.6003, + "step": 546 + }, + { + "epoch": 0.6807716241443684, + "grad_norm": 2.770317316055298, + "learning_rate": 4.328874538745388e-06, + "loss": 1.5576, + "step": 547 + }, + { + "epoch": 0.6820161792159303, + "grad_norm": 2.8631815910339355, + "learning_rate": 4.3265682656826575e-06, + "loss": 1.5308, + "step": 548 + }, + { + "epoch": 0.6832607342874922, + "grad_norm": 2.785576820373535, + "learning_rate": 4.324261992619926e-06, + "loss": 1.6113, + "step": 549 + }, + { + "epoch": 0.6845052893590542, + "grad_norm": 2.723919630050659, + "learning_rate": 4.321955719557196e-06, + "loss": 1.5606, + "step": 550 + }, + { + "epoch": 0.6845052893590542, + "eval_loss": 1.721803069114685, + "eval_runtime": 51.6036, + "eval_samples_per_second": 19.378, + "eval_steps_per_second": 0.814, + "step": 550 + }, + { + "epoch": 0.685749844430616, + "grad_norm": 3.081198215484619, + "learning_rate": 4.319649446494466e-06, + "loss": 1.5643, + "step": 551 + }, + { + "epoch": 0.686994399502178, + "grad_norm": 2.757392168045044, + "learning_rate": 4.317343173431734e-06, + "loss": 1.5909, + "step": 552 + }, + { + "epoch": 0.6882389545737398, + "grad_norm": 3.038753032684326, + "learning_rate": 4.315036900369004e-06, + "loss": 1.6023, + "step": 553 + }, + { + "epoch": 0.6894835096453018, + "grad_norm": 2.8246452808380127, + "learning_rate": 4.312730627306273e-06, + "loss": 1.5548, + "step": 554 + }, + { + "epoch": 0.6907280647168638, + "grad_norm": 2.589320182800293, + "learning_rate": 4.310424354243543e-06, + "loss": 1.5424, + "step": 555 + }, + { + "epoch": 0.6919726197884256, + "grad_norm": 2.6413373947143555, + "learning_rate": 4.308118081180812e-06, + "loss": 1.6243, + "step": 556 + }, + { + "epoch": 0.6932171748599876, + "grad_norm": 2.764784336090088, + "learning_rate": 4.3058118081180815e-06, + "loss": 1.589, + "step": 557 + }, + { + "epoch": 0.6944617299315494, + "grad_norm": 2.806821823120117, + "learning_rate": 4.303505535055351e-06, + "loss": 1.621, + "step": 558 + }, + { + "epoch": 0.6957062850031114, + "grad_norm": 2.718017578125, + "learning_rate": 4.30119926199262e-06, + "loss": 1.5773, + "step": 559 + }, + { + "epoch": 0.6969508400746733, + "grad_norm": 2.7620160579681396, + "learning_rate": 4.29889298892989e-06, + "loss": 1.5868, + "step": 560 + }, + { + "epoch": 0.6969508400746733, + "eval_loss": 1.7152249813079834, + "eval_runtime": 52.0463, + "eval_samples_per_second": 19.214, + "eval_steps_per_second": 0.807, + "step": 560 + }, + { + "epoch": 0.6981953951462352, + "grad_norm": 2.66686749458313, + "learning_rate": 4.296586715867159e-06, + "loss": 1.534, + "step": 561 + }, + { + "epoch": 0.6994399502177971, + "grad_norm": 2.812577247619629, + "learning_rate": 4.2942804428044286e-06, + "loss": 1.5501, + "step": 562 + }, + { + "epoch": 0.7006845052893591, + "grad_norm": 2.578508138656616, + "learning_rate": 4.291974169741698e-06, + "loss": 1.6286, + "step": 563 + }, + { + "epoch": 0.701929060360921, + "grad_norm": 2.524249315261841, + "learning_rate": 4.289667896678967e-06, + "loss": 1.5719, + "step": 564 + }, + { + "epoch": 0.7031736154324829, + "grad_norm": 2.827235460281372, + "learning_rate": 4.287361623616236e-06, + "loss": 1.5333, + "step": 565 + }, + { + "epoch": 0.7044181705040448, + "grad_norm": 2.6359963417053223, + "learning_rate": 4.285055350553506e-06, + "loss": 1.5849, + "step": 566 + }, + { + "epoch": 0.7056627255756067, + "grad_norm": 2.930530071258545, + "learning_rate": 4.282749077490776e-06, + "loss": 1.5672, + "step": 567 + }, + { + "epoch": 0.7069072806471687, + "grad_norm": 2.750102996826172, + "learning_rate": 4.280442804428044e-06, + "loss": 1.5696, + "step": 568 + }, + { + "epoch": 0.7081518357187305, + "grad_norm": 2.869690418243408, + "learning_rate": 4.278136531365314e-06, + "loss": 1.5689, + "step": 569 + }, + { + "epoch": 0.7093963907902925, + "grad_norm": 2.954852819442749, + "learning_rate": 4.275830258302583e-06, + "loss": 1.5931, + "step": 570 + }, + { + "epoch": 0.7093963907902925, + "eval_loss": 1.7143300771713257, + "eval_runtime": 52.7293, + "eval_samples_per_second": 18.965, + "eval_steps_per_second": 0.797, + "step": 570 + }, + { + "epoch": 0.7106409458618543, + "grad_norm": 2.702223539352417, + "learning_rate": 4.273523985239853e-06, + "loss": 1.5635, + "step": 571 + }, + { + "epoch": 0.7118855009334163, + "grad_norm": 2.689995050430298, + "learning_rate": 4.271217712177122e-06, + "loss": 1.5545, + "step": 572 + }, + { + "epoch": 0.7131300560049783, + "grad_norm": 2.68979811668396, + "learning_rate": 4.268911439114391e-06, + "loss": 1.5404, + "step": 573 + }, + { + "epoch": 0.7143746110765401, + "grad_norm": 2.7477986812591553, + "learning_rate": 4.266605166051661e-06, + "loss": 1.5719, + "step": 574 + }, + { + "epoch": 0.7156191661481021, + "grad_norm": 2.975778341293335, + "learning_rate": 4.26429889298893e-06, + "loss": 1.5587, + "step": 575 + }, + { + "epoch": 0.716863721219664, + "grad_norm": 2.658170700073242, + "learning_rate": 4.2619926199262e-06, + "loss": 1.5746, + "step": 576 + }, + { + "epoch": 0.7181082762912259, + "grad_norm": 2.8825011253356934, + "learning_rate": 4.259686346863469e-06, + "loss": 1.5246, + "step": 577 + }, + { + "epoch": 0.7193528313627878, + "grad_norm": 2.845280170440674, + "learning_rate": 4.2573800738007385e-06, + "loss": 1.5703, + "step": 578 + }, + { + "epoch": 0.7205973864343497, + "grad_norm": 2.660616636276245, + "learning_rate": 4.255073800738008e-06, + "loss": 1.5373, + "step": 579 + }, + { + "epoch": 0.7218419415059116, + "grad_norm": 2.749447822570801, + "learning_rate": 4.252767527675277e-06, + "loss": 1.59, + "step": 580 + }, + { + "epoch": 0.7218419415059116, + "eval_loss": 1.7149444818496704, + "eval_runtime": 52.2099, + "eval_samples_per_second": 19.153, + "eval_steps_per_second": 0.804, + "step": 580 + }, + { + "epoch": 0.7230864965774736, + "grad_norm": 2.813328742980957, + "learning_rate": 4.250461254612546e-06, + "loss": 1.5694, + "step": 581 + }, + { + "epoch": 0.7243310516490354, + "grad_norm": 2.823866844177246, + "learning_rate": 4.248154981549816e-06, + "loss": 1.5143, + "step": 582 + }, + { + "epoch": 0.7255756067205974, + "grad_norm": 2.71337890625, + "learning_rate": 4.245848708487086e-06, + "loss": 1.5915, + "step": 583 + }, + { + "epoch": 0.7268201617921594, + "grad_norm": 2.718085765838623, + "learning_rate": 4.243542435424355e-06, + "loss": 1.604, + "step": 584 + }, + { + "epoch": 0.7280647168637212, + "grad_norm": 2.5741796493530273, + "learning_rate": 4.241236162361624e-06, + "loss": 1.5398, + "step": 585 + }, + { + "epoch": 0.7293092719352832, + "grad_norm": 2.727114200592041, + "learning_rate": 4.238929889298893e-06, + "loss": 1.5886, + "step": 586 + }, + { + "epoch": 0.730553827006845, + "grad_norm": 2.6703338623046875, + "learning_rate": 4.236623616236163e-06, + "loss": 1.5959, + "step": 587 + }, + { + "epoch": 0.731798382078407, + "grad_norm": 2.750326633453369, + "learning_rate": 4.234317343173432e-06, + "loss": 1.5916, + "step": 588 + }, + { + "epoch": 0.7330429371499689, + "grad_norm": 2.6680715084075928, + "learning_rate": 4.232011070110701e-06, + "loss": 1.562, + "step": 589 + }, + { + "epoch": 0.7342874922215308, + "grad_norm": 2.6603832244873047, + "learning_rate": 4.229704797047971e-06, + "loss": 1.6121, + "step": 590 + }, + { + "epoch": 0.7342874922215308, + "eval_loss": 1.7144734859466553, + "eval_runtime": 50.5317, + "eval_samples_per_second": 19.79, + "eval_steps_per_second": 0.831, + "step": 590 + }, + { + "epoch": 0.7355320472930927, + "grad_norm": 2.7868523597717285, + "learning_rate": 4.22739852398524e-06, + "loss": 1.5488, + "step": 591 + }, + { + "epoch": 0.7367766023646546, + "grad_norm": 2.755270004272461, + "learning_rate": 4.2250922509225096e-06, + "loss": 1.5801, + "step": 592 + }, + { + "epoch": 0.7380211574362165, + "grad_norm": 2.7614126205444336, + "learning_rate": 4.222785977859779e-06, + "loss": 1.5507, + "step": 593 + }, + { + "epoch": 0.7392657125077785, + "grad_norm": 2.8674862384796143, + "learning_rate": 4.220479704797048e-06, + "loss": 1.5807, + "step": 594 + }, + { + "epoch": 0.7405102675793404, + "grad_norm": 2.7366859912872314, + "learning_rate": 4.218173431734318e-06, + "loss": 1.6594, + "step": 595 + }, + { + "epoch": 0.7417548226509023, + "grad_norm": 2.6231045722961426, + "learning_rate": 4.215867158671587e-06, + "loss": 1.5609, + "step": 596 + }, + { + "epoch": 0.7429993777224643, + "grad_norm": 2.7694146633148193, + "learning_rate": 4.213560885608857e-06, + "loss": 1.5889, + "step": 597 + }, + { + "epoch": 0.7442439327940261, + "grad_norm": 2.6275386810302734, + "learning_rate": 4.211254612546125e-06, + "loss": 1.575, + "step": 598 + }, + { + "epoch": 0.7454884878655881, + "grad_norm": 2.9413866996765137, + "learning_rate": 4.2089483394833955e-06, + "loss": 1.589, + "step": 599 + }, + { + "epoch": 0.7467330429371499, + "grad_norm": 2.768606185913086, + "learning_rate": 4.206642066420665e-06, + "loss": 1.5738, + "step": 600 + }, + { + "epoch": 0.7467330429371499, + "eval_loss": 1.7148027420043945, + "eval_runtime": 54.7031, + "eval_samples_per_second": 18.28, + "eval_steps_per_second": 0.768, + "step": 600 + } + ], + "logging_steps": 1, + "max_steps": 2409, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 100, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.870385645617152e+17, + "train_batch_size": 3, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/training_args.bin b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..6cdf93817e8f6d23b60d79888b226cc66ed2e88b --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a50e4918f3c49370326e14a7ed2f73b48b05e521f4b3399b53438fac6abee926 +size 6011 diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/zero_to_fp32.py b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..24cc342e78d1a006c782b3a4cd68d9ce786d8fd8 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/zero_to_fp32.py @@ -0,0 +1,604 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: python zero_to_fp32.py . pytorch_model.bin + +import argparse +import torch +import glob +import math +import os +import re +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + + total_files = len(files) + state_dicts = [] + for f in files: + state_dict = torch.load(f, map_location=device) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + if zero_stage <= 2: + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + elif zero_stage == 3: + # if there is more than one param group, there will be multiple flattened tensors - one + # flattened tensor per group - for simplicity merge them into a single tensor + # + # XXX: could make the script more memory efficient for when there are multiple groups - it + # will require matching the sub-lists of param_shapes for each param group flattened tensor + + fp32_flat_groups = [ + torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts)) + ] + + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = fp32_flat_groups[0].numel() * world_size + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + for name, shape in param_shapes.items(): + + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # XXX: memory usage doubles here + state_dict[name] = torch.cat( + tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)), + 0).narrow(0, 0, unpartitioned_numel).view(shape) + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + + Returns: + - pytorch ``state_dict`` + + Note: this approach may not work if your application doesn't have sufficient free CPU memory and + you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters) + print(f"Saving fp32 state dict to {output_file}") + torch.save(state_dict, output_file) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info(f"Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info(f"Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument( + "output_file", + type=str, + help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_file, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/artifacts/models--meta-llama--Llama-3.2-3B/snapshots/13afe5124825b4f3751f836b40dafda64c1ed062/model-00001-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/artifacts/models--meta-llama--Llama-3.2-3B/snapshots/13afe5124825b4f3751f836b40dafda64c1ed062/model-00001-of-00002.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e362b81bf93cef8c6d96b08171bab63416808c70 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/artifacts/models--meta-llama--Llama-3.2-3B/snapshots/13afe5124825b4f3751f836b40dafda64c1ed062/model-00001-of-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:584d8d3e3f82f7964955174dfe5e3b1cf117a9d859f022cfdf7fcb884856e002 +size 4965799096 diff --git a/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-1000/model-00002-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-1000/model-00002-of-00002.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7462b94ecaa532e7961db2bfc17426ab0d9120ac --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-1000/model-00002-of-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e92262222246eff6e04f0b22bdeb209774c53a1de22d95946813683624c0b97 +size 2247734920 diff --git a/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-1800/model-00002-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-1800/model-00002-of-00002.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ace27eaacde84b08823b905ce22937489692e412 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-1800/model-00002-of-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e09da666c31fcd5284e7f33e6aa466a7a1cdb9db0765a3b851ae5d80aa538bd +size 2247734920 diff --git a/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-200/model-00002-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-200/model-00002-of-00002.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3fd8918d8d7adeed6019825d61aa129bf3fb7edb --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-200/model-00002-of-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f30f6c265af06b6f2962fddd30a844e3c63d77b6e79a0041ed5b6fb06836fd2f +size 2247734920 diff --git a/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-2000/model-00002-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-2000/model-00002-of-00002.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..44ab65022b30120945c744c9eb31c40d71715fca --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-2000/model-00002-of-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5efd1fd5a64a1ceceba8b1f5b6dacdadd185d160edd46005b10a7b3a3d33cae1 +size 2247734920 diff --git a/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-2100/model-00002-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-2100/model-00002-of-00002.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..880165a3a92238aeb2a97ed0ab3296acea3f39a3 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-2100/model-00002-of-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b25b63a151123713387367b7810fb05583c6cf36cd8bd88c5bf536df1d21e92a +size 2247734920 diff --git a/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-2800/model-00002-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-2800/model-00002-of-00002.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b81e4875148a39ae0fa3d7a7c120a85d4cd98d3e --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-2800/model-00002-of-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48e04a6f0a7fabfa87e28ca954eec1f9d891c0390cfedf39f00158b4768f0926 +size 2247734920 diff --git a/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-2900/model-00002-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-2900/model-00002-of-00002.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e79b2014b1b181baddfe8b790220cb919b7b0ff8 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-2900/model-00002-of-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2128d9414fe5872b3a7ccf03939aba32eaba57bff6ffc8a91136a3de86811706 +size 2247734920 diff --git a/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-500/model-00002-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-500/model-00002-of-00002.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b89054a74804b49d8388b7e10b231f831cca2e71 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-500/model-00002-of-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:831df8fa51c51035352e592dbef4d94a6c69e539b2f77b67cd62abb579eb52fe +size 2247734920 diff --git a/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-700/model-00002-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-700/model-00002-of-00002.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f9aec303e4a1d5d4e260a0f36b07fae1e45d0c98 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-700/model-00002-of-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:330a62e65d99682591f1b76580595aa0bd8c26ebc2a29f3676b93daa9b88a952 +size 2247734920 diff --git a/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-900/model-00001-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-900/model-00001-of-00002.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3312a1cb3f3ad4495d38a84bb1152ee78b0f37ca --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-900/model-00001-of-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85d603c0c2e891aa3b2b5d80bb2218b9e1353955c268ea46c2839f622f07377d +size 4965798912 diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic21_10M_seed0/runs/checkpoint-2100/model-00001-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic21_10M_seed0/runs/checkpoint-2100/model-00001-of-00002.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ff589e693484cc408a57b1d801e4fb7c33523222 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic21_10M_seed0/runs/checkpoint-2100/model-00001-of-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbb1fd28f204ab1169220b716b571e17a501f2fd578eccfc167c805c0202405d +size 4965798912 diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic21_10M_seed0/runs/checkpoint-2200/model-00001-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic21_10M_seed0/runs/checkpoint-2200/model-00001-of-00002.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..99b8198b77e9003409d85dcb74642849e0f6f33a --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic21_10M_seed0/runs/checkpoint-2200/model-00001-of-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3445bb6e5a73f2ce5cbf58ca45cc2980515c777ee4dae0ea662c5169573e0981 +size 4965798912 diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic21_10M_seed0/runs/checkpoint-2300/model-00001-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic21_10M_seed0/runs/checkpoint-2300/model-00001-of-00002.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e393835de2a823b0cfae362c8647552c27020154 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic21_10M_seed0/runs/checkpoint-2300/model-00001-of-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6332d577806f91b1eacc159b79c02ee9eae187d98376aa0f7f930ca6e81b3a8c +size 4965798912 diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic21_10M_seed0/runs/checkpoint-2400/model-00001-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic21_10M_seed0/runs/checkpoint-2400/model-00001-of-00002.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f704bbf429ad119dc61ecc689182fe592fb0998b --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic21_10M_seed0/runs/checkpoint-2400/model-00001-of-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c852d50904cb3bb3da3aa3e78aec005aa1ba1a77643e6fc5141f6c8a1b2f2399 +size 4965798912 diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic21_10M_seed0/runs/checkpoint-2500/model-00001-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic21_10M_seed0/runs/checkpoint-2500/model-00001-of-00002.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4a3b1ea1f4c6f00d856ef771898c95ff4f322b70 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic21_10M_seed0/runs/checkpoint-2500/model-00001-of-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe90674a129f3a40775cfeb96b9bcfa5f48dc7a10b8ab508b587d5e5da1d917f +size 4965798912 diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic21_10M_seed0/runs/checkpoint-2600/model-00001-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic21_10M_seed0/runs/checkpoint-2600/model-00001-of-00002.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a954214d6c167230bfaf2e6c8d9df72b60e35130 --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic21_10M_seed0/runs/checkpoint-2600/model-00001-of-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1db2c32ea3abe940ae923c1e43ebd0fc8b3e8797d4c02e73b635acb00b7109f5 +size 4965798912 diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic21_10M_seed0/runs/checkpoint-300/model-00001-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic21_10M_seed0/runs/checkpoint-300/model-00001-of-00002.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b8e1e0d86285014ad3b6cd01d454cf9282122d9b --- /dev/null +++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic21_10M_seed0/runs/checkpoint-300/model-00001-of-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:343ddf2f2a95cd79e05644138bc05bcefa3f612392db23283df602a86ead0a58 +size 4965798912