Initial commit

README.md

@@ -0,0 +1,60 @@
+---
+license: other
+base_model: meta-llama/Meta-Llama-3-8B
+tags:
+- llama-factory
+- full
+- generated_from_trainer
+model-index:
+- name: train_2024-07-10-15-21-44_llama3
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# train_2024-07-10-15-21-44_llama3
+
+This model is a fine-tuned version of [meta-llama/Meta-Llama-3-8B](https://huggingface.co/meta-llama/Meta-Llama-3-8B) on the truth_train dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-06
+- train_batch_size: 4
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- gradient_accumulation_steps: 8
+- total_train_batch_size: 256
+- total_eval_batch_size: 64
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_steps: 600
+- num_epochs: 5.0
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.42.3
+- Pytorch 2.3.0a0+ebedce2
+- Datasets 2.20.0
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 4.951768488745981,
+    "num_input_tokens_seen": 5192736,
+    "total_flos": 2.3382655808988774e+17,
+    "train_loss": 0.7082552919200585,
+    "train_runtime": 5092.4527,
+    "train_samples_per_second": 19.519,
+    "train_steps_per_second": 0.076
+}

checkpoint-385/config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "meta-llama/Meta-Llama-3-8B",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 8192,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.42.3",
+  "use_cache": false,
+  "vocab_size": 128256
+}

checkpoint-385/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": 128001,
+  "max_length": 4096,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.42.3"
+}

checkpoint-385/global_step385/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1452cadca9e99e513aead937d68a06e4dd403bcebc6b0ae2107a9dc61602e286
+size 12045399116

checkpoint-385/global_step385/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:332130d0aaf9661efbdc8d63cfa2324b89b2f04ee89bdcc9dcc08feb296d38d0
+size 12045399948

checkpoint-385/global_step385/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ecfd854520db14d56ffa5b977c3479ad8f34214317cb8ce53045709be028ff6
+size 12045400204

checkpoint-385/global_step385/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e533b59643438fae3d510d4b42f8521df934ada18b2f6a743770065713bf3172
+size 12045399948

checkpoint-385/global_step385/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b17cdfb25369288f35fb479ad5732bcbf91ee4d690b5f2151f4ff37373e00408
+size 12045400204

checkpoint-385/global_step385/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b02a989a64476ddd93b2859abf8dee0985ba7510759591225582157595ce2e2
+size 12045400268

checkpoint-385/global_step385/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:438ff20a32901a038f604e1ab2f46a8fc48d6b5e7a0e239a711f2c0ff68497af
+size 12045399948

checkpoint-385/global_step385/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ed61b5ed7520962e8162628f611d10d8a17a54f2fbc511def6c1743fdcfe734
+size 12045398796

checkpoint-385/global_step385/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9908b515826fbc92424716b8e7dd94e5ce165e2a27df035c407a0f745a07f247
+size 16060610616

checkpoint-385/latest

@@ -0,0 +1 @@
+global_step385

checkpoint-385/model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95091273e88294fb364aa21ea26e19235d33e52c55bd3084a88c89fea8b26726
+size 4976698672

checkpoint-385/model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27524a69f85bdd427bbc9f3b06c352d75e57fb52dd8bbfaa72357a0aa776ec14
+size 4999802720

checkpoint-385/model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f3d18cfd3abd56f02f7d77fe84167a89940a20fad534fc60b9ccf8a13e458ba
+size 4915916176

checkpoint-385/model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7fd33cba41fd9bb770eab81315a245e1fd442a47ac61b49f563417d3882e8d6
+size 1168138808

checkpoint-385/model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 16060522496
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00004-of-00004.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

checkpoint-385/rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a0ef6f96a48e59aa52c4b471312c2a62378c19acc7ebbae839612b03a7d775a
+size 15984

checkpoint-385/rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab11d533c0fdad46ea8b8e295ba5fdb705e078eeb88cc28f37d82913508766e9
+size 15984

checkpoint-385/rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:615c168147e3465ce5bfab6da2ff4afc68566ce00ec0f0c6c9fc988038a58d0a
+size 15984

checkpoint-385/rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79f71e8f8674ecaef9f8cdcbf7ac457a8b8ff15b12694ba2a2fffcb4b43f0f08
+size 15984

checkpoint-385/rng_state_4.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88cf6d674dab5545c300a55135f08ca935730a3d35e2c419fb0b333f19482c19
+size 15984

checkpoint-385/rng_state_5.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2754f2cd8824702f027870d93748b3c0491b0ecd30f1e3d8e937116b2be6151f
+size 15984

checkpoint-385/rng_state_6.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1385124ac55604598f45ea6e2d141f29456647d3e7c10d12ca64ec93d312be8d
+size 15984

checkpoint-385/rng_state_7.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:416538efaec7391fa8fe782fb15146b83e5612d9e1961292c34c53e964806873
+size 15984

checkpoint-385/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5773c00222bef1db5ca810359f2616a1fbd633a740c194d6a3070d6b031a0320
+size 1064

checkpoint-385/special_tokens_map.json

@@ -0,0 +1,17 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|end_of_text|>"
+}

checkpoint-385/tokenizer_config.json

@@ -0,0 +1,2065 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_248|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_249|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_250|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% if system_message is defined %}{{ system_message + '\n' }}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ 'Human: ' + content + '\nAssistant:' }}{% elif message['role'] == 'assistant' %}{{ content + '<|end_of_text|>' + '\n' }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|end_of_text|>",
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|end_of_text|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

checkpoint-385/trainer_state.json

@@ -0,0 +1,3113 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 4.951768488745981,
+  "eval_steps": 500,
+  "global_step": 385,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.012861736334405145,
+      "grad_norm": 359.0199890136719,
+      "learning_rate": 8.333333333333335e-09,
+      "loss": 7.8034,
+      "num_input_tokens_seen": 13440,
+      "step": 1
+    },
+    {
+      "epoch": 0.02572347266881029,
+      "grad_norm": 359.18682861328125,
+      "learning_rate": 1.666666666666667e-08,
+      "loss": 7.7577,
+      "num_input_tokens_seen": 26976,
+      "step": 2
+    },
+    {
+      "epoch": 0.03858520900321544,
+      "grad_norm": 368.04339599609375,
+      "learning_rate": 2.5000000000000002e-08,
+      "loss": 7.8132,
+      "num_input_tokens_seen": 40800,
+      "step": 3
+    },
+    {
+      "epoch": 0.05144694533762058,
+      "grad_norm": 357.04132080078125,
+      "learning_rate": 3.333333333333334e-08,
+      "loss": 7.7906,
+      "num_input_tokens_seen": 54080,
+      "step": 4
+    },
+    {
+      "epoch": 0.06430868167202572,
+      "grad_norm": 361.8187255859375,
+      "learning_rate": 4.166666666666667e-08,
+      "loss": 7.7763,
+      "num_input_tokens_seen": 67552,
+      "step": 5
+    },
+    {
+      "epoch": 0.07717041800643087,
+      "grad_norm": 368.778076171875,
+      "learning_rate": 5.0000000000000004e-08,
+      "loss": 7.7824,
+      "num_input_tokens_seen": 81600,
+      "step": 6
+    },
+    {
+      "epoch": 0.09003215434083602,
+      "grad_norm": 367.5040588378906,
+      "learning_rate": 5.833333333333334e-08,
+      "loss": 7.833,
+      "num_input_tokens_seen": 94720,
+      "step": 7
+    },
+    {
+      "epoch": 0.10289389067524116,
+      "grad_norm": 366.2318420410156,
+      "learning_rate": 6.666666666666668e-08,
+      "loss": 7.6949,
+      "num_input_tokens_seen": 107488,
+      "step": 8
+    },
+    {
+      "epoch": 0.1157556270096463,
+      "grad_norm": 366.990966796875,
+      "learning_rate": 7.500000000000001e-08,
+      "loss": 7.8336,
+      "num_input_tokens_seen": 120992,
+      "step": 9
+    },
+    {
+      "epoch": 0.12861736334405144,
+      "grad_norm": 357.8764953613281,
+      "learning_rate": 8.333333333333334e-08,
+      "loss": 7.7282,
+      "num_input_tokens_seen": 134272,
+      "step": 10
+    },
+    {
+      "epoch": 0.1414790996784566,
+      "grad_norm": 363.4705505371094,
+      "learning_rate": 9.166666666666668e-08,
+      "loss": 7.6916,
+      "num_input_tokens_seen": 148096,
+      "step": 11
+    },
+    {
+      "epoch": 0.15434083601286175,
+      "grad_norm": 365.1483154296875,
+      "learning_rate": 1.0000000000000001e-07,
+      "loss": 7.7333,
+      "num_input_tokens_seen": 160736,
+      "step": 12
+    },
+    {
+      "epoch": 0.16720257234726688,
+      "grad_norm": 353.99609375,
+      "learning_rate": 1.0833333333333335e-07,
+      "loss": 7.6017,
+      "num_input_tokens_seen": 173376,
+      "step": 13
+    },
+    {
+      "epoch": 0.18006430868167203,
+      "grad_norm": 357.7409362792969,
+      "learning_rate": 1.1666666666666668e-07,
+      "loss": 7.644,
+      "num_input_tokens_seen": 187136,
+      "step": 14
+    },
+    {
+      "epoch": 0.19292604501607716,
+      "grad_norm": 357.3723449707031,
+      "learning_rate": 1.2500000000000002e-07,
+      "loss": 7.5965,
+      "num_input_tokens_seen": 200192,
+      "step": 15
+    },
+    {
+      "epoch": 0.2057877813504823,
+      "grad_norm": 352.3059997558594,
+      "learning_rate": 1.3333333333333336e-07,
+      "loss": 7.5883,
+      "num_input_tokens_seen": 213856,
+      "step": 16
+    },
+    {
+      "epoch": 0.21864951768488747,
+      "grad_norm": 349.4331970214844,
+      "learning_rate": 1.4166666666666668e-07,
+      "loss": 7.2464,
+      "num_input_tokens_seen": 226944,
+      "step": 17
+    },
+    {
+      "epoch": 0.2315112540192926,
+      "grad_norm": 346.9617004394531,
+      "learning_rate": 1.5000000000000002e-07,
+      "loss": 7.3133,
+      "num_input_tokens_seen": 240640,
+      "step": 18
+    },
+    {
+      "epoch": 0.24437299035369775,
+      "grad_norm": 344.31512451171875,
+      "learning_rate": 1.5833333333333336e-07,
+      "loss": 7.2133,
+      "num_input_tokens_seen": 253280,
+      "step": 19
+    },
+    {
+      "epoch": 0.2572347266881029,
+      "grad_norm": 348.57623291015625,
+      "learning_rate": 1.6666666666666668e-07,
+      "loss": 7.2431,
+      "num_input_tokens_seen": 266400,
+      "step": 20
+    },
+    {
+      "epoch": 0.27009646302250806,
+      "grad_norm": 345.67694091796875,
+      "learning_rate": 1.7500000000000002e-07,
+      "loss": 7.1875,
+      "num_input_tokens_seen": 280416,
+      "step": 21
+    },
+    {
+      "epoch": 0.2829581993569132,
+      "grad_norm": 336.11968994140625,
+      "learning_rate": 1.8333333333333336e-07,
+      "loss": 7.0659,
+      "num_input_tokens_seen": 294368,
+      "step": 22
+    },
+    {
+      "epoch": 0.2958199356913183,
+      "grad_norm": 348.02783203125,
+      "learning_rate": 1.9166666666666668e-07,
+      "loss": 6.3595,
+      "num_input_tokens_seen": 307840,
+      "step": 23
+    },
+    {
+      "epoch": 0.3086816720257235,
+      "grad_norm": 306.9109802246094,
+      "learning_rate": 2.0000000000000002e-07,
+      "loss": 6.0417,
+      "num_input_tokens_seen": 322144,
+      "step": 24
+    },
+    {
+      "epoch": 0.3215434083601286,
+      "grad_norm": 308.7346496582031,
+      "learning_rate": 2.0833333333333333e-07,
+      "loss": 5.9894,
+      "num_input_tokens_seen": 335424,
+      "step": 25
+    },
+    {
+      "epoch": 0.33440514469453375,
+      "grad_norm": 303.3129577636719,
+      "learning_rate": 2.166666666666667e-07,
+      "loss": 5.9259,
+      "num_input_tokens_seen": 349376,
+      "step": 26
+    },
+    {
+      "epoch": 0.34726688102893893,
+      "grad_norm": 305.47637939453125,
+      "learning_rate": 2.2500000000000002e-07,
+      "loss": 5.8983,
+      "num_input_tokens_seen": 362944,
+      "step": 27
+    },
+    {
+      "epoch": 0.36012861736334406,
+      "grad_norm": 306.3539123535156,
+      "learning_rate": 2.3333333333333336e-07,
+      "loss": 5.6848,
+      "num_input_tokens_seen": 376352,
+      "step": 28
+    },
+    {
+      "epoch": 0.3729903536977492,
+      "grad_norm": 300.50201416015625,
+      "learning_rate": 2.416666666666667e-07,
+      "loss": 5.5649,
+      "num_input_tokens_seen": 389312,
+      "step": 29
+    },
+    {
+      "epoch": 0.3858520900321543,
+      "grad_norm": 297.82757568359375,
+      "learning_rate": 2.5000000000000004e-07,
+      "loss": 5.4642,
+      "num_input_tokens_seen": 402848,
+      "step": 30
+    },
+    {
+      "epoch": 0.3987138263665595,
+      "grad_norm": 348.0853271484375,
+      "learning_rate": 2.5833333333333333e-07,
+      "loss": 4.7955,
+      "num_input_tokens_seen": 416352,
+      "step": 31
+    },
+    {
+      "epoch": 0.4115755627009646,
+      "grad_norm": 346.5870361328125,
+      "learning_rate": 2.666666666666667e-07,
+      "loss": 2.8339,
+      "num_input_tokens_seen": 429920,
+      "step": 32
+    },
+    {
+      "epoch": 0.42443729903536975,
+      "grad_norm": 327.6314697265625,
+      "learning_rate": 2.75e-07,
+      "loss": 2.4477,
+      "num_input_tokens_seen": 444064,
+      "step": 33
+    },
+    {
+      "epoch": 0.43729903536977494,
+      "grad_norm": 321.9542236328125,
+      "learning_rate": 2.8333333333333336e-07,
+      "loss": 2.3331,
+      "num_input_tokens_seen": 458304,
+      "step": 34
+    },
+    {
+      "epoch": 0.45016077170418006,
+      "grad_norm": 327.8713073730469,
+      "learning_rate": 2.916666666666667e-07,
+      "loss": 2.2143,
+      "num_input_tokens_seen": 471904,
+      "step": 35
+    },
+    {
+      "epoch": 0.4630225080385852,
+      "grad_norm": 317.32427978515625,
+      "learning_rate": 3.0000000000000004e-07,
+      "loss": 2.0067,
+      "num_input_tokens_seen": 484768,
+      "step": 36
+    },
+    {
+      "epoch": 0.4758842443729904,
+      "grad_norm": 251.88111877441406,
+      "learning_rate": 3.083333333333334e-07,
+      "loss": 1.7702,
+      "num_input_tokens_seen": 497696,
+      "step": 37
+    },
+    {
+      "epoch": 0.4887459807073955,
+      "grad_norm": 273.716064453125,
+      "learning_rate": 3.166666666666667e-07,
+      "loss": 1.5557,
+      "num_input_tokens_seen": 511072,
+      "step": 38
+    },
+    {
+      "epoch": 0.5016077170418006,
+      "grad_norm": 293.1105651855469,
+      "learning_rate": 3.25e-07,
+      "loss": 1.3024,
+      "num_input_tokens_seen": 524576,
+      "step": 39
+    },
+    {
+      "epoch": 0.5144694533762058,
+      "grad_norm": 243.40711975097656,
+      "learning_rate": 3.3333333333333335e-07,
+      "loss": 1.1652,
+      "num_input_tokens_seen": 538112,
+      "step": 40
+    },
+    {
+      "epoch": 0.5273311897106109,
+      "grad_norm": 111.22821807861328,
+      "learning_rate": 3.416666666666667e-07,
+      "loss": 0.6839,
+      "num_input_tokens_seen": 552096,
+      "step": 41
+    },
+    {
+      "epoch": 0.5401929260450161,
+      "grad_norm": 104.02100372314453,
+      "learning_rate": 3.5000000000000004e-07,
+      "loss": 0.4774,
+      "num_input_tokens_seen": 565920,
+      "step": 42
+    },
+    {
+      "epoch": 0.5530546623794212,
+      "grad_norm": 32.459312438964844,
+      "learning_rate": 3.583333333333334e-07,
+      "loss": 0.3841,
+      "num_input_tokens_seen": 579200,
+      "step": 43
+    },
+    {
+      "epoch": 0.5659163987138264,
+      "grad_norm": 15.353242874145508,
+      "learning_rate": 3.666666666666667e-07,
+      "loss": 0.3588,
+      "num_input_tokens_seen": 592864,
+      "step": 44
+    },
+    {
+      "epoch": 0.5787781350482315,
+      "grad_norm": 48.64632034301758,
+      "learning_rate": 3.75e-07,
+      "loss": 0.3628,
+      "num_input_tokens_seen": 606656,
+      "step": 45
+    },
+    {
+      "epoch": 0.5916398713826366,
+      "grad_norm": 34.02824020385742,
+      "learning_rate": 3.8333333333333335e-07,
+      "loss": 0.3426,
+      "num_input_tokens_seen": 619968,
+      "step": 46
+    },
+    {
+      "epoch": 0.6045016077170418,
+      "grad_norm": 15.54571533203125,
+      "learning_rate": 3.9166666666666675e-07,
+      "loss": 0.3279,
+      "num_input_tokens_seen": 633632,
+      "step": 47
+    },
+    {
+      "epoch": 0.617363344051447,
+      "grad_norm": 101.66536712646484,
+      "learning_rate": 4.0000000000000003e-07,
+      "loss": 0.3947,
+      "num_input_tokens_seen": 647360,
+      "step": 48
+    },
+    {
+      "epoch": 0.6302250803858521,
+      "grad_norm": 39.69755935668945,
+      "learning_rate": 4.083333333333334e-07,
+      "loss": 0.3075,
+      "num_input_tokens_seen": 661504,
+      "step": 49
+    },
+    {
+      "epoch": 0.6430868167202572,
+      "grad_norm": 23.356718063354492,
+      "learning_rate": 4.1666666666666667e-07,
+      "loss": 0.3236,
+      "num_input_tokens_seen": 675424,
+      "step": 50
+    },
+    {
+      "epoch": 0.6559485530546624,
+      "grad_norm": 74.00253295898438,
+      "learning_rate": 4.2500000000000006e-07,
+      "loss": 0.3557,
+      "num_input_tokens_seen": 688800,
+      "step": 51
+    },
+    {
+      "epoch": 0.6688102893890675,
+      "grad_norm": 109.3729248046875,
+      "learning_rate": 4.333333333333334e-07,
+      "loss": 0.4008,
+      "num_input_tokens_seen": 702208,
+      "step": 52
+    },
+    {
+      "epoch": 0.6816720257234726,
+      "grad_norm": 81.13125610351562,
+      "learning_rate": 4.416666666666667e-07,
+      "loss": 0.3586,
+      "num_input_tokens_seen": 716096,
+      "step": 53
+    },
+    {
+      "epoch": 0.6945337620578779,
+      "grad_norm": 24.126953125,
+      "learning_rate": 4.5000000000000003e-07,
+      "loss": 0.3023,
+      "num_input_tokens_seen": 728672,
+      "step": 54
+    },
+    {
+      "epoch": 0.707395498392283,
+      "grad_norm": 77.53913879394531,
+      "learning_rate": 4.583333333333333e-07,
+      "loss": 0.3547,
+      "num_input_tokens_seen": 742912,
+      "step": 55
+    },
+    {
+      "epoch": 0.7202572347266881,
+      "grad_norm": 95.46034240722656,
+      "learning_rate": 4.666666666666667e-07,
+      "loss": 0.3846,
+      "num_input_tokens_seen": 755808,
+      "step": 56
+    },
+    {
+      "epoch": 0.7331189710610932,
+      "grad_norm": 86.11170959472656,
+      "learning_rate": 4.7500000000000006e-07,
+      "loss": 0.3743,
+      "num_input_tokens_seen": 769696,
+      "step": 57
+    },
+    {
+      "epoch": 0.7459807073954984,
+      "grad_norm": 31.460582733154297,
+      "learning_rate": 4.833333333333334e-07,
+      "loss": 0.3091,
+      "num_input_tokens_seen": 783680,
+      "step": 58
+    },
+    {
+      "epoch": 0.7588424437299035,
+      "grad_norm": 34.459659576416016,
+      "learning_rate": 4.916666666666667e-07,
+      "loss": 0.3094,
+      "num_input_tokens_seen": 796672,
+      "step": 59
+    },
+    {
+      "epoch": 0.7717041800643086,
+      "grad_norm": 51.55731201171875,
+      "learning_rate": 5.000000000000001e-07,
+      "loss": 0.3309,
+      "num_input_tokens_seen": 810304,
+      "step": 60
+    },
+    {
+      "epoch": 0.7845659163987139,
+      "grad_norm": 35.633277893066406,
+      "learning_rate": 5.083333333333334e-07,
+      "loss": 0.3276,
+      "num_input_tokens_seen": 824160,
+      "step": 61
+    },
+    {
+      "epoch": 0.797427652733119,
+      "grad_norm": 9.230757713317871,
+      "learning_rate": 5.166666666666667e-07,
+      "loss": 0.3084,
+      "num_input_tokens_seen": 837312,
+      "step": 62
+    },
+    {
+      "epoch": 0.8102893890675241,
+      "grad_norm": 35.229454040527344,
+      "learning_rate": 5.250000000000001e-07,
+      "loss": 0.3182,
+      "num_input_tokens_seen": 851168,
+      "step": 63
+    },
+    {
+      "epoch": 0.8231511254019293,
+      "grad_norm": 47.88737487792969,
+      "learning_rate": 5.333333333333335e-07,
+      "loss": 0.3469,
+      "num_input_tokens_seen": 864096,
+      "step": 64
+    },
+    {
+      "epoch": 0.8360128617363344,
+      "grad_norm": 38.481746673583984,
+      "learning_rate": 5.416666666666667e-07,
+      "loss": 0.3253,
+      "num_input_tokens_seen": 878048,
+      "step": 65
+    },
+    {
+      "epoch": 0.8488745980707395,
+      "grad_norm": 11.656455993652344,
+      "learning_rate": 5.5e-07,
+      "loss": 0.2746,
+      "num_input_tokens_seen": 891904,
+      "step": 66
+    },
+    {
+      "epoch": 0.8617363344051447,
+      "grad_norm": 18.758018493652344,
+      "learning_rate": 5.583333333333333e-07,
+      "loss": 0.2893,
+      "num_input_tokens_seen": 906016,
+      "step": 67
+    },
+    {
+      "epoch": 0.8745980707395499,
+      "grad_norm": 19.316091537475586,
+      "learning_rate": 5.666666666666667e-07,
+      "loss": 0.2827,
+      "num_input_tokens_seen": 918944,
+      "step": 68
+    },
+    {
+      "epoch": 0.887459807073955,
+      "grad_norm": 32.92877960205078,
+      "learning_rate": 5.750000000000001e-07,
+      "loss": 0.2978,
+      "num_input_tokens_seen": 932672,
+      "step": 69
+    },
+    {
+      "epoch": 0.9003215434083601,
+      "grad_norm": 14.833833694458008,
+      "learning_rate": 5.833333333333334e-07,
+      "loss": 0.2703,
+      "num_input_tokens_seen": 945248,
+      "step": 70
+    },
+    {
+      "epoch": 0.9131832797427653,
+      "grad_norm": 23.44466209411621,
+      "learning_rate": 5.916666666666667e-07,
+      "loss": 0.2968,
+      "num_input_tokens_seen": 958784,
+      "step": 71
+    },
+    {
+      "epoch": 0.9260450160771704,
+      "grad_norm": 56.151981353759766,
+      "learning_rate": 6.000000000000001e-07,
+      "loss": 0.3035,
+      "num_input_tokens_seen": 972416,
+      "step": 72
+    },
+    {
+      "epoch": 0.9389067524115756,
+      "grad_norm": 77.62692260742188,
+      "learning_rate": 6.083333333333334e-07,
+      "loss": 0.3211,
+      "num_input_tokens_seen": 985888,
+      "step": 73
+    },
+    {
+      "epoch": 0.9517684887459807,
+      "grad_norm": 65.45875549316406,
+      "learning_rate": 6.166666666666668e-07,
+      "loss": 0.2913,
+      "num_input_tokens_seen": 999296,
+      "step": 74
+    },
+    {
+      "epoch": 0.9646302250803859,
+      "grad_norm": 16.159313201904297,
+      "learning_rate": 6.25e-07,
+      "loss": 0.2817,
+      "num_input_tokens_seen": 1012640,
+      "step": 75
+    },
+    {
+      "epoch": 0.977491961414791,
+      "grad_norm": 33.846290588378906,
+      "learning_rate": 6.333333333333334e-07,
+      "loss": 0.2827,
+      "num_input_tokens_seen": 1026112,
+      "step": 76
+    },
+    {
+      "epoch": 0.9903536977491961,
+      "grad_norm": 9.542298316955566,
+      "learning_rate": 6.416666666666667e-07,
+      "loss": 0.229,
+      "num_input_tokens_seen": 1039424,
+      "step": 77
+    },
+    {
+      "epoch": 1.0032154340836013,
+      "grad_norm": 10.284690856933594,
+      "learning_rate": 6.5e-07,
+      "loss": 0.2503,
+      "num_input_tokens_seen": 1053408,
+      "step": 78
+    },
+    {
+      "epoch": 1.0160771704180065,
+      "grad_norm": 19.490089416503906,
+      "learning_rate": 6.583333333333333e-07,
+      "loss": 0.2453,
+      "num_input_tokens_seen": 1067328,
+      "step": 79
+    },
+    {
+      "epoch": 1.0289389067524115,
+      "grad_norm": 18.234272003173828,
+      "learning_rate": 6.666666666666667e-07,
+      "loss": 0.2167,
+      "num_input_tokens_seen": 1080288,
+      "step": 80
+    },
+    {
+      "epoch": 1.0418006430868167,
+      "grad_norm": 9.240696907043457,
+      "learning_rate": 6.750000000000001e-07,
+      "loss": 0.2361,
+      "num_input_tokens_seen": 1093952,
+      "step": 81
+    },
+    {
+      "epoch": 1.0546623794212218,
+      "grad_norm": 22.255823135375977,
+      "learning_rate": 6.833333333333334e-07,
+      "loss": 0.2248,
+      "num_input_tokens_seen": 1107904,
+      "step": 82
+    },
+    {
+      "epoch": 1.067524115755627,
+      "grad_norm": 26.029281616210938,
+      "learning_rate": 6.916666666666668e-07,
+      "loss": 0.2491,
+      "num_input_tokens_seen": 1121120,
+      "step": 83
+    },
+    {
+      "epoch": 1.0803858520900322,
+      "grad_norm": 34.11492156982422,
+      "learning_rate": 7.000000000000001e-07,
+      "loss": 0.2352,
+      "num_input_tokens_seen": 1135040,
+      "step": 84
+    },
+    {
+      "epoch": 1.0932475884244373,
+      "grad_norm": 65.68599700927734,
+      "learning_rate": 7.083333333333334e-07,
+      "loss": 0.2365,
+      "num_input_tokens_seen": 1148992,
+      "step": 85
+    },
+    {
+      "epoch": 1.1061093247588425,
+      "grad_norm": 72.81002807617188,
+      "learning_rate": 7.166666666666668e-07,
+      "loss": 0.217,
+      "num_input_tokens_seen": 1162592,
+      "step": 86
+    },
+    {
+      "epoch": 1.1189710610932475,
+      "grad_norm": 21.043798446655273,
+      "learning_rate": 7.25e-07,
+      "loss": 0.2258,
+      "num_input_tokens_seen": 1175104,
+      "step": 87
+    },
+    {
+      "epoch": 1.1318327974276527,
+      "grad_norm": 71.2937240600586,
+      "learning_rate": 7.333333333333334e-07,
+      "loss": 0.245,
+      "num_input_tokens_seen": 1189376,
+      "step": 88
+    },
+    {
+      "epoch": 1.144694533762058,
+      "grad_norm": 106.12057495117188,
+      "learning_rate": 7.416666666666668e-07,
+      "loss": 0.3132,
+      "num_input_tokens_seen": 1202560,
+      "step": 89
+    },
+    {
+      "epoch": 1.157556270096463,
+      "grad_norm": 84.74403381347656,
+      "learning_rate": 7.5e-07,
+      "loss": 0.284,
+      "num_input_tokens_seen": 1216832,
+      "step": 90
+    },
+    {
+      "epoch": 1.1704180064308682,
+      "grad_norm": 10.08238410949707,
+      "learning_rate": 7.583333333333334e-07,
+      "loss": 0.1933,
+      "num_input_tokens_seen": 1230528,
+      "step": 91
+    },
+    {
+      "epoch": 1.1832797427652733,
+      "grad_norm": 56.27880859375,
+      "learning_rate": 7.666666666666667e-07,
+      "loss": 0.2154,
+      "num_input_tokens_seen": 1244352,
+      "step": 92
+    },
+    {
+      "epoch": 1.1961414790996785,
+      "grad_norm": 39.58720779418945,
+      "learning_rate": 7.750000000000001e-07,
+      "loss": 0.2064,
+      "num_input_tokens_seen": 1257472,
+      "step": 93
+    },
+    {
+      "epoch": 1.2090032154340835,
+      "grad_norm": 12.087499618530273,
+      "learning_rate": 7.833333333333335e-07,
+      "loss": 0.2038,
+      "num_input_tokens_seen": 1271392,
+      "step": 94
+    },
+    {
+      "epoch": 1.2218649517684887,
+      "grad_norm": 29.51178550720215,
+      "learning_rate": 7.916666666666667e-07,
+      "loss": 0.2152,
+      "num_input_tokens_seen": 1286432,
+      "step": 95
+    },
+    {
+      "epoch": 1.234726688102894,
+      "grad_norm": 12.675917625427246,
+      "learning_rate": 8.000000000000001e-07,
+      "loss": 0.1961,
+      "num_input_tokens_seen": 1300096,
+      "step": 96
+    },
+    {
+      "epoch": 1.247588424437299,
+      "grad_norm": 11.436767578125,
+      "learning_rate": 8.083333333333334e-07,
+      "loss": 0.1772,
+      "num_input_tokens_seen": 1313568,
+      "step": 97
+    },
+    {
+      "epoch": 1.2604501607717042,
+      "grad_norm": 25.415918350219727,
+      "learning_rate": 8.166666666666668e-07,
+      "loss": 0.1846,
+      "num_input_tokens_seen": 1327328,
+      "step": 98
+    },
+    {
+      "epoch": 1.2733118971061093,
+      "grad_norm": 11.161438941955566,
+      "learning_rate": 8.250000000000001e-07,
+      "loss": 0.1823,
+      "num_input_tokens_seen": 1340960,
+      "step": 99
+    },
+    {
+      "epoch": 1.2861736334405145,
+      "grad_norm": 7.966615200042725,
+      "learning_rate": 8.333333333333333e-07,
+      "loss": 0.1794,
+      "num_input_tokens_seen": 1353440,
+      "step": 100
+    },
+    {
+      "epoch": 1.2990353697749195,
+      "grad_norm": 32.296085357666016,
+      "learning_rate": 8.416666666666667e-07,
+      "loss": 0.2106,
+      "num_input_tokens_seen": 1367680,
+      "step": 101
+    },
+    {
+      "epoch": 1.3118971061093248,
+      "grad_norm": 26.521038055419922,
+      "learning_rate": 8.500000000000001e-07,
+      "loss": 0.2123,
+      "num_input_tokens_seen": 1380864,
+      "step": 102
+    },
+    {
+      "epoch": 1.32475884244373,
+      "grad_norm": 63.48737716674805,
+      "learning_rate": 8.583333333333334e-07,
+      "loss": 0.2413,
+      "num_input_tokens_seen": 1393888,
+      "step": 103
+    },
+    {
+      "epoch": 1.337620578778135,
+      "grad_norm": 55.781776428222656,
+      "learning_rate": 8.666666666666668e-07,
+      "loss": 0.2334,
+      "num_input_tokens_seen": 1406912,
+      "step": 104
+    },
+    {
+      "epoch": 1.3504823151125402,
+      "grad_norm": 13.036691665649414,
+      "learning_rate": 8.75e-07,
+      "loss": 0.2069,
+      "num_input_tokens_seen": 1420000,
+      "step": 105
+    },
+    {
+      "epoch": 1.3633440514469453,
+      "grad_norm": 45.98344421386719,
+      "learning_rate": 8.833333333333334e-07,
+      "loss": 0.2262,
+      "num_input_tokens_seen": 1433440,
+      "step": 106
+    },
+    {
+      "epoch": 1.3762057877813505,
+      "grad_norm": 34.49677276611328,
+      "learning_rate": 8.916666666666668e-07,
+      "loss": 0.1718,
+      "num_input_tokens_seen": 1446560,
+      "step": 107
+    },
+    {
+      "epoch": 1.3890675241157555,
+      "grad_norm": 21.14679718017578,
+      "learning_rate": 9.000000000000001e-07,
+      "loss": 0.204,
+      "num_input_tokens_seen": 1460320,
+      "step": 108
+    },
+    {
+      "epoch": 1.4019292604501608,
+      "grad_norm": 26.53287124633789,
+      "learning_rate": 9.083333333333335e-07,
+      "loss": 0.1849,
+      "num_input_tokens_seen": 1474272,
+      "step": 109
+    },
+    {
+      "epoch": 1.414790996784566,
+      "grad_norm": 34.011962890625,
+      "learning_rate": 9.166666666666666e-07,
+      "loss": 0.2028,
+      "num_input_tokens_seen": 1487904,
+      "step": 110
+    },
+    {
+      "epoch": 1.427652733118971,
+      "grad_norm": 17.4981689453125,
+      "learning_rate": 9.25e-07,
+      "loss": 0.179,
+      "num_input_tokens_seen": 1501664,
+      "step": 111
+    },
+    {
+      "epoch": 1.4405144694533762,
+      "grad_norm": 20.183582305908203,
+      "learning_rate": 9.333333333333334e-07,
+      "loss": 0.1813,
+      "num_input_tokens_seen": 1515264,
+      "step": 112
+    },
+    {
+      "epoch": 1.4533762057877815,
+      "grad_norm": 26.655994415283203,
+      "learning_rate": 9.416666666666667e-07,
+      "loss": 0.1955,
+      "num_input_tokens_seen": 1528640,
+      "step": 113
+    },
+    {
+      "epoch": 1.4662379421221865,
+      "grad_norm": 25.745370864868164,
+      "learning_rate": 9.500000000000001e-07,
+      "loss": 0.1577,
+      "num_input_tokens_seen": 1541632,
+      "step": 114
+    },
+    {
+      "epoch": 1.4790996784565915,
+      "grad_norm": 22.924352645874023,
+      "learning_rate": 9.583333333333334e-07,
+      "loss": 0.1509,
+      "num_input_tokens_seen": 1554592,
+      "step": 115
+    },
+    {
+      "epoch": 1.4919614147909968,
+      "grad_norm": 26.520856857299805,
+      "learning_rate": 9.666666666666668e-07,
+      "loss": 0.2052,
+      "num_input_tokens_seen": 1567296,
+      "step": 116
+    },
+    {
+      "epoch": 1.504823151125402,
+      "grad_norm": 23.989120483398438,
+      "learning_rate": 9.750000000000002e-07,
+      "loss": 0.1576,
+      "num_input_tokens_seen": 1580800,
+      "step": 117
+    },
+    {
+      "epoch": 1.517684887459807,
+      "grad_norm": 11.067609786987305,
+      "learning_rate": 9.833333333333334e-07,
+      "loss": 0.1459,
+      "num_input_tokens_seen": 1593792,
+      "step": 118
+    },
+    {
+      "epoch": 1.5305466237942122,
+      "grad_norm": 41.14239501953125,
+      "learning_rate": 9.916666666666668e-07,
+      "loss": 0.2694,
+      "num_input_tokens_seen": 1607648,
+      "step": 119
+    },
+    {
+      "epoch": 1.5434083601286175,
+      "grad_norm": 17.72841453552246,
+      "learning_rate": 1.0000000000000002e-06,
+      "loss": 0.1891,
+      "num_input_tokens_seen": 1621184,
+      "step": 120
+    },
+    {
+      "epoch": 1.5562700964630225,
+      "grad_norm": 13.838713645935059,
+      "learning_rate": 1.0083333333333333e-06,
+      "loss": 0.1655,
+      "num_input_tokens_seen": 1634240,
+      "step": 121
+    },
+    {
+      "epoch": 1.5691318327974275,
+      "grad_norm": 17.73114013671875,
+      "learning_rate": 1.0166666666666667e-06,
+      "loss": 0.1534,
+      "num_input_tokens_seen": 1647712,
+      "step": 122
+    },
+    {
+      "epoch": 1.5819935691318328,
+      "grad_norm": 7.044174671173096,
+      "learning_rate": 1.025e-06,
+      "loss": 0.1373,
+      "num_input_tokens_seen": 1661344,
+      "step": 123
+    },
+    {
+      "epoch": 1.594855305466238,
+      "grad_norm": 13.696454048156738,
+      "learning_rate": 1.0333333333333333e-06,
+      "loss": 0.1528,
+      "num_input_tokens_seen": 1674880,
+      "step": 124
+    },
+    {
+      "epoch": 1.607717041800643,
+      "grad_norm": 11.191144943237305,
+      "learning_rate": 1.0416666666666667e-06,
+      "loss": 0.2017,
+      "num_input_tokens_seen": 1688992,
+      "step": 125
+    },
+    {
+      "epoch": 1.6205787781350482,
+      "grad_norm": 10.479023933410645,
+      "learning_rate": 1.0500000000000001e-06,
+      "loss": 0.1554,
+      "num_input_tokens_seen": 1702944,
+      "step": 126
+    },
+    {
+      "epoch": 1.6334405144694535,
+      "grad_norm": 12.889703750610352,
+      "learning_rate": 1.0583333333333335e-06,
+      "loss": 0.1332,
+      "num_input_tokens_seen": 1716768,
+      "step": 127
+    },
+    {
+      "epoch": 1.6463022508038585,
+      "grad_norm": 8.173032760620117,
+      "learning_rate": 1.066666666666667e-06,
+      "loss": 0.115,
+      "num_input_tokens_seen": 1730848,
+      "step": 128
+    },
+    {
+      "epoch": 1.6591639871382635,
+      "grad_norm": 14.694462776184082,
+      "learning_rate": 1.075e-06,
+      "loss": 0.119,
+      "num_input_tokens_seen": 1743520,
+      "step": 129
+    },
+    {
+      "epoch": 1.6720257234726688,
+      "grad_norm": 8.472253799438477,
+      "learning_rate": 1.0833333333333335e-06,
+      "loss": 0.1164,
+      "num_input_tokens_seen": 1757696,
+      "step": 130
+    },
+    {
+      "epoch": 1.684887459807074,
+      "grad_norm": 20.757648468017578,
+      "learning_rate": 1.0916666666666667e-06,
+      "loss": 0.1981,
+      "num_input_tokens_seen": 1771264,
+      "step": 131
+    },
+    {
+      "epoch": 1.697749196141479,
+      "grad_norm": 20.149930953979492,
+      "learning_rate": 1.1e-06,
+      "loss": 0.168,
+      "num_input_tokens_seen": 1784480,
+      "step": 132
+    },
+    {
+      "epoch": 1.7106109324758842,
+      "grad_norm": 9.992751121520996,
+      "learning_rate": 1.1083333333333335e-06,
+      "loss": 0.0741,
+      "num_input_tokens_seen": 1798176,
+      "step": 133
+    },
+    {
+      "epoch": 1.7234726688102895,
+      "grad_norm": 26.80909538269043,
+      "learning_rate": 1.1166666666666666e-06,
+      "loss": 0.1847,
+      "num_input_tokens_seen": 1811264,
+      "step": 134
+    },
+    {
+      "epoch": 1.7363344051446945,
+      "grad_norm": 16.494613647460938,
+      "learning_rate": 1.125e-06,
+      "loss": 0.108,
+      "num_input_tokens_seen": 1825024,
+      "step": 135
+    },
+    {
+      "epoch": 1.7491961414790995,
+      "grad_norm": 18.742122650146484,
+      "learning_rate": 1.1333333333333334e-06,
+      "loss": 0.1214,
+      "num_input_tokens_seen": 1838624,
+      "step": 136
+    },
+    {
+      "epoch": 1.762057877813505,
+      "grad_norm": 10.102185249328613,
+      "learning_rate": 1.1416666666666668e-06,
+      "loss": 0.1252,
+      "num_input_tokens_seen": 1852288,
+      "step": 137
+    },
+    {
+      "epoch": 1.77491961414791,
+      "grad_norm": 23.612730026245117,
+      "learning_rate": 1.1500000000000002e-06,
+      "loss": 0.144,
+      "num_input_tokens_seen": 1865376,
+      "step": 138
+    },
+    {
+      "epoch": 1.787781350482315,
+      "grad_norm": 11.11936092376709,
+      "learning_rate": 1.1583333333333334e-06,
+      "loss": 0.1269,
+      "num_input_tokens_seen": 1878784,
+      "step": 139
+    },
+    {
+      "epoch": 1.8006430868167203,
+      "grad_norm": 7.341479778289795,
+      "learning_rate": 1.1666666666666668e-06,
+      "loss": 0.1283,
+      "num_input_tokens_seen": 1892320,
+      "step": 140
+    },
+    {
+      "epoch": 1.8135048231511255,
+      "grad_norm": 16.097875595092773,
+      "learning_rate": 1.175e-06,
+      "loss": 0.0929,
+      "num_input_tokens_seen": 1905728,
+      "step": 141
+    },
+    {
+      "epoch": 1.8263665594855305,
+      "grad_norm": 18.07818603515625,
+      "learning_rate": 1.1833333333333334e-06,
+      "loss": 0.1349,
+      "num_input_tokens_seen": 1918752,
+      "step": 142
+    },
+    {
+      "epoch": 1.8392282958199357,
+      "grad_norm": 18.349048614501953,
+      "learning_rate": 1.1916666666666668e-06,
+      "loss": 0.1277,
+      "num_input_tokens_seen": 1931808,
+      "step": 143
+    },
+    {
+      "epoch": 1.852090032154341,
+      "grad_norm": 12.308938980102539,
+      "learning_rate": 1.2000000000000002e-06,
+      "loss": 0.1585,
+      "num_input_tokens_seen": 1945312,
+      "step": 144
+    },
+    {
+      "epoch": 1.864951768488746,
+      "grad_norm": 19.95519256591797,
+      "learning_rate": 1.2083333333333333e-06,
+      "loss": 0.1468,
+      "num_input_tokens_seen": 1958016,
+      "step": 145
+    },
+    {
+      "epoch": 1.877813504823151,
+      "grad_norm": 11.419037818908691,
+      "learning_rate": 1.2166666666666667e-06,
+      "loss": 0.1049,
+      "num_input_tokens_seen": 1971392,
+      "step": 146
+    },
+    {
+      "epoch": 1.8906752411575563,
+      "grad_norm": 9.492958068847656,
+      "learning_rate": 1.2250000000000001e-06,
+      "loss": 0.1297,
+      "num_input_tokens_seen": 1984992,
+      "step": 147
+    },
+    {
+      "epoch": 1.9035369774919615,
+      "grad_norm": 6.134005546569824,
+      "learning_rate": 1.2333333333333335e-06,
+      "loss": 0.1111,
+      "num_input_tokens_seen": 1998144,
+      "step": 148
+    },
+    {
+      "epoch": 1.9163987138263665,
+      "grad_norm": 10.250520706176758,
+      "learning_rate": 1.2416666666666667e-06,
+      "loss": 0.1202,
+      "num_input_tokens_seen": 2012032,
+      "step": 149
+    },
+    {
+      "epoch": 1.9292604501607717,
+      "grad_norm": 10.018802642822266,
+      "learning_rate": 1.25e-06,
+      "loss": 0.0829,
+      "num_input_tokens_seen": 2025600,
+      "step": 150
+    },
+    {
+      "epoch": 1.942122186495177,
+      "grad_norm": 17.175769805908203,
+      "learning_rate": 1.2583333333333333e-06,
+      "loss": 0.1119,
+      "num_input_tokens_seen": 2040096,
+      "step": 151
+    },
+    {
+      "epoch": 1.954983922829582,
+      "grad_norm": 18.159189224243164,
+      "learning_rate": 1.2666666666666669e-06,
+      "loss": 0.1144,
+      "num_input_tokens_seen": 2053504,
+      "step": 152
+    },
+    {
+      "epoch": 1.967845659163987,
+      "grad_norm": 14.536725044250488,
+      "learning_rate": 1.275e-06,
+      "loss": 0.117,
+      "num_input_tokens_seen": 2065856,
+      "step": 153
+    },
+    {
+      "epoch": 1.9807073954983923,
+      "grad_norm": 10.24523639678955,
+      "learning_rate": 1.2833333333333335e-06,
+      "loss": 0.0998,
+      "num_input_tokens_seen": 2078656,
+      "step": 154
+    },
+    {
+      "epoch": 1.9935691318327975,
+      "grad_norm": 20.030460357666016,
+      "learning_rate": 1.2916666666666669e-06,
+      "loss": 0.1384,
+      "num_input_tokens_seen": 2092672,
+      "step": 155
+    },
+    {
+      "epoch": 2.0064308681672025,
+      "grad_norm": 12.654150009155273,
+      "learning_rate": 1.3e-06,
+      "loss": 0.1157,
+      "num_input_tokens_seen": 2105728,
+      "step": 156
+    },
+    {
+      "epoch": 2.0192926045016075,
+      "grad_norm": 5.51693058013916,
+      "learning_rate": 1.3083333333333334e-06,
+      "loss": 0.0696,
+      "num_input_tokens_seen": 2118976,
+      "step": 157
+    },
+    {
+      "epoch": 2.032154340836013,
+      "grad_norm": 9.432037353515625,
+      "learning_rate": 1.3166666666666666e-06,
+      "loss": 0.0665,
+      "num_input_tokens_seen": 2132352,
+      "step": 158
+    },
+    {
+      "epoch": 2.045016077170418,
+      "grad_norm": 15.199766159057617,
+      "learning_rate": 1.3250000000000002e-06,
+      "loss": 0.0783,
+      "num_input_tokens_seen": 2146048,
+      "step": 159
+    },
+    {
+      "epoch": 2.057877813504823,
+      "grad_norm": 10.740960121154785,
+      "learning_rate": 1.3333333333333334e-06,
+      "loss": 0.0749,
+      "num_input_tokens_seen": 2158752,
+      "step": 160
+    },
+    {
+      "epoch": 2.0707395498392285,
+      "grad_norm": 7.335105895996094,
+      "learning_rate": 1.3416666666666666e-06,
+      "loss": 0.0731,
+      "num_input_tokens_seen": 2171872,
+      "step": 161
+    },
+    {
+      "epoch": 2.0836012861736335,
+      "grad_norm": 14.225622177124023,
+      "learning_rate": 1.3500000000000002e-06,
+      "loss": 0.0913,
+      "num_input_tokens_seen": 2184800,
+      "step": 162
+    },
+    {
+      "epoch": 2.0964630225080385,
+      "grad_norm": 12.737798690795898,
+      "learning_rate": 1.3583333333333334e-06,
+      "loss": 0.0521,
+      "num_input_tokens_seen": 2198176,
+      "step": 163
+    },
+    {
+      "epoch": 2.1093247588424435,
+      "grad_norm": 10.609269142150879,
+      "learning_rate": 1.3666666666666668e-06,
+      "loss": 0.068,
+      "num_input_tokens_seen": 2211392,
+      "step": 164
+    },
+    {
+      "epoch": 2.122186495176849,
+      "grad_norm": 8.30199146270752,
+      "learning_rate": 1.3750000000000002e-06,
+      "loss": 0.0686,
+      "num_input_tokens_seen": 2224480,
+      "step": 165
+    },
+    {
+      "epoch": 2.135048231511254,
+      "grad_norm": 6.069125175476074,
+      "learning_rate": 1.3833333333333336e-06,
+      "loss": 0.0545,
+      "num_input_tokens_seen": 2237920,
+      "step": 166
+    },
+    {
+      "epoch": 2.147909967845659,
+      "grad_norm": 4.639097213745117,
+      "learning_rate": 1.3916666666666668e-06,
+      "loss": 0.0347,
+      "num_input_tokens_seen": 2251008,
+      "step": 167
+    },
+    {
+      "epoch": 2.1607717041800645,
+      "grad_norm": 9.093743324279785,
+      "learning_rate": 1.4000000000000001e-06,
+      "loss": 0.0993,
+      "num_input_tokens_seen": 2264448,
+      "step": 168
+    },
+    {
+      "epoch": 2.1736334405144695,
+      "grad_norm": 8.8851900100708,
+      "learning_rate": 1.4083333333333335e-06,
+      "loss": 0.1059,
+      "num_input_tokens_seen": 2277568,
+      "step": 169
+    },
+    {
+      "epoch": 2.1864951768488745,
+      "grad_norm": 10.823246955871582,
+      "learning_rate": 1.4166666666666667e-06,
+      "loss": 0.089,
+      "num_input_tokens_seen": 2291168,
+      "step": 170
+    },
+    {
+      "epoch": 2.19935691318328,
+      "grad_norm": 5.454392910003662,
+      "learning_rate": 1.425e-06,
+      "loss": 0.0379,
+      "num_input_tokens_seen": 2304192,
+      "step": 171
+    },
+    {
+      "epoch": 2.212218649517685,
+      "grad_norm": 6.2786030769348145,
+      "learning_rate": 1.4333333333333335e-06,
+      "loss": 0.0626,
+      "num_input_tokens_seen": 2317920,
+      "step": 172
+    },
+    {
+      "epoch": 2.22508038585209,
+      "grad_norm": 8.764283180236816,
+      "learning_rate": 1.4416666666666667e-06,
+      "loss": 0.0957,
+      "num_input_tokens_seen": 2332416,
+      "step": 173
+    },
+    {
+      "epoch": 2.237942122186495,
+      "grad_norm": 10.533244132995605,
+      "learning_rate": 1.45e-06,
+      "loss": 0.0636,
+      "num_input_tokens_seen": 2346112,
+      "step": 174
+    },
+    {
+      "epoch": 2.2508038585209005,
+      "grad_norm": 7.398101329803467,
+      "learning_rate": 1.4583333333333335e-06,
+      "loss": 0.074,
+      "num_input_tokens_seen": 2359488,
+      "step": 175
+    },
+    {
+      "epoch": 2.2636655948553055,
+      "grad_norm": 6.4459686279296875,
+      "learning_rate": 1.4666666666666669e-06,
+      "loss": 0.0685,
+      "num_input_tokens_seen": 2372928,
+      "step": 176
+    },
+    {
+      "epoch": 2.2765273311897105,
+      "grad_norm": 5.918321132659912,
+      "learning_rate": 1.475e-06,
+      "loss": 0.0574,
+      "num_input_tokens_seen": 2386016,
+      "step": 177
+    },
+    {
+      "epoch": 2.289389067524116,
+      "grad_norm": 7.045689582824707,
+      "learning_rate": 1.4833333333333337e-06,
+      "loss": 0.0619,
+      "num_input_tokens_seen": 2399936,
+      "step": 178
+    },
+    {
+      "epoch": 2.302250803858521,
+      "grad_norm": 9.59676742553711,
+      "learning_rate": 1.4916666666666669e-06,
+      "loss": 0.0683,
+      "num_input_tokens_seen": 2412480,
+      "step": 179
+    },
+    {
+      "epoch": 2.315112540192926,
+      "grad_norm": 7.824158668518066,
+      "learning_rate": 1.5e-06,
+      "loss": 0.07,
+      "num_input_tokens_seen": 2425888,
+      "step": 180
+    },
+    {
+      "epoch": 2.327974276527331,
+      "grad_norm": 10.920543670654297,
+      "learning_rate": 1.5083333333333336e-06,
+      "loss": 0.1154,
+      "num_input_tokens_seen": 2439168,
+      "step": 181
+    },
+    {
+      "epoch": 2.3408360128617365,
+      "grad_norm": 7.014737606048584,
+      "learning_rate": 1.5166666666666668e-06,
+      "loss": 0.0923,
+      "num_input_tokens_seen": 2452928,
+      "step": 182
+    },
+    {
+      "epoch": 2.3536977491961415,
+      "grad_norm": 7.072991371154785,
+      "learning_rate": 1.525e-06,
+      "loss": 0.0777,
+      "num_input_tokens_seen": 2465760,
+      "step": 183
+    },
+    {
+      "epoch": 2.3665594855305465,
+      "grad_norm": 13.426029205322266,
+      "learning_rate": 1.5333333333333334e-06,
+      "loss": 0.0754,
+      "num_input_tokens_seen": 2479744,
+      "step": 184
+    },
+    {
+      "epoch": 2.379421221864952,
+      "grad_norm": 9.223447799682617,
+      "learning_rate": 1.5416666666666668e-06,
+      "loss": 0.0704,
+      "num_input_tokens_seen": 2493024,
+      "step": 185
+    },
+    {
+      "epoch": 2.392282958199357,
+      "grad_norm": 6.772041320800781,
+      "learning_rate": 1.5500000000000002e-06,
+      "loss": 0.0915,
+      "num_input_tokens_seen": 2506720,
+      "step": 186
+    },
+    {
+      "epoch": 2.405144694533762,
+      "grad_norm": 7.140571117401123,
+      "learning_rate": 1.5583333333333334e-06,
+      "loss": 0.087,
+      "num_input_tokens_seen": 2519776,
+      "step": 187
+    },
+    {
+      "epoch": 2.418006430868167,
+      "grad_norm": 9.75447940826416,
+      "learning_rate": 1.566666666666667e-06,
+      "loss": 0.0566,
+      "num_input_tokens_seen": 2533536,
+      "step": 188
+    },
+    {
+      "epoch": 2.4308681672025725,
+      "grad_norm": 12.131728172302246,
+      "learning_rate": 1.5750000000000002e-06,
+      "loss": 0.1037,
+      "num_input_tokens_seen": 2547520,
+      "step": 189
+    },
+    {
+      "epoch": 2.4437299035369775,
+      "grad_norm": 10.324275970458984,
+      "learning_rate": 1.5833333333333333e-06,
+      "loss": 0.1143,
+      "num_input_tokens_seen": 2561280,
+      "step": 190
+    },
+    {
+      "epoch": 2.4565916398713825,
+      "grad_norm": 15.137426376342773,
+      "learning_rate": 1.591666666666667e-06,
+      "loss": 0.0829,
+      "num_input_tokens_seen": 2575136,
+      "step": 191
+    },
+    {
+      "epoch": 2.469453376205788,
+      "grad_norm": 4.877906322479248,
+      "learning_rate": 1.6000000000000001e-06,
+      "loss": 0.0422,
+      "num_input_tokens_seen": 2588736,
+      "step": 192
+    },
+    {
+      "epoch": 2.482315112540193,
+      "grad_norm": 6.23721170425415,
+      "learning_rate": 1.6083333333333333e-06,
+      "loss": 0.0727,
+      "num_input_tokens_seen": 2602400,
+      "step": 193
+    },
+    {
+      "epoch": 2.495176848874598,
+      "grad_norm": 10.653412818908691,
+      "learning_rate": 1.6166666666666667e-06,
+      "loss": 0.0836,
+      "num_input_tokens_seen": 2615648,
+      "step": 194
+    },
+    {
+      "epoch": 2.508038585209003,
+      "grad_norm": 8.391611099243164,
+      "learning_rate": 1.6250000000000001e-06,
+      "loss": 0.0803,
+      "num_input_tokens_seen": 2629024,
+      "step": 195
+    },
+    {
+      "epoch": 2.5209003215434085,
+      "grad_norm": 6.826237201690674,
+      "learning_rate": 1.6333333333333335e-06,
+      "loss": 0.0654,
+      "num_input_tokens_seen": 2642784,
+      "step": 196
+    },
+    {
+      "epoch": 2.5337620578778135,
+      "grad_norm": 7.795334339141846,
+      "learning_rate": 1.6416666666666667e-06,
+      "loss": 0.0587,
+      "num_input_tokens_seen": 2656064,
+      "step": 197
+    },
+    {
+      "epoch": 2.5466237942122185,
+      "grad_norm": 9.136542320251465,
+      "learning_rate": 1.6500000000000003e-06,
+      "loss": 0.0848,
+      "num_input_tokens_seen": 2669472,
+      "step": 198
+    },
+    {
+      "epoch": 2.559485530546624,
+      "grad_norm": 4.476882457733154,
+      "learning_rate": 1.6583333333333335e-06,
+      "loss": 0.0525,
+      "num_input_tokens_seen": 2682944,
+      "step": 199
+    },
+    {
+      "epoch": 2.572347266881029,
+      "grad_norm": 8.298385620117188,
+      "learning_rate": 1.6666666666666667e-06,
+      "loss": 0.0677,
+      "num_input_tokens_seen": 2695968,
+      "step": 200
+    },
+    {
+      "epoch": 2.585209003215434,
+      "grad_norm": 9.084216117858887,
+      "learning_rate": 1.6750000000000003e-06,
+      "loss": 0.062,
+      "num_input_tokens_seen": 2709504,
+      "step": 201
+    },
+    {
+      "epoch": 2.598070739549839,
+      "grad_norm": 10.124899864196777,
+      "learning_rate": 1.6833333333333335e-06,
+      "loss": 0.0674,
+      "num_input_tokens_seen": 2723264,
+      "step": 202
+    },
+    {
+      "epoch": 2.6109324758842445,
+      "grad_norm": 5.721729755401611,
+      "learning_rate": 1.6916666666666666e-06,
+      "loss": 0.0533,
+      "num_input_tokens_seen": 2736768,
+      "step": 203
+    },
+    {
+      "epoch": 2.6237942122186495,
+      "grad_norm": 6.966607570648193,
+      "learning_rate": 1.7000000000000002e-06,
+      "loss": 0.0757,
+      "num_input_tokens_seen": 2750816,
+      "step": 204
+    },
+    {
+      "epoch": 2.6366559485530545,
+      "grad_norm": 12.730990409851074,
+      "learning_rate": 1.7083333333333334e-06,
+      "loss": 0.0777,
+      "num_input_tokens_seen": 2764576,
+      "step": 205
+    },
+    {
+      "epoch": 2.64951768488746,
+      "grad_norm": 13.19457721710205,
+      "learning_rate": 1.7166666666666668e-06,
+      "loss": 0.0921,
+      "num_input_tokens_seen": 2778080,
+      "step": 206
+    },
+    {
+      "epoch": 2.662379421221865,
+      "grad_norm": 6.583987712860107,
+      "learning_rate": 1.725e-06,
+      "loss": 0.0378,
+      "num_input_tokens_seen": 2790976,
+      "step": 207
+    },
+    {
+      "epoch": 2.67524115755627,
+      "grad_norm": 6.989199161529541,
+      "learning_rate": 1.7333333333333336e-06,
+      "loss": 0.0671,
+      "num_input_tokens_seen": 2805152,
+      "step": 208
+    },
+    {
+      "epoch": 2.688102893890675,
+      "grad_norm": 8.903035163879395,
+      "learning_rate": 1.7416666666666668e-06,
+      "loss": 0.0664,
+      "num_input_tokens_seen": 2818688,
+      "step": 209
+    },
+    {
+      "epoch": 2.7009646302250805,
+      "grad_norm": 10.817940711975098,
+      "learning_rate": 1.75e-06,
+      "loss": 0.072,
+      "num_input_tokens_seen": 2831872,
+      "step": 210
+    },
+    {
+      "epoch": 2.7138263665594855,
+      "grad_norm": 9.068681716918945,
+      "learning_rate": 1.7583333333333336e-06,
+      "loss": 0.0883,
+      "num_input_tokens_seen": 2845984,
+      "step": 211
+    },
+    {
+      "epoch": 2.7266881028938905,
+      "grad_norm": 4.464389801025391,
+      "learning_rate": 1.7666666666666668e-06,
+      "loss": 0.0414,
+      "num_input_tokens_seen": 2859232,
+      "step": 212
+    },
+    {
+      "epoch": 2.739549839228296,
+      "grad_norm": 4.999591827392578,
+      "learning_rate": 1.7750000000000002e-06,
+      "loss": 0.031,
+      "num_input_tokens_seen": 2872192,
+      "step": 213
+    },
+    {
+      "epoch": 2.752411575562701,
+      "grad_norm": 6.336106300354004,
+      "learning_rate": 1.7833333333333336e-06,
+      "loss": 0.0634,
+      "num_input_tokens_seen": 2885312,
+      "step": 214
+    },
+    {
+      "epoch": 2.765273311897106,
+      "grad_norm": 7.798391819000244,
+      "learning_rate": 1.7916666666666667e-06,
+      "loss": 0.0837,
+      "num_input_tokens_seen": 2899072,
+      "step": 215
+    },
+    {
+      "epoch": 2.778135048231511,
+      "grad_norm": 8.36534595489502,
+      "learning_rate": 1.8000000000000001e-06,
+      "loss": 0.0855,
+      "num_input_tokens_seen": 2912448,
+      "step": 216
+    },
+    {
+      "epoch": 2.7909967845659165,
+      "grad_norm": 9.043126106262207,
+      "learning_rate": 1.8083333333333335e-06,
+      "loss": 0.0945,
+      "num_input_tokens_seen": 2925120,
+      "step": 217
+    },
+    {
+      "epoch": 2.8038585209003215,
+      "grad_norm": 5.886596202850342,
+      "learning_rate": 1.816666666666667e-06,
+      "loss": 0.078,
+      "num_input_tokens_seen": 2938336,
+      "step": 218
+    },
+    {
+      "epoch": 2.816720257234727,
+      "grad_norm": 6.200281620025635,
+      "learning_rate": 1.825e-06,
+      "loss": 0.0573,
+      "num_input_tokens_seen": 2951264,
+      "step": 219
+    },
+    {
+      "epoch": 2.829581993569132,
+      "grad_norm": 10.755176544189453,
+      "learning_rate": 1.8333333333333333e-06,
+      "loss": 0.0806,
+      "num_input_tokens_seen": 2965120,
+      "step": 220
+    },
+    {
+      "epoch": 2.842443729903537,
+      "grad_norm": 7.736689567565918,
+      "learning_rate": 1.8416666666666669e-06,
+      "loss": 0.0961,
+      "num_input_tokens_seen": 2978368,
+      "step": 221
+    },
+    {
+      "epoch": 2.855305466237942,
+      "grad_norm": 8.25919246673584,
+      "learning_rate": 1.85e-06,
+      "loss": 0.0732,
+      "num_input_tokens_seen": 2992000,
+      "step": 222
+    },
+    {
+      "epoch": 2.868167202572347,
+      "grad_norm": 9.59769344329834,
+      "learning_rate": 1.8583333333333335e-06,
+      "loss": 0.0957,
+      "num_input_tokens_seen": 3006048,
+      "step": 223
+    },
+    {
+      "epoch": 2.8810289389067525,
+      "grad_norm": 7.2752485275268555,
+      "learning_rate": 1.8666666666666669e-06,
+      "loss": 0.0774,
+      "num_input_tokens_seen": 3019744,
+      "step": 224
+    },
+    {
+      "epoch": 2.8938906752411575,
+      "grad_norm": 5.989022731781006,
+      "learning_rate": 1.8750000000000003e-06,
+      "loss": 0.0691,
+      "num_input_tokens_seen": 3033568,
+      "step": 225
+    },
+    {
+      "epoch": 2.906752411575563,
+      "grad_norm": 6.692146301269531,
+      "learning_rate": 1.8833333333333334e-06,
+      "loss": 0.0529,
+      "num_input_tokens_seen": 3047712,
+      "step": 226
+    },
+    {
+      "epoch": 2.919614147909968,
+      "grad_norm": 7.606103897094727,
+      "learning_rate": 1.8916666666666668e-06,
+      "loss": 0.0811,
+      "num_input_tokens_seen": 3060800,
+      "step": 227
+    },
+    {
+      "epoch": 2.932475884244373,
+      "grad_norm": 8.168295860290527,
+      "learning_rate": 1.9000000000000002e-06,
+      "loss": 0.1211,
+      "num_input_tokens_seen": 3074720,
+      "step": 228
+    },
+    {
+      "epoch": 2.945337620578778,
+      "grad_norm": 5.379516124725342,
+      "learning_rate": 1.9083333333333334e-06,
+      "loss": 0.0489,
+      "num_input_tokens_seen": 3088448,
+      "step": 229
+    },
+    {
+      "epoch": 2.958199356913183,
+      "grad_norm": 11.40697956085205,
+      "learning_rate": 1.916666666666667e-06,
+      "loss": 0.0947,
+      "num_input_tokens_seen": 3103008,
+      "step": 230
+    },
+    {
+      "epoch": 2.9710610932475885,
+      "grad_norm": 6.254599571228027,
+      "learning_rate": 1.925e-06,
+      "loss": 0.0561,
+      "num_input_tokens_seen": 3116320,
+      "step": 231
+    },
+    {
+      "epoch": 2.9839228295819935,
+      "grad_norm": 6.7563300132751465,
+      "learning_rate": 1.9333333333333336e-06,
+      "loss": 0.0629,
+      "num_input_tokens_seen": 3130592,
+      "step": 232
+    },
+    {
+      "epoch": 2.996784565916399,
+      "grad_norm": 3.745004177093506,
+      "learning_rate": 1.9416666666666666e-06,
+      "loss": 0.0579,
+      "num_input_tokens_seen": 3144000,
+      "step": 233
+    },
+    {
+      "epoch": 3.009646302250804,
+      "grad_norm": 3.5915324687957764,
+      "learning_rate": 1.9500000000000004e-06,
+      "loss": 0.0285,
+      "num_input_tokens_seen": 3157472,
+      "step": 234
+    },
+    {
+      "epoch": 3.022508038585209,
+      "grad_norm": 2.564159631729126,
+      "learning_rate": 1.9583333333333334e-06,
+      "loss": 0.0256,
+      "num_input_tokens_seen": 3170656,
+      "step": 235
+    },
+    {
+      "epoch": 3.035369774919614,
+      "grad_norm": 5.488891124725342,
+      "learning_rate": 1.9666666666666668e-06,
+      "loss": 0.0247,
+      "num_input_tokens_seen": 3184320,
+      "step": 236
+    },
+    {
+      "epoch": 3.0482315112540195,
+      "grad_norm": 4.964622497558594,
+      "learning_rate": 1.975e-06,
+      "loss": 0.0325,
+      "num_input_tokens_seen": 3197216,
+      "step": 237
+    },
+    {
+      "epoch": 3.0610932475884245,
+      "grad_norm": 5.399533748626709,
+      "learning_rate": 1.9833333333333335e-06,
+      "loss": 0.0172,
+      "num_input_tokens_seen": 3211008,
+      "step": 238
+    },
+    {
+      "epoch": 3.0739549839228295,
+      "grad_norm": 5.841439247131348,
+      "learning_rate": 1.991666666666667e-06,
+      "loss": 0.05,
+      "num_input_tokens_seen": 3225152,
+      "step": 239
+    },
+    {
+      "epoch": 3.0868167202572345,
+      "grad_norm": 3.352426052093506,
+      "learning_rate": 2.0000000000000003e-06,
+      "loss": 0.0134,
+      "num_input_tokens_seen": 3238752,
+      "step": 240
+    },
+    {
+      "epoch": 3.09967845659164,
+      "grad_norm": 8.727036476135254,
+      "learning_rate": 2.0083333333333337e-06,
+      "loss": 0.0434,
+      "num_input_tokens_seen": 3252224,
+      "step": 241
+    },
+    {
+      "epoch": 3.112540192926045,
+      "grad_norm": 6.6712727546691895,
+      "learning_rate": 2.0166666666666667e-06,
+      "loss": 0.0186,
+      "num_input_tokens_seen": 3264832,
+      "step": 242
+    },
+    {
+      "epoch": 3.12540192926045,
+      "grad_norm": 5.2496490478515625,
+      "learning_rate": 2.025e-06,
+      "loss": 0.0341,
+      "num_input_tokens_seen": 3278048,
+      "step": 243
+    },
+    {
+      "epoch": 3.1382636655948555,
+      "grad_norm": 6.601498126983643,
+      "learning_rate": 2.0333333333333335e-06,
+      "loss": 0.0386,
+      "num_input_tokens_seen": 3291680,
+      "step": 244
+    },
+    {
+      "epoch": 3.1511254019292605,
+      "grad_norm": 5.705418109893799,
+      "learning_rate": 2.041666666666667e-06,
+      "loss": 0.0389,
+      "num_input_tokens_seen": 3305440,
+      "step": 245
+    },
+    {
+      "epoch": 3.1639871382636655,
+      "grad_norm": 4.315848350524902,
+      "learning_rate": 2.05e-06,
+      "loss": 0.0227,
+      "num_input_tokens_seen": 3319360,
+      "step": 246
+    },
+    {
+      "epoch": 3.176848874598071,
+      "grad_norm": 3.5968470573425293,
+      "learning_rate": 2.0583333333333337e-06,
+      "loss": 0.0317,
+      "num_input_tokens_seen": 3333088,
+      "step": 247
+    },
+    {
+      "epoch": 3.189710610932476,
+      "grad_norm": 3.23504638671875,
+      "learning_rate": 2.0666666666666666e-06,
+      "loss": 0.0335,
+      "num_input_tokens_seen": 3345856,
+      "step": 248
+    },
+    {
+      "epoch": 3.202572347266881,
+      "grad_norm": 3.6574056148529053,
+      "learning_rate": 2.075e-06,
+      "loss": 0.0257,
+      "num_input_tokens_seen": 3358880,
+      "step": 249
+    },
+    {
+      "epoch": 3.215434083601286,
+      "grad_norm": 2.903745651245117,
+      "learning_rate": 2.0833333333333334e-06,
+      "loss": 0.0244,
+      "num_input_tokens_seen": 3373312,
+      "step": 250
+    },
+    {
+      "epoch": 3.2282958199356915,
+      "grad_norm": 5.933866024017334,
+      "learning_rate": 2.091666666666667e-06,
+      "loss": 0.0285,
+      "num_input_tokens_seen": 3386336,
+      "step": 251
+    },
+    {
+      "epoch": 3.2411575562700965,
+      "grad_norm": 2.2281062602996826,
+      "learning_rate": 2.1000000000000002e-06,
+      "loss": 0.0093,
+      "num_input_tokens_seen": 3399904,
+      "step": 252
+    },
+    {
+      "epoch": 3.2540192926045015,
+      "grad_norm": 5.6479573249816895,
+      "learning_rate": 2.1083333333333336e-06,
+      "loss": 0.0415,
+      "num_input_tokens_seen": 3412896,
+      "step": 253
+    },
+    {
+      "epoch": 3.266881028938907,
+      "grad_norm": 3.5148093700408936,
+      "learning_rate": 2.116666666666667e-06,
+      "loss": 0.0239,
+      "num_input_tokens_seen": 3426336,
+      "step": 254
+    },
+    {
+      "epoch": 3.279742765273312,
+      "grad_norm": 8.129992485046387,
+      "learning_rate": 2.125e-06,
+      "loss": 0.0412,
+      "num_input_tokens_seen": 3439680,
+      "step": 255
+    },
+    {
+      "epoch": 3.292604501607717,
+      "grad_norm": 10.018027305603027,
+      "learning_rate": 2.133333333333334e-06,
+      "loss": 0.0503,
+      "num_input_tokens_seen": 3453760,
+      "step": 256
+    },
+    {
+      "epoch": 3.305466237942122,
+      "grad_norm": 1.7141941785812378,
+      "learning_rate": 2.1416666666666668e-06,
+      "loss": 0.0046,
+      "num_input_tokens_seen": 3466496,
+      "step": 257
+    },
+    {
+      "epoch": 3.3183279742765275,
+      "grad_norm": 4.564700126647949,
+      "learning_rate": 2.15e-06,
+      "loss": 0.041,
+      "num_input_tokens_seen": 3480352,
+      "step": 258
+    },
+    {
+      "epoch": 3.3311897106109325,
+      "grad_norm": 5.49994421005249,
+      "learning_rate": 2.1583333333333336e-06,
+      "loss": 0.0257,
+      "num_input_tokens_seen": 3494112,
+      "step": 259
+    },
+    {
+      "epoch": 3.3440514469453375,
+      "grad_norm": 4.59380578994751,
+      "learning_rate": 2.166666666666667e-06,
+      "loss": 0.0168,
+      "num_input_tokens_seen": 3507520,
+      "step": 260
+    },
+    {
+      "epoch": 3.356913183279743,
+      "grad_norm": 5.633969306945801,
+      "learning_rate": 2.1750000000000004e-06,
+      "loss": 0.0439,
+      "num_input_tokens_seen": 3520352,
+      "step": 261
+    },
+    {
+      "epoch": 3.369774919614148,
+      "grad_norm": 7.95892333984375,
+      "learning_rate": 2.1833333333333333e-06,
+      "loss": 0.0204,
+      "num_input_tokens_seen": 3533984,
+      "step": 262
+    },
+    {
+      "epoch": 3.382636655948553,
+      "grad_norm": 4.728748798370361,
+      "learning_rate": 2.191666666666667e-06,
+      "loss": 0.0284,
+      "num_input_tokens_seen": 3547072,
+      "step": 263
+    },
+    {
+      "epoch": 3.395498392282958,
+      "grad_norm": 6.725067615509033,
+      "learning_rate": 2.2e-06,
+      "loss": 0.0684,
+      "num_input_tokens_seen": 3559872,
+      "step": 264
+    },
+    {
+      "epoch": 3.4083601286173635,
+      "grad_norm": 10.331790924072266,
+      "learning_rate": 2.2083333333333335e-06,
+      "loss": 0.0479,
+      "num_input_tokens_seen": 3572832,
+      "step": 265
+    },
+    {
+      "epoch": 3.4212218649517685,
+      "grad_norm": 5.8449296951293945,
+      "learning_rate": 2.216666666666667e-06,
+      "loss": 0.0434,
+      "num_input_tokens_seen": 3585792,
+      "step": 266
+    },
+    {
+      "epoch": 3.4340836012861735,
+      "grad_norm": 4.554612636566162,
+      "learning_rate": 2.2250000000000003e-06,
+      "loss": 0.0213,
+      "num_input_tokens_seen": 3599392,
+      "step": 267
+    },
+    {
+      "epoch": 3.446945337620579,
+      "grad_norm": 7.6923828125,
+      "learning_rate": 2.2333333333333333e-06,
+      "loss": 0.0415,
+      "num_input_tokens_seen": 3613088,
+      "step": 268
+    },
+    {
+      "epoch": 3.459807073954984,
+      "grad_norm": 5.714204788208008,
+      "learning_rate": 2.2416666666666667e-06,
+      "loss": 0.0404,
+      "num_input_tokens_seen": 3626304,
+      "step": 269
+    },
+    {
+      "epoch": 3.472668810289389,
+      "grad_norm": 5.449560165405273,
+      "learning_rate": 2.25e-06,
+      "loss": 0.0566,
+      "num_input_tokens_seen": 3639968,
+      "step": 270
+    },
+    {
+      "epoch": 3.485530546623794,
+      "grad_norm": 4.743899345397949,
+      "learning_rate": 2.2583333333333335e-06,
+      "loss": 0.0509,
+      "num_input_tokens_seen": 3653472,
+      "step": 271
+    },
+    {
+      "epoch": 3.4983922829581995,
+      "grad_norm": 4.767292022705078,
+      "learning_rate": 2.266666666666667e-06,
+      "loss": 0.0385,
+      "num_input_tokens_seen": 3667104,
+      "step": 272
+    },
+    {
+      "epoch": 3.5112540192926045,
+      "grad_norm": 2.342560291290283,
+      "learning_rate": 2.2750000000000002e-06,
+      "loss": 0.0225,
+      "num_input_tokens_seen": 3680480,
+      "step": 273
+    },
+    {
+      "epoch": 3.5241157556270095,
+      "grad_norm": 3.9172496795654297,
+      "learning_rate": 2.2833333333333336e-06,
+      "loss": 0.0255,
+      "num_input_tokens_seen": 3693568,
+      "step": 274
+    },
+    {
+      "epoch": 3.536977491961415,
+      "grad_norm": 6.793405532836914,
+      "learning_rate": 2.2916666666666666e-06,
+      "loss": 0.0531,
+      "num_input_tokens_seen": 3706496,
+      "step": 275
+    },
+    {
+      "epoch": 3.54983922829582,
+      "grad_norm": 2.142134189605713,
+      "learning_rate": 2.3000000000000004e-06,
+      "loss": 0.0095,
+      "num_input_tokens_seen": 3720352,
+      "step": 276
+    },
+    {
+      "epoch": 3.562700964630225,
+      "grad_norm": 6.340362071990967,
+      "learning_rate": 2.3083333333333334e-06,
+      "loss": 0.0229,
+      "num_input_tokens_seen": 3734176,
+      "step": 277
+    },
+    {
+      "epoch": 3.57556270096463,
+      "grad_norm": 5.204097747802734,
+      "learning_rate": 2.316666666666667e-06,
+      "loss": 0.038,
+      "num_input_tokens_seen": 3748544,
+      "step": 278
+    },
+    {
+      "epoch": 3.5884244372990355,
+      "grad_norm": 6.532016754150391,
+      "learning_rate": 2.325e-06,
+      "loss": 0.0316,
+      "num_input_tokens_seen": 3761984,
+      "step": 279
+    },
+    {
+      "epoch": 3.6012861736334405,
+      "grad_norm": 10.49596881866455,
+      "learning_rate": 2.3333333333333336e-06,
+      "loss": 0.0861,
+      "num_input_tokens_seen": 3774304,
+      "step": 280
+    },
+    {
+      "epoch": 3.6141479099678455,
+      "grad_norm": 5.68178129196167,
+      "learning_rate": 2.341666666666667e-06,
+      "loss": 0.0566,
+      "num_input_tokens_seen": 3788416,
+      "step": 281
+    },
+    {
+      "epoch": 3.627009646302251,
+      "grad_norm": 6.790673732757568,
+      "learning_rate": 2.35e-06,
+      "loss": 0.0804,
+      "num_input_tokens_seen": 3802112,
+      "step": 282
+    },
+    {
+      "epoch": 3.639871382636656,
+      "grad_norm": 7.294980049133301,
+      "learning_rate": 2.3583333333333338e-06,
+      "loss": 0.046,
+      "num_input_tokens_seen": 3815968,
+      "step": 283
+    },
+    {
+      "epoch": 3.652733118971061,
+      "grad_norm": 10.37634563446045,
+      "learning_rate": 2.3666666666666667e-06,
+      "loss": 0.0693,
+      "num_input_tokens_seen": 3829248,
+      "step": 284
+    },
+    {
+      "epoch": 3.665594855305466,
+      "grad_norm": 4.692931652069092,
+      "learning_rate": 2.375e-06,
+      "loss": 0.0342,
+      "num_input_tokens_seen": 3843072,
+      "step": 285
+    },
+    {
+      "epoch": 3.6784565916398715,
+      "grad_norm": 3.5910987854003906,
+      "learning_rate": 2.3833333333333335e-06,
+      "loss": 0.0479,
+      "num_input_tokens_seen": 3856288,
+      "step": 286
+    },
+    {
+      "epoch": 3.6913183279742765,
+      "grad_norm": 5.858832359313965,
+      "learning_rate": 2.391666666666667e-06,
+      "loss": 0.0388,
+      "num_input_tokens_seen": 3870240,
+      "step": 287
+    },
+    {
+      "epoch": 3.7041800643086815,
+      "grad_norm": 4.469853401184082,
+      "learning_rate": 2.4000000000000003e-06,
+      "loss": 0.0274,
+      "num_input_tokens_seen": 3884096,
+      "step": 288
+    },
+    {
+      "epoch": 3.717041800643087,
+      "grad_norm": 4.967989921569824,
+      "learning_rate": 2.4083333333333337e-06,
+      "loss": 0.0259,
+      "num_input_tokens_seen": 3898368,
+      "step": 289
+    },
+    {
+      "epoch": 3.729903536977492,
+      "grad_norm": 6.155540943145752,
+      "learning_rate": 2.4166666666666667e-06,
+      "loss": 0.0367,
+      "num_input_tokens_seen": 3911200,
+      "step": 290
+    },
+    {
+      "epoch": 3.742765273311897,
+      "grad_norm": 6.5608367919921875,
+      "learning_rate": 2.425e-06,
+      "loss": 0.0661,
+      "num_input_tokens_seen": 3924320,
+      "step": 291
+    },
+    {
+      "epoch": 3.755627009646302,
+      "grad_norm": 6.416184902191162,
+      "learning_rate": 2.4333333333333335e-06,
+      "loss": 0.0466,
+      "num_input_tokens_seen": 3937632,
+      "step": 292
+    },
+    {
+      "epoch": 3.7684887459807075,
+      "grad_norm": 5.672483444213867,
+      "learning_rate": 2.441666666666667e-06,
+      "loss": 0.0286,
+      "num_input_tokens_seen": 3951040,
+      "step": 293
+    },
+    {
+      "epoch": 3.7813504823151125,
+      "grad_norm": 9.215237617492676,
+      "learning_rate": 2.4500000000000003e-06,
+      "loss": 0.0586,
+      "num_input_tokens_seen": 3963936,
+      "step": 294
+    },
+    {
+      "epoch": 3.7942122186495175,
+      "grad_norm": 5.630943298339844,
+      "learning_rate": 2.4583333333333332e-06,
+      "loss": 0.0329,
+      "num_input_tokens_seen": 3976832,
+      "step": 295
+    },
+    {
+      "epoch": 3.807073954983923,
+      "grad_norm": 8.257805824279785,
+      "learning_rate": 2.466666666666667e-06,
+      "loss": 0.0582,
+      "num_input_tokens_seen": 3990592,
+      "step": 296
+    },
+    {
+      "epoch": 3.819935691318328,
+      "grad_norm": 3.351003408432007,
+      "learning_rate": 2.475e-06,
+      "loss": 0.0312,
+      "num_input_tokens_seen": 4004288,
+      "step": 297
+    },
+    {
+      "epoch": 3.832797427652733,
+      "grad_norm": 3.2983040809631348,
+      "learning_rate": 2.4833333333333334e-06,
+      "loss": 0.0329,
+      "num_input_tokens_seen": 4017856,
+      "step": 298
+    },
+    {
+      "epoch": 3.845659163987138,
+      "grad_norm": 4.082300662994385,
+      "learning_rate": 2.491666666666667e-06,
+      "loss": 0.0206,
+      "num_input_tokens_seen": 4031168,
+      "step": 299
+    },
+    {
+      "epoch": 3.8585209003215435,
+      "grad_norm": 3.222336530685425,
+      "learning_rate": 2.5e-06,
+      "loss": 0.0426,
+      "num_input_tokens_seen": 4044416,
+      "step": 300
+    },
+    {
+      "epoch": 3.8713826366559485,
+      "grad_norm": 3.699768304824829,
+      "learning_rate": 2.5083333333333336e-06,
+      "loss": 0.0179,
+      "num_input_tokens_seen": 4057824,
+      "step": 301
+    },
+    {
+      "epoch": 3.884244372990354,
+      "grad_norm": 2.819528341293335,
+      "learning_rate": 2.5166666666666666e-06,
+      "loss": 0.0289,
+      "num_input_tokens_seen": 4071936,
+      "step": 302
+    },
+    {
+      "epoch": 3.897106109324759,
+      "grad_norm": 4.0458478927612305,
+      "learning_rate": 2.5250000000000004e-06,
+      "loss": 0.0303,
+      "num_input_tokens_seen": 4085536,
+      "step": 303
+    },
+    {
+      "epoch": 3.909967845659164,
+      "grad_norm": 11.214617729187012,
+      "learning_rate": 2.5333333333333338e-06,
+      "loss": 0.046,
+      "num_input_tokens_seen": 4099552,
+      "step": 304
+    },
+    {
+      "epoch": 3.922829581993569,
+      "grad_norm": 6.758171081542969,
+      "learning_rate": 2.5416666666666668e-06,
+      "loss": 0.0523,
+      "num_input_tokens_seen": 4112736,
+      "step": 305
+    },
+    {
+      "epoch": 3.935691318327974,
+      "grad_norm": 5.622665882110596,
+      "learning_rate": 2.55e-06,
+      "loss": 0.0329,
+      "num_input_tokens_seen": 4126080,
+      "step": 306
+    },
+    {
+      "epoch": 3.9485530546623795,
+      "grad_norm": 1.4951086044311523,
+      "learning_rate": 2.558333333333334e-06,
+      "loss": 0.0072,
+      "num_input_tokens_seen": 4138912,
+      "step": 307
+    },
+    {
+      "epoch": 3.9614147909967845,
+      "grad_norm": 5.639418601989746,
+      "learning_rate": 2.566666666666667e-06,
+      "loss": 0.0415,
+      "num_input_tokens_seen": 4152768,
+      "step": 308
+    },
+    {
+      "epoch": 3.97427652733119,
+      "grad_norm": 7.496403694152832,
+      "learning_rate": 2.5750000000000003e-06,
+      "loss": 0.0233,
+      "num_input_tokens_seen": 4166528,
+      "step": 309
+    },
+    {
+      "epoch": 3.987138263665595,
+      "grad_norm": 8.74985408782959,
+      "learning_rate": 2.5833333333333337e-06,
+      "loss": 0.0423,
+      "num_input_tokens_seen": 4180832,
+      "step": 310
+    },
+    {
+      "epoch": 4.0,
+      "grad_norm": 6.229910850524902,
+      "learning_rate": 2.5916666666666667e-06,
+      "loss": 0.0295,
+      "num_input_tokens_seen": 4194048,
+      "step": 311
+    },
+    {
+      "epoch": 4.012861736334405,
+      "grad_norm": 3.404331922531128,
+      "learning_rate": 2.6e-06,
+      "loss": 0.0327,
+      "num_input_tokens_seen": 4207040,
+      "step": 312
+    },
+    {
+      "epoch": 4.02572347266881,
+      "grad_norm": 4.214801788330078,
+      "learning_rate": 2.608333333333333e-06,
+      "loss": 0.0301,
+      "num_input_tokens_seen": 4220000,
+      "step": 313
+    },
+    {
+      "epoch": 4.038585209003215,
+      "grad_norm": 6.241875171661377,
+      "learning_rate": 2.616666666666667e-06,
+      "loss": 0.0301,
+      "num_input_tokens_seen": 4233344,
+      "step": 314
+    },
+    {
+      "epoch": 4.051446945337621,
+      "grad_norm": 3.2021894454956055,
+      "learning_rate": 2.6250000000000003e-06,
+      "loss": 0.0281,
+      "num_input_tokens_seen": 4246144,
+      "step": 315
+    },
+    {
+      "epoch": 4.064308681672026,
+      "grad_norm": 3.5197951793670654,
+      "learning_rate": 2.6333333333333332e-06,
+      "loss": 0.0136,
+      "num_input_tokens_seen": 4260992,
+      "step": 316
+    },
+    {
+      "epoch": 4.077170418006431,
+      "grad_norm": 4.2946319580078125,
+      "learning_rate": 2.6416666666666666e-06,
+      "loss": 0.0219,
+      "num_input_tokens_seen": 4274112,
+      "step": 317
+    },
+    {
+      "epoch": 4.090032154340836,
+      "grad_norm": 1.3583834171295166,
+      "learning_rate": 2.6500000000000005e-06,
+      "loss": 0.0044,
+      "num_input_tokens_seen": 4287904,
+      "step": 318
+    },
+    {
+      "epoch": 4.102893890675241,
+      "grad_norm": 5.484881401062012,
+      "learning_rate": 2.6583333333333334e-06,
+      "loss": 0.0335,
+      "num_input_tokens_seen": 4301568,
+      "step": 319
+    },
+    {
+      "epoch": 4.115755627009646,
+      "grad_norm": 3.5873327255249023,
+      "learning_rate": 2.666666666666667e-06,
+      "loss": 0.0053,
+      "num_input_tokens_seen": 4314848,
+      "step": 320
+    },
+    {
+      "epoch": 4.128617363344051,
+      "grad_norm": 2.954930067062378,
+      "learning_rate": 2.6750000000000002e-06,
+      "loss": 0.0196,
+      "num_input_tokens_seen": 4327456,
+      "step": 321
+    },
+    {
+      "epoch": 4.141479099678457,
+      "grad_norm": 2.628854274749756,
+      "learning_rate": 2.683333333333333e-06,
+      "loss": 0.0309,
+      "num_input_tokens_seen": 4341024,
+      "step": 322
+    },
+    {
+      "epoch": 4.154340836012862,
+      "grad_norm": 6.691638469696045,
+      "learning_rate": 2.691666666666667e-06,
+      "loss": 0.0382,
+      "num_input_tokens_seen": 4353824,
+      "step": 323
+    },
+    {
+      "epoch": 4.167202572347267,
+      "grad_norm": 6.866171360015869,
+      "learning_rate": 2.7000000000000004e-06,
+      "loss": 0.046,
+      "num_input_tokens_seen": 4367616,
+      "step": 324
+    },
+    {
+      "epoch": 4.180064308681672,
+      "grad_norm": 4.016347408294678,
+      "learning_rate": 2.7083333333333334e-06,
+      "loss": 0.0133,
+      "num_input_tokens_seen": 4380448,
+      "step": 325
+    },
+    {
+      "epoch": 4.192926045016077,
+      "grad_norm": 4.948652744293213,
+      "learning_rate": 2.7166666666666668e-06,
+      "loss": 0.0265,
+      "num_input_tokens_seen": 4393920,
+      "step": 326
+    },
+    {
+      "epoch": 4.205787781350482,
+      "grad_norm": 2.7615582942962646,
+      "learning_rate": 2.7250000000000006e-06,
+      "loss": 0.0084,
+      "num_input_tokens_seen": 4407264,
+      "step": 327
+    },
+    {
+      "epoch": 4.218649517684887,
+      "grad_norm": 3.0052027702331543,
+      "learning_rate": 2.7333333333333336e-06,
+      "loss": 0.0382,
+      "num_input_tokens_seen": 4420352,
+      "step": 328
+    },
+    {
+      "epoch": 4.231511254019293,
+      "grad_norm": 3.0679566860198975,
+      "learning_rate": 2.741666666666667e-06,
+      "loss": 0.0101,
+      "num_input_tokens_seen": 4434688,
+      "step": 329
+    },
+    {
+      "epoch": 4.244372990353698,
+      "grad_norm": 4.612461566925049,
+      "learning_rate": 2.7500000000000004e-06,
+      "loss": 0.0174,
+      "num_input_tokens_seen": 4448160,
+      "step": 330
+    },
+    {
+      "epoch": 4.257234726688103,
+      "grad_norm": 4.299385070800781,
+      "learning_rate": 2.7583333333333333e-06,
+      "loss": 0.023,
+      "num_input_tokens_seen": 4461184,
+      "step": 331
+    },
+    {
+      "epoch": 4.270096463022508,
+      "grad_norm": 1.5578008890151978,
+      "learning_rate": 2.766666666666667e-06,
+      "loss": 0.0162,
+      "num_input_tokens_seen": 4475168,
+      "step": 332
+    },
+    {
+      "epoch": 4.282958199356913,
+      "grad_norm": 3.8595075607299805,
+      "learning_rate": 2.7750000000000005e-06,
+      "loss": 0.0261,
+      "num_input_tokens_seen": 4487872,
+      "step": 333
+    },
+    {
+      "epoch": 4.295819935691318,
+      "grad_norm": 8.00913143157959,
+      "learning_rate": 2.7833333333333335e-06,
+      "loss": 0.0266,
+      "num_input_tokens_seen": 4501216,
+      "step": 334
+    },
+    {
+      "epoch": 4.308681672025724,
+      "grad_norm": 2.2975409030914307,
+      "learning_rate": 2.791666666666667e-06,
+      "loss": 0.0194,
+      "num_input_tokens_seen": 4514336,
+      "step": 335
+    },
+    {
+      "epoch": 4.321543408360129,
+      "grad_norm": 1.9800870418548584,
+      "learning_rate": 2.8000000000000003e-06,
+      "loss": 0.0058,
+      "num_input_tokens_seen": 4528288,
+      "step": 336
+    },
+    {
+      "epoch": 4.334405144694534,
+      "grad_norm": 6.778807163238525,
+      "learning_rate": 2.8083333333333333e-06,
+      "loss": 0.0065,
+      "num_input_tokens_seen": 4541728,
+      "step": 337
+    },
+    {
+      "epoch": 4.347266881028939,
+      "grad_norm": 7.885656833648682,
+      "learning_rate": 2.816666666666667e-06,
+      "loss": 0.0202,
+      "num_input_tokens_seen": 4555552,
+      "step": 338
+    },
+    {
+      "epoch": 4.360128617363344,
+      "grad_norm": 9.400197982788086,
+      "learning_rate": 2.825e-06,
+      "loss": 0.0135,
+      "num_input_tokens_seen": 4568768,
+      "step": 339
+    },
+    {
+      "epoch": 4.372990353697749,
+      "grad_norm": 5.712286472320557,
+      "learning_rate": 2.8333333333333335e-06,
+      "loss": 0.01,
+      "num_input_tokens_seen": 4582400,
+      "step": 340
+    },
+    {
+      "epoch": 4.385852090032154,
+      "grad_norm": 4.751468181610107,
+      "learning_rate": 2.841666666666667e-06,
+      "loss": 0.0051,
+      "num_input_tokens_seen": 4595360,
+      "step": 341
+    },
+    {
+      "epoch": 4.39871382636656,
+      "grad_norm": 2.9044384956359863,
+      "learning_rate": 2.85e-06,
+      "loss": 0.0293,
+      "num_input_tokens_seen": 4609632,
+      "step": 342
+    },
+    {
+      "epoch": 4.411575562700965,
+      "grad_norm": 5.8322224617004395,
+      "learning_rate": 2.8583333333333336e-06,
+      "loss": 0.046,
+      "num_input_tokens_seen": 4623232,
+      "step": 343
+    },
+    {
+      "epoch": 4.42443729903537,
+      "grad_norm": 0.7529265284538269,
+      "learning_rate": 2.866666666666667e-06,
+      "loss": 0.0024,
+      "num_input_tokens_seen": 4636864,
+      "step": 344
+    },
+    {
+      "epoch": 4.437299035369775,
+      "grad_norm": 4.490261554718018,
+      "learning_rate": 2.875e-06,
+      "loss": 0.0211,
+      "num_input_tokens_seen": 4649600,
+      "step": 345
+    },
+    {
+      "epoch": 4.45016077170418,
+      "grad_norm": 9.859882354736328,
+      "learning_rate": 2.8833333333333334e-06,
+      "loss": 0.0229,
+      "num_input_tokens_seen": 4663072,
+      "step": 346
+    },
+    {
+      "epoch": 4.463022508038585,
+      "grad_norm": 3.8769121170043945,
+      "learning_rate": 2.8916666666666672e-06,
+      "loss": 0.0103,
+      "num_input_tokens_seen": 4676960,
+      "step": 347
+    },
+    {
+      "epoch": 4.47588424437299,
+      "grad_norm": 6.608347415924072,
+      "learning_rate": 2.9e-06,
+      "loss": 0.0262,
+      "num_input_tokens_seen": 4690080,
+      "step": 348
+    },
+    {
+      "epoch": 4.488745980707396,
+      "grad_norm": 3.058494806289673,
+      "learning_rate": 2.9083333333333336e-06,
+      "loss": 0.0295,
+      "num_input_tokens_seen": 4704064,
+      "step": 349
+    },
+    {
+      "epoch": 4.501607717041801,
+      "grad_norm": 4.893054485321045,
+      "learning_rate": 2.916666666666667e-06,
+      "loss": 0.0149,
+      "num_input_tokens_seen": 4717536,
+      "step": 350
+    },
+    {
+      "epoch": 4.514469453376206,
+      "grad_norm": 5.85437536239624,
+      "learning_rate": 2.925e-06,
+      "loss": 0.0337,
+      "num_input_tokens_seen": 4731040,
+      "step": 351
+    },
+    {
+      "epoch": 4.527331189710611,
+      "grad_norm": 5.814053058624268,
+      "learning_rate": 2.9333333333333338e-06,
+      "loss": 0.0318,
+      "num_input_tokens_seen": 4744864,
+      "step": 352
+    },
+    {
+      "epoch": 4.540192926045016,
+      "grad_norm": 6.330677509307861,
+      "learning_rate": 2.941666666666667e-06,
+      "loss": 0.0213,
+      "num_input_tokens_seen": 4758240,
+      "step": 353
+    },
+    {
+      "epoch": 4.553054662379421,
+      "grad_norm": 0.9806466698646545,
+      "learning_rate": 2.95e-06,
+      "loss": 0.0048,
+      "num_input_tokens_seen": 4770368,
+      "step": 354
+    },
+    {
+      "epoch": 4.565916398713826,
+      "grad_norm": 3.046319007873535,
+      "learning_rate": 2.9583333333333335e-06,
+      "loss": 0.0326,
+      "num_input_tokens_seen": 4783328,
+      "step": 355
+    },
+    {
+      "epoch": 4.578778135048232,
+      "grad_norm": 5.004611015319824,
+      "learning_rate": 2.9666666666666673e-06,
+      "loss": 0.013,
+      "num_input_tokens_seen": 4797088,
+      "step": 356
+    },
+    {
+      "epoch": 4.591639871382637,
+      "grad_norm": 5.062610626220703,
+      "learning_rate": 2.9750000000000003e-06,
+      "loss": 0.0293,
+      "num_input_tokens_seen": 4810464,
+      "step": 357
+    },
+    {
+      "epoch": 4.604501607717042,
+      "grad_norm": 9.05587100982666,
+      "learning_rate": 2.9833333333333337e-06,
+      "loss": 0.0411,
+      "num_input_tokens_seen": 4823744,
+      "step": 358
+    },
+    {
+      "epoch": 4.617363344051447,
+      "grad_norm": 7.2998175621032715,
+      "learning_rate": 2.991666666666667e-06,
+      "loss": 0.0389,
+      "num_input_tokens_seen": 4837824,
+      "step": 359
+    },
+    {
+      "epoch": 4.630225080385852,
+      "grad_norm": 6.108879566192627,
+      "learning_rate": 3e-06,
+      "loss": 0.0395,
+      "num_input_tokens_seen": 4852096,
+      "step": 360
+    },
+    {
+      "epoch": 4.643086816720257,
+      "grad_norm": 4.677525043487549,
+      "learning_rate": 3.0083333333333335e-06,
+      "loss": 0.0065,
+      "num_input_tokens_seen": 4865600,
+      "step": 361
+    },
+    {
+      "epoch": 4.655948553054662,
+      "grad_norm": 9.179205894470215,
+      "learning_rate": 3.0166666666666673e-06,
+      "loss": 0.0294,
+      "num_input_tokens_seen": 4878592,
+      "step": 362
+    },
+    {
+      "epoch": 4.668810289389068,
+      "grad_norm": 3.478533983230591,
+      "learning_rate": 3.0250000000000003e-06,
+      "loss": 0.0192,
+      "num_input_tokens_seen": 4892064,
+      "step": 363
+    },
+    {
+      "epoch": 4.681672025723473,
+      "grad_norm": 6.015811443328857,
+      "learning_rate": 3.0333333333333337e-06,
+      "loss": 0.0179,
+      "num_input_tokens_seen": 4906304,
+      "step": 364
+    },
+    {
+      "epoch": 4.694533762057878,
+      "grad_norm": 3.1433732509613037,
+      "learning_rate": 3.0416666666666666e-06,
+      "loss": 0.0131,
+      "num_input_tokens_seen": 4920480,
+      "step": 365
+    },
+    {
+      "epoch": 4.707395498392283,
+      "grad_norm": 3.894927501678467,
+      "learning_rate": 3.05e-06,
+      "loss": 0.0216,
+      "num_input_tokens_seen": 4933376,
+      "step": 366
+    },
+    {
+      "epoch": 4.720257234726688,
+      "grad_norm": 3.4421629905700684,
+      "learning_rate": 3.058333333333334e-06,
+      "loss": 0.0171,
+      "num_input_tokens_seen": 4947872,
+      "step": 367
+    },
+    {
+      "epoch": 4.733118971061093,
+      "grad_norm": 2.935985803604126,
+      "learning_rate": 3.066666666666667e-06,
+      "loss": 0.0129,
+      "num_input_tokens_seen": 4961280,
+      "step": 368
+    },
+    {
+      "epoch": 4.745980707395498,
+      "grad_norm": 7.003020763397217,
+      "learning_rate": 3.075e-06,
+      "loss": 0.0268,
+      "num_input_tokens_seen": 4975072,
+      "step": 369
+    },
+    {
+      "epoch": 4.758842443729904,
+      "grad_norm": 5.833902835845947,
+      "learning_rate": 3.0833333333333336e-06,
+      "loss": 0.0313,
+      "num_input_tokens_seen": 4988928,
+      "step": 370
+    },
+    {
+      "epoch": 4.771704180064309,
+      "grad_norm": 4.902740001678467,
+      "learning_rate": 3.0916666666666666e-06,
+      "loss": 0.0197,
+      "num_input_tokens_seen": 5002656,
+      "step": 371
+    },
+    {
+      "epoch": 4.784565916398714,
+      "grad_norm": 1.6855580806732178,
+      "learning_rate": 3.1000000000000004e-06,
+      "loss": 0.0051,
+      "num_input_tokens_seen": 5016928,
+      "step": 372
+    },
+    {
+      "epoch": 4.797427652733119,
+      "grad_norm": 3.116506814956665,
+      "learning_rate": 3.1083333333333338e-06,
+      "loss": 0.0107,
+      "num_input_tokens_seen": 5030432,
+      "step": 373
+    },
+    {
+      "epoch": 4.810289389067524,
+      "grad_norm": 2.606302261352539,
+      "learning_rate": 3.1166666666666668e-06,
+      "loss": 0.0299,
+      "num_input_tokens_seen": 5044128,
+      "step": 374
+    },
+    {
+      "epoch": 4.823151125401929,
+      "grad_norm": 5.198400497436523,
+      "learning_rate": 3.125e-06,
+      "loss": 0.0549,
+      "num_input_tokens_seen": 5058592,
+      "step": 375
+    },
+    {
+      "epoch": 4.836012861736334,
+      "grad_norm": 3.7756950855255127,
+      "learning_rate": 3.133333333333334e-06,
+      "loss": 0.0163,
+      "num_input_tokens_seen": 5071680,
+      "step": 376
+    },
+    {
+      "epoch": 4.84887459807074,
+      "grad_norm": 3.266502618789673,
+      "learning_rate": 3.141666666666667e-06,
+      "loss": 0.0133,
+      "num_input_tokens_seen": 5085760,
+      "step": 377
+    },
+    {
+      "epoch": 4.861736334405145,
+      "grad_norm": 3.7333478927612305,
+      "learning_rate": 3.1500000000000003e-06,
+      "loss": 0.0338,
+      "num_input_tokens_seen": 5099168,
+      "step": 378
+    },
+    {
+      "epoch": 4.87459807073955,
+      "grad_norm": 6.106029033660889,
+      "learning_rate": 3.1583333333333337e-06,
+      "loss": 0.0219,
+      "num_input_tokens_seen": 5112512,
+      "step": 379
+    },
+    {
+      "epoch": 4.887459807073955,
+      "grad_norm": 3.018552541732788,
+      "learning_rate": 3.1666666666666667e-06,
+      "loss": 0.0113,
+      "num_input_tokens_seen": 5125696,
+      "step": 380
+    },
+    {
+      "epoch": 4.90032154340836,
+      "grad_norm": 5.034224987030029,
+      "learning_rate": 3.175e-06,
+      "loss": 0.0297,
+      "num_input_tokens_seen": 5138816,
+      "step": 381
+    },
+    {
+      "epoch": 4.913183279742765,
+      "grad_norm": 6.260331630706787,
+      "learning_rate": 3.183333333333334e-06,
+      "loss": 0.0417,
+      "num_input_tokens_seen": 5152320,
+      "step": 382
+    },
+    {
+      "epoch": 4.92604501607717,
+      "grad_norm": 4.149762153625488,
+      "learning_rate": 3.191666666666667e-06,
+      "loss": 0.027,
+      "num_input_tokens_seen": 5165696,
+      "step": 383
+    },
+    {
+      "epoch": 4.938906752411576,
+      "grad_norm": 2.9361073970794678,
+      "learning_rate": 3.2000000000000003e-06,
+      "loss": 0.0271,
+      "num_input_tokens_seen": 5178368,
+      "step": 384
+    },
+    {
+      "epoch": 4.951768488745981,
+      "grad_norm": 5.155781269073486,
+      "learning_rate": 3.2083333333333337e-06,
+      "loss": 0.0207,
+      "num_input_tokens_seen": 5192736,
+      "step": 385
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 385,
+  "num_input_tokens_seen": 5192736,
+  "num_train_epochs": 5,
+  "save_steps": 1000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.3382655808988774e+17,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-385/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e3de8a8e741121a5f4fde13b7396c2d104a757f5aaccd4dd320366c51b2f0ea
+size 6584

checkpoint-385/zero_to_fp32.py

@@ -0,0 +1,604 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example: python zero_to_fp32.py . pytorch_model.bin
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+
+    total_files = len(files)
+    state_dicts = []
+    for f in files:
+        state_dict = torch.load(f, map_location=device)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    if zero_stage <= 2:
+        fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    elif zero_stage == 3:
+        # if there is more than one param group, there will be multiple flattened tensors - one
+        # flattened tensor per group - for simplicity merge them into a single tensor
+        #
+        # XXX: could make the script more memory efficient for when there are multiple groups - it
+        # will require matching the sub-lists of param_shapes for each param group flattened tensor
+
+        fp32_flat_groups = [
+            torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts))
+        ]
+
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    model_files = get_model_state_files(ds_checkpoint_dir)
+
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = fp32_flat_groups[0].numel() * world_size
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    for name, shape in param_shapes.items():
+
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # XXX: memory usage doubles here
+        state_dict[name] = torch.cat(
+            tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)),
+            0).narrow(0, 0, unpartitioned_numel).view(shape)
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+
+    Returns:
+        - pytorch ``state_dict``
+
+    Note: this approach may not work if your application doesn't have sufficient free CPU memory and
+    you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters)
+    print(f"Saving fp32 state dict to {output_file}")
+    torch.save(state_dict, output_file)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info(f"Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info(f"Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument(
+        "output_file",
+        type=str,
+        help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_file,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)

config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "meta-llama/Meta-Llama-3-8B",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 8192,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.42.3",
+  "use_cache": false,
+  "vocab_size": 128256
+}

generation_config.json

@@ -0,0 +1,9 @@
+{
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": 128001,
+  "max_length": 4096,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.42.3"
+}

llamaboard_config.yaml

@@ -0,0 +1,65 @@
+top.booster: auto
+top.checkpoint_path: null
+top.finetuning_type: full
+top.model_name: LLaMA3-8B
+top.quantization_bit: none
+top.quantization_method: bitsandbytes
+top.rope_scaling: none
+top.template: default
+top.visual_inputs: false
+train.additional_target: ''
+train.badam_mode: layer
+train.badam_switch_interval: 50
+train.badam_switch_mode: ascending
+train.badam_update_ratio: 0.05
+train.batch_size: 4
+train.compute_type: bf16
+train.create_new_adapter: false
+train.cutoff_len: 1024
+train.dataset:
+- truth_train
+train.dataset_dir: data
+train.ds_offload: false
+train.ds_stage: '2'
+train.freeze_extra_modules: ''
+train.freeze_trainable_layers: 2
+train.freeze_trainable_modules: all
+train.galore_rank: 16
+train.galore_scale: 0.25
+train.galore_target: all
+train.galore_update_interval: 200
+train.gradient_accumulation_steps: 8
+train.learning_rate: 5e-6
+train.logging_steps: 1
+train.lora_alpha: 16
+train.lora_dropout: 0
+train.lora_rank: 8
+train.lora_target: ''
+train.loraplus_lr_ratio: 0
+train.lr_scheduler_type: cosine
+train.max_grad_norm: '1.0'
+train.max_samples: '100000'
+train.neat_packing: false
+train.neftune_alpha: 0
+train.num_train_epochs: '5.0'
+train.optim: adamw_torch
+train.packing: false
+train.ppo_score_norm: false
+train.ppo_whiten_rewards: false
+train.pref_beta: 0.1
+train.pref_ftx: 0
+train.pref_loss: sigmoid
+train.report_to: false
+train.resize_vocab: false
+train.reward_model: null
+train.save_steps: 1000
+train.shift_attn: false
+train.training_stage: Supervised Fine-Tuning
+train.use_badam: false
+train.use_dora: false
+train.use_galore: false
+train.use_llama_pro: false
+train.use_pissa: false
+train.use_rslora: false
+train.val_size: 0
+train.warmup_steps: 600

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95091273e88294fb364aa21ea26e19235d33e52c55bd3084a88c89fea8b26726
+size 4976698672

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27524a69f85bdd427bbc9f3b06c352d75e57fb52dd8bbfaa72357a0aa776ec14
+size 4999802720

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f3d18cfd3abd56f02f7d77fe84167a89940a20fad534fc60b9ccf8a13e458ba
+size 4915916176

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7fd33cba41fd9bb770eab81315a245e1fd442a47ac61b49f563417d3882e8d6
+size 1168138808

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 16060522496
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00004-of-00004.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

running_log.txt

@@ -0,0 +1,1038 @@
+07/10/2024 15:30:43 - INFO - llamafactory.hparams.parser - Process rank: 2, device: cuda:2, n_gpu: 1, distributed training: True, compute dtype: torch.bfloat16
+
+07/10/2024 15:30:43 - INFO - llamafactory.hparams.parser - Process rank: 3, device: cuda:3, n_gpu: 1, distributed training: True, compute dtype: torch.bfloat16
+
+[INFO|parser.py:325] 2024-07-10 15:30:43,487 >> Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: True, compute dtype: torch.bfloat16
+
+07/10/2024 15:30:43 - INFO - llamafactory.hparams.parser - Process rank: 4, device: cuda:4, n_gpu: 1, distributed training: True, compute dtype: torch.bfloat16
+
+07/10/2024 15:30:43 - INFO - llamafactory.hparams.parser - Process rank: 1, device: cuda:1, n_gpu: 1, distributed training: True, compute dtype: torch.bfloat16
+
+07/10/2024 15:30:43 - INFO - llamafactory.hparams.parser - Process rank: 7, device: cuda:7, n_gpu: 1, distributed training: True, compute dtype: torch.bfloat16
+
+07/10/2024 15:30:43 - INFO - llamafactory.hparams.parser - Process rank: 5, device: cuda:5, n_gpu: 1, distributed training: True, compute dtype: torch.bfloat16
+
+07/10/2024 15:30:43 - INFO - llamafactory.hparams.parser - Process rank: 6, device: cuda:6, n_gpu: 1, distributed training: True, compute dtype: torch.bfloat16
+
+07/10/2024 15:30:43 - WARNING - transformers.tokenization_utils_base - Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
+
+07/10/2024 15:30:43 - INFO - llamafactory.data.template - Add pad token: <|end_of_text|>
+
+07/10/2024 15:30:44 - WARNING - transformers.tokenization_utils_base - Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
+
+07/10/2024 15:30:44 - INFO - llamafactory.data.template - Add pad token: <|end_of_text|>
+
+[INFO|tokenization_utils_base.py:2161] 2024-07-10 15:30:43,965 >> loading file tokenizer.json from cache at /root/.cache/huggingface/hub/models--meta-llama--Meta-Llama-3-8B/snapshots/62bd457b6fe961a42a631306577e622c83876cb6/tokenizer.json
+
+[INFO|tokenization_utils_base.py:2161] 2024-07-10 15:30:43,965 >> loading file added_tokens.json from cache at None
+
+[INFO|tokenization_utils_base.py:2161] 2024-07-10 15:30:43,965 >> loading file special_tokens_map.json from cache at /root/.cache/huggingface/hub/models--meta-llama--Meta-Llama-3-8B/snapshots/62bd457b6fe961a42a631306577e622c83876cb6/special_tokens_map.json
+
+[INFO|tokenization_utils_base.py:2161] 2024-07-10 15:30:43,965 >> loading file tokenizer_config.json from cache at /root/.cache/huggingface/hub/models--meta-llama--Meta-Llama-3-8B/snapshots/62bd457b6fe961a42a631306577e622c83876cb6/tokenizer_config.json
+
+[WARNING|logging.py:313] 2024-07-10 15:30:44,255 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
+
+[INFO|template.py:372] 2024-07-10 15:30:44,255 >> Add pad token: <|end_of_text|>
+
+[INFO|loader.py:50] 2024-07-10 15:30:44,255 >> Loading dataset train_output.json...
+
+07/10/2024 15:30:44 - WARNING - transformers.tokenization_utils_base - Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
+
+07/10/2024 15:30:44 - INFO - llamafactory.data.template - Add pad token: <|end_of_text|>
+
+07/10/2024 15:30:44 - WARNING - transformers.tokenization_utils_base - Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
+
+07/10/2024 15:30:44 - INFO - llamafactory.data.template - Add pad token: <|end_of_text|>
+
+07/10/2024 15:30:44 - WARNING - transformers.tokenization_utils_base - Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
+
+07/10/2024 15:30:44 - INFO - llamafactory.data.template - Add pad token: <|end_of_text|>
+
+07/10/2024 15:30:44 - WARNING - transformers.tokenization_utils_base - Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
+
+07/10/2024 15:30:44 - INFO - llamafactory.data.template - Add pad token: <|end_of_text|>
+
+07/10/2024 15:30:44 - WARNING - transformers.tokenization_utils_base - Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
+
+07/10/2024 15:30:44 - INFO - llamafactory.data.template - Add pad token: <|end_of_text|>
+
+07/10/2024 15:30:45 - INFO - llamafactory.data.loader - Loading dataset train_output.json...
+
+07/10/2024 15:30:45 - INFO - llamafactory.data.loader - Loading dataset train_output.json...
+
+07/10/2024 15:30:45 - INFO - llamafactory.data.loader - Loading dataset train_output.json...
+
+07/10/2024 15:30:45 - INFO - llamafactory.data.loader - Loading dataset train_output.json...
+
+07/10/2024 15:30:45 - INFO - llamafactory.data.loader - Loading dataset train_output.json...
+
+07/10/2024 15:30:45 - INFO - llamafactory.data.loader - Loading dataset train_output.json...
+
+07/10/2024 15:30:45 - INFO - llamafactory.data.loader - Loading dataset train_output.json...
+
+[INFO|configuration_utils.py:733] 2024-07-10 15:30:46,457 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--meta-llama--Meta-Llama-3-8B/snapshots/62bd457b6fe961a42a631306577e622c83876cb6/config.json
+
+[INFO|configuration_utils.py:800] 2024-07-10 15:30:46,458 >> Model config LlamaConfig {
+  "_name_or_path": "meta-llama/Meta-Llama-3-8B",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 8192,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.42.3",
+  "use_cache": true,
+  "vocab_size": 128256
+}
+
+
+[INFO|modeling_utils.py:3556] 2024-07-10 15:30:46,480 >> loading weights file model.safetensors from cache at /root/.cache/huggingface/hub/models--meta-llama--Meta-Llama-3-8B/snapshots/62bd457b6fe961a42a631306577e622c83876cb6/model.safetensors.index.json
+
+[INFO|modeling_utils.py:1531] 2024-07-10 15:30:46,481 >> Instantiating LlamaForCausalLM model under default dtype torch.bfloat16.
+
+[INFO|configuration_utils.py:1000] 2024-07-10 15:30:46,482 >> Generate config GenerationConfig {
+  "bos_token_id": 128000,
+  "eos_token_id": 128001
+}
+
+
+[INFO|modeling_utils.py:4364] 2024-07-10 15:30:50,121 >> All model checkpoint weights were used when initializing LlamaForCausalLM.
+
+
+[INFO|modeling_utils.py:4372] 2024-07-10 15:30:50,121 >> All the weights of LlamaForCausalLM were initialized from the model checkpoint at meta-llama/Meta-Llama-3-8B.
+If your task is similar to the task the model of the checkpoint was trained on, you can already use LlamaForCausalLM for predictions without further training.
+
+[INFO|configuration_utils.py:955] 2024-07-10 15:30:50,295 >> loading configuration file generation_config.json from cache at /root/.cache/huggingface/hub/models--meta-llama--Meta-Llama-3-8B/snapshots/62bd457b6fe961a42a631306577e622c83876cb6/generation_config.json
+
+[INFO|configuration_utils.py:1000] 2024-07-10 15:30:50,295 >> Generate config GenerationConfig {
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": 128001,
+  "max_length": 4096,
+  "temperature": 0.6,
+  "top_p": 0.9
+}
+
+
+[INFO|checkpointing.py:103] 2024-07-10 15:30:50,302 >> Gradient checkpointing enabled.
+
+[INFO|attention.py:80] 2024-07-10 15:30:50,303 >> Using torch SDPA for faster training and inference.
+
+[INFO|adapter.py:302] 2024-07-10 15:30:50,303 >> Upcasting trainable params to float32.
+
+[INFO|adapter.py:48] 2024-07-10 15:30:50,303 >> Fine-tuning method: Full
+
+[INFO|loader.py:196] 2024-07-10 15:30:50,345 >> trainable params: 8,030,261,248 || all params: 8,030,261,248 || trainable%: 100.0000
+
+07/10/2024 15:30:50 - INFO - llamafactory.model.model_utils.checkpointing - Gradient checkpointing enabled.
+
+07/10/2024 15:30:50 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
+
+07/10/2024 15:30:50 - INFO - llamafactory.model.adapter - Upcasting trainable params to float32.
+
+07/10/2024 15:30:50 - INFO - llamafactory.model.adapter - Fine-tuning method: Full
+
+07/10/2024 15:30:50 - INFO - llamafactory.model.loader - trainable params: 8,030,261,248 || all params: 8,030,261,248 || trainable%: 100.0000
+
+07/10/2024 15:30:50 - INFO - llamafactory.model.model_utils.checkpointing - Gradient checkpointing enabled.
+
+07/10/2024 15:30:50 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
+
+07/10/2024 15:30:50 - INFO - llamafactory.model.adapter - Upcasting trainable params to float32.
+
+07/10/2024 15:30:50 - INFO - llamafactory.model.adapter - Fine-tuning method: Full
+
+[INFO|trainer.py:642] 2024-07-10 15:30:50,350 >> Using auto half precision backend
+
+07/10/2024 15:30:50 - INFO - llamafactory.model.loader - trainable params: 8,030,261,248 || all params: 8,030,261,248 || trainable%: 100.0000
+
+07/10/2024 15:30:50 - INFO - llamafactory.model.model_utils.checkpointing - Gradient checkpointing enabled.
+
+07/10/2024 15:30:50 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
+
+07/10/2024 15:30:50 - INFO - llamafactory.model.adapter - Upcasting trainable params to float32.
+
+07/10/2024 15:30:50 - INFO - llamafactory.model.adapter - Fine-tuning method: Full
+
+07/10/2024 15:30:50 - INFO - llamafactory.model.loader - trainable params: 8,030,261,248 || all params: 8,030,261,248 || trainable%: 100.0000
+
+07/10/2024 15:30:50 - INFO - llamafactory.model.model_utils.checkpointing - Gradient checkpointing enabled.
+
+07/10/2024 15:30:50 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
+
+07/10/2024 15:30:50 - INFO - llamafactory.model.adapter - Upcasting trainable params to float32.
+
+07/10/2024 15:30:50 - INFO - llamafactory.model.adapter - Fine-tuning method: Full
+
+07/10/2024 15:30:50 - INFO - llamafactory.model.model_utils.checkpointing - Gradient checkpointing enabled.
+
+07/10/2024 15:30:50 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
+
+07/10/2024 15:30:50 - INFO - llamafactory.model.adapter - Upcasting trainable params to float32.
+
+07/10/2024 15:30:50 - INFO - llamafactory.model.adapter - Fine-tuning method: Full
+
+07/10/2024 15:30:50 - INFO - llamafactory.model.loader - trainable params: 8,030,261,248 || all params: 8,030,261,248 || trainable%: 100.0000
+
+07/10/2024 15:30:50 - INFO - llamafactory.model.loader - trainable params: 8,030,261,248 || all params: 8,030,261,248 || trainable%: 100.0000
+
+07/10/2024 15:30:51 - INFO - llamafactory.model.model_utils.checkpointing - Gradient checkpointing enabled.
+
+07/10/2024 15:30:51 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
+
+07/10/2024 15:30:51 - INFO - llamafactory.model.adapter - Upcasting trainable params to float32.
+
+07/10/2024 15:30:51 - INFO - llamafactory.model.adapter - Fine-tuning method: Full
+
+07/10/2024 15:30:51 - INFO - llamafactory.model.loader - trainable params: 8,030,261,248 || all params: 8,030,261,248 || trainable%: 100.0000
+
+07/10/2024 15:30:51 - INFO - llamafactory.model.model_utils.checkpointing - Gradient checkpointing enabled.
+
+07/10/2024 15:30:51 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
+
+07/10/2024 15:30:51 - INFO - llamafactory.model.adapter - Upcasting trainable params to float32.
+
+07/10/2024 15:30:51 - INFO - llamafactory.model.adapter - Fine-tuning method: Full
+
+07/10/2024 15:30:51 - INFO - llamafactory.model.loader - trainable params: 8,030,261,248 || all params: 8,030,261,248 || trainable%: 100.0000
+
+[INFO|trainer.py:2128] 2024-07-10 15:31:12,003 >> ***** Running training *****
+
+[INFO|trainer.py:2129] 2024-07-10 15:31:12,003 >>   Num examples = 19,880
+
+[INFO|trainer.py:2130] 2024-07-10 15:31:12,003 >>   Num Epochs = 5
+
+[INFO|trainer.py:2131] 2024-07-10 15:31:12,003 >>   Instantaneous batch size per device = 4
+
+[INFO|trainer.py:2134] 2024-07-10 15:31:12,003 >>   Total train batch size (w. parallel, distributed & accumulation) = 256
+
+[INFO|trainer.py:2135] 2024-07-10 15:31:12,003 >>   Gradient Accumulation steps = 8
+
+[INFO|trainer.py:2136] 2024-07-10 15:31:12,003 >>   Total optimization steps = 385
+
+[INFO|trainer.py:2137] 2024-07-10 15:31:12,004 >>   Number of trainable parameters = 8,030,261,248
+
+[INFO|callbacks.py:310] 2024-07-10 15:31:28,728 >> {'loss': 7.8034, 'learning_rate': 8.3333e-09, 'epoch': 0.01, 'throughput': 803.70}
+
+[INFO|callbacks.py:310] 2024-07-10 15:31:41,798 >> {'loss': 7.7577, 'learning_rate': 1.6667e-08, 'epoch': 0.03, 'throughput': 905.46}
+
+[INFO|callbacks.py:310] 2024-07-10 15:31:54,840 >> {'loss': 7.8132, 'learning_rate': 2.5000e-08, 'epoch': 0.04, 'throughput': 952.51}
+
+[INFO|callbacks.py:310] 2024-07-10 15:32:07,885 >> {'loss': 7.7906, 'learning_rate': 3.3333e-08, 'epoch': 0.05, 'throughput': 967.80}
+
+[INFO|callbacks.py:310] 2024-07-10 15:32:20,925 >> {'loss': 7.7763, 'learning_rate': 4.1667e-08, 'epoch': 0.06, 'throughput': 980.16}
+
+[INFO|callbacks.py:310] 2024-07-10 15:32:33,999 >> {'loss': 7.7824, 'learning_rate': 5.0000e-08, 'epoch': 0.08, 'throughput': 995.20}
+
+[INFO|callbacks.py:310] 2024-07-10 15:32:47,054 >> {'loss': 7.8330, 'learning_rate': 5.8333e-08, 'epoch': 0.09, 'throughput': 996.54}
+
+[INFO|callbacks.py:310] 2024-07-10 15:33:00,116 >> {'loss': 7.6949, 'learning_rate': 6.6667e-08, 'epoch': 0.10, 'throughput': 994.24}
+
+[INFO|callbacks.py:310] 2024-07-10 15:33:13,214 >> {'loss': 7.8336, 'learning_rate': 7.5000e-08, 'epoch': 0.12, 'throughput': 998.22}
+
+[INFO|callbacks.py:310] 2024-07-10 15:33:26,241 >> {'loss': 7.7282, 'learning_rate': 8.3333e-08, 'epoch': 0.13, 'throughput': 1000.27}
+
+[INFO|callbacks.py:310] 2024-07-10 15:33:39,293 >> {'loss': 7.6916, 'learning_rate': 9.1667e-08, 'epoch': 0.14, 'throughput': 1005.49}
+
+[INFO|callbacks.py:310] 2024-07-10 15:33:52,303 >> {'loss': 7.7333, 'learning_rate': 1.0000e-07, 'epoch': 0.15, 'throughput': 1002.74}
+
+[INFO|callbacks.py:310] 2024-07-10 15:34:05,346 >> {'loss': 7.6017, 'learning_rate': 1.0833e-07, 'epoch': 0.17, 'throughput': 1000.21}
+
+[INFO|callbacks.py:310] 2024-07-10 15:34:18,384 >> {'loss': 7.6440, 'learning_rate': 1.1667e-07, 'epoch': 0.18, 'throughput': 1004.07}
+
+[INFO|callbacks.py:310] 2024-07-10 15:34:31,502 >> {'loss': 7.5965, 'learning_rate': 1.2500e-07, 'epoch': 0.19, 'throughput': 1003.49}
+
+[INFO|callbacks.py:310] 2024-07-10 15:34:44,580 >> {'loss': 7.5883, 'learning_rate': 1.3333e-07, 'epoch': 0.21, 'throughput': 1006.03}
+
+[INFO|callbacks.py:310] 2024-07-10 15:34:57,629 >> {'loss': 7.2464, 'learning_rate': 1.4167e-07, 'epoch': 0.22, 'throughput': 1005.85}
+
+[INFO|callbacks.py:310] 2024-07-10 15:35:10,663 >> {'loss': 7.3133, 'learning_rate': 1.5000e-07, 'epoch': 0.23, 'throughput': 1008.31}
+
+[INFO|callbacks.py:310] 2024-07-10 15:35:23,706 >> {'loss': 7.2133, 'learning_rate': 1.5833e-07, 'epoch': 0.24, 'throughput': 1006.28}
+
+[INFO|callbacks.py:310] 2024-07-10 15:35:36,722 >> {'loss': 7.2431, 'learning_rate': 1.6667e-07, 'epoch': 0.26, 'throughput': 1006.36}
+
+[INFO|callbacks.py:310] 2024-07-10 15:35:49,807 >> {'loss': 7.1875, 'learning_rate': 1.7500e-07, 'epoch': 0.27, 'throughput': 1009.41}
+
+[INFO|callbacks.py:310] 2024-07-10 15:36:02,888 >> {'loss': 7.0659, 'learning_rate': 1.8333e-07, 'epoch': 0.28, 'throughput': 1011.98}
+
+[INFO|callbacks.py:310] 2024-07-10 15:36:16,015 >> {'loss': 6.3595, 'learning_rate': 1.9167e-07, 'epoch': 0.30, 'throughput': 1012.60}
+
+[INFO|callbacks.py:310] 2024-07-10 15:36:29,093 >> {'loss': 6.0417, 'learning_rate': 2.0000e-07, 'epoch': 0.31, 'throughput': 1015.95}
+
+[INFO|callbacks.py:310] 2024-07-10 15:36:42,120 >> {'loss': 5.9894, 'learning_rate': 2.0833e-07, 'epoch': 0.32, 'throughput': 1016.08}
+
+[INFO|callbacks.py:310] 2024-07-10 15:36:55,176 >> {'loss': 5.9259, 'learning_rate': 2.1667e-07, 'epoch': 0.33, 'throughput': 1018.08}
+
+[INFO|callbacks.py:310] 2024-07-10 15:37:08,224 >> {'loss': 5.8983, 'learning_rate': 2.2500e-07, 'epoch': 0.35, 'throughput': 1018.88}
+
+[INFO|callbacks.py:310] 2024-07-10 15:37:21,315 >> {'loss': 5.6848, 'learning_rate': 2.3333e-07, 'epoch': 0.36, 'throughput': 1019.07}
+
+[INFO|callbacks.py:310] 2024-07-10 15:37:34,363 >> {'loss': 5.5649, 'learning_rate': 2.4167e-07, 'epoch': 0.37, 'throughput': 1018.19}
+
+[INFO|callbacks.py:310] 2024-07-10 15:37:47,442 >> {'loss': 5.4642, 'learning_rate': 2.5000e-07, 'epoch': 0.39, 'throughput': 1018.74}
+
+[INFO|callbacks.py:310] 2024-07-10 15:38:00,541 >> {'loss': 4.7955, 'learning_rate': 2.5833e-07, 'epoch': 0.40, 'throughput': 1019.13}
+
+[INFO|callbacks.py:310] 2024-07-10 15:38:13,587 >> {'loss': 2.8339, 'learning_rate': 2.6667e-07, 'epoch': 0.41, 'throughput': 1019.78}
+
+[INFO|callbacks.py:310] 2024-07-10 15:38:26,680 >> {'loss': 2.4477, 'learning_rate': 2.7500e-07, 'epoch': 0.42, 'throughput': 1021.60}
+
+[INFO|callbacks.py:310] 2024-07-10 15:38:39,757 >> {'loss': 2.3331, 'learning_rate': 2.8333e-07, 'epoch': 0.44, 'throughput': 1023.57}
+
+[INFO|callbacks.py:310] 2024-07-10 15:38:52,796 >> {'loss': 2.2143, 'learning_rate': 2.9167e-07, 'epoch': 0.45, 'throughput': 1024.12}
+
+[INFO|callbacks.py:310] 2024-07-10 15:39:05,831 >> {'loss': 2.0067, 'learning_rate': 3.0000e-07, 'epoch': 0.46, 'throughput': 1023.09}
+
+[INFO|callbacks.py:310] 2024-07-10 15:39:18,925 >> {'loss': 1.7702, 'learning_rate': 3.0833e-07, 'epoch': 0.48, 'throughput': 1022.13}
+
+[INFO|callbacks.py:310] 2024-07-10 15:39:31,996 >> {'loss': 1.5557, 'learning_rate': 3.1667e-07, 'epoch': 0.49, 'throughput': 1022.16}
+
+[INFO|callbacks.py:310] 2024-07-10 15:39:45,035 >> {'loss': 1.3024, 'learning_rate': 3.2500e-07, 'epoch': 0.50, 'throughput': 1022.51}
+
+[INFO|callbacks.py:310] 2024-07-10 15:39:58,045 >> {'loss': 1.1652, 'learning_rate': 3.3333e-07, 'epoch': 0.51, 'throughput': 1022.95}
+
+[INFO|callbacks.py:310] 2024-07-10 15:40:11,137 >> {'loss': 0.6839, 'learning_rate': 3.4167e-07, 'epoch': 0.53, 'throughput': 1024.05}
+
+[INFO|callbacks.py:310] 2024-07-10 15:40:24,186 >> {'loss': 0.4774, 'learning_rate': 3.5000e-07, 'epoch': 0.54, 'throughput': 1024.88}
+
+[INFO|callbacks.py:310] 2024-07-10 15:40:37,254 >> {'loss': 0.3841, 'learning_rate': 3.5833e-07, 'epoch': 0.55, 'throughput': 1024.68}
+
+[INFO|callbacks.py:310] 2024-07-10 15:40:50,317 >> {'loss': 0.3588, 'learning_rate': 3.6667e-07, 'epoch': 0.57, 'throughput': 1025.16}
+
+[INFO|callbacks.py:310] 2024-07-10 15:41:03,428 >> {'loss': 0.3628, 'learning_rate': 3.7500e-07, 'epoch': 0.58, 'throughput': 1025.76}
+
+[INFO|callbacks.py:310] 2024-07-10 15:41:16,474 >> {'loss': 0.3426, 'learning_rate': 3.8333e-07, 'epoch': 0.59, 'throughput': 1025.64}
+
+[INFO|callbacks.py:310] 2024-07-10 15:41:29,543 >> {'loss': 0.3279, 'learning_rate': 3.9167e-07, 'epoch': 0.60, 'throughput': 1026.06}
+
+[INFO|callbacks.py:310] 2024-07-10 15:41:42,615 >> {'loss': 0.3947, 'learning_rate': 4.0000e-07, 'epoch': 0.62, 'throughput': 1026.56}
+
+[INFO|callbacks.py:310] 2024-07-10 15:41:55,704 >> {'loss': 0.3075, 'learning_rate': 4.0833e-07, 'epoch': 0.63, 'throughput': 1027.66}
+
+[INFO|callbacks.py:310] 2024-07-10 15:42:08,756 >> {'loss': 0.3236, 'learning_rate': 4.1667e-07, 'epoch': 0.64, 'throughput': 1028.43}
+
+[INFO|callbacks.py:310] 2024-07-10 15:42:21,803 >> {'loss': 0.3557, 'learning_rate': 4.2500e-07, 'epoch': 0.66, 'throughput': 1028.37}
+
+[INFO|callbacks.py:310] 2024-07-10 15:42:34,889 >> {'loss': 0.4008, 'learning_rate': 4.3333e-07, 'epoch': 0.67, 'throughput': 1028.30}
+
+[INFO|callbacks.py:310] 2024-07-10 15:42:47,969 >> {'loss': 0.3586, 'learning_rate': 4.4167e-07, 'epoch': 0.68, 'throughput': 1028.93}
+
+[INFO|callbacks.py:310] 2024-07-10 15:43:00,979 >> {'loss': 0.3023, 'learning_rate': 4.5000e-07, 'epoch': 0.69, 'throughput': 1027.78}
+
+[INFO|callbacks.py:310] 2024-07-10 15:43:14,044 >> {'loss': 0.3547, 'learning_rate': 4.5833e-07, 'epoch': 0.71, 'throughput': 1028.91}
+
+[INFO|callbacks.py:310] 2024-07-10 15:43:27,066 >> {'loss': 0.3846, 'learning_rate': 4.6667e-07, 'epoch': 0.72, 'throughput': 1028.23}
+
+[INFO|callbacks.py:310] 2024-07-10 15:43:40,122 >> {'loss': 0.3743, 'learning_rate': 4.7500e-07, 'epoch': 0.73, 'throughput': 1028.85}
+
+[INFO|callbacks.py:310] 2024-07-10 15:43:53,200 >> {'loss': 0.3091, 'learning_rate': 4.8333e-07, 'epoch': 0.75, 'throughput': 1029.54}
+
+[INFO|callbacks.py:310] 2024-07-10 15:44:06,285 >> {'loss': 0.3094, 'learning_rate': 4.9167e-07, 'epoch': 0.76, 'throughput': 1028.92}
+
+[INFO|callbacks.py:310] 2024-07-10 15:44:19,363 >> {'loss': 0.3309, 'learning_rate': 5.0000e-07, 'epoch': 0.77, 'throughput': 1029.14}
+
+[INFO|callbacks.py:310] 2024-07-10 15:44:32,413 >> {'loss': 0.3276, 'learning_rate': 5.0833e-07, 'epoch': 0.78, 'throughput': 1029.68}
+
+[INFO|callbacks.py:310] 2024-07-10 15:44:45,451 >> {'loss': 0.3084, 'learning_rate': 5.1667e-07, 'epoch': 0.80, 'throughput': 1029.34}
+
+[INFO|callbacks.py:310] 2024-07-10 15:44:58,517 >> {'loss': 0.3182, 'learning_rate': 5.2500e-07, 'epoch': 0.81, 'throughput': 1029.83}
+
+[INFO|callbacks.py:310] 2024-07-10 15:45:11,542 >> {'loss': 0.3469, 'learning_rate': 5.3333e-07, 'epoch': 0.82, 'throughput': 1029.25}
+
+[INFO|callbacks.py:310] 2024-07-10 15:45:24,606 >> {'loss': 0.3253, 'learning_rate': 5.4167e-07, 'epoch': 0.84, 'throughput': 1029.85}
+
+[INFO|callbacks.py:310] 2024-07-10 15:45:37,676 >> {'loss': 0.2746, 'learning_rate': 5.5000e-07, 'epoch': 0.85, 'throughput': 1030.30}
+
+[INFO|callbacks.py:310] 2024-07-10 15:45:50,772 >> {'loss': 0.2893, 'learning_rate': 5.5833e-07, 'epoch': 0.86, 'throughput': 1031.01}
+
+[INFO|callbacks.py:310] 2024-07-10 15:46:03,815 >> {'loss': 0.2827, 'learning_rate': 5.6667e-07, 'epoch': 0.87, 'throughput': 1030.43}
+
+[INFO|callbacks.py:310] 2024-07-10 15:46:16,892 >> {'loss': 0.2978, 'learning_rate': 5.7500e-07, 'epoch': 0.89, 'throughput': 1030.71}
+
+[INFO|callbacks.py:310] 2024-07-10 15:46:29,914 >> {'loss': 0.2703, 'learning_rate': 5.8333e-07, 'epoch': 0.90, 'throughput': 1029.78}
+
+[INFO|callbacks.py:310] 2024-07-10 15:46:42,975 >> {'loss': 0.2968, 'learning_rate': 5.9167e-07, 'epoch': 0.91, 'throughput': 1029.88}
+
+[INFO|callbacks.py:310] 2024-07-10 15:46:56,043 >> {'loss': 0.3035, 'learning_rate': 6.0000e-07, 'epoch': 0.93, 'throughput': 1030.06}
+
+[INFO|callbacks.py:310] 2024-07-10 15:47:09,118 >> {'loss': 0.3211, 'learning_rate': 6.0833e-07, 'epoch': 0.94, 'throughput': 1030.07}
+
+[INFO|callbacks.py:310] 2024-07-10 15:47:22,193 >> {'loss': 0.2913, 'learning_rate': 6.1667e-07, 'epoch': 0.95, 'throughput': 1030.00}
+
+[INFO|callbacks.py:310] 2024-07-10 15:47:35,264 >> {'loss': 0.2817, 'learning_rate': 6.2500e-07, 'epoch': 0.96, 'throughput': 1029.88}
+
+[INFO|callbacks.py:310] 2024-07-10 15:47:48,300 >> {'loss': 0.2827, 'learning_rate': 6.3333e-07, 'epoch': 0.98, 'throughput': 1029.93}
+
+[INFO|callbacks.py:310] 2024-07-10 15:48:01,345 >> {'loss': 0.2290, 'learning_rate': 6.4167e-07, 'epoch': 0.99, 'throughput': 1029.81}
+
+[INFO|callbacks.py:310] 2024-07-10 15:48:14,403 >> {'loss': 0.2503, 'learning_rate': 6.5000e-07, 'epoch': 1.00, 'throughput': 1030.33}
+
+[INFO|callbacks.py:310] 2024-07-10 15:48:27,482 >> {'loss': 0.2453, 'learning_rate': 6.5833e-07, 'epoch': 1.02, 'throughput': 1030.76}
+
+[INFO|callbacks.py:310] 2024-07-10 15:48:40,533 >> {'loss': 0.2167, 'learning_rate': 6.6667e-07, 'epoch': 1.03, 'throughput': 1030.29}
+
+[INFO|callbacks.py:310] 2024-07-10 15:48:53,615 >> {'loss': 0.2361, 'learning_rate': 6.7500e-07, 'epoch': 1.04, 'throughput': 1030.47}
+
+[INFO|callbacks.py:310] 2024-07-10 15:49:06,694 >> {'loss': 0.2248, 'learning_rate': 6.8333e-07, 'epoch': 1.05, 'throughput': 1030.91}
+
+[INFO|callbacks.py:310] 2024-07-10 15:49:19,720 >> {'loss': 0.2491, 'learning_rate': 6.9167e-07, 'epoch': 1.07, 'throughput': 1030.71}
+
+[INFO|callbacks.py:310] 2024-07-10 15:49:32,815 >> {'loss': 0.2352, 'learning_rate': 7.0000e-07, 'epoch': 1.08, 'throughput': 1031.10}
+
+[INFO|callbacks.py:310] 2024-07-10 15:49:45,904 >> {'loss': 0.2365, 'learning_rate': 7.0833e-07, 'epoch': 1.09, 'throughput': 1031.51}
+
+[INFO|callbacks.py:310] 2024-07-10 15:49:58,930 >> {'loss': 0.2170, 'learning_rate': 7.1667e-07, 'epoch': 1.11, 'throughput': 1031.65}
+
+[INFO|callbacks.py:310] 2024-07-10 15:50:11,963 >> {'loss': 0.2258, 'learning_rate': 7.2500e-07, 'epoch': 1.12, 'throughput': 1030.83}
+
+[INFO|callbacks.py:310] 2024-07-10 15:50:25,032 >> {'loss': 0.2450, 'learning_rate': 7.3333e-07, 'epoch': 1.13, 'throughput': 1031.53}
+
+[INFO|callbacks.py:310] 2024-07-10 15:50:38,116 >> {'loss': 0.3132, 'learning_rate': 7.4167e-07, 'epoch': 1.14, 'throughput': 1031.26}
+
+[INFO|callbacks.py:310] 2024-07-10 15:50:51,182 >> {'loss': 0.2840, 'learning_rate': 7.5000e-07, 'epoch': 1.16, 'throughput': 1031.93}
+
+[INFO|callbacks.py:310] 2024-07-10 15:51:04,209 >> {'loss': 0.1933, 'learning_rate': 7.5833e-07, 'epoch': 1.17, 'throughput': 1032.15}
+
+[INFO|callbacks.py:310] 2024-07-10 15:51:17,279 >> {'loss': 0.2154, 'learning_rate': 7.6667e-07, 'epoch': 1.18, 'throughput': 1032.42}
+
+[INFO|callbacks.py:310] 2024-07-10 15:51:30,278 >> {'loss': 0.2064, 'learning_rate': 7.7500e-07, 'epoch': 1.20, 'throughput': 1032.18}
+
+[INFO|callbacks.py:310] 2024-07-10 15:51:43,357 >> {'loss': 0.2038, 'learning_rate': 7.8333e-07, 'epoch': 1.21, 'throughput': 1032.52}
+
+[INFO|callbacks.py:310] 2024-07-10 15:51:56,455 >> {'loss': 0.2152, 'learning_rate': 7.9167e-07, 'epoch': 1.22, 'throughput': 1033.74}
+
+[INFO|callbacks.py:310] 2024-07-10 15:52:09,536 >> {'loss': 0.1961, 'learning_rate': 8.0000e-07, 'epoch': 1.23, 'throughput': 1033.85}
+
+[INFO|callbacks.py:310] 2024-07-10 15:52:22,616 >> {'loss': 0.1772, 'learning_rate': 8.0833e-07, 'epoch': 1.25, 'throughput': 1033.81}
+
+[INFO|callbacks.py:310] 2024-07-10 15:52:35,723 >> {'loss': 0.1846, 'learning_rate': 8.1667e-07, 'epoch': 1.26, 'throughput': 1033.97}
+
+[INFO|callbacks.py:310] 2024-07-10 15:52:48,763 >> {'loss': 0.1823, 'learning_rate': 8.2500e-07, 'epoch': 1.27, 'throughput': 1034.09}
+
+[INFO|callbacks.py:310] 2024-07-10 15:53:01,762 >> {'loss': 0.1794, 'learning_rate': 8.3333e-07, 'epoch': 1.29, 'throughput': 1033.35}
+
+[INFO|callbacks.py:310] 2024-07-10 15:53:14,835 >> {'loss': 0.2106, 'learning_rate': 8.4167e-07, 'epoch': 1.30, 'throughput': 1033.90}
+
+[INFO|callbacks.py:310] 2024-07-10 15:53:27,876 >> {'loss': 0.2123, 'learning_rate': 8.5000e-07, 'epoch': 1.31, 'throughput': 1033.68}
+
+[INFO|callbacks.py:310] 2024-07-10 15:53:40,937 >> {'loss': 0.2413, 'learning_rate': 8.5833e-07, 'epoch': 1.32, 'throughput': 1033.33}
+
+[INFO|callbacks.py:310] 2024-07-10 15:53:54,008 >> {'loss': 0.2334, 'learning_rate': 8.6667e-07, 'epoch': 1.34, 'throughput': 1032.97}
+
+[INFO|callbacks.py:310] 2024-07-10 15:54:07,043 >> {'loss': 0.2069, 'learning_rate': 8.7500e-07, 'epoch': 1.35, 'throughput': 1032.70}
+
+[INFO|callbacks.py:310] 2024-07-10 15:54:20,100 >> {'loss': 0.2262, 'learning_rate': 8.8333e-07, 'epoch': 1.36, 'throughput': 1032.67}
+
+[INFO|callbacks.py:310] 2024-07-10 15:54:33,175 >> {'loss': 0.1718, 'learning_rate': 8.9167e-07, 'epoch': 1.38, 'throughput': 1032.39}
+
+[INFO|callbacks.py:310] 2024-07-10 15:54:46,250 >> {'loss': 0.2040, 'learning_rate': 9.0000e-07, 'epoch': 1.39, 'throughput': 1032.58}
+
+[INFO|callbacks.py:310] 2024-07-10 15:54:59,311 >> {'loss': 0.1849, 'learning_rate': 9.0833e-07, 'epoch': 1.40, 'throughput': 1032.91}
+
+[INFO|callbacks.py:310] 2024-07-10 15:55:12,379 >> {'loss': 0.2028, 'learning_rate': 9.1667e-07, 'epoch': 1.41, 'throughput': 1033.00}
+
+[INFO|callbacks.py:310] 2024-07-10 15:55:25,480 >> {'loss': 0.1790, 'learning_rate': 9.2500e-07, 'epoch': 1.43, 'throughput': 1033.15}
+
+[INFO|callbacks.py:310] 2024-07-10 15:55:38,555 >> {'loss': 0.1813, 'learning_rate': 9.3333e-07, 'epoch': 1.44, 'throughput': 1033.22}
+
+[INFO|callbacks.py:310] 2024-07-10 15:55:51,609 >> {'loss': 0.1955, 'learning_rate': 9.4167e-07, 'epoch': 1.45, 'throughput': 1033.14}
+
+[INFO|callbacks.py:310] 2024-07-10 15:56:04,654 >> {'loss': 0.1577, 'learning_rate': 9.5000e-07, 'epoch': 1.47, 'throughput': 1032.82}
+
+[INFO|callbacks.py:310] 2024-07-10 15:56:17,693 >> {'loss': 0.1509, 'learning_rate': 9.5833e-07, 'epoch': 1.48, 'throughput': 1032.48}
+
+[INFO|callbacks.py:310] 2024-07-10 15:56:30,731 >> {'loss': 0.2052, 'learning_rate': 9.6667e-07, 'epoch': 1.49, 'throughput': 1031.98}
+
+[INFO|callbacks.py:310] 2024-07-10 15:56:43,807 >> {'loss': 0.1576, 'learning_rate': 9.7500e-07, 'epoch': 1.50, 'throughput': 1031.99}
+
+[INFO|callbacks.py:310] 2024-07-10 15:56:56,876 >> {'loss': 0.1459, 'learning_rate': 9.8333e-07, 'epoch': 1.52, 'throughput': 1031.67}
+
+[INFO|callbacks.py:310] 2024-07-10 15:57:09,932 >> {'loss': 0.2694, 'learning_rate': 9.9167e-07, 'epoch': 1.53, 'throughput': 1031.92}
+
+[INFO|callbacks.py:310] 2024-07-10 15:57:22,991 >> {'loss': 0.1891, 'learning_rate': 1.0000e-06, 'epoch': 1.54, 'throughput': 1031.95}
+
+[INFO|callbacks.py:310] 2024-07-10 15:57:36,004 >> {'loss': 0.1655, 'learning_rate': 1.0083e-06, 'epoch': 1.56, 'throughput': 1031.72}
+
+[INFO|callbacks.py:310] 2024-07-10 15:57:49,058 >> {'loss': 0.1534, 'learning_rate': 1.0167e-06, 'epoch': 1.57, 'throughput': 1031.72}
+
+[INFO|callbacks.py:310] 2024-07-10 15:58:02,124 >> {'loss': 0.1373, 'learning_rate': 1.0250e-06, 'epoch': 1.58, 'throughput': 1031.81}
+
+[INFO|callbacks.py:310] 2024-07-10 15:58:15,198 >> {'loss': 0.1528, 'learning_rate': 1.0333e-06, 'epoch': 1.59, 'throughput': 1031.84}
+
+[INFO|callbacks.py:310] 2024-07-10 15:58:28,311 >> {'loss': 0.2017, 'learning_rate': 1.0417e-06, 'epoch': 1.61, 'throughput': 1032.20}
+
+[INFO|callbacks.py:310] 2024-07-10 15:58:41,392 >> {'loss': 0.1554, 'learning_rate': 1.0500e-06, 'epoch': 1.62, 'throughput': 1032.47}
+
+[INFO|callbacks.py:310] 2024-07-10 15:58:54,437 >> {'loss': 0.1332, 'learning_rate': 1.0583e-06, 'epoch': 1.63, 'throughput': 1032.68}
+
+[INFO|callbacks.py:310] 2024-07-10 15:59:07,498 >> {'loss': 0.1150, 'learning_rate': 1.0667e-06, 'epoch': 1.65, 'throughput': 1033.04}
+
+[INFO|callbacks.py:310] 2024-07-10 15:59:20,535 >> {'loss': 0.1190, 'learning_rate': 1.0750e-06, 'epoch': 1.66, 'throughput': 1032.57}
+
+[INFO|callbacks.py:310] 2024-07-10 15:59:33,580 >> {'loss': 0.1164, 'learning_rate': 1.0833e-06, 'epoch': 1.67, 'throughput': 1032.98}
+
+[INFO|callbacks.py:310] 2024-07-10 15:59:46,641 >> {'loss': 0.1981, 'learning_rate': 1.0917e-06, 'epoch': 1.68, 'throughput': 1033.03}
+
+[INFO|callbacks.py:310] 2024-07-10 15:59:59,719 >> {'loss': 0.1680, 'learning_rate': 1.1000e-06, 'epoch': 1.70, 'throughput': 1032.86}
+
+[INFO|callbacks.py:310] 2024-07-10 16:00:12,811 >> {'loss': 0.0741, 'learning_rate': 1.1083e-06, 'epoch': 1.71, 'throughput': 1032.96}
+
+[INFO|callbacks.py:310] 2024-07-10 16:00:25,847 >> {'loss': 0.1847, 'learning_rate': 1.1167e-06, 'epoch': 1.72, 'throughput': 1032.74}
+
+[INFO|callbacks.py:310] 2024-07-10 16:00:38,940 >> {'loss': 0.1080, 'learning_rate': 1.1250e-06, 'epoch': 1.74, 'throughput': 1032.88}
+
+[INFO|callbacks.py:310] 2024-07-10 16:00:51,967 >> {'loss': 0.1214, 'learning_rate': 1.1333e-06, 'epoch': 1.75, 'throughput': 1032.96}
+
+[INFO|callbacks.py:310] 2024-07-10 16:01:05,023 >> {'loss': 0.1252, 'learning_rate': 1.1417e-06, 'epoch': 1.76, 'throughput': 1033.06}
+
+[INFO|callbacks.py:310] 2024-07-10 16:01:18,067 >> {'loss': 0.1440, 'learning_rate': 1.1500e-06, 'epoch': 1.77, 'throughput': 1032.84}
+
+[INFO|callbacks.py:310] 2024-07-10 16:01:31,137 >> {'loss': 0.1269, 'learning_rate': 1.1583e-06, 'epoch': 1.79, 'throughput': 1032.79}
+
+[INFO|callbacks.py:310] 2024-07-10 16:01:44,213 >> {'loss': 0.1283, 'learning_rate': 1.1667e-06, 'epoch': 1.80, 'throughput': 1032.81}
+
+[INFO|callbacks.py:310] 2024-07-10 16:01:57,241 >> {'loss': 0.0929, 'learning_rate': 1.1750e-06, 'epoch': 1.81, 'throughput': 1032.78}
+
+[INFO|callbacks.py:310] 2024-07-10 16:02:10,258 >> {'loss': 0.1349, 'learning_rate': 1.1833e-06, 'epoch': 1.83, 'throughput': 1032.56}
+
+[INFO|callbacks.py:310] 2024-07-10 16:02:23,316 >> {'loss': 0.1277, 'learning_rate': 1.1917e-06, 'epoch': 1.84, 'throughput': 1032.33}
+
+[INFO|callbacks.py:310] 2024-07-10 16:02:36,363 >> {'loss': 0.1585, 'learning_rate': 1.2000e-06, 'epoch': 1.85, 'throughput': 1032.35}
+
+[INFO|callbacks.py:310] 2024-07-10 16:02:49,387 >> {'loss': 0.1468, 'learning_rate': 1.2083e-06, 'epoch': 1.86, 'throughput': 1031.96}
+
+[INFO|callbacks.py:310] 2024-07-10 16:03:02,421 >> {'loss': 0.1049, 'learning_rate': 1.2167e-06, 'epoch': 1.88, 'throughput': 1031.92}
+
+[INFO|callbacks.py:310] 2024-07-10 16:03:15,527 >> {'loss': 0.1297, 'learning_rate': 1.2250e-06, 'epoch': 1.89, 'throughput': 1031.96}
+
+[INFO|callbacks.py:310] 2024-07-10 16:03:28,588 >> {'loss': 0.1111, 'learning_rate': 1.2333e-06, 'epoch': 1.90, 'throughput': 1031.79}
+
+[INFO|callbacks.py:310] 2024-07-10 16:03:41,633 >> {'loss': 0.1202, 'learning_rate': 1.2417e-06, 'epoch': 1.92, 'throughput': 1032.01}
+
+[INFO|callbacks.py:310] 2024-07-10 16:03:54,708 >> {'loss': 0.0829, 'learning_rate': 1.2500e-06, 'epoch': 1.93, 'throughput': 1032.05}
+
+[INFO|callbacks.py:310] 2024-07-10 16:04:07,790 >> {'loss': 0.1119, 'learning_rate': 1.2583e-06, 'epoch': 1.94, 'throughput': 1032.55}
+
+[INFO|callbacks.py:310] 2024-07-10 16:04:20,848 >> {'loss': 0.1144, 'learning_rate': 1.2667e-06, 'epoch': 1.95, 'throughput': 1032.51}
+
+[INFO|callbacks.py:310] 2024-07-10 16:04:33,876 >> {'loss': 0.1170, 'learning_rate': 1.2750e-06, 'epoch': 1.97, 'throughput': 1031.96}
+
+[INFO|callbacks.py:310] 2024-07-10 16:04:46,937 >> {'loss': 0.0998, 'learning_rate': 1.2833e-06, 'epoch': 1.98, 'throughput': 1031.63}
+
+[INFO|callbacks.py:310] 2024-07-10 16:05:00,031 >> {'loss': 0.1384, 'learning_rate': 1.2917e-06, 'epoch': 1.99, 'throughput': 1031.88}
+
+[INFO|callbacks.py:310] 2024-07-10 16:05:13,054 >> {'loss': 0.1157, 'learning_rate': 1.3000e-06, 'epoch': 2.01, 'throughput': 1031.69}
+
+[INFO|callbacks.py:310] 2024-07-10 16:05:26,109 >> {'loss': 0.0696, 'learning_rate': 1.3083e-06, 'epoch': 2.02, 'throughput': 1031.58}
+
+[INFO|callbacks.py:310] 2024-07-10 16:05:39,158 >> {'loss': 0.0665, 'learning_rate': 1.3167e-06, 'epoch': 2.03, 'throughput': 1031.54}
+
+[INFO|callbacks.py:310] 2024-07-10 16:05:52,234 >> {'loss': 0.0783, 'learning_rate': 1.3250e-06, 'epoch': 2.05, 'throughput': 1031.64}
+
+[INFO|callbacks.py:310] 2024-07-10 16:06:05,266 >> {'loss': 0.0749, 'learning_rate': 1.3333e-06, 'epoch': 2.06, 'throughput': 1031.29}
+
+[INFO|callbacks.py:310] 2024-07-10 16:06:18,321 >> {'loss': 0.0731, 'learning_rate': 1.3417e-06, 'epoch': 2.07, 'throughput': 1031.12}
+
+[INFO|callbacks.py:310] 2024-07-10 16:06:31,379 >> {'loss': 0.0913, 'learning_rate': 1.3500e-06, 'epoch': 2.08, 'throughput': 1030.87}
+
+[INFO|callbacks.py:310] 2024-07-10 16:06:44,451 >> {'loss': 0.0521, 'learning_rate': 1.3583e-06, 'epoch': 2.10, 'throughput': 1030.82}
+
+[INFO|callbacks.py:310] 2024-07-10 16:06:57,506 >> {'loss': 0.0680, 'learning_rate': 1.3667e-06, 'epoch': 2.11, 'throughput': 1030.71}
+
+[INFO|callbacks.py:310] 2024-07-10 16:07:10,589 >> {'loss': 0.0686, 'learning_rate': 1.3750e-06, 'epoch': 2.12, 'throughput': 1030.53}
+
+[INFO|callbacks.py:310] 2024-07-10 16:07:23,614 >> {'loss': 0.0545, 'learning_rate': 1.3833e-06, 'epoch': 2.14, 'throughput': 1030.54}
+
+[INFO|callbacks.py:310] 2024-07-10 16:07:36,662 >> {'loss': 0.0347, 'learning_rate': 1.3917e-06, 'epoch': 2.15, 'throughput': 1030.37}
+
+[INFO|callbacks.py:310] 2024-07-10 16:07:49,739 >> {'loss': 0.0993, 'learning_rate': 1.4000e-06, 'epoch': 2.16, 'throughput': 1030.36}
+
+[INFO|callbacks.py:310] 2024-07-10 16:08:02,845 >> {'loss': 0.1059, 'learning_rate': 1.4083e-06, 'epoch': 2.17, 'throughput': 1030.18}
+
+[INFO|callbacks.py:310] 2024-07-10 16:08:15,921 >> {'loss': 0.0890, 'learning_rate': 1.4167e-06, 'epoch': 2.19, 'throughput': 1030.24}
+
+[INFO|callbacks.py:310] 2024-07-10 16:08:28,943 >> {'loss': 0.0379, 'learning_rate': 1.4250e-06, 'epoch': 2.20, 'throughput': 1030.07}
+
+[INFO|callbacks.py:310] 2024-07-10 16:08:41,971 >> {'loss': 0.0626, 'learning_rate': 1.4333e-06, 'epoch': 2.21, 'throughput': 1030.20}
+
+[INFO|callbacks.py:310] 2024-07-10 16:08:55,075 >> {'loss': 0.0957, 'learning_rate': 1.4417e-06, 'epoch': 2.23, 'throughput': 1030.64}
+
+[INFO|callbacks.py:310] 2024-07-10 16:09:08,128 >> {'loss': 0.0636, 'learning_rate': 1.4500e-06, 'epoch': 2.24, 'throughput': 1030.75}
+
+[INFO|callbacks.py:310] 2024-07-10 16:09:21,197 >> {'loss': 0.0740, 'learning_rate': 1.4583e-06, 'epoch': 2.25, 'throughput': 1030.71}
+
+[INFO|callbacks.py:310] 2024-07-10 16:09:34,272 >> {'loss': 0.0685, 'learning_rate': 1.4667e-06, 'epoch': 2.26, 'throughput': 1030.69}
+
+[INFO|callbacks.py:310] 2024-07-10 16:09:47,345 >> {'loss': 0.0574, 'learning_rate': 1.4750e-06, 'epoch': 2.28, 'throughput': 1030.53}
+
+[INFO|callbacks.py:310] 2024-07-10 16:10:00,383 >> {'loss': 0.0619, 'learning_rate': 1.4833e-06, 'epoch': 2.29, 'throughput': 1030.73}
+
+[INFO|callbacks.py:310] 2024-07-10 16:10:13,410 >> {'loss': 0.0683, 'learning_rate': 1.4917e-06, 'epoch': 2.30, 'throughput': 1030.36}
+
+[INFO|callbacks.py:310] 2024-07-10 16:10:26,483 >> {'loss': 0.0700, 'learning_rate': 1.5000e-06, 'epoch': 2.32, 'throughput': 1030.33}
+
+[INFO|callbacks.py:310] 2024-07-10 16:10:39,532 >> {'loss': 0.1154, 'learning_rate': 1.5083e-06, 'epoch': 2.33, 'throughput': 1030.26}
+
+[INFO|callbacks.py:310] 2024-07-10 16:10:52,574 >> {'loss': 0.0923, 'learning_rate': 1.5167e-06, 'epoch': 2.34, 'throughput': 1030.40}
+
+[INFO|callbacks.py:310] 2024-07-10 16:11:05,633 >> {'loss': 0.0777, 'learning_rate': 1.5250e-06, 'epoch': 2.35, 'throughput': 1030.14}
+
+[INFO|callbacks.py:310] 2024-07-10 16:11:18,722 >> {'loss': 0.0754, 'learning_rate': 1.5333e-06, 'epoch': 2.37, 'throughput': 1030.34}
+
+[INFO|callbacks.py:310] 2024-07-10 16:11:31,778 >> {'loss': 0.0704, 'learning_rate': 1.5417e-06, 'epoch': 2.38, 'throughput': 1030.27}
+
+[INFO|callbacks.py:310] 2024-07-10 16:11:44,816 >> {'loss': 0.0915, 'learning_rate': 1.5500e-06, 'epoch': 2.39, 'throughput': 1030.38}
+
+[INFO|callbacks.py:310] 2024-07-10 16:11:57,871 >> {'loss': 0.0870, 'learning_rate': 1.5583e-06, 'epoch': 2.41, 'throughput': 1030.22}
+
+[INFO|callbacks.py:310] 2024-07-10 16:12:10,914 >> {'loss': 0.0566, 'learning_rate': 1.5667e-06, 'epoch': 2.42, 'throughput': 1030.35}
+
+[INFO|callbacks.py:310] 2024-07-10 16:12:23,986 >> {'loss': 0.1037, 'learning_rate': 1.5750e-06, 'epoch': 2.43, 'throughput': 1030.56}
+
+[INFO|callbacks.py:310] 2024-07-10 16:12:37,049 >> {'loss': 0.1143, 'learning_rate': 1.5833e-06, 'epoch': 2.44, 'throughput': 1030.68}
+
+[INFO|callbacks.py:310] 2024-07-10 16:12:50,149 >> {'loss': 0.0829, 'learning_rate': 1.5917e-06, 'epoch': 2.46, 'throughput': 1030.82}
+
+[INFO|callbacks.py:310] 2024-07-10 16:13:03,221 >> {'loss': 0.0422, 'learning_rate': 1.6000e-06, 'epoch': 2.47, 'throughput': 1030.87}
+
+[INFO|callbacks.py:310] 2024-07-10 16:13:16,270 >> {'loss': 0.0727, 'learning_rate': 1.6083e-06, 'epoch': 2.48, 'throughput': 1030.95}
+
+[INFO|callbacks.py:310] 2024-07-10 16:13:29,306 >> {'loss': 0.0836, 'learning_rate': 1.6167e-06, 'epoch': 2.50, 'throughput': 1030.88}
+
+[INFO|callbacks.py:310] 2024-07-10 16:13:42,353 >> {'loss': 0.0803, 'learning_rate': 1.6250e-06, 'epoch': 2.51, 'throughput': 1030.85}
+
+[INFO|callbacks.py:310] 2024-07-10 16:13:55,421 >> {'loss': 0.0654, 'learning_rate': 1.6333e-06, 'epoch': 2.52, 'throughput': 1030.96}
+
+[INFO|callbacks.py:310] 2024-07-10 16:14:08,471 >> {'loss': 0.0587, 'learning_rate': 1.6417e-06, 'epoch': 2.53, 'throughput': 1030.89}
+
+[INFO|callbacks.py:310] 2024-07-10 16:14:21,530 >> {'loss': 0.0848, 'learning_rate': 1.6500e-06, 'epoch': 2.55, 'throughput': 1030.87}
+
+[INFO|callbacks.py:310] 2024-07-10 16:14:34,591 >> {'loss': 0.0525, 'learning_rate': 1.6583e-06, 'epoch': 2.56, 'throughput': 1030.88}
+
+[INFO|callbacks.py:310] 2024-07-10 16:14:47,595 >> {'loss': 0.0677, 'learning_rate': 1.6667e-06, 'epoch': 2.57, 'throughput': 1030.73}
+
+[INFO|callbacks.py:310] 2024-07-10 16:15:00,635 >> {'loss': 0.0620, 'learning_rate': 1.6750e-06, 'epoch': 2.59, 'throughput': 1030.77}
+
+[INFO|callbacks.py:310] 2024-07-10 16:15:13,672 >> {'loss': 0.0674, 'learning_rate': 1.6833e-06, 'epoch': 2.60, 'throughput': 1030.89}
+
+[INFO|callbacks.py:310] 2024-07-10 16:15:26,718 >> {'loss': 0.0533, 'learning_rate': 1.6917e-06, 'epoch': 2.61, 'throughput': 1030.91}
+
+[INFO|callbacks.py:310] 2024-07-10 16:15:39,798 >> {'loss': 0.0757, 'learning_rate': 1.7000e-06, 'epoch': 2.62, 'throughput': 1031.12}
+
+[INFO|callbacks.py:310] 2024-07-10 16:15:52,886 >> {'loss': 0.0777, 'learning_rate': 1.7083e-06, 'epoch': 2.64, 'throughput': 1031.22}
+
+[INFO|callbacks.py:310] 2024-07-10 16:16:05,977 >> {'loss': 0.0921, 'learning_rate': 1.7167e-06, 'epoch': 2.65, 'throughput': 1031.22}
+
+[INFO|callbacks.py:310] 2024-07-10 16:16:19,012 >> {'loss': 0.0378, 'learning_rate': 1.7250e-06, 'epoch': 2.66, 'throughput': 1031.02}
+
+[INFO|callbacks.py:310] 2024-07-10 16:16:32,072 >> {'loss': 0.0671, 'learning_rate': 1.7333e-06, 'epoch': 2.68, 'throughput': 1031.28}
+
+[INFO|callbacks.py:310] 2024-07-10 16:16:45,154 >> {'loss': 0.0664, 'learning_rate': 1.7417e-06, 'epoch': 2.69, 'throughput': 1031.30}
+
+[INFO|callbacks.py:310] 2024-07-10 16:16:58,186 >> {'loss': 0.0720, 'learning_rate': 1.7500e-06, 'epoch': 2.70, 'throughput': 1031.20}
+
+[INFO|callbacks.py:310] 2024-07-10 16:17:11,271 >> {'loss': 0.0883, 'learning_rate': 1.7583e-06, 'epoch': 2.71, 'throughput': 1031.43}
+
+[INFO|callbacks.py:310] 2024-07-10 16:17:24,320 >> {'loss': 0.0414, 'learning_rate': 1.7667e-06, 'epoch': 2.73, 'throughput': 1031.35}
+
+[INFO|callbacks.py:310] 2024-07-10 16:17:37,394 >> {'loss': 0.0310, 'learning_rate': 1.7750e-06, 'epoch': 2.74, 'throughput': 1031.16}
+
+[INFO|callbacks.py:310] 2024-07-10 16:17:50,443 >> {'loss': 0.0634, 'learning_rate': 1.7833e-06, 'epoch': 2.75, 'throughput': 1031.04}
+
+[INFO|callbacks.py:310] 2024-07-10 16:18:03,511 >> {'loss': 0.0837, 'learning_rate': 1.7917e-06, 'epoch': 2.77, 'throughput': 1031.15}
+
+[INFO|callbacks.py:310] 2024-07-10 16:18:16,574 >> {'loss': 0.0855, 'learning_rate': 1.8000e-06, 'epoch': 2.78, 'throughput': 1031.11}
+
+[INFO|callbacks.py:310] 2024-07-10 16:18:29,592 >> {'loss': 0.0945, 'learning_rate': 1.8083e-06, 'epoch': 2.79, 'throughput': 1030.85}
+
+[INFO|callbacks.py:310] 2024-07-10 16:18:42,612 >> {'loss': 0.0780, 'learning_rate': 1.8167e-06, 'epoch': 2.80, 'throughput': 1030.78}
+
+[INFO|callbacks.py:310] 2024-07-10 16:18:55,683 >> {'loss': 0.0573, 'learning_rate': 1.8250e-06, 'epoch': 2.82, 'throughput': 1030.59}
+
+[INFO|callbacks.py:310] 2024-07-10 16:19:08,787 >> {'loss': 0.0806, 'learning_rate': 1.8333e-06, 'epoch': 2.83, 'throughput': 1030.71}
+
+[INFO|callbacks.py:310] 2024-07-10 16:19:21,866 >> {'loss': 0.0961, 'learning_rate': 1.8417e-06, 'epoch': 2.84, 'throughput': 1030.63}
+
+[INFO|callbacks.py:310] 2024-07-10 16:19:34,886 >> {'loss': 0.0732, 'learning_rate': 1.8500e-06, 'epoch': 2.86, 'throughput': 1030.70}
+
+[INFO|callbacks.py:310] 2024-07-10 16:19:47,985 >> {'loss': 0.0957, 'learning_rate': 1.8583e-06, 'epoch': 2.87, 'throughput': 1030.89}
+
+[INFO|callbacks.py:310] 2024-07-10 16:20:01,036 >> {'loss': 0.0774, 'learning_rate': 1.8667e-06, 'epoch': 2.88, 'throughput': 1030.97}
+
+[INFO|callbacks.py:310] 2024-07-10 16:20:14,106 >> {'loss': 0.0691, 'learning_rate': 1.8750e-06, 'epoch': 2.89, 'throughput': 1031.09}
+
+[INFO|callbacks.py:310] 2024-07-10 16:20:27,180 >> {'loss': 0.0529, 'learning_rate': 1.8833e-06, 'epoch': 2.91, 'throughput': 1031.31}
+
+[INFO|callbacks.py:310] 2024-07-10 16:20:40,264 >> {'loss': 0.0811, 'learning_rate': 1.8917e-06, 'epoch': 2.92, 'throughput': 1031.18}
+
+[INFO|callbacks.py:310] 2024-07-10 16:20:53,356 >> {'loss': 0.1211, 'learning_rate': 1.9000e-06, 'epoch': 2.93, 'throughput': 1031.32}
+
+[INFO|callbacks.py:310] 2024-07-10 16:21:06,425 >> {'loss': 0.0489, 'learning_rate': 1.9083e-06, 'epoch': 2.95, 'throughput': 1031.40}
+
+[INFO|callbacks.py:310] 2024-07-10 16:21:19,473 >> {'loss': 0.0947, 'learning_rate': 1.9167e-06, 'epoch': 2.96, 'throughput': 1031.77}
+
+[INFO|callbacks.py:310] 2024-07-10 16:21:32,529 >> {'loss': 0.0561, 'learning_rate': 1.9250e-06, 'epoch': 2.97, 'throughput': 1031.72}
+
+[INFO|callbacks.py:310] 2024-07-10 16:21:45,617 >> {'loss': 0.0629, 'learning_rate': 1.9333e-06, 'epoch': 2.98, 'throughput': 1031.97}
+
+[INFO|callbacks.py:310] 2024-07-10 16:21:58,684 >> {'loss': 0.0579, 'learning_rate': 1.9417e-06, 'epoch': 3.00, 'throughput': 1031.94}
+
+[INFO|callbacks.py:310] 2024-07-10 16:22:11,759 >> {'loss': 0.0285, 'learning_rate': 1.9500e-06, 'epoch': 3.01, 'throughput': 1031.94}
+
+[INFO|callbacks.py:310] 2024-07-10 16:22:24,852 >> {'loss': 0.0256, 'learning_rate': 1.9583e-06, 'epoch': 3.02, 'throughput': 1031.83}
+
+[INFO|callbacks.py:310] 2024-07-10 16:22:37,918 >> {'loss': 0.0247, 'learning_rate': 1.9667e-06, 'epoch': 3.04, 'throughput': 1031.89}
+
+[INFO|callbacks.py:310] 2024-07-10 16:22:50,948 >> {'loss': 0.0325, 'learning_rate': 1.9750e-06, 'epoch': 3.05, 'throughput': 1031.71}
+
+[INFO|callbacks.py:310] 2024-07-10 16:23:03,975 >> {'loss': 0.0172, 'learning_rate': 1.9833e-06, 'epoch': 3.06, 'throughput': 1031.83}
+
+[INFO|callbacks.py:310] 2024-07-10 16:23:17,052 >> {'loss': 0.0500, 'learning_rate': 1.9917e-06, 'epoch': 3.07, 'throughput': 1032.03}
+
+[INFO|callbacks.py:310] 2024-07-10 16:23:30,104 >> {'loss': 0.0134, 'learning_rate': 2.0000e-06, 'epoch': 3.09, 'throughput': 1032.07}
+
+[INFO|callbacks.py:310] 2024-07-10 16:23:43,194 >> {'loss': 0.0434, 'learning_rate': 2.0083e-06, 'epoch': 3.10, 'throughput': 1032.06}
+
+[INFO|callbacks.py:310] 2024-07-10 16:23:56,261 >> {'loss': 0.0186, 'learning_rate': 2.0167e-06, 'epoch': 3.11, 'throughput': 1031.79}
+
+[INFO|callbacks.py:310] 2024-07-10 16:24:09,346 >> {'loss': 0.0341, 'learning_rate': 2.0250e-06, 'epoch': 3.13, 'throughput': 1031.70}
+
+[INFO|callbacks.py:310] 2024-07-10 16:24:22,384 >> {'loss': 0.0386, 'learning_rate': 2.0333e-06, 'epoch': 3.14, 'throughput': 1031.75}
+
+[INFO|callbacks.py:310] 2024-07-10 16:24:35,442 >> {'loss': 0.0389, 'learning_rate': 2.0417e-06, 'epoch': 3.15, 'throughput': 1031.84}
+
+[INFO|callbacks.py:310] 2024-07-10 16:24:48,496 >> {'loss': 0.0227, 'learning_rate': 2.0500e-06, 'epoch': 3.16, 'throughput': 1031.98}
+
+[INFO|callbacks.py:310] 2024-07-10 16:25:01,559 >> {'loss': 0.0317, 'learning_rate': 2.0583e-06, 'epoch': 3.18, 'throughput': 1032.06}
+
+[INFO|callbacks.py:310] 2024-07-10 16:25:14,587 >> {'loss': 0.0335, 'learning_rate': 2.0667e-06, 'epoch': 3.19, 'throughput': 1031.85}
+
+[INFO|callbacks.py:310] 2024-07-10 16:25:27,658 >> {'loss': 0.0257, 'learning_rate': 2.0750e-06, 'epoch': 3.20, 'throughput': 1031.71}
+
+[INFO|callbacks.py:310] 2024-07-10 16:25:40,758 >> {'loss': 0.0244, 'learning_rate': 2.0833e-06, 'epoch': 3.22, 'throughput': 1031.99}
+
+[INFO|callbacks.py:310] 2024-07-10 16:25:53,802 >> {'loss': 0.0285, 'learning_rate': 2.0917e-06, 'epoch': 3.23, 'throughput': 1031.85}
+
+[INFO|callbacks.py:310] 2024-07-10 16:26:06,837 >> {'loss': 0.0093, 'learning_rate': 2.1000e-06, 'epoch': 3.24, 'throughput': 1031.89}
+
+[INFO|callbacks.py:310] 2024-07-10 16:26:19,879 >> {'loss': 0.0415, 'learning_rate': 2.1083e-06, 'epoch': 3.25, 'throughput': 1031.75}
+
+[INFO|callbacks.py:310] 2024-07-10 16:26:32,890 >> {'loss': 0.0239, 'learning_rate': 2.1167e-06, 'epoch': 3.27, 'throughput': 1031.75}
+
+[INFO|callbacks.py:310] 2024-07-10 16:26:45,971 >> {'loss': 0.0412, 'learning_rate': 2.1250e-06, 'epoch': 3.28, 'throughput': 1031.71}
+
+[INFO|callbacks.py:310] 2024-07-10 16:26:59,041 >> {'loss': 0.0503, 'learning_rate': 2.1333e-06, 'epoch': 3.29, 'throughput': 1031.89}
+
+[INFO|callbacks.py:310] 2024-07-10 16:27:12,121 >> {'loss': 0.0046, 'learning_rate': 2.1417e-06, 'epoch': 3.31, 'throughput': 1031.66}
+
+[INFO|callbacks.py:310] 2024-07-10 16:27:25,205 >> {'loss': 0.0410, 'learning_rate': 2.1500e-06, 'epoch': 3.32, 'throughput': 1031.77}
+
+[INFO|callbacks.py:310] 2024-07-10 16:27:38,269 >> {'loss': 0.0257, 'learning_rate': 2.1583e-06, 'epoch': 3.33, 'throughput': 1031.85}
+
+[INFO|callbacks.py:310] 2024-07-10 16:27:51,299 >> {'loss': 0.0168, 'learning_rate': 2.1667e-06, 'epoch': 3.34, 'throughput': 1031.84}
+
+[INFO|callbacks.py:310] 2024-07-10 16:28:04,359 >> {'loss': 0.0439, 'learning_rate': 2.1750e-06, 'epoch': 3.36, 'throughput': 1031.65}
+
+[INFO|callbacks.py:310] 2024-07-10 16:28:17,402 >> {'loss': 0.0204, 'learning_rate': 2.1833e-06, 'epoch': 3.37, 'throughput': 1031.70}
+
+[INFO|callbacks.py:310] 2024-07-10 16:28:30,454 >> {'loss': 0.0284, 'learning_rate': 2.1917e-06, 'epoch': 3.38, 'throughput': 1031.59}
+
+[INFO|callbacks.py:310] 2024-07-10 16:28:43,535 >> {'loss': 0.0684, 'learning_rate': 2.2000e-06, 'epoch': 3.40, 'throughput': 1031.39}
+
+[INFO|callbacks.py:310] 2024-07-10 16:28:56,599 >> {'loss': 0.0479, 'learning_rate': 2.2083e-06, 'epoch': 3.41, 'throughput': 1031.24}
+
+[INFO|callbacks.py:310] 2024-07-10 16:29:09,608 >> {'loss': 0.0434, 'learning_rate': 2.2167e-06, 'epoch': 3.42, 'throughput': 1031.11}
+
+[INFO|callbacks.py:310] 2024-07-10 16:29:22,660 >> {'loss': 0.0213, 'learning_rate': 2.2250e-06, 'epoch': 3.43, 'throughput': 1031.15}
+
+[INFO|callbacks.py:310] 2024-07-10 16:29:35,701 >> {'loss': 0.0415, 'learning_rate': 2.2333e-06, 'epoch': 3.45, 'throughput': 1031.22}
+
+[INFO|callbacks.py:310] 2024-07-10 16:29:48,780 >> {'loss': 0.0404, 'learning_rate': 2.2417e-06, 'epoch': 3.46, 'throughput': 1031.14}
+
+[INFO|callbacks.py:310] 2024-07-10 16:30:01,859 >> {'loss': 0.0566, 'learning_rate': 2.2500e-06, 'epoch': 3.47, 'throughput': 1031.20}
+
+[INFO|callbacks.py:310] 2024-07-10 16:30:14,945 >> {'loss': 0.0509, 'learning_rate': 2.2583e-06, 'epoch': 3.49, 'throughput': 1031.20}
+
+[INFO|callbacks.py:310] 2024-07-10 16:30:28,041 >> {'loss': 0.0385, 'learning_rate': 2.2667e-06, 'epoch': 3.50, 'throughput': 1031.23}
+
+[INFO|callbacks.py:310] 2024-07-10 16:30:41,117 >> {'loss': 0.0225, 'learning_rate': 2.2750e-06, 'epoch': 3.51, 'throughput': 1031.20}
+
+[INFO|callbacks.py:310] 2024-07-10 16:30:54,140 >> {'loss': 0.0255, 'learning_rate': 2.2833e-06, 'epoch': 3.52, 'throughput': 1031.11}
+
+[INFO|callbacks.py:310] 2024-07-10 16:31:07,183 >> {'loss': 0.0531, 'learning_rate': 2.2917e-06, 'epoch': 3.54, 'throughput': 1030.96}
+
+[INFO|callbacks.py:310] 2024-07-10 16:31:20,244 >> {'loss': 0.0095, 'learning_rate': 2.3000e-06, 'epoch': 3.55, 'throughput': 1031.07}
+
+[INFO|callbacks.py:310] 2024-07-10 16:31:33,302 >> {'loss': 0.0229, 'learning_rate': 2.3083e-06, 'epoch': 3.56, 'throughput': 1031.17}
+
+[INFO|callbacks.py:310] 2024-07-10 16:31:46,386 >> {'loss': 0.0380, 'learning_rate': 2.3167e-06, 'epoch': 3.58, 'throughput': 1031.41}
+
+[INFO|callbacks.py:310] 2024-07-10 16:31:59,473 >> {'loss': 0.0316, 'learning_rate': 2.3250e-06, 'epoch': 3.59, 'throughput': 1031.40}
+
+[INFO|callbacks.py:310] 2024-07-10 16:32:12,500 >> {'loss': 0.0861, 'learning_rate': 2.3333e-06, 'epoch': 3.60, 'throughput': 1031.09}
+
+[INFO|callbacks.py:310] 2024-07-10 16:32:25,573 >> {'loss': 0.0566, 'learning_rate': 2.3417e-06, 'epoch': 3.61, 'throughput': 1031.26}
+
+[INFO|callbacks.py:310] 2024-07-10 16:32:38,614 >> {'loss': 0.0804, 'learning_rate': 2.3500e-06, 'epoch': 3.63, 'throughput': 1031.33}
+
+[INFO|callbacks.py:310] 2024-07-10 16:32:51,675 >> {'loss': 0.0460, 'learning_rate': 2.3583e-06, 'epoch': 3.64, 'throughput': 1031.43}
+
+[INFO|callbacks.py:310] 2024-07-10 16:33:04,743 >> {'loss': 0.0693, 'learning_rate': 2.3667e-06, 'epoch': 3.65, 'throughput': 1031.38}
+
+[INFO|callbacks.py:310] 2024-07-10 16:33:17,816 >> {'loss': 0.0342, 'learning_rate': 2.3750e-06, 'epoch': 3.67, 'throughput': 1031.47}
+
+[INFO|callbacks.py:310] 2024-07-10 16:33:30,909 >> {'loss': 0.0479, 'learning_rate': 2.3833e-06, 'epoch': 3.68, 'throughput': 1031.40}
+
+[INFO|callbacks.py:310] 2024-07-10 16:33:44,006 >> {'loss': 0.0388, 'learning_rate': 2.3917e-06, 'epoch': 3.69, 'throughput': 1031.51}
+
+[INFO|callbacks.py:310] 2024-07-10 16:33:57,059 >> {'loss': 0.0274, 'learning_rate': 2.4000e-06, 'epoch': 3.70, 'throughput': 1031.62}
+
+[INFO|callbacks.py:310] 2024-07-10 16:34:10,124 >> {'loss': 0.0259, 'learning_rate': 2.4083e-06, 'epoch': 3.72, 'throughput': 1031.83}
+
+[INFO|callbacks.py:310] 2024-07-10 16:34:23,174 >> {'loss': 0.0367, 'learning_rate': 2.4167e-06, 'epoch': 3.73, 'throughput': 1031.66}
+
+[INFO|callbacks.py:310] 2024-07-10 16:34:36,226 >> {'loss': 0.0661, 'learning_rate': 2.4250e-06, 'epoch': 3.74, 'throughput': 1031.57}
+
+[INFO|callbacks.py:310] 2024-07-10 16:34:49,271 >> {'loss': 0.0466, 'learning_rate': 2.4333e-06, 'epoch': 3.76, 'throughput': 1031.53}
+
+[INFO|callbacks.py:310] 2024-07-10 16:35:02,361 >> {'loss': 0.0286, 'learning_rate': 2.4417e-06, 'epoch': 3.77, 'throughput': 1031.51}
+
+[INFO|callbacks.py:310] 2024-07-10 16:35:15,418 >> {'loss': 0.0586, 'learning_rate': 2.4500e-06, 'epoch': 3.78, 'throughput': 1031.36}
+
+[INFO|callbacks.py:310] 2024-07-10 16:35:28,427 >> {'loss': 0.0329, 'learning_rate': 2.4583e-06, 'epoch': 3.79, 'throughput': 1031.22}
+
+[INFO|callbacks.py:310] 2024-07-10 16:35:41,482 >> {'loss': 0.0582, 'learning_rate': 2.4667e-06, 'epoch': 3.81, 'throughput': 1031.30}
+
+[INFO|callbacks.py:310] 2024-07-10 16:35:54,541 >> {'loss': 0.0312, 'learning_rate': 2.4750e-06, 'epoch': 3.82, 'throughput': 1031.36}
+
+[INFO|callbacks.py:310] 2024-07-10 16:36:07,569 >> {'loss': 0.0329, 'learning_rate': 2.4833e-06, 'epoch': 3.83, 'throughput': 1031.39}
+
+[INFO|callbacks.py:310] 2024-07-10 16:36:20,626 >> {'loss': 0.0206, 'learning_rate': 2.4917e-06, 'epoch': 3.85, 'throughput': 1031.35}
+
+[INFO|callbacks.py:310] 2024-07-10 16:36:33,678 >> {'loss': 0.0426, 'learning_rate': 2.5000e-06, 'epoch': 3.86, 'throughput': 1031.30}
+
+[INFO|callbacks.py:310] 2024-07-10 16:36:46,762 >> {'loss': 0.0179, 'learning_rate': 2.5083e-06, 'epoch': 3.87, 'throughput': 1031.28}
+
+[INFO|callbacks.py:310] 2024-07-10 16:36:59,795 >> {'loss': 0.0289, 'learning_rate': 2.5167e-06, 'epoch': 3.88, 'throughput': 1031.45}
+
+[INFO|callbacks.py:310] 2024-07-10 16:37:12,850 >> {'loss': 0.0303, 'learning_rate': 2.5250e-06, 'epoch': 3.90, 'throughput': 1031.48}
+
+[INFO|callbacks.py:310] 2024-07-10 16:37:25,916 >> {'loss': 0.0460, 'learning_rate': 2.5333e-06, 'epoch': 3.91, 'throughput': 1031.62}
+
+[INFO|callbacks.py:310] 2024-07-10 16:37:38,945 >> {'loss': 0.0523, 'learning_rate': 2.5417e-06, 'epoch': 3.92, 'throughput': 1031.55}
+
+[INFO|callbacks.py:310] 2024-07-10 16:37:51,996 >> {'loss': 0.0329, 'learning_rate': 2.5500e-06, 'epoch': 3.94, 'throughput': 1031.52}
+
+[INFO|callbacks.py:310] 2024-07-10 16:38:05,038 >> {'loss': 0.0072, 'learning_rate': 2.5583e-06, 'epoch': 3.95, 'throughput': 1031.37}
+
+[INFO|callbacks.py:310] 2024-07-10 16:38:18,108 >> {'loss': 0.0415, 'learning_rate': 2.5667e-06, 'epoch': 3.96, 'throughput': 1031.46}
+
+[INFO|callbacks.py:310] 2024-07-10 16:38:31,175 >> {'loss': 0.0233, 'learning_rate': 2.5750e-06, 'epoch': 3.97, 'throughput': 1031.53}
+
+[INFO|callbacks.py:310] 2024-07-10 16:38:44,257 >> {'loss': 0.0423, 'learning_rate': 2.5833e-06, 'epoch': 3.99, 'throughput': 1031.73}
+
+[INFO|callbacks.py:310] 2024-07-10 16:38:57,319 >> {'loss': 0.0295, 'learning_rate': 2.5917e-06, 'epoch': 4.00, 'throughput': 1031.67}
+
+[INFO|callbacks.py:310] 2024-07-10 16:39:10,340 >> {'loss': 0.0327, 'learning_rate': 2.6000e-06, 'epoch': 4.01, 'throughput': 1031.56}
+
+[INFO|callbacks.py:310] 2024-07-10 16:39:23,403 >> {'loss': 0.0301, 'learning_rate': 2.6083e-06, 'epoch': 4.03, 'throughput': 1031.43}
+
+[INFO|callbacks.py:310] 2024-07-10 16:39:36,458 >> {'loss': 0.0301, 'learning_rate': 2.6167e-06, 'epoch': 4.04, 'throughput': 1031.40}
+
+[INFO|callbacks.py:310] 2024-07-10 16:39:49,518 >> {'loss': 0.0281, 'learning_rate': 2.6250e-06, 'epoch': 4.05, 'throughput': 1031.24}
+
+[INFO|callbacks.py:310] 2024-07-10 16:40:02,644 >> {'loss': 0.0136, 'learning_rate': 2.6333e-06, 'epoch': 4.06, 'throughput': 1031.56}
+
+[INFO|callbacks.py:310] 2024-07-10 16:40:15,692 >> {'loss': 0.0219, 'learning_rate': 2.6417e-06, 'epoch': 4.08, 'throughput': 1031.48}
+
+[INFO|callbacks.py:310] 2024-07-10 16:40:28,769 >> {'loss': 0.0044, 'learning_rate': 2.6500e-06, 'epoch': 4.09, 'throughput': 1031.55}
+
+[INFO|callbacks.py:310] 2024-07-10 16:40:41,845 >> {'loss': 0.0335, 'learning_rate': 2.6583e-06, 'epoch': 4.10, 'throughput': 1031.59}
+
+[INFO|callbacks.py:310] 2024-07-10 16:40:54,879 >> {'loss': 0.0053, 'learning_rate': 2.6667e-06, 'epoch': 4.12, 'throughput': 1031.55}
+
+[INFO|callbacks.py:310] 2024-07-10 16:41:07,940 >> {'loss': 0.0196, 'learning_rate': 2.6750e-06, 'epoch': 4.13, 'throughput': 1031.35}
+
+[INFO|callbacks.py:310] 2024-07-10 16:41:21,035 >> {'loss': 0.0309, 'learning_rate': 2.6833e-06, 'epoch': 4.14, 'throughput': 1031.36}
+
+[INFO|callbacks.py:310] 2024-07-10 16:41:34,100 >> {'loss': 0.0382, 'learning_rate': 2.6917e-06, 'epoch': 4.15, 'throughput': 1031.20}
+
+[INFO|callbacks.py:310] 2024-07-10 16:41:47,159 >> {'loss': 0.0460, 'learning_rate': 2.7000e-06, 'epoch': 4.17, 'throughput': 1031.28}
+
+[INFO|callbacks.py:310] 2024-07-10 16:42:00,206 >> {'loss': 0.0133, 'learning_rate': 2.7083e-06, 'epoch': 4.18, 'throughput': 1031.13}
+
+[INFO|callbacks.py:310] 2024-07-10 16:42:13,260 >> {'loss': 0.0265, 'learning_rate': 2.7167e-06, 'epoch': 4.19, 'throughput': 1031.13}
+
+[INFO|callbacks.py:310] 2024-07-10 16:42:26,301 >> {'loss': 0.0084, 'learning_rate': 2.7250e-06, 'epoch': 4.21, 'throughput': 1031.11}
+
+[INFO|callbacks.py:310] 2024-07-10 16:42:39,360 >> {'loss': 0.0382, 'learning_rate': 2.7333e-06, 'epoch': 4.22, 'throughput': 1031.02}
+
+[INFO|callbacks.py:310] 2024-07-10 16:42:52,463 >> {'loss': 0.0101, 'learning_rate': 2.7417e-06, 'epoch': 4.23, 'throughput': 1031.21}
+
+[INFO|callbacks.py:310] 2024-07-10 16:43:05,548 >> {'loss': 0.0174, 'learning_rate': 2.7500e-06, 'epoch': 4.24, 'throughput': 1031.21}
+
+[INFO|callbacks.py:310] 2024-07-10 16:43:18,594 >> {'loss': 0.0230, 'learning_rate': 2.7583e-06, 'epoch': 4.26, 'throughput': 1031.11}
+
+[INFO|callbacks.py:310] 2024-07-10 16:43:31,646 >> {'loss': 0.0162, 'learning_rate': 2.7667e-06, 'epoch': 4.27, 'throughput': 1031.23}
+
+[INFO|callbacks.py:310] 2024-07-10 16:43:44,663 >> {'loss': 0.0261, 'learning_rate': 2.7750e-06, 'epoch': 4.28, 'throughput': 1031.06}
+
+[INFO|callbacks.py:310] 2024-07-10 16:43:57,701 >> {'loss': 0.0266, 'learning_rate': 2.7833e-06, 'epoch': 4.30, 'throughput': 1031.04}
+
+[INFO|callbacks.py:310] 2024-07-10 16:44:10,756 >> {'loss': 0.0194, 'learning_rate': 2.7917e-06, 'epoch': 4.31, 'throughput': 1030.96}
+
+[INFO|callbacks.py:310] 2024-07-10 16:44:23,810 >> {'loss': 0.0058, 'learning_rate': 2.8000e-06, 'epoch': 4.32, 'throughput': 1031.08}
+
+[INFO|callbacks.py:310] 2024-07-10 16:44:36,914 >> {'loss': 0.0065, 'learning_rate': 2.8083e-06, 'epoch': 4.33, 'throughput': 1031.06}
+
+[INFO|callbacks.py:310] 2024-07-10 16:44:50,018 >> {'loss': 0.0202, 'learning_rate': 2.8167e-06, 'epoch': 4.35, 'throughput': 1031.13}
+
+[INFO|callbacks.py:310] 2024-07-10 16:45:03,057 >> {'loss': 0.0135, 'learning_rate': 2.8250e-06, 'epoch': 4.36, 'throughput': 1031.08}
+
+[INFO|callbacks.py:310] 2024-07-10 16:45:16,126 >> {'loss': 0.0100, 'learning_rate': 2.8333e-06, 'epoch': 4.37, 'throughput': 1031.12}
+
+[INFO|callbacks.py:310] 2024-07-10 16:45:29,168 >> {'loss': 0.0051, 'learning_rate': 2.8417e-06, 'epoch': 4.39, 'throughput': 1031.01}
+
+[INFO|callbacks.py:310] 2024-07-10 16:45:42,203 >> {'loss': 0.0293, 'learning_rate': 2.8500e-06, 'epoch': 4.40, 'throughput': 1031.19}
+
+[INFO|callbacks.py:310] 2024-07-10 16:45:55,247 >> {'loss': 0.0460, 'learning_rate': 2.8583e-06, 'epoch': 4.41, 'throughput': 1031.23}
+
+[INFO|callbacks.py:310] 2024-07-10 16:46:08,301 >> {'loss': 0.0024, 'learning_rate': 2.8667e-06, 'epoch': 4.42, 'throughput': 1031.26}
+
+[INFO|callbacks.py:310] 2024-07-10 16:46:21,356 >> {'loss': 0.0211, 'learning_rate': 2.8750e-06, 'epoch': 4.44, 'throughput': 1031.10}
+
+[INFO|callbacks.py:310] 2024-07-10 16:46:34,393 >> {'loss': 0.0229, 'learning_rate': 2.8833e-06, 'epoch': 4.45, 'throughput': 1031.11}
+
+[INFO|callbacks.py:310] 2024-07-10 16:46:47,487 >> {'loss': 0.0103, 'learning_rate': 2.8917e-06, 'epoch': 4.46, 'throughput': 1031.19}
+
+[INFO|callbacks.py:310] 2024-07-10 16:47:00,516 >> {'loss': 0.0262, 'learning_rate': 2.9000e-06, 'epoch': 4.48, 'throughput': 1031.12}
+
+[INFO|callbacks.py:310] 2024-07-10 16:47:13,566 >> {'loss': 0.0295, 'learning_rate': 2.9083e-06, 'epoch': 4.49, 'throughput': 1031.24}
+
+[INFO|callbacks.py:310] 2024-07-10 16:47:26,605 >> {'loss': 0.0149, 'learning_rate': 2.9167e-06, 'epoch': 4.50, 'throughput': 1031.25}
+
+[INFO|callbacks.py:310] 2024-07-10 16:47:39,682 >> {'loss': 0.0337, 'learning_rate': 2.9250e-06, 'epoch': 4.51, 'throughput': 1031.25}
+
+[INFO|callbacks.py:310] 2024-07-10 16:47:52,786 >> {'loss': 0.0318, 'learning_rate': 2.9333e-06, 'epoch': 4.53, 'throughput': 1031.32}
+
+[INFO|callbacks.py:310] 2024-07-10 16:48:05,841 >> {'loss': 0.0213, 'learning_rate': 2.9417e-06, 'epoch': 4.54, 'throughput': 1031.30}
+
+[INFO|callbacks.py:310] 2024-07-10 16:48:18,816 >> {'loss': 0.0048, 'learning_rate': 2.9500e-06, 'epoch': 4.55, 'throughput': 1031.03}
+
+[INFO|callbacks.py:310] 2024-07-10 16:48:31,878 >> {'loss': 0.0326, 'learning_rate': 2.9583e-06, 'epoch': 4.57, 'throughput': 1030.92}
+
+[INFO|callbacks.py:310] 2024-07-10 16:48:44,907 >> {'loss': 0.0130, 'learning_rate': 2.9667e-06, 'epoch': 4.58, 'throughput': 1030.99}
+
+[INFO|callbacks.py:310] 2024-07-10 16:48:57,961 >> {'loss': 0.0293, 'learning_rate': 2.9750e-06, 'epoch': 4.59, 'throughput': 1030.97}
+
+[INFO|callbacks.py:310] 2024-07-10 16:49:11,009 >> {'loss': 0.0411, 'learning_rate': 2.9833e-06, 'epoch': 4.60, 'throughput': 1030.93}
+
+[INFO|callbacks.py:310] 2024-07-10 16:49:24,110 >> {'loss': 0.0389, 'learning_rate': 2.9917e-06, 'epoch': 4.62, 'throughput': 1031.06}
+
+[INFO|callbacks.py:310] 2024-07-10 16:49:37,190 >> {'loss': 0.0395, 'learning_rate': 3.0000e-06, 'epoch': 4.63, 'throughput': 1031.22}
+
+[INFO|callbacks.py:310] 2024-07-10 16:49:50,206 >> {'loss': 0.0065, 'learning_rate': 3.0083e-06, 'epoch': 4.64, 'throughput': 1031.24}
+
+[INFO|callbacks.py:310] 2024-07-10 16:50:03,225 >> {'loss': 0.0294, 'learning_rate': 3.0167e-06, 'epoch': 4.66, 'throughput': 1031.15}
+
+[INFO|callbacks.py:310] 2024-07-10 16:50:16,282 >> {'loss': 0.0192, 'learning_rate': 3.0250e-06, 'epoch': 4.67, 'throughput': 1031.15}
+
+[INFO|callbacks.py:310] 2024-07-10 16:50:29,374 >> {'loss': 0.0179, 'learning_rate': 3.0333e-06, 'epoch': 4.68, 'throughput': 1031.31}
+
+[INFO|callbacks.py:310] 2024-07-10 16:50:42,461 >> {'loss': 0.0131, 'learning_rate': 3.0417e-06, 'epoch': 4.69, 'throughput': 1031.45}
+
+[INFO|callbacks.py:310] 2024-07-10 16:50:55,506 >> {'loss': 0.0216, 'learning_rate': 3.0500e-06, 'epoch': 4.71, 'throughput': 1031.33}
+
+[INFO|callbacks.py:310] 2024-07-10 16:51:08,603 >> {'loss': 0.0171, 'learning_rate': 3.0583e-06, 'epoch': 4.72, 'throughput': 1031.54}
+
+[INFO|callbacks.py:310] 2024-07-10 16:51:21,642 >> {'loss': 0.0129, 'learning_rate': 3.0667e-06, 'epoch': 4.73, 'throughput': 1031.53}
+
+[INFO|callbacks.py:310] 2024-07-10 16:51:34,663 >> {'loss': 0.0268, 'learning_rate': 3.0750e-06, 'epoch': 4.75, 'throughput': 1031.60}
+
+[INFO|callbacks.py:310] 2024-07-10 16:51:47,770 >> {'loss': 0.0313, 'learning_rate': 3.0833e-06, 'epoch': 4.76, 'throughput': 1031.67}
+
+[INFO|callbacks.py:310] 2024-07-10 16:52:00,829 >> {'loss': 0.0197, 'learning_rate': 3.0917e-06, 'epoch': 4.77, 'throughput': 1031.73}
+
+[INFO|callbacks.py:310] 2024-07-10 16:52:13,901 >> {'loss': 0.0051, 'learning_rate': 3.1000e-06, 'epoch': 4.78, 'throughput': 1031.89}
+
+[INFO|callbacks.py:310] 2024-07-10 16:52:27,006 >> {'loss': 0.0107, 'learning_rate': 3.1083e-06, 'epoch': 4.80, 'throughput': 1031.88}
+
+[INFO|callbacks.py:310] 2024-07-10 16:52:40,104 >> {'loss': 0.0299, 'learning_rate': 3.1167e-06, 'epoch': 4.81, 'throughput': 1031.92}
+
+[INFO|callbacks.py:310] 2024-07-10 16:52:53,184 >> {'loss': 0.0549, 'learning_rate': 3.1250e-06, 'epoch': 4.82, 'throughput': 1032.12}
+
+[INFO|callbacks.py:310] 2024-07-10 16:53:06,195 >> {'loss': 0.0163, 'learning_rate': 3.1333e-06, 'epoch': 4.84, 'throughput': 1032.05}
+
+[INFO|callbacks.py:310] 2024-07-10 16:53:19,295 >> {'loss': 0.0133, 'learning_rate': 3.1417e-06, 'epoch': 4.85, 'throughput': 1032.16}
+
+[INFO|callbacks.py:310] 2024-07-10 16:53:32,359 >> {'loss': 0.0338, 'learning_rate': 3.1500e-06, 'epoch': 4.86, 'throughput': 1032.15}
+
+[INFO|callbacks.py:310] 2024-07-10 16:53:45,415 >> {'loss': 0.0219, 'learning_rate': 3.1583e-06, 'epoch': 4.87, 'throughput': 1032.12}
+
+[INFO|callbacks.py:310] 2024-07-10 16:53:58,454 >> {'loss': 0.0113, 'learning_rate': 3.1667e-06, 'epoch': 4.89, 'throughput': 1032.06}
+
+[INFO|callbacks.py:310] 2024-07-10 16:54:11,538 >> {'loss': 0.0297, 'learning_rate': 3.1750e-06, 'epoch': 4.90, 'throughput': 1031.99}
+
+[INFO|callbacks.py:310] 2024-07-10 16:54:24,604 >> {'loss': 0.0417, 'learning_rate': 3.1833e-06, 'epoch': 4.91, 'throughput': 1031.99}
+
+[INFO|callbacks.py:310] 2024-07-10 16:54:37,636 >> {'loss': 0.0270, 'learning_rate': 3.1917e-06, 'epoch': 4.93, 'throughput': 1031.98}
+
+[INFO|callbacks.py:310] 2024-07-10 16:54:50,650 >> {'loss': 0.0271, 'learning_rate': 3.2000e-06, 'epoch': 4.94, 'throughput': 1031.83}
+
+[INFO|callbacks.py:310] 2024-07-10 16:55:03,728 >> {'loss': 0.0207, 'learning_rate': 3.2083e-06, 'epoch': 4.95, 'throughput': 1032.00}
+
+[INFO|trainer.py:3478] 2024-07-10 16:55:11,454 >> Saving model checkpoint to saves/LLaMA3-8B/full/train_2024-07-10-15-21-44_llama3/checkpoint-385
+
+[INFO|configuration_utils.py:472] 2024-07-10 16:55:11,458 >> Configuration saved in saves/LLaMA3-8B/full/train_2024-07-10-15-21-44_llama3/checkpoint-385/config.json
+
+[INFO|configuration_utils.py:769] 2024-07-10 16:55:11,458 >> Configuration saved in saves/LLaMA3-8B/full/train_2024-07-10-15-21-44_llama3/checkpoint-385/generation_config.json
+
+[INFO|modeling_utils.py:2698] 2024-07-10 16:55:27,632 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 4 checkpoint shards. You can find where each parameters has been saved in the index located at saves/LLaMA3-8B/full/train_2024-07-10-15-21-44_llama3/checkpoint-385/model.safetensors.index.json.
+
+[INFO|tokenization_utils_base.py:2574] 2024-07-10 16:55:27,635 >> tokenizer config file saved in saves/LLaMA3-8B/full/train_2024-07-10-15-21-44_llama3/checkpoint-385/tokenizer_config.json
+
+[INFO|tokenization_utils_base.py:2583] 2024-07-10 16:55:27,636 >> Special tokens file saved in saves/LLaMA3-8B/full/train_2024-07-10-15-21-44_llama3/checkpoint-385/special_tokens_map.json
+
+[INFO|trainer.py:2383] 2024-07-10 16:56:04,457 >> 
+
+Training completed. Do not forget to share your model on huggingface.co/models =)
+
+
+
+[INFO|trainer.py:3478] 2024-07-10 16:56:12,165 >> Saving model checkpoint to saves/LLaMA3-8B/full/train_2024-07-10-15-21-44_llama3
+
+[INFO|configuration_utils.py:472] 2024-07-10 16:56:12,167 >> Configuration saved in saves/LLaMA3-8B/full/train_2024-07-10-15-21-44_llama3/config.json
+
+[INFO|configuration_utils.py:769] 2024-07-10 16:56:12,168 >> Configuration saved in saves/LLaMA3-8B/full/train_2024-07-10-15-21-44_llama3/generation_config.json
+
+[INFO|modeling_utils.py:2698] 2024-07-10 16:56:28,747 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 4 checkpoint shards. You can find where each parameters has been saved in the index located at saves/LLaMA3-8B/full/train_2024-07-10-15-21-44_llama3/model.safetensors.index.json.
+
+[INFO|tokenization_utils_base.py:2574] 2024-07-10 16:56:28,750 >> tokenizer config file saved in saves/LLaMA3-8B/full/train_2024-07-10-15-21-44_llama3/tokenizer_config.json
+
+[INFO|tokenization_utils_base.py:2583] 2024-07-10 16:56:28,750 >> Special tokens file saved in saves/LLaMA3-8B/full/train_2024-07-10-15-21-44_llama3/special_tokens_map.json
+
+[WARNING|ploting.py:89] 2024-07-10 16:56:30,068 >> No metric eval_loss to plot.
+
+[WARNING|ploting.py:89] 2024-07-10 16:56:30,068 >> No metric eval_accuracy to plot.
+
+[INFO|modelcard.py:449] 2024-07-10 16:56:30,069 >> Dropping the following result as it does not have all the necessary fields:
+{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}}
+

special_tokens_map.json

@@ -0,0 +1,17 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|end_of_text|>"
+}

tokenizer_config.json

@@ -0,0 +1,2065 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_248|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_249|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_250|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% if system_message is defined %}{{ system_message + '\n' }}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ 'Human: ' + content + '\nAssistant:' }}{% elif message['role'] == 'assistant' %}{{ content + '<|end_of_text|>' + '\n' }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|end_of_text|>",
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|end_of_text|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 4.951768488745981,
+    "num_input_tokens_seen": 5192736,
+    "total_flos": 2.3382655808988774e+17,
+    "train_loss": 0.7082552919200585,
+    "train_runtime": 5092.4527,
+    "train_samples_per_second": 19.519,
+    "train_steps_per_second": 0.076
+}

trainer_log.jsonl

@@ -0,0 +1,386 @@
+{"current_steps": 1, "total_steps": 385, "loss": 7.8034, "learning_rate": 8.333333333333335e-09, "epoch": 0.012861736334405145, "percentage": 0.26, "elapsed_time": "0:00:16", "remaining_time": "1:47:01", "throughput": "803.70", "total_tokens": 13440}
+{"current_steps": 2, "total_steps": 385, "loss": 7.7577, "learning_rate": 1.666666666666667e-08, "epoch": 0.02572347266881029, "percentage": 0.52, "elapsed_time": "0:00:29", "remaining_time": "1:35:05", "throughput": "905.46", "total_tokens": 26976}
+{"current_steps": 3, "total_steps": 385, "loss": 7.8132, "learning_rate": 2.5000000000000002e-08, "epoch": 0.03858520900321544, "percentage": 0.78, "elapsed_time": "0:00:42", "remaining_time": "1:30:54", "throughput": "952.51", "total_tokens": 40800}
+{"current_steps": 4, "total_steps": 385, "loss": 7.7906, "learning_rate": 3.333333333333334e-08, "epoch": 0.05144694533762058, "percentage": 1.04, "elapsed_time": "0:00:55", "remaining_time": "1:28:42", "throughput": "967.80", "total_tokens": 54080}
+{"current_steps": 5, "total_steps": 385, "loss": 7.7763, "learning_rate": 4.166666666666667e-08, "epoch": 0.06430868167202572, "percentage": 1.3, "elapsed_time": "0:01:08", "remaining_time": "1:27:17", "throughput": "980.16", "total_tokens": 67552}
+{"current_steps": 6, "total_steps": 385, "loss": 7.7824, "learning_rate": 5.0000000000000004e-08, "epoch": 0.07717041800643087, "percentage": 1.56, "elapsed_time": "0:01:21", "remaining_time": "1:26:19", "throughput": "995.20", "total_tokens": 81600}
+{"current_steps": 7, "total_steps": 385, "loss": 7.833, "learning_rate": 5.833333333333334e-08, "epoch": 0.09003215434083602, "percentage": 1.82, "elapsed_time": "0:01:35", "remaining_time": "1:25:32", "throughput": "996.54", "total_tokens": 94720}
+{"current_steps": 8, "total_steps": 385, "loss": 7.6949, "learning_rate": 6.666666666666668e-08, "epoch": 0.10289389067524116, "percentage": 2.08, "elapsed_time": "0:01:48", "remaining_time": "1:24:54", "throughput": "994.24", "total_tokens": 107488}
+{"current_steps": 9, "total_steps": 385, "loss": 7.8336, "learning_rate": 7.500000000000001e-08, "epoch": 0.1157556270096463, "percentage": 2.34, "elapsed_time": "0:02:01", "remaining_time": "1:24:23", "throughput": "998.22", "total_tokens": 120992}
+{"current_steps": 10, "total_steps": 385, "loss": 7.7282, "learning_rate": 8.333333333333334e-08, "epoch": 0.12861736334405144, "percentage": 2.6, "elapsed_time": "0:02:14", "remaining_time": "1:23:53", "throughput": "1000.27", "total_tokens": 134272}
+{"current_steps": 11, "total_steps": 385, "loss": 7.6916, "learning_rate": 9.166666666666668e-08, "epoch": 0.1414790996784566, "percentage": 2.86, "elapsed_time": "0:02:27", "remaining_time": "1:23:27", "throughput": "1005.49", "total_tokens": 148096}
+{"current_steps": 12, "total_steps": 385, "loss": 7.7333, "learning_rate": 1.0000000000000001e-07, "epoch": 0.15434083601286175, "percentage": 3.12, "elapsed_time": "0:02:40", "remaining_time": "1:23:02", "throughput": "1002.74", "total_tokens": 160736}
+{"current_steps": 13, "total_steps": 385, "loss": 7.6017, "learning_rate": 1.0833333333333335e-07, "epoch": 0.16720257234726688, "percentage": 3.38, "elapsed_time": "0:02:53", "remaining_time": "1:22:40", "throughput": "1000.21", "total_tokens": 173376}
+{"current_steps": 14, "total_steps": 385, "loss": 7.644, "learning_rate": 1.1666666666666668e-07, "epoch": 0.18006430868167203, "percentage": 3.64, "elapsed_time": "0:03:06", "remaining_time": "1:22:19", "throughput": "1004.07", "total_tokens": 187136}
+{"current_steps": 15, "total_steps": 385, "loss": 7.5965, "learning_rate": 1.2500000000000002e-07, "epoch": 0.19292604501607716, "percentage": 3.9, "elapsed_time": "0:03:19", "remaining_time": "1:22:00", "throughput": "1003.49", "total_tokens": 200192}
+{"current_steps": 16, "total_steps": 385, "loss": 7.5883, "learning_rate": 1.3333333333333336e-07, "epoch": 0.2057877813504823, "percentage": 4.16, "elapsed_time": "0:03:32", "remaining_time": "1:21:42", "throughput": "1006.03", "total_tokens": 213856}
+{"current_steps": 17, "total_steps": 385, "loss": 7.2464, "learning_rate": 1.4166666666666668e-07, "epoch": 0.21864951768488747, "percentage": 4.42, "elapsed_time": "0:03:45", "remaining_time": "1:21:24", "throughput": "1005.85", "total_tokens": 226944}
+{"current_steps": 18, "total_steps": 385, "loss": 7.3133, "learning_rate": 1.5000000000000002e-07, "epoch": 0.2315112540192926, "percentage": 4.68, "elapsed_time": "0:03:58", "remaining_time": "1:21:05", "throughput": "1008.31", "total_tokens": 240640}
+{"current_steps": 19, "total_steps": 385, "loss": 7.2133, "learning_rate": 1.5833333333333336e-07, "epoch": 0.24437299035369775, "percentage": 4.94, "elapsed_time": "0:04:11", "remaining_time": "1:20:48", "throughput": "1006.28", "total_tokens": 253280}
+{"current_steps": 20, "total_steps": 385, "loss": 7.2431, "learning_rate": 1.6666666666666668e-07, "epoch": 0.2572347266881029, "percentage": 5.19, "elapsed_time": "0:04:24", "remaining_time": "1:20:31", "throughput": "1006.36", "total_tokens": 266400}
+{"current_steps": 21, "total_steps": 385, "loss": 7.1875, "learning_rate": 1.7500000000000002e-07, "epoch": 0.27009646302250806, "percentage": 5.45, "elapsed_time": "0:04:37", "remaining_time": "1:20:15", "throughput": "1009.41", "total_tokens": 280416}
+{"current_steps": 22, "total_steps": 385, "loss": 7.0659, "learning_rate": 1.8333333333333336e-07, "epoch": 0.2829581993569132, "percentage": 5.71, "elapsed_time": "0:04:50", "remaining_time": "1:19:59", "throughput": "1011.98", "total_tokens": 294368}
+{"current_steps": 23, "total_steps": 385, "loss": 6.3595, "learning_rate": 1.9166666666666668e-07, "epoch": 0.2958199356913183, "percentage": 5.97, "elapsed_time": "0:05:04", "remaining_time": "1:19:44", "throughput": "1012.60", "total_tokens": 307840}
+{"current_steps": 24, "total_steps": 385, "loss": 6.0417, "learning_rate": 2.0000000000000002e-07, "epoch": 0.3086816720257235, "percentage": 6.23, "elapsed_time": "0:05:17", "remaining_time": "1:19:29", "throughput": "1015.95", "total_tokens": 322144}
+{"current_steps": 25, "total_steps": 385, "loss": 5.9894, "learning_rate": 2.0833333333333333e-07, "epoch": 0.3215434083601286, "percentage": 6.49, "elapsed_time": "0:05:30", "remaining_time": "1:19:13", "throughput": "1016.08", "total_tokens": 335424}
+{"current_steps": 26, "total_steps": 385, "loss": 5.9259, "learning_rate": 2.166666666666667e-07, "epoch": 0.33440514469453375, "percentage": 6.75, "elapsed_time": "0:05:43", "remaining_time": "1:18:58", "throughput": "1018.08", "total_tokens": 349376}
+{"current_steps": 27, "total_steps": 385, "loss": 5.8983, "learning_rate": 2.2500000000000002e-07, "epoch": 0.34726688102893893, "percentage": 7.01, "elapsed_time": "0:05:56", "remaining_time": "1:18:43", "throughput": "1018.88", "total_tokens": 362944}
+{"current_steps": 28, "total_steps": 385, "loss": 5.6848, "learning_rate": 2.3333333333333336e-07, "epoch": 0.36012861736334406, "percentage": 7.27, "elapsed_time": "0:06:09", "remaining_time": "1:18:28", "throughput": "1019.07", "total_tokens": 376352}
+{"current_steps": 29, "total_steps": 385, "loss": 5.5649, "learning_rate": 2.416666666666667e-07, "epoch": 0.3729903536977492, "percentage": 7.53, "elapsed_time": "0:06:22", "remaining_time": "1:18:13", "throughput": "1018.19", "total_tokens": 389312}
+{"current_steps": 30, "total_steps": 385, "loss": 5.4642, "learning_rate": 2.5000000000000004e-07, "epoch": 0.3858520900321543, "percentage": 7.79, "elapsed_time": "0:06:35", "remaining_time": "1:17:59", "throughput": "1018.74", "total_tokens": 402848}
+{"current_steps": 31, "total_steps": 385, "loss": 4.7955, "learning_rate": 2.5833333333333333e-07, "epoch": 0.3987138263665595, "percentage": 8.05, "elapsed_time": "0:06:48", "remaining_time": "1:17:45", "throughput": "1019.13", "total_tokens": 416352}
+{"current_steps": 32, "total_steps": 385, "loss": 2.8339, "learning_rate": 2.666666666666667e-07, "epoch": 0.4115755627009646, "percentage": 8.31, "elapsed_time": "0:07:01", "remaining_time": "1:17:30", "throughput": "1019.78", "total_tokens": 429920}
+{"current_steps": 33, "total_steps": 385, "loss": 2.4477, "learning_rate": 2.75e-07, "epoch": 0.42443729903536975, "percentage": 8.57, "elapsed_time": "0:07:14", "remaining_time": "1:17:16", "throughput": "1021.60", "total_tokens": 444064}
+{"current_steps": 34, "total_steps": 385, "loss": 2.3331, "learning_rate": 2.8333333333333336e-07, "epoch": 0.43729903536977494, "percentage": 8.83, "elapsed_time": "0:07:27", "remaining_time": "1:17:02", "throughput": "1023.57", "total_tokens": 458304}
+{"current_steps": 35, "total_steps": 385, "loss": 2.2143, "learning_rate": 2.916666666666667e-07, "epoch": 0.45016077170418006, "percentage": 9.09, "elapsed_time": "0:07:40", "remaining_time": "1:16:47", "throughput": "1024.12", "total_tokens": 471904}
+{"current_steps": 36, "total_steps": 385, "loss": 2.0067, "learning_rate": 3.0000000000000004e-07, "epoch": 0.4630225080385852, "percentage": 9.35, "elapsed_time": "0:07:53", "remaining_time": "1:16:33", "throughput": "1023.09", "total_tokens": 484768}
+{"current_steps": 37, "total_steps": 385, "loss": 1.7702, "learning_rate": 3.083333333333334e-07, "epoch": 0.4758842443729904, "percentage": 9.61, "elapsed_time": "0:08:06", "remaining_time": "1:16:19", "throughput": "1022.13", "total_tokens": 497696}
+{"current_steps": 38, "total_steps": 385, "loss": 1.5557, "learning_rate": 3.166666666666667e-07, "epoch": 0.4887459807073955, "percentage": 9.87, "elapsed_time": "0:08:19", "remaining_time": "1:16:05", "throughput": "1022.16", "total_tokens": 511072}
+{"current_steps": 39, "total_steps": 385, "loss": 1.3024, "learning_rate": 3.25e-07, "epoch": 0.5016077170418006, "percentage": 10.13, "elapsed_time": "0:08:33", "remaining_time": "1:15:51", "throughput": "1022.51", "total_tokens": 524576}
+{"current_steps": 40, "total_steps": 385, "loss": 1.1652, "learning_rate": 3.3333333333333335e-07, "epoch": 0.5144694533762058, "percentage": 10.39, "elapsed_time": "0:08:46", "remaining_time": "1:15:37", "throughput": "1022.95", "total_tokens": 538112}
+{"current_steps": 41, "total_steps": 385, "loss": 0.6839, "learning_rate": 3.416666666666667e-07, "epoch": 0.5273311897106109, "percentage": 10.65, "elapsed_time": "0:08:59", "remaining_time": "1:15:23", "throughput": "1024.05", "total_tokens": 552096}
+{"current_steps": 42, "total_steps": 385, "loss": 0.4774, "learning_rate": 3.5000000000000004e-07, "epoch": 0.5401929260450161, "percentage": 10.91, "elapsed_time": "0:09:12", "remaining_time": "1:15:09", "throughput": "1024.88", "total_tokens": 565920}
+{"current_steps": 43, "total_steps": 385, "loss": 0.3841, "learning_rate": 3.583333333333334e-07, "epoch": 0.5530546623794212, "percentage": 11.17, "elapsed_time": "0:09:25", "remaining_time": "1:14:55", "throughput": "1024.68", "total_tokens": 579200}
+{"current_steps": 44, "total_steps": 385, "loss": 0.3588, "learning_rate": 3.666666666666667e-07, "epoch": 0.5659163987138264, "percentage": 11.43, "elapsed_time": "0:09:38", "remaining_time": "1:14:41", "throughput": "1025.16", "total_tokens": 592864}
+{"current_steps": 45, "total_steps": 385, "loss": 0.3628, "learning_rate": 3.75e-07, "epoch": 0.5787781350482315, "percentage": 11.69, "elapsed_time": "0:09:51", "remaining_time": "1:14:28", "throughput": "1025.76", "total_tokens": 606656}
+{"current_steps": 46, "total_steps": 385, "loss": 0.3426, "learning_rate": 3.8333333333333335e-07, "epoch": 0.5916398713826366, "percentage": 11.95, "elapsed_time": "0:10:04", "remaining_time": "1:14:14", "throughput": "1025.64", "total_tokens": 619968}
+{"current_steps": 47, "total_steps": 385, "loss": 0.3279, "learning_rate": 3.9166666666666675e-07, "epoch": 0.6045016077170418, "percentage": 12.21, "elapsed_time": "0:10:17", "remaining_time": "1:14:01", "throughput": "1026.06", "total_tokens": 633632}
+{"current_steps": 48, "total_steps": 385, "loss": 0.3947, "learning_rate": 4.0000000000000003e-07, "epoch": 0.617363344051447, "percentage": 12.47, "elapsed_time": "0:10:30", "remaining_time": "1:13:47", "throughput": "1026.56", "total_tokens": 647360}
+{"current_steps": 49, "total_steps": 385, "loss": 0.3075, "learning_rate": 4.083333333333334e-07, "epoch": 0.6302250803858521, "percentage": 12.73, "elapsed_time": "0:10:43", "remaining_time": "1:13:33", "throughput": "1027.66", "total_tokens": 661504}
+{"current_steps": 50, "total_steps": 385, "loss": 0.3236, "learning_rate": 4.1666666666666667e-07, "epoch": 0.6430868167202572, "percentage": 12.99, "elapsed_time": "0:10:56", "remaining_time": "1:13:20", "throughput": "1028.43", "total_tokens": 675424}
+{"current_steps": 51, "total_steps": 385, "loss": 0.3557, "learning_rate": 4.2500000000000006e-07, "epoch": 0.6559485530546624, "percentage": 13.25, "elapsed_time": "0:11:09", "remaining_time": "1:13:06", "throughput": "1028.37", "total_tokens": 688800}
+{"current_steps": 52, "total_steps": 385, "loss": 0.4008, "learning_rate": 4.333333333333334e-07, "epoch": 0.6688102893890675, "percentage": 13.51, "elapsed_time": "0:11:22", "remaining_time": "1:12:53", "throughput": "1028.30", "total_tokens": 702208}
+{"current_steps": 53, "total_steps": 385, "loss": 0.3586, "learning_rate": 4.416666666666667e-07, "epoch": 0.6816720257234726, "percentage": 13.77, "elapsed_time": "0:11:35", "remaining_time": "1:12:39", "throughput": "1028.93", "total_tokens": 716096}
+{"current_steps": 54, "total_steps": 385, "loss": 0.3023, "learning_rate": 4.5000000000000003e-07, "epoch": 0.6945337620578779, "percentage": 14.03, "elapsed_time": "0:11:48", "remaining_time": "1:12:25", "throughput": "1027.78", "total_tokens": 728672}
+{"current_steps": 55, "total_steps": 385, "loss": 0.3547, "learning_rate": 4.583333333333333e-07, "epoch": 0.707395498392283, "percentage": 14.29, "elapsed_time": "0:12:02", "remaining_time": "1:12:12", "throughput": "1028.91", "total_tokens": 742912}
+{"current_steps": 56, "total_steps": 385, "loss": 0.3846, "learning_rate": 4.666666666666667e-07, "epoch": 0.7202572347266881, "percentage": 14.55, "elapsed_time": "0:12:15", "remaining_time": "1:11:58", "throughput": "1028.23", "total_tokens": 755808}
+{"current_steps": 57, "total_steps": 385, "loss": 0.3743, "learning_rate": 4.7500000000000006e-07, "epoch": 0.7331189710610932, "percentage": 14.81, "elapsed_time": "0:12:28", "remaining_time": "1:11:44", "throughput": "1028.85", "total_tokens": 769696}
+{"current_steps": 58, "total_steps": 385, "loss": 0.3091, "learning_rate": 4.833333333333334e-07, "epoch": 0.7459807073954984, "percentage": 15.06, "elapsed_time": "0:12:41", "remaining_time": "1:11:31", "throughput": "1029.54", "total_tokens": 783680}
+{"current_steps": 59, "total_steps": 385, "loss": 0.3094, "learning_rate": 4.916666666666667e-07, "epoch": 0.7588424437299035, "percentage": 15.32, "elapsed_time": "0:12:54", "remaining_time": "1:11:18", "throughput": "1028.92", "total_tokens": 796672}
+{"current_steps": 60, "total_steps": 385, "loss": 0.3309, "learning_rate": 5.000000000000001e-07, "epoch": 0.7717041800643086, "percentage": 15.58, "elapsed_time": "0:13:07", "remaining_time": "1:11:04", "throughput": "1029.14", "total_tokens": 810304}
+{"current_steps": 61, "total_steps": 385, "loss": 0.3276, "learning_rate": 5.083333333333334e-07, "epoch": 0.7845659163987139, "percentage": 15.84, "elapsed_time": "0:13:20", "remaining_time": "1:10:51", "throughput": "1029.68", "total_tokens": 824160}
+{"current_steps": 62, "total_steps": 385, "loss": 0.3084, "learning_rate": 5.166666666666667e-07, "epoch": 0.797427652733119, "percentage": 16.1, "elapsed_time": "0:13:33", "remaining_time": "1:10:37", "throughput": "1029.34", "total_tokens": 837312}
+{"current_steps": 63, "total_steps": 385, "loss": 0.3182, "learning_rate": 5.250000000000001e-07, "epoch": 0.8102893890675241, "percentage": 16.36, "elapsed_time": "0:13:46", "remaining_time": "1:10:24", "throughput": "1029.83", "total_tokens": 851168}
+{"current_steps": 64, "total_steps": 385, "loss": 0.3469, "learning_rate": 5.333333333333335e-07, "epoch": 0.8231511254019293, "percentage": 16.62, "elapsed_time": "0:13:59", "remaining_time": "1:10:10", "throughput": "1029.25", "total_tokens": 864096}
+{"current_steps": 65, "total_steps": 385, "loss": 0.3253, "learning_rate": 5.416666666666667e-07, "epoch": 0.8360128617363344, "percentage": 16.88, "elapsed_time": "0:14:12", "remaining_time": "1:09:57", "throughput": "1029.85", "total_tokens": 878048}
+{"current_steps": 66, "total_steps": 385, "loss": 0.2746, "learning_rate": 5.5e-07, "epoch": 0.8488745980707395, "percentage": 17.14, "elapsed_time": "0:14:25", "remaining_time": "1:09:44", "throughput": "1030.30", "total_tokens": 891904}
+{"current_steps": 67, "total_steps": 385, "loss": 0.2893, "learning_rate": 5.583333333333333e-07, "epoch": 0.8617363344051447, "percentage": 17.4, "elapsed_time": "0:14:38", "remaining_time": "1:09:30", "throughput": "1031.01", "total_tokens": 906016}
+{"current_steps": 68, "total_steps": 385, "loss": 0.2827, "learning_rate": 5.666666666666667e-07, "epoch": 0.8745980707395499, "percentage": 17.66, "elapsed_time": "0:14:51", "remaining_time": "1:09:17", "throughput": "1030.43", "total_tokens": 918944}
+{"current_steps": 69, "total_steps": 385, "loss": 0.2978, "learning_rate": 5.750000000000001e-07, "epoch": 0.887459807073955, "percentage": 17.92, "elapsed_time": "0:15:04", "remaining_time": "1:09:04", "throughput": "1030.71", "total_tokens": 932672}
+{"current_steps": 70, "total_steps": 385, "loss": 0.2703, "learning_rate": 5.833333333333334e-07, "epoch": 0.9003215434083601, "percentage": 18.18, "elapsed_time": "0:15:17", "remaining_time": "1:08:50", "throughput": "1029.78", "total_tokens": 945248}
+{"current_steps": 71, "total_steps": 385, "loss": 0.2968, "learning_rate": 5.916666666666667e-07, "epoch": 0.9131832797427653, "percentage": 18.44, "elapsed_time": "0:15:30", "remaining_time": "1:08:37", "throughput": "1029.88", "total_tokens": 958784}
+{"current_steps": 72, "total_steps": 385, "loss": 0.3035, "learning_rate": 6.000000000000001e-07, "epoch": 0.9260450160771704, "percentage": 18.7, "elapsed_time": "0:15:44", "remaining_time": "1:08:23", "throughput": "1030.06", "total_tokens": 972416}
+{"current_steps": 73, "total_steps": 385, "loss": 0.3211, "learning_rate": 6.083333333333334e-07, "epoch": 0.9389067524115756, "percentage": 18.96, "elapsed_time": "0:15:57", "remaining_time": "1:08:10", "throughput": "1030.07", "total_tokens": 985888}
+{"current_steps": 74, "total_steps": 385, "loss": 0.2913, "learning_rate": 6.166666666666668e-07, "epoch": 0.9517684887459807, "percentage": 19.22, "elapsed_time": "0:16:10", "remaining_time": "1:07:57", "throughput": "1030.00", "total_tokens": 999296}
+{"current_steps": 75, "total_steps": 385, "loss": 0.2817, "learning_rate": 6.25e-07, "epoch": 0.9646302250803859, "percentage": 19.48, "elapsed_time": "0:16:23", "remaining_time": "1:07:44", "throughput": "1029.88", "total_tokens": 1012640}
+{"current_steps": 76, "total_steps": 385, "loss": 0.2827, "learning_rate": 6.333333333333334e-07, "epoch": 0.977491961414791, "percentage": 19.74, "elapsed_time": "0:16:36", "remaining_time": "1:07:30", "throughput": "1029.93", "total_tokens": 1026112}
+{"current_steps": 77, "total_steps": 385, "loss": 0.229, "learning_rate": 6.416666666666667e-07, "epoch": 0.9903536977491961, "percentage": 20.0, "elapsed_time": "0:16:49", "remaining_time": "1:07:17", "throughput": "1029.81", "total_tokens": 1039424}
+{"current_steps": 78, "total_steps": 385, "loss": 0.2503, "learning_rate": 6.5e-07, "epoch": 1.0032154340836013, "percentage": 20.26, "elapsed_time": "0:17:02", "remaining_time": "1:07:04", "throughput": "1030.33", "total_tokens": 1053408}
+{"current_steps": 79, "total_steps": 385, "loss": 0.2453, "learning_rate": 6.583333333333333e-07, "epoch": 1.0160771704180065, "percentage": 20.52, "elapsed_time": "0:17:15", "remaining_time": "1:06:50", "throughput": "1030.76", "total_tokens": 1067328}
+{"current_steps": 80, "total_steps": 385, "loss": 0.2167, "learning_rate": 6.666666666666667e-07, "epoch": 1.0289389067524115, "percentage": 20.78, "elapsed_time": "0:17:28", "remaining_time": "1:06:37", "throughput": "1030.29", "total_tokens": 1080288}
+{"current_steps": 81, "total_steps": 385, "loss": 0.2361, "learning_rate": 6.750000000000001e-07, "epoch": 1.0418006430868167, "percentage": 21.04, "elapsed_time": "0:17:41", "remaining_time": "1:06:24", "throughput": "1030.47", "total_tokens": 1093952}
+{"current_steps": 82, "total_steps": 385, "loss": 0.2248, "learning_rate": 6.833333333333334e-07, "epoch": 1.0546623794212218, "percentage": 21.3, "elapsed_time": "0:17:54", "remaining_time": "1:06:11", "throughput": "1030.91", "total_tokens": 1107904}
+{"current_steps": 83, "total_steps": 385, "loss": 0.2491, "learning_rate": 6.916666666666668e-07, "epoch": 1.067524115755627, "percentage": 21.56, "elapsed_time": "0:18:07", "remaining_time": "1:05:57", "throughput": "1030.71", "total_tokens": 1121120}
+{"current_steps": 84, "total_steps": 385, "loss": 0.2352, "learning_rate": 7.000000000000001e-07, "epoch": 1.0803858520900322, "percentage": 21.82, "elapsed_time": "0:18:20", "remaining_time": "1:05:44", "throughput": "1031.10", "total_tokens": 1135040}
+{"current_steps": 85, "total_steps": 385, "loss": 0.2365, "learning_rate": 7.083333333333334e-07, "epoch": 1.0932475884244373, "percentage": 22.08, "elapsed_time": "0:18:33", "remaining_time": "1:05:31", "throughput": "1031.51", "total_tokens": 1148992}
+{"current_steps": 86, "total_steps": 385, "loss": 0.217, "learning_rate": 7.166666666666668e-07, "epoch": 1.1061093247588425, "percentage": 22.34, "elapsed_time": "0:18:46", "remaining_time": "1:05:18", "throughput": "1031.65", "total_tokens": 1162592}
+{"current_steps": 87, "total_steps": 385, "loss": 0.2258, "learning_rate": 7.25e-07, "epoch": 1.1189710610932475, "percentage": 22.6, "elapsed_time": "0:18:59", "remaining_time": "1:05:04", "throughput": "1030.83", "total_tokens": 1175104}
+{"current_steps": 88, "total_steps": 385, "loss": 0.245, "learning_rate": 7.333333333333334e-07, "epoch": 1.1318327974276527, "percentage": 22.86, "elapsed_time": "0:19:13", "remaining_time": "1:04:51", "throughput": "1031.53", "total_tokens": 1189376}
+{"current_steps": 89, "total_steps": 385, "loss": 0.3132, "learning_rate": 7.416666666666668e-07, "epoch": 1.144694533762058, "percentage": 23.12, "elapsed_time": "0:19:26", "remaining_time": "1:04:38", "throughput": "1031.26", "total_tokens": 1202560}
+{"current_steps": 90, "total_steps": 385, "loss": 0.284, "learning_rate": 7.5e-07, "epoch": 1.157556270096463, "percentage": 23.38, "elapsed_time": "0:19:39", "remaining_time": "1:04:25", "throughput": "1031.93", "total_tokens": 1216832}
+{"current_steps": 91, "total_steps": 385, "loss": 0.1933, "learning_rate": 7.583333333333334e-07, "epoch": 1.1704180064308682, "percentage": 23.64, "elapsed_time": "0:19:52", "remaining_time": "1:04:11", "throughput": "1032.15", "total_tokens": 1230528}
+{"current_steps": 92, "total_steps": 385, "loss": 0.2154, "learning_rate": 7.666666666666667e-07, "epoch": 1.1832797427652733, "percentage": 23.9, "elapsed_time": "0:20:05", "remaining_time": "1:03:58", "throughput": "1032.42", "total_tokens": 1244352}
+{"current_steps": 93, "total_steps": 385, "loss": 0.2064, "learning_rate": 7.750000000000001e-07, "epoch": 1.1961414790996785, "percentage": 24.16, "elapsed_time": "0:20:18", "remaining_time": "1:03:45", "throughput": "1032.18", "total_tokens": 1257472}
+{"current_steps": 94, "total_steps": 385, "loss": 0.2038, "learning_rate": 7.833333333333335e-07, "epoch": 1.2090032154340835, "percentage": 24.42, "elapsed_time": "0:20:31", "remaining_time": "1:03:31", "throughput": "1032.52", "total_tokens": 1271392}
+{"current_steps": 95, "total_steps": 385, "loss": 0.2152, "learning_rate": 7.916666666666667e-07, "epoch": 1.2218649517684887, "percentage": 24.68, "elapsed_time": "0:20:44", "remaining_time": "1:03:18", "throughput": "1033.74", "total_tokens": 1286432}
+{"current_steps": 96, "total_steps": 385, "loss": 0.1961, "learning_rate": 8.000000000000001e-07, "epoch": 1.234726688102894, "percentage": 24.94, "elapsed_time": "0:20:57", "remaining_time": "1:03:05", "throughput": "1033.85", "total_tokens": 1300096}
+{"current_steps": 97, "total_steps": 385, "loss": 0.1772, "learning_rate": 8.083333333333334e-07, "epoch": 1.247588424437299, "percentage": 25.19, "elapsed_time": "0:21:10", "remaining_time": "1:02:52", "throughput": "1033.81", "total_tokens": 1313568}
+{"current_steps": 98, "total_steps": 385, "loss": 0.1846, "learning_rate": 8.166666666666668e-07, "epoch": 1.2604501607717042, "percentage": 25.45, "elapsed_time": "0:21:23", "remaining_time": "1:02:39", "throughput": "1033.97", "total_tokens": 1327328}
+{"current_steps": 99, "total_steps": 385, "loss": 0.1823, "learning_rate": 8.250000000000001e-07, "epoch": 1.2733118971061093, "percentage": 25.71, "elapsed_time": "0:21:36", "remaining_time": "1:02:26", "throughput": "1034.09", "total_tokens": 1340960}
+{"current_steps": 100, "total_steps": 385, "loss": 0.1794, "learning_rate": 8.333333333333333e-07, "epoch": 1.2861736334405145, "percentage": 25.97, "elapsed_time": "0:21:49", "remaining_time": "1:02:12", "throughput": "1033.35", "total_tokens": 1353440}
+{"current_steps": 101, "total_steps": 385, "loss": 0.2106, "learning_rate": 8.416666666666667e-07, "epoch": 1.2990353697749195, "percentage": 26.23, "elapsed_time": "0:22:02", "remaining_time": "1:01:59", "throughput": "1033.90", "total_tokens": 1367680}
+{"current_steps": 102, "total_steps": 385, "loss": 0.2123, "learning_rate": 8.500000000000001e-07, "epoch": 1.3118971061093248, "percentage": 26.49, "elapsed_time": "0:22:15", "remaining_time": "1:01:46", "throughput": "1033.68", "total_tokens": 1380864}
+{"current_steps": 103, "total_steps": 385, "loss": 0.2413, "learning_rate": 8.583333333333334e-07, "epoch": 1.32475884244373, "percentage": 26.75, "elapsed_time": "0:22:28", "remaining_time": "1:01:33", "throughput": "1033.33", "total_tokens": 1393888}
+{"current_steps": 104, "total_steps": 385, "loss": 0.2334, "learning_rate": 8.666666666666668e-07, "epoch": 1.337620578778135, "percentage": 27.01, "elapsed_time": "0:22:42", "remaining_time": "1:01:20", "throughput": "1032.97", "total_tokens": 1406912}
+{"current_steps": 105, "total_steps": 385, "loss": 0.2069, "learning_rate": 8.75e-07, "epoch": 1.3504823151125402, "percentage": 27.27, "elapsed_time": "0:22:55", "remaining_time": "1:01:06", "throughput": "1032.70", "total_tokens": 1420000}
+{"current_steps": 106, "total_steps": 385, "loss": 0.2262, "learning_rate": 8.833333333333334e-07, "epoch": 1.3633440514469453, "percentage": 27.53, "elapsed_time": "0:23:08", "remaining_time": "1:00:53", "throughput": "1032.67", "total_tokens": 1433440}
+{"current_steps": 107, "total_steps": 385, "loss": 0.1718, "learning_rate": 8.916666666666668e-07, "epoch": 1.3762057877813505, "percentage": 27.79, "elapsed_time": "0:23:21", "remaining_time": "1:00:40", "throughput": "1032.39", "total_tokens": 1446560}
+{"current_steps": 108, "total_steps": 385, "loss": 0.204, "learning_rate": 9.000000000000001e-07, "epoch": 1.3890675241157555, "percentage": 28.05, "elapsed_time": "0:23:34", "remaining_time": "1:00:27", "throughput": "1032.58", "total_tokens": 1460320}
+{"current_steps": 109, "total_steps": 385, "loss": 0.1849, "learning_rate": 9.083333333333335e-07, "epoch": 1.4019292604501608, "percentage": 28.31, "elapsed_time": "0:23:47", "remaining_time": "1:00:14", "throughput": "1032.91", "total_tokens": 1474272}
+{"current_steps": 110, "total_steps": 385, "loss": 0.2028, "learning_rate": 9.166666666666666e-07, "epoch": 1.414790996784566, "percentage": 28.57, "elapsed_time": "0:24:00", "remaining_time": "1:00:00", "throughput": "1033.00", "total_tokens": 1487904}
+{"current_steps": 111, "total_steps": 385, "loss": 0.179, "learning_rate": 9.25e-07, "epoch": 1.427652733118971, "percentage": 28.83, "elapsed_time": "0:24:13", "remaining_time": "0:59:47", "throughput": "1033.15", "total_tokens": 1501664}
+{"current_steps": 112, "total_steps": 385, "loss": 0.1813, "learning_rate": 9.333333333333334e-07, "epoch": 1.4405144694533762, "percentage": 29.09, "elapsed_time": "0:24:26", "remaining_time": "0:59:34", "throughput": "1033.22", "total_tokens": 1515264}
+{"current_steps": 113, "total_steps": 385, "loss": 0.1955, "learning_rate": 9.416666666666667e-07, "epoch": 1.4533762057877815, "percentage": 29.35, "elapsed_time": "0:24:39", "remaining_time": "0:59:21", "throughput": "1033.14", "total_tokens": 1528640}
+{"current_steps": 114, "total_steps": 385, "loss": 0.1577, "learning_rate": 9.500000000000001e-07, "epoch": 1.4662379421221865, "percentage": 29.61, "elapsed_time": "0:24:52", "remaining_time": "0:59:08", "throughput": "1032.82", "total_tokens": 1541632}
+{"current_steps": 115, "total_steps": 385, "loss": 0.1509, "learning_rate": 9.583333333333334e-07, "epoch": 1.4790996784565915, "percentage": 29.87, "elapsed_time": "0:25:05", "remaining_time": "0:58:55", "throughput": "1032.48", "total_tokens": 1554592}
+{"current_steps": 116, "total_steps": 385, "loss": 0.2052, "learning_rate": 9.666666666666668e-07, "epoch": 1.4919614147909968, "percentage": 30.13, "elapsed_time": "0:25:18", "remaining_time": "0:58:41", "throughput": "1031.98", "total_tokens": 1567296}
+{"current_steps": 117, "total_steps": 385, "loss": 0.1576, "learning_rate": 9.750000000000002e-07, "epoch": 1.504823151125402, "percentage": 30.39, "elapsed_time": "0:25:31", "remaining_time": "0:58:28", "throughput": "1031.99", "total_tokens": 1580800}
+{"current_steps": 118, "total_steps": 385, "loss": 0.1459, "learning_rate": 9.833333333333334e-07, "epoch": 1.517684887459807, "percentage": 30.65, "elapsed_time": "0:25:44", "remaining_time": "0:58:15", "throughput": "1031.67", "total_tokens": 1593792}
+{"current_steps": 119, "total_steps": 385, "loss": 0.2694, "learning_rate": 9.916666666666668e-07, "epoch": 1.5305466237942122, "percentage": 30.91, "elapsed_time": "0:25:57", "remaining_time": "0:58:02", "throughput": "1031.92", "total_tokens": 1607648}
+{"current_steps": 120, "total_steps": 385, "loss": 0.1891, "learning_rate": 1.0000000000000002e-06, "epoch": 1.5434083601286175, "percentage": 31.17, "elapsed_time": "0:26:10", "remaining_time": "0:57:49", "throughput": "1031.95", "total_tokens": 1621184}
+{"current_steps": 121, "total_steps": 385, "loss": 0.1655, "learning_rate": 1.0083333333333333e-06, "epoch": 1.5562700964630225, "percentage": 31.43, "elapsed_time": "0:26:23", "remaining_time": "0:57:35", "throughput": "1031.72", "total_tokens": 1634240}
+{"current_steps": 122, "total_steps": 385, "loss": 0.1534, "learning_rate": 1.0166666666666667e-06, "epoch": 1.5691318327974275, "percentage": 31.69, "elapsed_time": "0:26:37", "remaining_time": "0:57:22", "throughput": "1031.72", "total_tokens": 1647712}
+{"current_steps": 123, "total_steps": 385, "loss": 0.1373, "learning_rate": 1.025e-06, "epoch": 1.5819935691318328, "percentage": 31.95, "elapsed_time": "0:26:50", "remaining_time": "0:57:09", "throughput": "1031.81", "total_tokens": 1661344}
+{"current_steps": 124, "total_steps": 385, "loss": 0.1528, "learning_rate": 1.0333333333333333e-06, "epoch": 1.594855305466238, "percentage": 32.21, "elapsed_time": "0:27:03", "remaining_time": "0:56:56", "throughput": "1031.84", "total_tokens": 1674880}
+{"current_steps": 125, "total_steps": 385, "loss": 0.2017, "learning_rate": 1.0416666666666667e-06, "epoch": 1.607717041800643, "percentage": 32.47, "elapsed_time": "0:27:16", "remaining_time": "0:56:43", "throughput": "1032.20", "total_tokens": 1688992}
+{"current_steps": 126, "total_steps": 385, "loss": 0.1554, "learning_rate": 1.0500000000000001e-06, "epoch": 1.6205787781350482, "percentage": 32.73, "elapsed_time": "0:27:29", "remaining_time": "0:56:30", "throughput": "1032.47", "total_tokens": 1702944}
+{"current_steps": 127, "total_steps": 385, "loss": 0.1332, "learning_rate": 1.0583333333333335e-06, "epoch": 1.6334405144694535, "percentage": 32.99, "elapsed_time": "0:27:42", "remaining_time": "0:56:17", "throughput": "1032.68", "total_tokens": 1716768}
+{"current_steps": 128, "total_steps": 385, "loss": 0.115, "learning_rate": 1.066666666666667e-06, "epoch": 1.6463022508038585, "percentage": 33.25, "elapsed_time": "0:27:55", "remaining_time": "0:56:04", "throughput": "1033.04", "total_tokens": 1730848}
+{"current_steps": 129, "total_steps": 385, "loss": 0.119, "learning_rate": 1.075e-06, "epoch": 1.6591639871382635, "percentage": 33.51, "elapsed_time": "0:28:08", "remaining_time": "0:55:50", "throughput": "1032.57", "total_tokens": 1743520}
+{"current_steps": 130, "total_steps": 385, "loss": 0.1164, "learning_rate": 1.0833333333333335e-06, "epoch": 1.6720257234726688, "percentage": 33.77, "elapsed_time": "0:28:21", "remaining_time": "0:55:37", "throughput": "1032.98", "total_tokens": 1757696}
+{"current_steps": 131, "total_steps": 385, "loss": 0.1981, "learning_rate": 1.0916666666666667e-06, "epoch": 1.684887459807074, "percentage": 34.03, "elapsed_time": "0:28:34", "remaining_time": "0:55:24", "throughput": "1033.03", "total_tokens": 1771264}
+{"current_steps": 132, "total_steps": 385, "loss": 0.168, "learning_rate": 1.1e-06, "epoch": 1.697749196141479, "percentage": 34.29, "elapsed_time": "0:28:47", "remaining_time": "0:55:11", "throughput": "1032.86", "total_tokens": 1784480}
+{"current_steps": 133, "total_steps": 385, "loss": 0.0741, "learning_rate": 1.1083333333333335e-06, "epoch": 1.7106109324758842, "percentage": 34.55, "elapsed_time": "0:29:00", "remaining_time": "0:54:58", "throughput": "1032.96", "total_tokens": 1798176}
+{"current_steps": 134, "total_steps": 385, "loss": 0.1847, "learning_rate": 1.1166666666666666e-06, "epoch": 1.7234726688102895, "percentage": 34.81, "elapsed_time": "0:29:13", "remaining_time": "0:54:45", "throughput": "1032.74", "total_tokens": 1811264}
+{"current_steps": 135, "total_steps": 385, "loss": 0.108, "learning_rate": 1.125e-06, "epoch": 1.7363344051446945, "percentage": 35.06, "elapsed_time": "0:29:26", "remaining_time": "0:54:32", "throughput": "1032.88", "total_tokens": 1825024}
+{"current_steps": 136, "total_steps": 385, "loss": 0.1214, "learning_rate": 1.1333333333333334e-06, "epoch": 1.7491961414790995, "percentage": 35.32, "elapsed_time": "0:29:39", "remaining_time": "0:54:18", "throughput": "1032.96", "total_tokens": 1838624}
+{"current_steps": 137, "total_steps": 385, "loss": 0.1252, "learning_rate": 1.1416666666666668e-06, "epoch": 1.762057877813505, "percentage": 35.58, "elapsed_time": "0:29:53", "remaining_time": "0:54:05", "throughput": "1033.06", "total_tokens": 1852288}
+{"current_steps": 138, "total_steps": 385, "loss": 0.144, "learning_rate": 1.1500000000000002e-06, "epoch": 1.77491961414791, "percentage": 35.84, "elapsed_time": "0:30:06", "remaining_time": "0:53:52", "throughput": "1032.84", "total_tokens": 1865376}
+{"current_steps": 139, "total_steps": 385, "loss": 0.1269, "learning_rate": 1.1583333333333334e-06, "epoch": 1.787781350482315, "percentage": 36.1, "elapsed_time": "0:30:19", "remaining_time": "0:53:39", "throughput": "1032.79", "total_tokens": 1878784}
+{"current_steps": 140, "total_steps": 385, "loss": 0.1283, "learning_rate": 1.1666666666666668e-06, "epoch": 1.8006430868167203, "percentage": 36.36, "elapsed_time": "0:30:32", "remaining_time": "0:53:26", "throughput": "1032.81", "total_tokens": 1892320}
+{"current_steps": 141, "total_steps": 385, "loss": 0.0929, "learning_rate": 1.175e-06, "epoch": 1.8135048231511255, "percentage": 36.62, "elapsed_time": "0:30:45", "remaining_time": "0:53:13", "throughput": "1032.78", "total_tokens": 1905728}
+{"current_steps": 142, "total_steps": 385, "loss": 0.1349, "learning_rate": 1.1833333333333334e-06, "epoch": 1.8263665594855305, "percentage": 36.88, "elapsed_time": "0:30:58", "remaining_time": "0:52:59", "throughput": "1032.56", "total_tokens": 1918752}
+{"current_steps": 143, "total_steps": 385, "loss": 0.1277, "learning_rate": 1.1916666666666668e-06, "epoch": 1.8392282958199357, "percentage": 37.14, "elapsed_time": "0:31:11", "remaining_time": "0:52:46", "throughput": "1032.33", "total_tokens": 1931808}
+{"current_steps": 144, "total_steps": 385, "loss": 0.1585, "learning_rate": 1.2000000000000002e-06, "epoch": 1.852090032154341, "percentage": 37.4, "elapsed_time": "0:31:24", "remaining_time": "0:52:33", "throughput": "1032.35", "total_tokens": 1945312}
+{"current_steps": 145, "total_steps": 385, "loss": 0.1468, "learning_rate": 1.2083333333333333e-06, "epoch": 1.864951768488746, "percentage": 37.66, "elapsed_time": "0:31:37", "remaining_time": "0:52:20", "throughput": "1031.96", "total_tokens": 1958016}
+{"current_steps": 146, "total_steps": 385, "loss": 0.1049, "learning_rate": 1.2166666666666667e-06, "epoch": 1.877813504823151, "percentage": 37.92, "elapsed_time": "0:31:50", "remaining_time": "0:52:07", "throughput": "1031.92", "total_tokens": 1971392}
+{"current_steps": 147, "total_steps": 385, "loss": 0.1297, "learning_rate": 1.2250000000000001e-06, "epoch": 1.8906752411575563, "percentage": 38.18, "elapsed_time": "0:32:03", "remaining_time": "0:51:54", "throughput": "1031.96", "total_tokens": 1984992}
+{"current_steps": 148, "total_steps": 385, "loss": 0.1111, "learning_rate": 1.2333333333333335e-06, "epoch": 1.9035369774919615, "percentage": 38.44, "elapsed_time": "0:32:16", "remaining_time": "0:51:41", "throughput": "1031.79", "total_tokens": 1998144}
+{"current_steps": 149, "total_steps": 385, "loss": 0.1202, "learning_rate": 1.2416666666666667e-06, "epoch": 1.9163987138263665, "percentage": 38.7, "elapsed_time": "0:32:29", "remaining_time": "0:51:28", "throughput": "1032.01", "total_tokens": 2012032}
+{"current_steps": 150, "total_steps": 385, "loss": 0.0829, "learning_rate": 1.25e-06, "epoch": 1.9292604501607717, "percentage": 38.96, "elapsed_time": "0:32:42", "remaining_time": "0:51:14", "throughput": "1032.05", "total_tokens": 2025600}
+{"current_steps": 151, "total_steps": 385, "loss": 0.1119, "learning_rate": 1.2583333333333333e-06, "epoch": 1.942122186495177, "percentage": 39.22, "elapsed_time": "0:32:55", "remaining_time": "0:51:01", "throughput": "1032.55", "total_tokens": 2040096}
+{"current_steps": 152, "total_steps": 385, "loss": 0.1144, "learning_rate": 1.2666666666666669e-06, "epoch": 1.954983922829582, "percentage": 39.48, "elapsed_time": "0:33:08", "remaining_time": "0:50:48", "throughput": "1032.51", "total_tokens": 2053504}
+{"current_steps": 153, "total_steps": 385, "loss": 0.117, "learning_rate": 1.275e-06, "epoch": 1.967845659163987, "percentage": 39.74, "elapsed_time": "0:33:21", "remaining_time": "0:50:35", "throughput": "1031.96", "total_tokens": 2065856}
+{"current_steps": 154, "total_steps": 385, "loss": 0.0998, "learning_rate": 1.2833333333333335e-06, "epoch": 1.9807073954983923, "percentage": 40.0, "elapsed_time": "0:33:34", "remaining_time": "0:50:22", "throughput": "1031.63", "total_tokens": 2078656}
+{"current_steps": 155, "total_steps": 385, "loss": 0.1384, "learning_rate": 1.2916666666666669e-06, "epoch": 1.9935691318327975, "percentage": 40.26, "elapsed_time": "0:33:48", "remaining_time": "0:50:09", "throughput": "1031.88", "total_tokens": 2092672}
+{"current_steps": 156, "total_steps": 385, "loss": 0.1157, "learning_rate": 1.3e-06, "epoch": 2.0064308681672025, "percentage": 40.52, "elapsed_time": "0:34:01", "remaining_time": "0:49:56", "throughput": "1031.69", "total_tokens": 2105728}
+{"current_steps": 157, "total_steps": 385, "loss": 0.0696, "learning_rate": 1.3083333333333334e-06, "epoch": 2.0192926045016075, "percentage": 40.78, "elapsed_time": "0:34:14", "remaining_time": "0:49:43", "throughput": "1031.58", "total_tokens": 2118976}
+{"current_steps": 158, "total_steps": 385, "loss": 0.0665, "learning_rate": 1.3166666666666666e-06, "epoch": 2.032154340836013, "percentage": 41.04, "elapsed_time": "0:34:27", "remaining_time": "0:49:29", "throughput": "1031.54", "total_tokens": 2132352}
+{"current_steps": 159, "total_steps": 385, "loss": 0.0783, "learning_rate": 1.3250000000000002e-06, "epoch": 2.045016077170418, "percentage": 41.3, "elapsed_time": "0:34:40", "remaining_time": "0:49:16", "throughput": "1031.64", "total_tokens": 2146048}
+{"current_steps": 160, "total_steps": 385, "loss": 0.0749, "learning_rate": 1.3333333333333334e-06, "epoch": 2.057877813504823, "percentage": 41.56, "elapsed_time": "0:34:53", "remaining_time": "0:49:03", "throughput": "1031.29", "total_tokens": 2158752}
+{"current_steps": 161, "total_steps": 385, "loss": 0.0731, "learning_rate": 1.3416666666666666e-06, "epoch": 2.0707395498392285, "percentage": 41.82, "elapsed_time": "0:35:06", "remaining_time": "0:48:50", "throughput": "1031.12", "total_tokens": 2171872}
+{"current_steps": 162, "total_steps": 385, "loss": 0.0913, "learning_rate": 1.3500000000000002e-06, "epoch": 2.0836012861736335, "percentage": 42.08, "elapsed_time": "0:35:19", "remaining_time": "0:48:37", "throughput": "1030.87", "total_tokens": 2184800}
+{"current_steps": 163, "total_steps": 385, "loss": 0.0521, "learning_rate": 1.3583333333333334e-06, "epoch": 2.0964630225080385, "percentage": 42.34, "elapsed_time": "0:35:32", "remaining_time": "0:48:24", "throughput": "1030.82", "total_tokens": 2198176}
+{"current_steps": 164, "total_steps": 385, "loss": 0.068, "learning_rate": 1.3666666666666668e-06, "epoch": 2.1093247588424435, "percentage": 42.6, "elapsed_time": "0:35:45", "remaining_time": "0:48:11", "throughput": "1030.71", "total_tokens": 2211392}
+{"current_steps": 165, "total_steps": 385, "loss": 0.0686, "learning_rate": 1.3750000000000002e-06, "epoch": 2.122186495176849, "percentage": 42.86, "elapsed_time": "0:35:58", "remaining_time": "0:47:58", "throughput": "1030.53", "total_tokens": 2224480}
+{"current_steps": 166, "total_steps": 385, "loss": 0.0545, "learning_rate": 1.3833333333333336e-06, "epoch": 2.135048231511254, "percentage": 43.12, "elapsed_time": "0:36:11", "remaining_time": "0:47:44", "throughput": "1030.54", "total_tokens": 2237920}
+{"current_steps": 167, "total_steps": 385, "loss": 0.0347, "learning_rate": 1.3916666666666668e-06, "epoch": 2.147909967845659, "percentage": 43.38, "elapsed_time": "0:36:24", "remaining_time": "0:47:31", "throughput": "1030.37", "total_tokens": 2251008}
+{"current_steps": 168, "total_steps": 385, "loss": 0.0993, "learning_rate": 1.4000000000000001e-06, "epoch": 2.1607717041800645, "percentage": 43.64, "elapsed_time": "0:36:37", "remaining_time": "0:47:18", "throughput": "1030.36", "total_tokens": 2264448}
+{"current_steps": 169, "total_steps": 385, "loss": 0.1059, "learning_rate": 1.4083333333333335e-06, "epoch": 2.1736334405144695, "percentage": 43.9, "elapsed_time": "0:36:50", "remaining_time": "0:47:05", "throughput": "1030.18", "total_tokens": 2277568}
+{"current_steps": 170, "total_steps": 385, "loss": 0.089, "learning_rate": 1.4166666666666667e-06, "epoch": 2.1864951768488745, "percentage": 44.16, "elapsed_time": "0:37:03", "remaining_time": "0:46:52", "throughput": "1030.24", "total_tokens": 2291168}
+{"current_steps": 171, "total_steps": 385, "loss": 0.0379, "learning_rate": 1.425e-06, "epoch": 2.19935691318328, "percentage": 44.42, "elapsed_time": "0:37:16", "remaining_time": "0:46:39", "throughput": "1030.07", "total_tokens": 2304192}
+{"current_steps": 172, "total_steps": 385, "loss": 0.0626, "learning_rate": 1.4333333333333335e-06, "epoch": 2.212218649517685, "percentage": 44.68, "elapsed_time": "0:37:29", "remaining_time": "0:46:26", "throughput": "1030.20", "total_tokens": 2317920}
+{"current_steps": 173, "total_steps": 385, "loss": 0.0957, "learning_rate": 1.4416666666666667e-06, "epoch": 2.22508038585209, "percentage": 44.94, "elapsed_time": "0:37:43", "remaining_time": "0:46:13", "throughput": "1030.64", "total_tokens": 2332416}
+{"current_steps": 174, "total_steps": 385, "loss": 0.0636, "learning_rate": 1.45e-06, "epoch": 2.237942122186495, "percentage": 45.19, "elapsed_time": "0:37:56", "remaining_time": "0:46:00", "throughput": "1030.75", "total_tokens": 2346112}
+{"current_steps": 175, "total_steps": 385, "loss": 0.074, "learning_rate": 1.4583333333333335e-06, "epoch": 2.2508038585209005, "percentage": 45.45, "elapsed_time": "0:38:09", "remaining_time": "0:45:47", "throughput": "1030.71", "total_tokens": 2359488}
+{"current_steps": 176, "total_steps": 385, "loss": 0.0685, "learning_rate": 1.4666666666666669e-06, "epoch": 2.2636655948553055, "percentage": 45.71, "elapsed_time": "0:38:22", "remaining_time": "0:45:33", "throughput": "1030.69", "total_tokens": 2372928}
+{"current_steps": 177, "total_steps": 385, "loss": 0.0574, "learning_rate": 1.475e-06, "epoch": 2.2765273311897105, "percentage": 45.97, "elapsed_time": "0:38:35", "remaining_time": "0:45:20", "throughput": "1030.53", "total_tokens": 2386016}
+{"current_steps": 178, "total_steps": 385, "loss": 0.0619, "learning_rate": 1.4833333333333337e-06, "epoch": 2.289389067524116, "percentage": 46.23, "elapsed_time": "0:38:48", "remaining_time": "0:45:07", "throughput": "1030.73", "total_tokens": 2399936}
+{"current_steps": 179, "total_steps": 385, "loss": 0.0683, "learning_rate": 1.4916666666666669e-06, "epoch": 2.302250803858521, "percentage": 46.49, "elapsed_time": "0:39:01", "remaining_time": "0:44:54", "throughput": "1030.36", "total_tokens": 2412480}
+{"current_steps": 180, "total_steps": 385, "loss": 0.07, "learning_rate": 1.5e-06, "epoch": 2.315112540192926, "percentage": 46.75, "elapsed_time": "0:39:14", "remaining_time": "0:44:41", "throughput": "1030.33", "total_tokens": 2425888}
+{"current_steps": 181, "total_steps": 385, "loss": 0.1154, "learning_rate": 1.5083333333333336e-06, "epoch": 2.327974276527331, "percentage": 47.01, "elapsed_time": "0:39:27", "remaining_time": "0:44:28", "throughput": "1030.26", "total_tokens": 2439168}
+{"current_steps": 182, "total_steps": 385, "loss": 0.0923, "learning_rate": 1.5166666666666668e-06, "epoch": 2.3408360128617365, "percentage": 47.27, "elapsed_time": "0:39:40", "remaining_time": "0:44:15", "throughput": "1030.40", "total_tokens": 2452928}
+{"current_steps": 183, "total_steps": 385, "loss": 0.0777, "learning_rate": 1.525e-06, "epoch": 2.3536977491961415, "percentage": 47.53, "elapsed_time": "0:39:53", "remaining_time": "0:44:02", "throughput": "1030.14", "total_tokens": 2465760}
+{"current_steps": 184, "total_steps": 385, "loss": 0.0754, "learning_rate": 1.5333333333333334e-06, "epoch": 2.3665594855305465, "percentage": 47.79, "elapsed_time": "0:40:06", "remaining_time": "0:43:49", "throughput": "1030.34", "total_tokens": 2479744}
+{"current_steps": 185, "total_steps": 385, "loss": 0.0704, "learning_rate": 1.5416666666666668e-06, "epoch": 2.379421221864952, "percentage": 48.05, "elapsed_time": "0:40:19", "remaining_time": "0:43:35", "throughput": "1030.27", "total_tokens": 2493024}
+{"current_steps": 186, "total_steps": 385, "loss": 0.0915, "learning_rate": 1.5500000000000002e-06, "epoch": 2.392282958199357, "percentage": 48.31, "elapsed_time": "0:40:32", "remaining_time": "0:43:22", "throughput": "1030.38", "total_tokens": 2506720}
+{"current_steps": 187, "total_steps": 385, "loss": 0.087, "learning_rate": 1.5583333333333334e-06, "epoch": 2.405144694533762, "percentage": 48.57, "elapsed_time": "0:40:45", "remaining_time": "0:43:09", "throughput": "1030.22", "total_tokens": 2519776}
+{"current_steps": 188, "total_steps": 385, "loss": 0.0566, "learning_rate": 1.566666666666667e-06, "epoch": 2.418006430868167, "percentage": 48.83, "elapsed_time": "0:40:58", "remaining_time": "0:42:56", "throughput": "1030.35", "total_tokens": 2533536}
+{"current_steps": 189, "total_steps": 385, "loss": 0.1037, "learning_rate": 1.5750000000000002e-06, "epoch": 2.4308681672025725, "percentage": 49.09, "elapsed_time": "0:41:11", "remaining_time": "0:42:43", "throughput": "1030.56", "total_tokens": 2547520}
+{"current_steps": 190, "total_steps": 385, "loss": 0.1143, "learning_rate": 1.5833333333333333e-06, "epoch": 2.4437299035369775, "percentage": 49.35, "elapsed_time": "0:41:25", "remaining_time": "0:42:30", "throughput": "1030.68", "total_tokens": 2561280}
+{"current_steps": 191, "total_steps": 385, "loss": 0.0829, "learning_rate": 1.591666666666667e-06, "epoch": 2.4565916398713825, "percentage": 49.61, "elapsed_time": "0:41:38", "remaining_time": "0:42:17", "throughput": "1030.82", "total_tokens": 2575136}
+{"current_steps": 192, "total_steps": 385, "loss": 0.0422, "learning_rate": 1.6000000000000001e-06, "epoch": 2.469453376205788, "percentage": 49.87, "elapsed_time": "0:41:51", "remaining_time": "0:42:04", "throughput": "1030.87", "total_tokens": 2588736}
+{"current_steps": 193, "total_steps": 385, "loss": 0.0727, "learning_rate": 1.6083333333333333e-06, "epoch": 2.482315112540193, "percentage": 50.13, "elapsed_time": "0:42:04", "remaining_time": "0:41:51", "throughput": "1030.95", "total_tokens": 2602400}
+{"current_steps": 194, "total_steps": 385, "loss": 0.0836, "learning_rate": 1.6166666666666667e-06, "epoch": 2.495176848874598, "percentage": 50.39, "elapsed_time": "0:42:17", "remaining_time": "0:41:38", "throughput": "1030.88", "total_tokens": 2615648}
+{"current_steps": 195, "total_steps": 385, "loss": 0.0803, "learning_rate": 1.6250000000000001e-06, "epoch": 2.508038585209003, "percentage": 50.65, "elapsed_time": "0:42:30", "remaining_time": "0:41:24", "throughput": "1030.85", "total_tokens": 2629024}
+{"current_steps": 196, "total_steps": 385, "loss": 0.0654, "learning_rate": 1.6333333333333335e-06, "epoch": 2.5209003215434085, "percentage": 50.91, "elapsed_time": "0:42:43", "remaining_time": "0:41:11", "throughput": "1030.96", "total_tokens": 2642784}
+{"current_steps": 197, "total_steps": 385, "loss": 0.0587, "learning_rate": 1.6416666666666667e-06, "epoch": 2.5337620578778135, "percentage": 51.17, "elapsed_time": "0:42:56", "remaining_time": "0:40:58", "throughput": "1030.89", "total_tokens": 2656064}
+{"current_steps": 198, "total_steps": 385, "loss": 0.0848, "learning_rate": 1.6500000000000003e-06, "epoch": 2.5466237942122185, "percentage": 51.43, "elapsed_time": "0:43:09", "remaining_time": "0:40:45", "throughput": "1030.87", "total_tokens": 2669472}
+{"current_steps": 199, "total_steps": 385, "loss": 0.0525, "learning_rate": 1.6583333333333335e-06, "epoch": 2.559485530546624, "percentage": 51.69, "elapsed_time": "0:43:22", "remaining_time": "0:40:32", "throughput": "1030.88", "total_tokens": 2682944}
+{"current_steps": 200, "total_steps": 385, "loss": 0.0677, "learning_rate": 1.6666666666666667e-06, "epoch": 2.572347266881029, "percentage": 51.95, "elapsed_time": "0:43:35", "remaining_time": "0:40:19", "throughput": "1030.73", "total_tokens": 2695968}
+{"current_steps": 201, "total_steps": 385, "loss": 0.062, "learning_rate": 1.6750000000000003e-06, "epoch": 2.585209003215434, "percentage": 52.21, "elapsed_time": "0:43:48", "remaining_time": "0:40:06", "throughput": "1030.77", "total_tokens": 2709504}
+{"current_steps": 202, "total_steps": 385, "loss": 0.0674, "learning_rate": 1.6833333333333335e-06, "epoch": 2.598070739549839, "percentage": 52.47, "elapsed_time": "0:44:01", "remaining_time": "0:39:53", "throughput": "1030.89", "total_tokens": 2723264}
+{"current_steps": 203, "total_steps": 385, "loss": 0.0533, "learning_rate": 1.6916666666666666e-06, "epoch": 2.6109324758842445, "percentage": 52.73, "elapsed_time": "0:44:14", "remaining_time": "0:39:40", "throughput": "1030.91", "total_tokens": 2736768}
+{"current_steps": 204, "total_steps": 385, "loss": 0.0757, "learning_rate": 1.7000000000000002e-06, "epoch": 2.6237942122186495, "percentage": 52.99, "elapsed_time": "0:44:27", "remaining_time": "0:39:27", "throughput": "1031.12", "total_tokens": 2750816}
+{"current_steps": 205, "total_steps": 385, "loss": 0.0777, "learning_rate": 1.7083333333333334e-06, "epoch": 2.6366559485530545, "percentage": 53.25, "elapsed_time": "0:44:40", "remaining_time": "0:39:13", "throughput": "1031.22", "total_tokens": 2764576}
+{"current_steps": 206, "total_steps": 385, "loss": 0.0921, "learning_rate": 1.7166666666666668e-06, "epoch": 2.64951768488746, "percentage": 53.51, "elapsed_time": "0:44:53", "remaining_time": "0:39:00", "throughput": "1031.22", "total_tokens": 2778080}
+{"current_steps": 207, "total_steps": 385, "loss": 0.0378, "learning_rate": 1.725e-06, "epoch": 2.662379421221865, "percentage": 53.77, "elapsed_time": "0:45:07", "remaining_time": "0:38:47", "throughput": "1031.02", "total_tokens": 2790976}
+{"current_steps": 208, "total_steps": 385, "loss": 0.0671, "learning_rate": 1.7333333333333336e-06, "epoch": 2.67524115755627, "percentage": 54.03, "elapsed_time": "0:45:20", "remaining_time": "0:38:34", "throughput": "1031.28", "total_tokens": 2805152}
+{"current_steps": 209, "total_steps": 385, "loss": 0.0664, "learning_rate": 1.7416666666666668e-06, "epoch": 2.688102893890675, "percentage": 54.29, "elapsed_time": "0:45:33", "remaining_time": "0:38:21", "throughput": "1031.30", "total_tokens": 2818688}
+{"current_steps": 210, "total_steps": 385, "loss": 0.072, "learning_rate": 1.75e-06, "epoch": 2.7009646302250805, "percentage": 54.55, "elapsed_time": "0:45:46", "remaining_time": "0:38:08", "throughput": "1031.20", "total_tokens": 2831872}
+{"current_steps": 211, "total_steps": 385, "loss": 0.0883, "learning_rate": 1.7583333333333336e-06, "epoch": 2.7138263665594855, "percentage": 54.81, "elapsed_time": "0:45:59", "remaining_time": "0:37:55", "throughput": "1031.43", "total_tokens": 2845984}
+{"current_steps": 212, "total_steps": 385, "loss": 0.0414, "learning_rate": 1.7666666666666668e-06, "epoch": 2.7266881028938905, "percentage": 55.06, "elapsed_time": "0:46:12", "remaining_time": "0:37:42", "throughput": "1031.35", "total_tokens": 2859232}
+{"current_steps": 213, "total_steps": 385, "loss": 0.031, "learning_rate": 1.7750000000000002e-06, "epoch": 2.739549839228296, "percentage": 55.32, "elapsed_time": "0:46:25", "remaining_time": "0:37:29", "throughput": "1031.16", "total_tokens": 2872192}
+{"current_steps": 214, "total_steps": 385, "loss": 0.0634, "learning_rate": 1.7833333333333336e-06, "epoch": 2.752411575562701, "percentage": 55.58, "elapsed_time": "0:46:38", "remaining_time": "0:37:16", "throughput": "1031.04", "total_tokens": 2885312}
+{"current_steps": 215, "total_steps": 385, "loss": 0.0837, "learning_rate": 1.7916666666666667e-06, "epoch": 2.765273311897106, "percentage": 55.84, "elapsed_time": "0:46:51", "remaining_time": "0:37:03", "throughput": "1031.15", "total_tokens": 2899072}
+{"current_steps": 216, "total_steps": 385, "loss": 0.0855, "learning_rate": 1.8000000000000001e-06, "epoch": 2.778135048231511, "percentage": 56.1, "elapsed_time": "0:47:04", "remaining_time": "0:36:49", "throughput": "1031.11", "total_tokens": 2912448}
+{"current_steps": 217, "total_steps": 385, "loss": 0.0945, "learning_rate": 1.8083333333333335e-06, "epoch": 2.7909967845659165, "percentage": 56.36, "elapsed_time": "0:47:17", "remaining_time": "0:36:36", "throughput": "1030.85", "total_tokens": 2925120}
+{"current_steps": 218, "total_steps": 385, "loss": 0.078, "learning_rate": 1.816666666666667e-06, "epoch": 2.8038585209003215, "percentage": 56.62, "elapsed_time": "0:47:30", "remaining_time": "0:36:23", "throughput": "1030.78", "total_tokens": 2938336}
+{"current_steps": 219, "total_steps": 385, "loss": 0.0573, "learning_rate": 1.825e-06, "epoch": 2.816720257234727, "percentage": 56.88, "elapsed_time": "0:47:43", "remaining_time": "0:36:10", "throughput": "1030.59", "total_tokens": 2951264}
+{"current_steps": 220, "total_steps": 385, "loss": 0.0806, "learning_rate": 1.8333333333333333e-06, "epoch": 2.829581993569132, "percentage": 57.14, "elapsed_time": "0:47:56", "remaining_time": "0:35:57", "throughput": "1030.71", "total_tokens": 2965120}
+{"current_steps": 221, "total_steps": 385, "loss": 0.0961, "learning_rate": 1.8416666666666669e-06, "epoch": 2.842443729903537, "percentage": 57.4, "elapsed_time": "0:48:09", "remaining_time": "0:35:44", "throughput": "1030.63", "total_tokens": 2978368}
+{"current_steps": 222, "total_steps": 385, "loss": 0.0732, "learning_rate": 1.85e-06, "epoch": 2.855305466237942, "percentage": 57.66, "elapsed_time": "0:48:22", "remaining_time": "0:35:31", "throughput": "1030.70", "total_tokens": 2992000}
+{"current_steps": 223, "total_steps": 385, "loss": 0.0957, "learning_rate": 1.8583333333333335e-06, "epoch": 2.868167202572347, "percentage": 57.92, "elapsed_time": "0:48:35", "remaining_time": "0:35:18", "throughput": "1030.89", "total_tokens": 3006048}
+{"current_steps": 224, "total_steps": 385, "loss": 0.0774, "learning_rate": 1.8666666666666669e-06, "epoch": 2.8810289389067525, "percentage": 58.18, "elapsed_time": "0:48:49", "remaining_time": "0:35:05", "throughput": "1030.97", "total_tokens": 3019744}
+{"current_steps": 225, "total_steps": 385, "loss": 0.0691, "learning_rate": 1.8750000000000003e-06, "epoch": 2.8938906752411575, "percentage": 58.44, "elapsed_time": "0:49:02", "remaining_time": "0:34:52", "throughput": "1031.09", "total_tokens": 3033568}
+{"current_steps": 226, "total_steps": 385, "loss": 0.0529, "learning_rate": 1.8833333333333334e-06, "epoch": 2.906752411575563, "percentage": 58.7, "elapsed_time": "0:49:15", "remaining_time": "0:34:39", "throughput": "1031.31", "total_tokens": 3047712}
+{"current_steps": 227, "total_steps": 385, "loss": 0.0811, "learning_rate": 1.8916666666666668e-06, "epoch": 2.919614147909968, "percentage": 58.96, "elapsed_time": "0:49:28", "remaining_time": "0:34:26", "throughput": "1031.18", "total_tokens": 3060800}
+{"current_steps": 228, "total_steps": 385, "loss": 0.1211, "learning_rate": 1.9000000000000002e-06, "epoch": 2.932475884244373, "percentage": 59.22, "elapsed_time": "0:49:41", "remaining_time": "0:34:12", "throughput": "1031.32", "total_tokens": 3074720}
+{"current_steps": 229, "total_steps": 385, "loss": 0.0489, "learning_rate": 1.9083333333333334e-06, "epoch": 2.945337620578778, "percentage": 59.48, "elapsed_time": "0:49:54", "remaining_time": "0:33:59", "throughput": "1031.40", "total_tokens": 3088448}
+{"current_steps": 230, "total_steps": 385, "loss": 0.0947, "learning_rate": 1.916666666666667e-06, "epoch": 2.958199356913183, "percentage": 59.74, "elapsed_time": "0:50:07", "remaining_time": "0:33:46", "throughput": "1031.77", "total_tokens": 3103008}
+{"current_steps": 231, "total_steps": 385, "loss": 0.0561, "learning_rate": 1.925e-06, "epoch": 2.9710610932475885, "percentage": 60.0, "elapsed_time": "0:50:20", "remaining_time": "0:33:33", "throughput": "1031.72", "total_tokens": 3116320}
+{"current_steps": 232, "total_steps": 385, "loss": 0.0629, "learning_rate": 1.9333333333333336e-06, "epoch": 2.9839228295819935, "percentage": 60.26, "elapsed_time": "0:50:33", "remaining_time": "0:33:20", "throughput": "1031.97", "total_tokens": 3130592}
+{"current_steps": 233, "total_steps": 385, "loss": 0.0579, "learning_rate": 1.9416666666666666e-06, "epoch": 2.996784565916399, "percentage": 60.52, "elapsed_time": "0:50:46", "remaining_time": "0:33:07", "throughput": "1031.94", "total_tokens": 3144000}
+{"current_steps": 234, "total_steps": 385, "loss": 0.0285, "learning_rate": 1.9500000000000004e-06, "epoch": 3.009646302250804, "percentage": 60.78, "elapsed_time": "0:50:59", "remaining_time": "0:32:54", "throughput": "1031.94", "total_tokens": 3157472}
+{"current_steps": 235, "total_steps": 385, "loss": 0.0256, "learning_rate": 1.9583333333333334e-06, "epoch": 3.022508038585209, "percentage": 61.04, "elapsed_time": "0:51:12", "remaining_time": "0:32:41", "throughput": "1031.83", "total_tokens": 3170656}
+{"current_steps": 236, "total_steps": 385, "loss": 0.0247, "learning_rate": 1.9666666666666668e-06, "epoch": 3.035369774919614, "percentage": 61.3, "elapsed_time": "0:51:25", "remaining_time": "0:32:28", "throughput": "1031.89", "total_tokens": 3184320}
+{"current_steps": 237, "total_steps": 385, "loss": 0.0325, "learning_rate": 1.975e-06, "epoch": 3.0482315112540195, "percentage": 61.56, "elapsed_time": "0:51:38", "remaining_time": "0:32:15", "throughput": "1031.71", "total_tokens": 3197216}
+{"current_steps": 238, "total_steps": 385, "loss": 0.0172, "learning_rate": 1.9833333333333335e-06, "epoch": 3.0610932475884245, "percentage": 61.82, "elapsed_time": "0:51:51", "remaining_time": "0:32:02", "throughput": "1031.83", "total_tokens": 3211008}
+{"current_steps": 239, "total_steps": 385, "loss": 0.05, "learning_rate": 1.991666666666667e-06, "epoch": 3.0739549839228295, "percentage": 62.08, "elapsed_time": "0:52:05", "remaining_time": "0:31:49", "throughput": "1032.03", "total_tokens": 3225152}
+{"current_steps": 240, "total_steps": 385, "loss": 0.0134, "learning_rate": 2.0000000000000003e-06, "epoch": 3.0868167202572345, "percentage": 62.34, "elapsed_time": "0:52:18", "remaining_time": "0:31:35", "throughput": "1032.07", "total_tokens": 3238752}
+{"current_steps": 241, "total_steps": 385, "loss": 0.0434, "learning_rate": 2.0083333333333337e-06, "epoch": 3.09967845659164, "percentage": 62.6, "elapsed_time": "0:52:31", "remaining_time": "0:31:22", "throughput": "1032.06", "total_tokens": 3252224}
+{"current_steps": 242, "total_steps": 385, "loss": 0.0186, "learning_rate": 2.0166666666666667e-06, "epoch": 3.112540192926045, "percentage": 62.86, "elapsed_time": "0:52:44", "remaining_time": "0:31:09", "throughput": "1031.79", "total_tokens": 3264832}
+{"current_steps": 243, "total_steps": 385, "loss": 0.0341, "learning_rate": 2.025e-06, "epoch": 3.12540192926045, "percentage": 63.12, "elapsed_time": "0:52:57", "remaining_time": "0:30:56", "throughput": "1031.70", "total_tokens": 3278048}
+{"current_steps": 244, "total_steps": 385, "loss": 0.0386, "learning_rate": 2.0333333333333335e-06, "epoch": 3.1382636655948555, "percentage": 63.38, "elapsed_time": "0:53:10", "remaining_time": "0:30:43", "throughput": "1031.75", "total_tokens": 3291680}
+{"current_steps": 245, "total_steps": 385, "loss": 0.0389, "learning_rate": 2.041666666666667e-06, "epoch": 3.1511254019292605, "percentage": 63.64, "elapsed_time": "0:53:23", "remaining_time": "0:30:30", "throughput": "1031.84", "total_tokens": 3305440}
+{"current_steps": 246, "total_steps": 385, "loss": 0.0227, "learning_rate": 2.05e-06, "epoch": 3.1639871382636655, "percentage": 63.9, "elapsed_time": "0:53:36", "remaining_time": "0:30:17", "throughput": "1031.98", "total_tokens": 3319360}
+{"current_steps": 247, "total_steps": 385, "loss": 0.0317, "learning_rate": 2.0583333333333337e-06, "epoch": 3.176848874598071, "percentage": 64.16, "elapsed_time": "0:53:49", "remaining_time": "0:30:04", "throughput": "1032.06", "total_tokens": 3333088}
+{"current_steps": 248, "total_steps": 385, "loss": 0.0335, "learning_rate": 2.0666666666666666e-06, "epoch": 3.189710610932476, "percentage": 64.42, "elapsed_time": "0:54:02", "remaining_time": "0:29:51", "throughput": "1031.85", "total_tokens": 3345856}
+{"current_steps": 249, "total_steps": 385, "loss": 0.0257, "learning_rate": 2.075e-06, "epoch": 3.202572347266881, "percentage": 64.68, "elapsed_time": "0:54:15", "remaining_time": "0:29:38", "throughput": "1031.71", "total_tokens": 3358880}
+{"current_steps": 250, "total_steps": 385, "loss": 0.0244, "learning_rate": 2.0833333333333334e-06, "epoch": 3.215434083601286, "percentage": 64.94, "elapsed_time": "0:54:28", "remaining_time": "0:29:25", "throughput": "1031.99", "total_tokens": 3373312}
+{"current_steps": 251, "total_steps": 385, "loss": 0.0285, "learning_rate": 2.091666666666667e-06, "epoch": 3.2282958199356915, "percentage": 65.19, "elapsed_time": "0:54:41", "remaining_time": "0:29:12", "throughput": "1031.85", "total_tokens": 3386336}
+{"current_steps": 252, "total_steps": 385, "loss": 0.0093, "learning_rate": 2.1000000000000002e-06, "epoch": 3.2411575562700965, "percentage": 65.45, "elapsed_time": "0:54:54", "remaining_time": "0:28:58", "throughput": "1031.89", "total_tokens": 3399904}
+{"current_steps": 253, "total_steps": 385, "loss": 0.0415, "learning_rate": 2.1083333333333336e-06, "epoch": 3.2540192926045015, "percentage": 65.71, "elapsed_time": "0:55:07", "remaining_time": "0:28:45", "throughput": "1031.75", "total_tokens": 3412896}
+{"current_steps": 254, "total_steps": 385, "loss": 0.0239, "learning_rate": 2.116666666666667e-06, "epoch": 3.266881028938907, "percentage": 65.97, "elapsed_time": "0:55:20", "remaining_time": "0:28:32", "throughput": "1031.75", "total_tokens": 3426336}
+{"current_steps": 255, "total_steps": 385, "loss": 0.0412, "learning_rate": 2.125e-06, "epoch": 3.279742765273312, "percentage": 66.23, "elapsed_time": "0:55:33", "remaining_time": "0:28:19", "throughput": "1031.71", "total_tokens": 3439680}
+{"current_steps": 256, "total_steps": 385, "loss": 0.0503, "learning_rate": 2.133333333333334e-06, "epoch": 3.292604501607717, "percentage": 66.49, "elapsed_time": "0:55:47", "remaining_time": "0:28:06", "throughput": "1031.89", "total_tokens": 3453760}
+{"current_steps": 257, "total_steps": 385, "loss": 0.0046, "learning_rate": 2.1416666666666668e-06, "epoch": 3.305466237942122, "percentage": 66.75, "elapsed_time": "0:56:00", "remaining_time": "0:27:53", "throughput": "1031.66", "total_tokens": 3466496}
+{"current_steps": 258, "total_steps": 385, "loss": 0.041, "learning_rate": 2.15e-06, "epoch": 3.3183279742765275, "percentage": 67.01, "elapsed_time": "0:56:13", "remaining_time": "0:27:40", "throughput": "1031.77", "total_tokens": 3480352}
+{"current_steps": 259, "total_steps": 385, "loss": 0.0257, "learning_rate": 2.1583333333333336e-06, "epoch": 3.3311897106109325, "percentage": 67.27, "elapsed_time": "0:56:26", "remaining_time": "0:27:27", "throughput": "1031.85", "total_tokens": 3494112}
+{"current_steps": 260, "total_steps": 385, "loss": 0.0168, "learning_rate": 2.166666666666667e-06, "epoch": 3.3440514469453375, "percentage": 67.53, "elapsed_time": "0:56:39", "remaining_time": "0:27:14", "throughput": "1031.84", "total_tokens": 3507520}
+{"current_steps": 261, "total_steps": 385, "loss": 0.0439, "learning_rate": 2.1750000000000004e-06, "epoch": 3.356913183279743, "percentage": 67.79, "elapsed_time": "0:56:52", "remaining_time": "0:27:01", "throughput": "1031.65", "total_tokens": 3520352}
+{"current_steps": 262, "total_steps": 385, "loss": 0.0204, "learning_rate": 2.1833333333333333e-06, "epoch": 3.369774919614148, "percentage": 68.05, "elapsed_time": "0:57:05", "remaining_time": "0:26:48", "throughput": "1031.70", "total_tokens": 3533984}
+{"current_steps": 263, "total_steps": 385, "loss": 0.0284, "learning_rate": 2.191666666666667e-06, "epoch": 3.382636655948553, "percentage": 68.31, "elapsed_time": "0:57:18", "remaining_time": "0:26:35", "throughput": "1031.59", "total_tokens": 3547072}
+{"current_steps": 264, "total_steps": 385, "loss": 0.0684, "learning_rate": 2.2e-06, "epoch": 3.395498392282958, "percentage": 68.57, "elapsed_time": "0:57:31", "remaining_time": "0:26:21", "throughput": "1031.39", "total_tokens": 3559872}
+{"current_steps": 265, "total_steps": 385, "loss": 0.0479, "learning_rate": 2.2083333333333335e-06, "epoch": 3.4083601286173635, "percentage": 68.83, "elapsed_time": "0:57:44", "remaining_time": "0:26:08", "throughput": "1031.24", "total_tokens": 3572832}
+{"current_steps": 266, "total_steps": 385, "loss": 0.0434, "learning_rate": 2.216666666666667e-06, "epoch": 3.4212218649517685, "percentage": 69.09, "elapsed_time": "0:57:57", "remaining_time": "0:25:55", "throughput": "1031.11", "total_tokens": 3585792}
+{"current_steps": 267, "total_steps": 385, "loss": 0.0213, "learning_rate": 2.2250000000000003e-06, "epoch": 3.4340836012861735, "percentage": 69.35, "elapsed_time": "0:58:10", "remaining_time": "0:25:42", "throughput": "1031.15", "total_tokens": 3599392}
+{"current_steps": 268, "total_steps": 385, "loss": 0.0415, "learning_rate": 2.2333333333333333e-06, "epoch": 3.446945337620579, "percentage": 69.61, "elapsed_time": "0:58:23", "remaining_time": "0:25:29", "throughput": "1031.22", "total_tokens": 3613088}
+{"current_steps": 269, "total_steps": 385, "loss": 0.0404, "learning_rate": 2.2416666666666667e-06, "epoch": 3.459807073954984, "percentage": 69.87, "elapsed_time": "0:58:36", "remaining_time": "0:25:16", "throughput": "1031.14", "total_tokens": 3626304}
+{"current_steps": 270, "total_steps": 385, "loss": 0.0566, "learning_rate": 2.25e-06, "epoch": 3.472668810289389, "percentage": 70.13, "elapsed_time": "0:58:49", "remaining_time": "0:25:03", "throughput": "1031.20", "total_tokens": 3639968}
+{"current_steps": 271, "total_steps": 385, "loss": 0.0509, "learning_rate": 2.2583333333333335e-06, "epoch": 3.485530546623794, "percentage": 70.39, "elapsed_time": "0:59:02", "remaining_time": "0:24:50", "throughput": "1031.20", "total_tokens": 3653472}
+{"current_steps": 272, "total_steps": 385, "loss": 0.0385, "learning_rate": 2.266666666666667e-06, "epoch": 3.4983922829581995, "percentage": 70.65, "elapsed_time": "0:59:16", "remaining_time": "0:24:37", "throughput": "1031.23", "total_tokens": 3667104}
+{"current_steps": 273, "total_steps": 385, "loss": 0.0225, "learning_rate": 2.2750000000000002e-06, "epoch": 3.5112540192926045, "percentage": 70.91, "elapsed_time": "0:59:29", "remaining_time": "0:24:24", "throughput": "1031.20", "total_tokens": 3680480}
+{"current_steps": 274, "total_steps": 385, "loss": 0.0255, "learning_rate": 2.2833333333333336e-06, "epoch": 3.5241157556270095, "percentage": 71.17, "elapsed_time": "0:59:42", "remaining_time": "0:24:11", "throughput": "1031.11", "total_tokens": 3693568}
+{"current_steps": 275, "total_steps": 385, "loss": 0.0531, "learning_rate": 2.2916666666666666e-06, "epoch": 3.536977491961415, "percentage": 71.43, "elapsed_time": "0:59:55", "remaining_time": "0:23:58", "throughput": "1030.96", "total_tokens": 3706496}
+{"current_steps": 276, "total_steps": 385, "loss": 0.0095, "learning_rate": 2.3000000000000004e-06, "epoch": 3.54983922829582, "percentage": 71.69, "elapsed_time": "1:00:08", "remaining_time": "0:23:44", "throughput": "1031.07", "total_tokens": 3720352}
+{"current_steps": 277, "total_steps": 385, "loss": 0.0229, "learning_rate": 2.3083333333333334e-06, "epoch": 3.562700964630225, "percentage": 71.95, "elapsed_time": "1:00:21", "remaining_time": "0:23:31", "throughput": "1031.17", "total_tokens": 3734176}
+{"current_steps": 278, "total_steps": 385, "loss": 0.038, "learning_rate": 2.316666666666667e-06, "epoch": 3.57556270096463, "percentage": 72.21, "elapsed_time": "1:00:34", "remaining_time": "0:23:18", "throughput": "1031.41", "total_tokens": 3748544}
+{"current_steps": 279, "total_steps": 385, "loss": 0.0316, "learning_rate": 2.325e-06, "epoch": 3.5884244372990355, "percentage": 72.47, "elapsed_time": "1:00:47", "remaining_time": "0:23:05", "throughput": "1031.40", "total_tokens": 3761984}
+{"current_steps": 280, "total_steps": 385, "loss": 0.0861, "learning_rate": 2.3333333333333336e-06, "epoch": 3.6012861736334405, "percentage": 72.73, "elapsed_time": "1:01:00", "remaining_time": "0:22:52", "throughput": "1031.09", "total_tokens": 3774304}
+{"current_steps": 281, "total_steps": 385, "loss": 0.0566, "learning_rate": 2.341666666666667e-06, "epoch": 3.6141479099678455, "percentage": 72.99, "elapsed_time": "1:01:13", "remaining_time": "0:22:39", "throughput": "1031.26", "total_tokens": 3788416}
+{"current_steps": 282, "total_steps": 385, "loss": 0.0804, "learning_rate": 2.35e-06, "epoch": 3.627009646302251, "percentage": 73.25, "elapsed_time": "1:01:26", "remaining_time": "0:22:26", "throughput": "1031.33", "total_tokens": 3802112}
+{"current_steps": 283, "total_steps": 385, "loss": 0.046, "learning_rate": 2.3583333333333338e-06, "epoch": 3.639871382636656, "percentage": 73.51, "elapsed_time": "1:01:39", "remaining_time": "0:22:13", "throughput": "1031.43", "total_tokens": 3815968}
+{"current_steps": 284, "total_steps": 385, "loss": 0.0693, "learning_rate": 2.3666666666666667e-06, "epoch": 3.652733118971061, "percentage": 73.77, "elapsed_time": "1:01:52", "remaining_time": "0:22:00", "throughput": "1031.38", "total_tokens": 3829248}
+{"current_steps": 285, "total_steps": 385, "loss": 0.0342, "learning_rate": 2.375e-06, "epoch": 3.665594855305466, "percentage": 74.03, "elapsed_time": "1:02:05", "remaining_time": "0:21:47", "throughput": "1031.47", "total_tokens": 3843072}
+{"current_steps": 286, "total_steps": 385, "loss": 0.0479, "learning_rate": 2.3833333333333335e-06, "epoch": 3.6784565916398715, "percentage": 74.29, "elapsed_time": "1:02:18", "remaining_time": "0:21:34", "throughput": "1031.40", "total_tokens": 3856288}
+{"current_steps": 287, "total_steps": 385, "loss": 0.0388, "learning_rate": 2.391666666666667e-06, "epoch": 3.6913183279742765, "percentage": 74.55, "elapsed_time": "1:02:32", "remaining_time": "0:21:21", "throughput": "1031.51", "total_tokens": 3870240}
+{"current_steps": 288, "total_steps": 385, "loss": 0.0274, "learning_rate": 2.4000000000000003e-06, "epoch": 3.7041800643086815, "percentage": 74.81, "elapsed_time": "1:02:45", "remaining_time": "0:21:08", "throughput": "1031.62", "total_tokens": 3884096}
+{"current_steps": 289, "total_steps": 385, "loss": 0.0259, "learning_rate": 2.4083333333333337e-06, "epoch": 3.717041800643087, "percentage": 75.06, "elapsed_time": "1:02:58", "remaining_time": "0:20:55", "throughput": "1031.83", "total_tokens": 3898368}
+{"current_steps": 290, "total_steps": 385, "loss": 0.0367, "learning_rate": 2.4166666666666667e-06, "epoch": 3.729903536977492, "percentage": 75.32, "elapsed_time": "1:03:11", "remaining_time": "0:20:41", "throughput": "1031.66", "total_tokens": 3911200}
+{"current_steps": 291, "total_steps": 385, "loss": 0.0661, "learning_rate": 2.425e-06, "epoch": 3.742765273311897, "percentage": 75.58, "elapsed_time": "1:03:24", "remaining_time": "0:20:28", "throughput": "1031.57", "total_tokens": 3924320}
+{"current_steps": 292, "total_steps": 385, "loss": 0.0466, "learning_rate": 2.4333333333333335e-06, "epoch": 3.755627009646302, "percentage": 75.84, "elapsed_time": "1:03:37", "remaining_time": "0:20:15", "throughput": "1031.53", "total_tokens": 3937632}
+{"current_steps": 293, "total_steps": 385, "loss": 0.0286, "learning_rate": 2.441666666666667e-06, "epoch": 3.7684887459807075, "percentage": 76.1, "elapsed_time": "1:03:50", "remaining_time": "0:20:02", "throughput": "1031.51", "total_tokens": 3951040}
+{"current_steps": 294, "total_steps": 385, "loss": 0.0586, "learning_rate": 2.4500000000000003e-06, "epoch": 3.7813504823151125, "percentage": 76.36, "elapsed_time": "1:04:03", "remaining_time": "0:19:49", "throughput": "1031.36", "total_tokens": 3963936}
+{"current_steps": 295, "total_steps": 385, "loss": 0.0329, "learning_rate": 2.4583333333333332e-06, "epoch": 3.7942122186495175, "percentage": 76.62, "elapsed_time": "1:04:16", "remaining_time": "0:19:36", "throughput": "1031.22", "total_tokens": 3976832}
+{"current_steps": 296, "total_steps": 385, "loss": 0.0582, "learning_rate": 2.466666666666667e-06, "epoch": 3.807073954983923, "percentage": 76.88, "elapsed_time": "1:04:29", "remaining_time": "0:19:23", "throughput": "1031.30", "total_tokens": 3990592}
+{"current_steps": 297, "total_steps": 385, "loss": 0.0312, "learning_rate": 2.475e-06, "epoch": 3.819935691318328, "percentage": 77.14, "elapsed_time": "1:04:42", "remaining_time": "0:19:10", "throughput": "1031.36", "total_tokens": 4004288}
+{"current_steps": 298, "total_steps": 385, "loss": 0.0329, "learning_rate": 2.4833333333333334e-06, "epoch": 3.832797427652733, "percentage": 77.4, "elapsed_time": "1:04:55", "remaining_time": "0:18:57", "throughput": "1031.39", "total_tokens": 4017856}
+{"current_steps": 299, "total_steps": 385, "loss": 0.0206, "learning_rate": 2.491666666666667e-06, "epoch": 3.845659163987138, "percentage": 77.66, "elapsed_time": "1:05:08", "remaining_time": "0:18:44", "throughput": "1031.35", "total_tokens": 4031168}
+{"current_steps": 300, "total_steps": 385, "loss": 0.0426, "learning_rate": 2.5e-06, "epoch": 3.8585209003215435, "percentage": 77.92, "elapsed_time": "1:05:21", "remaining_time": "0:18:31", "throughput": "1031.30", "total_tokens": 4044416}
+{"current_steps": 301, "total_steps": 385, "loss": 0.0179, "learning_rate": 2.5083333333333336e-06, "epoch": 3.8713826366559485, "percentage": 78.18, "elapsed_time": "1:05:34", "remaining_time": "0:18:18", "throughput": "1031.28", "total_tokens": 4057824}
+{"current_steps": 302, "total_steps": 385, "loss": 0.0289, "learning_rate": 2.5166666666666666e-06, "epoch": 3.884244372990354, "percentage": 78.44, "elapsed_time": "1:05:47", "remaining_time": "0:18:04", "throughput": "1031.45", "total_tokens": 4071936}
+{"current_steps": 303, "total_steps": 385, "loss": 0.0303, "learning_rate": 2.5250000000000004e-06, "epoch": 3.897106109324759, "percentage": 78.7, "elapsed_time": "1:06:00", "remaining_time": "0:17:51", "throughput": "1031.48", "total_tokens": 4085536}
+{"current_steps": 304, "total_steps": 385, "loss": 0.046, "learning_rate": 2.5333333333333338e-06, "epoch": 3.909967845659164, "percentage": 78.96, "elapsed_time": "1:06:13", "remaining_time": "0:17:38", "throughput": "1031.62", "total_tokens": 4099552}
+{"current_steps": 305, "total_steps": 385, "loss": 0.0523, "learning_rate": 2.5416666666666668e-06, "epoch": 3.922829581993569, "percentage": 79.22, "elapsed_time": "1:06:26", "remaining_time": "0:17:25", "throughput": "1031.55", "total_tokens": 4112736}
+{"current_steps": 306, "total_steps": 385, "loss": 0.0329, "learning_rate": 2.55e-06, "epoch": 3.935691318327974, "percentage": 79.48, "elapsed_time": "1:06:39", "remaining_time": "0:17:12", "throughput": "1031.52", "total_tokens": 4126080}
+{"current_steps": 307, "total_steps": 385, "loss": 0.0072, "learning_rate": 2.558333333333334e-06, "epoch": 3.9485530546623795, "percentage": 79.74, "elapsed_time": "1:06:53", "remaining_time": "0:16:59", "throughput": "1031.37", "total_tokens": 4138912}
+{"current_steps": 308, "total_steps": 385, "loss": 0.0415, "learning_rate": 2.566666666666667e-06, "epoch": 3.9614147909967845, "percentage": 80.0, "elapsed_time": "1:07:06", "remaining_time": "0:16:46", "throughput": "1031.46", "total_tokens": 4152768}
+{"current_steps": 309, "total_steps": 385, "loss": 0.0233, "learning_rate": 2.5750000000000003e-06, "epoch": 3.97427652733119, "percentage": 80.26, "elapsed_time": "1:07:19", "remaining_time": "0:16:33", "throughput": "1031.53", "total_tokens": 4166528}
+{"current_steps": 310, "total_steps": 385, "loss": 0.0423, "learning_rate": 2.5833333333333337e-06, "epoch": 3.987138263665595, "percentage": 80.52, "elapsed_time": "1:07:32", "remaining_time": "0:16:20", "throughput": "1031.73", "total_tokens": 4180832}
+{"current_steps": 311, "total_steps": 385, "loss": 0.0295, "learning_rate": 2.5916666666666667e-06, "epoch": 4.0, "percentage": 80.78, "elapsed_time": "1:07:45", "remaining_time": "0:16:07", "throughput": "1031.67", "total_tokens": 4194048}
+{"current_steps": 312, "total_steps": 385, "loss": 0.0327, "learning_rate": 2.6e-06, "epoch": 4.012861736334405, "percentage": 81.04, "elapsed_time": "1:07:58", "remaining_time": "0:15:54", "throughput": "1031.56", "total_tokens": 4207040}
+{"current_steps": 313, "total_steps": 385, "loss": 0.0301, "learning_rate": 2.608333333333333e-06, "epoch": 4.02572347266881, "percentage": 81.3, "elapsed_time": "1:08:11", "remaining_time": "0:15:41", "throughput": "1031.43", "total_tokens": 4220000}
+{"current_steps": 314, "total_steps": 385, "loss": 0.0301, "learning_rate": 2.616666666666667e-06, "epoch": 4.038585209003215, "percentage": 81.56, "elapsed_time": "1:08:24", "remaining_time": "0:15:28", "throughput": "1031.40", "total_tokens": 4233344}
+{"current_steps": 315, "total_steps": 385, "loss": 0.0281, "learning_rate": 2.6250000000000003e-06, "epoch": 4.051446945337621, "percentage": 81.82, "elapsed_time": "1:08:37", "remaining_time": "0:15:15", "throughput": "1031.24", "total_tokens": 4246144}
+{"current_steps": 316, "total_steps": 385, "loss": 0.0136, "learning_rate": 2.6333333333333332e-06, "epoch": 4.064308681672026, "percentage": 82.08, "elapsed_time": "1:08:50", "remaining_time": "0:15:01", "throughput": "1031.56", "total_tokens": 4260992}
+{"current_steps": 317, "total_steps": 385, "loss": 0.0219, "learning_rate": 2.6416666666666666e-06, "epoch": 4.077170418006431, "percentage": 82.34, "elapsed_time": "1:09:03", "remaining_time": "0:14:48", "throughput": "1031.48", "total_tokens": 4274112}
+{"current_steps": 318, "total_steps": 385, "loss": 0.0044, "learning_rate": 2.6500000000000005e-06, "epoch": 4.090032154340836, "percentage": 82.6, "elapsed_time": "1:09:16", "remaining_time": "0:14:35", "throughput": "1031.55", "total_tokens": 4287904}
+{"current_steps": 319, "total_steps": 385, "loss": 0.0335, "learning_rate": 2.6583333333333334e-06, "epoch": 4.102893890675241, "percentage": 82.86, "elapsed_time": "1:09:29", "remaining_time": "0:14:22", "throughput": "1031.59", "total_tokens": 4301568}
+{"current_steps": 320, "total_steps": 385, "loss": 0.0053, "learning_rate": 2.666666666666667e-06, "epoch": 4.115755627009646, "percentage": 83.12, "elapsed_time": "1:09:42", "remaining_time": "0:14:09", "throughput": "1031.55", "total_tokens": 4314848}
+{"current_steps": 321, "total_steps": 385, "loss": 0.0196, "learning_rate": 2.6750000000000002e-06, "epoch": 4.128617363344051, "percentage": 83.38, "elapsed_time": "1:09:55", "remaining_time": "0:13:56", "throughput": "1031.35", "total_tokens": 4327456}
+{"current_steps": 322, "total_steps": 385, "loss": 0.0309, "learning_rate": 2.683333333333333e-06, "epoch": 4.141479099678457, "percentage": 83.64, "elapsed_time": "1:10:09", "remaining_time": "0:13:43", "throughput": "1031.36", "total_tokens": 4341024}
+{"current_steps": 323, "total_steps": 385, "loss": 0.0382, "learning_rate": 2.691666666666667e-06, "epoch": 4.154340836012862, "percentage": 83.9, "elapsed_time": "1:10:22", "remaining_time": "0:13:30", "throughput": "1031.20", "total_tokens": 4353824}
+{"current_steps": 324, "total_steps": 385, "loss": 0.046, "learning_rate": 2.7000000000000004e-06, "epoch": 4.167202572347267, "percentage": 84.16, "elapsed_time": "1:10:35", "remaining_time": "0:13:17", "throughput": "1031.28", "total_tokens": 4367616}
+{"current_steps": 325, "total_steps": 385, "loss": 0.0133, "learning_rate": 2.7083333333333334e-06, "epoch": 4.180064308681672, "percentage": 84.42, "elapsed_time": "1:10:48", "remaining_time": "0:13:04", "throughput": "1031.13", "total_tokens": 4380448}
+{"current_steps": 326, "total_steps": 385, "loss": 0.0265, "learning_rate": 2.7166666666666668e-06, "epoch": 4.192926045016077, "percentage": 84.68, "elapsed_time": "1:11:01", "remaining_time": "0:12:51", "throughput": "1031.13", "total_tokens": 4393920}
+{"current_steps": 327, "total_steps": 385, "loss": 0.0084, "learning_rate": 2.7250000000000006e-06, "epoch": 4.205787781350482, "percentage": 84.94, "elapsed_time": "1:11:14", "remaining_time": "0:12:38", "throughput": "1031.11", "total_tokens": 4407264}
+{"current_steps": 328, "total_steps": 385, "loss": 0.0382, "learning_rate": 2.7333333333333336e-06, "epoch": 4.218649517684887, "percentage": 85.19, "elapsed_time": "1:11:27", "remaining_time": "0:12:25", "throughput": "1031.02", "total_tokens": 4420352}
+{"current_steps": 329, "total_steps": 385, "loss": 0.0101, "learning_rate": 2.741666666666667e-06, "epoch": 4.231511254019293, "percentage": 85.45, "elapsed_time": "1:11:40", "remaining_time": "0:12:11", "throughput": "1031.21", "total_tokens": 4434688}
+{"current_steps": 330, "total_steps": 385, "loss": 0.0174, "learning_rate": 2.7500000000000004e-06, "epoch": 4.244372990353698, "percentage": 85.71, "elapsed_time": "1:11:53", "remaining_time": "0:11:58", "throughput": "1031.21", "total_tokens": 4448160}
+{"current_steps": 331, "total_steps": 385, "loss": 0.023, "learning_rate": 2.7583333333333333e-06, "epoch": 4.257234726688103, "percentage": 85.97, "elapsed_time": "1:12:06", "remaining_time": "0:11:45", "throughput": "1031.11", "total_tokens": 4461184}
+{"current_steps": 332, "total_steps": 385, "loss": 0.0162, "learning_rate": 2.766666666666667e-06, "epoch": 4.270096463022508, "percentage": 86.23, "elapsed_time": "1:12:19", "remaining_time": "0:11:32", "throughput": "1031.23", "total_tokens": 4475168}
+{"current_steps": 333, "total_steps": 385, "loss": 0.0261, "learning_rate": 2.7750000000000005e-06, "epoch": 4.282958199356913, "percentage": 86.49, "elapsed_time": "1:12:32", "remaining_time": "0:11:19", "throughput": "1031.06", "total_tokens": 4487872}
+{"current_steps": 334, "total_steps": 385, "loss": 0.0266, "learning_rate": 2.7833333333333335e-06, "epoch": 4.295819935691318, "percentage": 86.75, "elapsed_time": "1:12:45", "remaining_time": "0:11:06", "throughput": "1031.04", "total_tokens": 4501216}
+{"current_steps": 335, "total_steps": 385, "loss": 0.0194, "learning_rate": 2.791666666666667e-06, "epoch": 4.308681672025724, "percentage": 87.01, "elapsed_time": "1:12:58", "remaining_time": "0:10:53", "throughput": "1030.96", "total_tokens": 4514336}
+{"current_steps": 336, "total_steps": 385, "loss": 0.0058, "learning_rate": 2.8000000000000003e-06, "epoch": 4.321543408360129, "percentage": 87.27, "elapsed_time": "1:13:11", "remaining_time": "0:10:40", "throughput": "1031.08", "total_tokens": 4528288}
+{"current_steps": 337, "total_steps": 385, "loss": 0.0065, "learning_rate": 2.8083333333333333e-06, "epoch": 4.334405144694534, "percentage": 87.53, "elapsed_time": "1:13:24", "remaining_time": "0:10:27", "throughput": "1031.06", "total_tokens": 4541728}
+{"current_steps": 338, "total_steps": 385, "loss": 0.0202, "learning_rate": 2.816666666666667e-06, "epoch": 4.347266881028939, "percentage": 87.79, "elapsed_time": "1:13:38", "remaining_time": "0:10:14", "throughput": "1031.13", "total_tokens": 4555552}
+{"current_steps": 339, "total_steps": 385, "loss": 0.0135, "learning_rate": 2.825e-06, "epoch": 4.360128617363344, "percentage": 88.05, "elapsed_time": "1:13:51", "remaining_time": "0:10:01", "throughput": "1031.08", "total_tokens": 4568768}
+{"current_steps": 340, "total_steps": 385, "loss": 0.01, "learning_rate": 2.8333333333333335e-06, "epoch": 4.372990353697749, "percentage": 88.31, "elapsed_time": "1:14:04", "remaining_time": "0:09:48", "throughput": "1031.12", "total_tokens": 4582400}
+{"current_steps": 341, "total_steps": 385, "loss": 0.0051, "learning_rate": 2.841666666666667e-06, "epoch": 4.385852090032154, "percentage": 88.57, "elapsed_time": "1:14:17", "remaining_time": "0:09:35", "throughput": "1031.01", "total_tokens": 4595360}
+{"current_steps": 342, "total_steps": 385, "loss": 0.0293, "learning_rate": 2.85e-06, "epoch": 4.39871382636656, "percentage": 88.83, "elapsed_time": "1:14:30", "remaining_time": "0:09:22", "throughput": "1031.19", "total_tokens": 4609632}
+{"current_steps": 343, "total_steps": 385, "loss": 0.046, "learning_rate": 2.8583333333333336e-06, "epoch": 4.411575562700965, "percentage": 89.09, "elapsed_time": "1:14:43", "remaining_time": "0:09:08", "throughput": "1031.23", "total_tokens": 4623232}
+{"current_steps": 344, "total_steps": 385, "loss": 0.0024, "learning_rate": 2.866666666666667e-06, "epoch": 4.42443729903537, "percentage": 89.35, "elapsed_time": "1:14:56", "remaining_time": "0:08:55", "throughput": "1031.26", "total_tokens": 4636864}
+{"current_steps": 345, "total_steps": 385, "loss": 0.0211, "learning_rate": 2.875e-06, "epoch": 4.437299035369775, "percentage": 89.61, "elapsed_time": "1:15:09", "remaining_time": "0:08:42", "throughput": "1031.10", "total_tokens": 4649600}
+{"current_steps": 346, "total_steps": 385, "loss": 0.0229, "learning_rate": 2.8833333333333334e-06, "epoch": 4.45016077170418, "percentage": 89.87, "elapsed_time": "1:15:22", "remaining_time": "0:08:29", "throughput": "1031.11", "total_tokens": 4663072}
+{"current_steps": 347, "total_steps": 385, "loss": 0.0103, "learning_rate": 2.8916666666666672e-06, "epoch": 4.463022508038585, "percentage": 90.13, "elapsed_time": "1:15:35", "remaining_time": "0:08:16", "throughput": "1031.19", "total_tokens": 4676960}
+{"current_steps": 348, "total_steps": 385, "loss": 0.0262, "learning_rate": 2.9e-06, "epoch": 4.47588424437299, "percentage": 90.39, "elapsed_time": "1:15:48", "remaining_time": "0:08:03", "throughput": "1031.12", "total_tokens": 4690080}
+{"current_steps": 349, "total_steps": 385, "loss": 0.0295, "learning_rate": 2.9083333333333336e-06, "epoch": 4.488745980707396, "percentage": 90.65, "elapsed_time": "1:16:01", "remaining_time": "0:07:50", "throughput": "1031.24", "total_tokens": 4704064}
+{"current_steps": 350, "total_steps": 385, "loss": 0.0149, "learning_rate": 2.916666666666667e-06, "epoch": 4.501607717041801, "percentage": 90.91, "elapsed_time": "1:16:14", "remaining_time": "0:07:37", "throughput": "1031.25", "total_tokens": 4717536}
+{"current_steps": 351, "total_steps": 385, "loss": 0.0337, "learning_rate": 2.925e-06, "epoch": 4.514469453376206, "percentage": 91.17, "elapsed_time": "1:16:27", "remaining_time": "0:07:24", "throughput": "1031.25", "total_tokens": 4731040}
+{"current_steps": 352, "total_steps": 385, "loss": 0.0318, "learning_rate": 2.9333333333333338e-06, "epoch": 4.527331189710611, "percentage": 91.43, "elapsed_time": "1:16:40", "remaining_time": "0:07:11", "throughput": "1031.32", "total_tokens": 4744864}
+{"current_steps": 353, "total_steps": 385, "loss": 0.0213, "learning_rate": 2.941666666666667e-06, "epoch": 4.540192926045016, "percentage": 91.69, "elapsed_time": "1:16:53", "remaining_time": "0:06:58", "throughput": "1031.30", "total_tokens": 4758240}
+{"current_steps": 354, "total_steps": 385, "loss": 0.0048, "learning_rate": 2.95e-06, "epoch": 4.553054662379421, "percentage": 91.95, "elapsed_time": "1:17:06", "remaining_time": "0:06:45", "throughput": "1031.03", "total_tokens": 4770368}
+{"current_steps": 355, "total_steps": 385, "loss": 0.0326, "learning_rate": 2.9583333333333335e-06, "epoch": 4.565916398713826, "percentage": 92.21, "elapsed_time": "1:17:19", "remaining_time": "0:06:32", "throughput": "1030.92", "total_tokens": 4783328}
+{"current_steps": 356, "total_steps": 385, "loss": 0.013, "learning_rate": 2.9666666666666673e-06, "epoch": 4.578778135048232, "percentage": 92.47, "elapsed_time": "1:17:32", "remaining_time": "0:06:19", "throughput": "1030.99", "total_tokens": 4797088}
+{"current_steps": 357, "total_steps": 385, "loss": 0.0293, "learning_rate": 2.9750000000000003e-06, "epoch": 4.591639871382637, "percentage": 92.73, "elapsed_time": "1:17:45", "remaining_time": "0:06:05", "throughput": "1030.97", "total_tokens": 4810464}
+{"current_steps": 358, "total_steps": 385, "loss": 0.0411, "learning_rate": 2.9833333333333337e-06, "epoch": 4.604501607717042, "percentage": 92.99, "elapsed_time": "1:17:59", "remaining_time": "0:05:52", "throughput": "1030.93", "total_tokens": 4823744}
+{"current_steps": 359, "total_steps": 385, "loss": 0.0389, "learning_rate": 2.991666666666667e-06, "epoch": 4.617363344051447, "percentage": 93.25, "elapsed_time": "1:18:12", "remaining_time": "0:05:39", "throughput": "1031.06", "total_tokens": 4837824}
+{"current_steps": 360, "total_steps": 385, "loss": 0.0395, "learning_rate": 3e-06, "epoch": 4.630225080385852, "percentage": 93.51, "elapsed_time": "1:18:25", "remaining_time": "0:05:26", "throughput": "1031.22", "total_tokens": 4852096}
+{"current_steps": 361, "total_steps": 385, "loss": 0.0065, "learning_rate": 3.0083333333333335e-06, "epoch": 4.643086816720257, "percentage": 93.77, "elapsed_time": "1:18:38", "remaining_time": "0:05:13", "throughput": "1031.24", "total_tokens": 4865600}
+{"current_steps": 362, "total_steps": 385, "loss": 0.0294, "learning_rate": 3.0166666666666673e-06, "epoch": 4.655948553054662, "percentage": 94.03, "elapsed_time": "1:18:51", "remaining_time": "0:05:00", "throughput": "1031.15", "total_tokens": 4878592}
+{"current_steps": 363, "total_steps": 385, "loss": 0.0192, "learning_rate": 3.0250000000000003e-06, "epoch": 4.668810289389068, "percentage": 94.29, "elapsed_time": "1:19:04", "remaining_time": "0:04:47", "throughput": "1031.15", "total_tokens": 4892064}
+{"current_steps": 364, "total_steps": 385, "loss": 0.0179, "learning_rate": 3.0333333333333337e-06, "epoch": 4.681672025723473, "percentage": 94.55, "elapsed_time": "1:19:17", "remaining_time": "0:04:34", "throughput": "1031.31", "total_tokens": 4906304}
+{"current_steps": 365, "total_steps": 385, "loss": 0.0131, "learning_rate": 3.0416666666666666e-06, "epoch": 4.694533762057878, "percentage": 94.81, "elapsed_time": "1:19:30", "remaining_time": "0:04:21", "throughput": "1031.45", "total_tokens": 4920480}
+{"current_steps": 366, "total_steps": 385, "loss": 0.0216, "learning_rate": 3.05e-06, "epoch": 4.707395498392283, "percentage": 95.06, "elapsed_time": "1:19:43", "remaining_time": "0:04:08", "throughput": "1031.33", "total_tokens": 4933376}
+{"current_steps": 367, "total_steps": 385, "loss": 0.0171, "learning_rate": 3.058333333333334e-06, "epoch": 4.720257234726688, "percentage": 95.32, "elapsed_time": "1:19:56", "remaining_time": "0:03:55", "throughput": "1031.54", "total_tokens": 4947872}
+{"current_steps": 368, "total_steps": 385, "loss": 0.0129, "learning_rate": 3.066666666666667e-06, "epoch": 4.733118971061093, "percentage": 95.58, "elapsed_time": "1:20:09", "remaining_time": "0:03:42", "throughput": "1031.53", "total_tokens": 4961280}
+{"current_steps": 369, "total_steps": 385, "loss": 0.0268, "learning_rate": 3.075e-06, "epoch": 4.745980707395498, "percentage": 95.84, "elapsed_time": "1:20:22", "remaining_time": "0:03:29", "throughput": "1031.60", "total_tokens": 4975072}
+{"current_steps": 370, "total_steps": 385, "loss": 0.0313, "learning_rate": 3.0833333333333336e-06, "epoch": 4.758842443729904, "percentage": 96.1, "elapsed_time": "1:20:35", "remaining_time": "0:03:16", "throughput": "1031.67", "total_tokens": 4988928}
+{"current_steps": 371, "total_steps": 385, "loss": 0.0197, "learning_rate": 3.0916666666666666e-06, "epoch": 4.771704180064309, "percentage": 96.36, "elapsed_time": "1:20:48", "remaining_time": "0:03:02", "throughput": "1031.73", "total_tokens": 5002656}
+{"current_steps": 372, "total_steps": 385, "loss": 0.0051, "learning_rate": 3.1000000000000004e-06, "epoch": 4.784565916398714, "percentage": 96.62, "elapsed_time": "1:21:01", "remaining_time": "0:02:49", "throughput": "1031.89", "total_tokens": 5016928}
+{"current_steps": 373, "total_steps": 385, "loss": 0.0107, "learning_rate": 3.1083333333333338e-06, "epoch": 4.797427652733119, "percentage": 96.88, "elapsed_time": "1:21:15", "remaining_time": "0:02:36", "throughput": "1031.88", "total_tokens": 5030432}
+{"current_steps": 374, "total_steps": 385, "loss": 0.0299, "learning_rate": 3.1166666666666668e-06, "epoch": 4.810289389067524, "percentage": 97.14, "elapsed_time": "1:21:28", "remaining_time": "0:02:23", "throughput": "1031.92", "total_tokens": 5044128}
+{"current_steps": 375, "total_steps": 385, "loss": 0.0549, "learning_rate": 3.125e-06, "epoch": 4.823151125401929, "percentage": 97.4, "elapsed_time": "1:21:41", "remaining_time": "0:02:10", "throughput": "1032.12", "total_tokens": 5058592}
+{"current_steps": 376, "total_steps": 385, "loss": 0.0163, "learning_rate": 3.133333333333334e-06, "epoch": 4.836012861736334, "percentage": 97.66, "elapsed_time": "1:21:54", "remaining_time": "0:01:57", "throughput": "1032.05", "total_tokens": 5071680}
+{"current_steps": 377, "total_steps": 385, "loss": 0.0133, "learning_rate": 3.141666666666667e-06, "epoch": 4.84887459807074, "percentage": 97.92, "elapsed_time": "1:22:07", "remaining_time": "0:01:44", "throughput": "1032.16", "total_tokens": 5085760}
+{"current_steps": 378, "total_steps": 385, "loss": 0.0338, "learning_rate": 3.1500000000000003e-06, "epoch": 4.861736334405145, "percentage": 98.18, "elapsed_time": "1:22:20", "remaining_time": "0:01:31", "throughput": "1032.15", "total_tokens": 5099168}
+{"current_steps": 379, "total_steps": 385, "loss": 0.0219, "learning_rate": 3.1583333333333337e-06, "epoch": 4.87459807073955, "percentage": 98.44, "elapsed_time": "1:22:33", "remaining_time": "0:01:18", "throughput": "1032.12", "total_tokens": 5112512}
+{"current_steps": 380, "total_steps": 385, "loss": 0.0113, "learning_rate": 3.1666666666666667e-06, "epoch": 4.887459807073955, "percentage": 98.7, "elapsed_time": "1:22:46", "remaining_time": "0:01:05", "throughput": "1032.06", "total_tokens": 5125696}
+{"current_steps": 381, "total_steps": 385, "loss": 0.0297, "learning_rate": 3.175e-06, "epoch": 4.90032154340836, "percentage": 98.96, "elapsed_time": "1:22:59", "remaining_time": "0:00:52", "throughput": "1031.99", "total_tokens": 5138816}
+{"current_steps": 382, "total_steps": 385, "loss": 0.0417, "learning_rate": 3.183333333333334e-06, "epoch": 4.913183279742765, "percentage": 99.22, "elapsed_time": "1:23:12", "remaining_time": "0:00:39", "throughput": "1031.99", "total_tokens": 5152320}
+{"current_steps": 383, "total_steps": 385, "loss": 0.027, "learning_rate": 3.191666666666667e-06, "epoch": 4.92604501607717, "percentage": 99.48, "elapsed_time": "1:23:25", "remaining_time": "0:00:26", "throughput": "1031.98", "total_tokens": 5165696}
+{"current_steps": 384, "total_steps": 385, "loss": 0.0271, "learning_rate": 3.2000000000000003e-06, "epoch": 4.938906752411576, "percentage": 99.74, "elapsed_time": "1:23:38", "remaining_time": "0:00:13", "throughput": "1031.83", "total_tokens": 5178368}
+{"current_steps": 385, "total_steps": 385, "loss": 0.0207, "learning_rate": 3.2083333333333337e-06, "epoch": 4.951768488745981, "percentage": 100.0, "elapsed_time": "1:23:51", "remaining_time": "0:00:00", "throughput": "1032.00", "total_tokens": 5192736}
+{"current_steps": 385, "total_steps": 385, "epoch": 4.951768488745981, "percentage": 100.0, "elapsed_time": "1:24:52", "remaining_time": "0:00:00", "throughput": "1019.69", "total_tokens": 5192736}

trainer_state.json

@@ -0,0 +1,3123 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 4.951768488745981,
+  "eval_steps": 500,
+  "global_step": 385,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.012861736334405145,
+      "grad_norm": 359.0199890136719,
+      "learning_rate": 8.333333333333335e-09,
+      "loss": 7.8034,
+      "num_input_tokens_seen": 13440,
+      "step": 1
+    },
+    {
+      "epoch": 0.02572347266881029,
+      "grad_norm": 359.18682861328125,
+      "learning_rate": 1.666666666666667e-08,
+      "loss": 7.7577,
+      "num_input_tokens_seen": 26976,
+      "step": 2
+    },
+    {
+      "epoch": 0.03858520900321544,
+      "grad_norm": 368.04339599609375,
+      "learning_rate": 2.5000000000000002e-08,
+      "loss": 7.8132,
+      "num_input_tokens_seen": 40800,
+      "step": 3
+    },
+    {
+      "epoch": 0.05144694533762058,
+      "grad_norm": 357.04132080078125,
+      "learning_rate": 3.333333333333334e-08,
+      "loss": 7.7906,
+      "num_input_tokens_seen": 54080,
+      "step": 4
+    },
+    {
+      "epoch": 0.06430868167202572,
+      "grad_norm": 361.8187255859375,
+      "learning_rate": 4.166666666666667e-08,
+      "loss": 7.7763,
+      "num_input_tokens_seen": 67552,
+      "step": 5
+    },
+    {
+      "epoch": 0.07717041800643087,
+      "grad_norm": 368.778076171875,
+      "learning_rate": 5.0000000000000004e-08,
+      "loss": 7.7824,
+      "num_input_tokens_seen": 81600,
+      "step": 6
+    },
+    {
+      "epoch": 0.09003215434083602,
+      "grad_norm": 367.5040588378906,
+      "learning_rate": 5.833333333333334e-08,
+      "loss": 7.833,
+      "num_input_tokens_seen": 94720,
+      "step": 7
+    },
+    {
+      "epoch": 0.10289389067524116,
+      "grad_norm": 366.2318420410156,
+      "learning_rate": 6.666666666666668e-08,
+      "loss": 7.6949,
+      "num_input_tokens_seen": 107488,
+      "step": 8
+    },
+    {
+      "epoch": 0.1157556270096463,
+      "grad_norm": 366.990966796875,
+      "learning_rate": 7.500000000000001e-08,
+      "loss": 7.8336,
+      "num_input_tokens_seen": 120992,
+      "step": 9
+    },
+    {
+      "epoch": 0.12861736334405144,
+      "grad_norm": 357.8764953613281,
+      "learning_rate": 8.333333333333334e-08,
+      "loss": 7.7282,
+      "num_input_tokens_seen": 134272,
+      "step": 10
+    },
+    {
+      "epoch": 0.1414790996784566,
+      "grad_norm": 363.4705505371094,
+      "learning_rate": 9.166666666666668e-08,
+      "loss": 7.6916,
+      "num_input_tokens_seen": 148096,
+      "step": 11
+    },
+    {
+      "epoch": 0.15434083601286175,
+      "grad_norm": 365.1483154296875,
+      "learning_rate": 1.0000000000000001e-07,
+      "loss": 7.7333,
+      "num_input_tokens_seen": 160736,
+      "step": 12
+    },
+    {
+      "epoch": 0.16720257234726688,
+      "grad_norm": 353.99609375,
+      "learning_rate": 1.0833333333333335e-07,
+      "loss": 7.6017,
+      "num_input_tokens_seen": 173376,
+      "step": 13
+    },
+    {
+      "epoch": 0.18006430868167203,
+      "grad_norm": 357.7409362792969,
+      "learning_rate": 1.1666666666666668e-07,
+      "loss": 7.644,
+      "num_input_tokens_seen": 187136,
+      "step": 14
+    },
+    {
+      "epoch": 0.19292604501607716,
+      "grad_norm": 357.3723449707031,
+      "learning_rate": 1.2500000000000002e-07,
+      "loss": 7.5965,
+      "num_input_tokens_seen": 200192,
+      "step": 15
+    },
+    {
+      "epoch": 0.2057877813504823,
+      "grad_norm": 352.3059997558594,
+      "learning_rate": 1.3333333333333336e-07,
+      "loss": 7.5883,
+      "num_input_tokens_seen": 213856,
+      "step": 16
+    },
+    {
+      "epoch": 0.21864951768488747,
+      "grad_norm": 349.4331970214844,
+      "learning_rate": 1.4166666666666668e-07,
+      "loss": 7.2464,
+      "num_input_tokens_seen": 226944,
+      "step": 17
+    },
+    {
+      "epoch": 0.2315112540192926,
+      "grad_norm": 346.9617004394531,
+      "learning_rate": 1.5000000000000002e-07,
+      "loss": 7.3133,
+      "num_input_tokens_seen": 240640,
+      "step": 18
+    },
+    {
+      "epoch": 0.24437299035369775,
+      "grad_norm": 344.31512451171875,
+      "learning_rate": 1.5833333333333336e-07,
+      "loss": 7.2133,
+      "num_input_tokens_seen": 253280,
+      "step": 19
+    },
+    {
+      "epoch": 0.2572347266881029,
+      "grad_norm": 348.57623291015625,
+      "learning_rate": 1.6666666666666668e-07,
+      "loss": 7.2431,
+      "num_input_tokens_seen": 266400,
+      "step": 20
+    },
+    {
+      "epoch": 0.27009646302250806,
+      "grad_norm": 345.67694091796875,
+      "learning_rate": 1.7500000000000002e-07,
+      "loss": 7.1875,
+      "num_input_tokens_seen": 280416,
+      "step": 21
+    },
+    {
+      "epoch": 0.2829581993569132,
+      "grad_norm": 336.11968994140625,
+      "learning_rate": 1.8333333333333336e-07,
+      "loss": 7.0659,
+      "num_input_tokens_seen": 294368,
+      "step": 22
+    },
+    {
+      "epoch": 0.2958199356913183,
+      "grad_norm": 348.02783203125,
+      "learning_rate": 1.9166666666666668e-07,
+      "loss": 6.3595,
+      "num_input_tokens_seen": 307840,
+      "step": 23
+    },
+    {
+      "epoch": 0.3086816720257235,
+      "grad_norm": 306.9109802246094,
+      "learning_rate": 2.0000000000000002e-07,
+      "loss": 6.0417,
+      "num_input_tokens_seen": 322144,
+      "step": 24
+    },
+    {
+      "epoch": 0.3215434083601286,
+      "grad_norm": 308.7346496582031,
+      "learning_rate": 2.0833333333333333e-07,
+      "loss": 5.9894,
+      "num_input_tokens_seen": 335424,
+      "step": 25
+    },
+    {
+      "epoch": 0.33440514469453375,
+      "grad_norm": 303.3129577636719,
+      "learning_rate": 2.166666666666667e-07,
+      "loss": 5.9259,
+      "num_input_tokens_seen": 349376,
+      "step": 26
+    },
+    {
+      "epoch": 0.34726688102893893,
+      "grad_norm": 305.47637939453125,
+      "learning_rate": 2.2500000000000002e-07,
+      "loss": 5.8983,
+      "num_input_tokens_seen": 362944,
+      "step": 27
+    },
+    {
+      "epoch": 0.36012861736334406,
+      "grad_norm": 306.3539123535156,
+      "learning_rate": 2.3333333333333336e-07,
+      "loss": 5.6848,
+      "num_input_tokens_seen": 376352,
+      "step": 28
+    },
+    {
+      "epoch": 0.3729903536977492,
+      "grad_norm": 300.50201416015625,
+      "learning_rate": 2.416666666666667e-07,
+      "loss": 5.5649,
+      "num_input_tokens_seen": 389312,
+      "step": 29
+    },
+    {
+      "epoch": 0.3858520900321543,
+      "grad_norm": 297.82757568359375,
+      "learning_rate": 2.5000000000000004e-07,
+      "loss": 5.4642,
+      "num_input_tokens_seen": 402848,
+      "step": 30
+    },
+    {
+      "epoch": 0.3987138263665595,
+      "grad_norm": 348.0853271484375,
+      "learning_rate": 2.5833333333333333e-07,
+      "loss": 4.7955,
+      "num_input_tokens_seen": 416352,
+      "step": 31
+    },
+    {
+      "epoch": 0.4115755627009646,
+      "grad_norm": 346.5870361328125,
+      "learning_rate": 2.666666666666667e-07,
+      "loss": 2.8339,
+      "num_input_tokens_seen": 429920,
+      "step": 32
+    },
+    {
+      "epoch": 0.42443729903536975,
+      "grad_norm": 327.6314697265625,
+      "learning_rate": 2.75e-07,
+      "loss": 2.4477,
+      "num_input_tokens_seen": 444064,
+      "step": 33
+    },
+    {
+      "epoch": 0.43729903536977494,
+      "grad_norm": 321.9542236328125,
+      "learning_rate": 2.8333333333333336e-07,
+      "loss": 2.3331,
+      "num_input_tokens_seen": 458304,
+      "step": 34
+    },
+    {
+      "epoch": 0.45016077170418006,
+      "grad_norm": 327.8713073730469,
+      "learning_rate": 2.916666666666667e-07,
+      "loss": 2.2143,
+      "num_input_tokens_seen": 471904,
+      "step": 35
+    },
+    {
+      "epoch": 0.4630225080385852,
+      "grad_norm": 317.32427978515625,
+      "learning_rate": 3.0000000000000004e-07,
+      "loss": 2.0067,
+      "num_input_tokens_seen": 484768,
+      "step": 36
+    },
+    {
+      "epoch": 0.4758842443729904,
+      "grad_norm": 251.88111877441406,
+      "learning_rate": 3.083333333333334e-07,
+      "loss": 1.7702,
+      "num_input_tokens_seen": 497696,
+      "step": 37
+    },
+    {
+      "epoch": 0.4887459807073955,
+      "grad_norm": 273.716064453125,
+      "learning_rate": 3.166666666666667e-07,
+      "loss": 1.5557,
+      "num_input_tokens_seen": 511072,
+      "step": 38
+    },
+    {
+      "epoch": 0.5016077170418006,
+      "grad_norm": 293.1105651855469,
+      "learning_rate": 3.25e-07,
+      "loss": 1.3024,
+      "num_input_tokens_seen": 524576,
+      "step": 39
+    },
+    {
+      "epoch": 0.5144694533762058,
+      "grad_norm": 243.40711975097656,
+      "learning_rate": 3.3333333333333335e-07,
+      "loss": 1.1652,
+      "num_input_tokens_seen": 538112,
+      "step": 40
+    },
+    {
+      "epoch": 0.5273311897106109,
+      "grad_norm": 111.22821807861328,
+      "learning_rate": 3.416666666666667e-07,
+      "loss": 0.6839,
+      "num_input_tokens_seen": 552096,
+      "step": 41
+    },
+    {
+      "epoch": 0.5401929260450161,
+      "grad_norm": 104.02100372314453,
+      "learning_rate": 3.5000000000000004e-07,
+      "loss": 0.4774,
+      "num_input_tokens_seen": 565920,
+      "step": 42
+    },
+    {
+      "epoch": 0.5530546623794212,
+      "grad_norm": 32.459312438964844,
+      "learning_rate": 3.583333333333334e-07,
+      "loss": 0.3841,
+      "num_input_tokens_seen": 579200,
+      "step": 43
+    },
+    {
+      "epoch": 0.5659163987138264,
+      "grad_norm": 15.353242874145508,
+      "learning_rate": 3.666666666666667e-07,
+      "loss": 0.3588,
+      "num_input_tokens_seen": 592864,
+      "step": 44
+    },
+    {
+      "epoch": 0.5787781350482315,
+      "grad_norm": 48.64632034301758,
+      "learning_rate": 3.75e-07,
+      "loss": 0.3628,
+      "num_input_tokens_seen": 606656,
+      "step": 45
+    },
+    {
+      "epoch": 0.5916398713826366,
+      "grad_norm": 34.02824020385742,
+      "learning_rate": 3.8333333333333335e-07,
+      "loss": 0.3426,
+      "num_input_tokens_seen": 619968,
+      "step": 46
+    },
+    {
+      "epoch": 0.6045016077170418,
+      "grad_norm": 15.54571533203125,
+      "learning_rate": 3.9166666666666675e-07,
+      "loss": 0.3279,
+      "num_input_tokens_seen": 633632,
+      "step": 47
+    },
+    {
+      "epoch": 0.617363344051447,
+      "grad_norm": 101.66536712646484,
+      "learning_rate": 4.0000000000000003e-07,
+      "loss": 0.3947,
+      "num_input_tokens_seen": 647360,
+      "step": 48
+    },
+    {
+      "epoch": 0.6302250803858521,
+      "grad_norm": 39.69755935668945,
+      "learning_rate": 4.083333333333334e-07,
+      "loss": 0.3075,
+      "num_input_tokens_seen": 661504,
+      "step": 49
+    },
+    {
+      "epoch": 0.6430868167202572,
+      "grad_norm": 23.356718063354492,
+      "learning_rate": 4.1666666666666667e-07,
+      "loss": 0.3236,
+      "num_input_tokens_seen": 675424,
+      "step": 50
+    },
+    {
+      "epoch": 0.6559485530546624,
+      "grad_norm": 74.00253295898438,
+      "learning_rate": 4.2500000000000006e-07,
+      "loss": 0.3557,
+      "num_input_tokens_seen": 688800,
+      "step": 51
+    },
+    {
+      "epoch": 0.6688102893890675,
+      "grad_norm": 109.3729248046875,
+      "learning_rate": 4.333333333333334e-07,
+      "loss": 0.4008,
+      "num_input_tokens_seen": 702208,
+      "step": 52
+    },
+    {
+      "epoch": 0.6816720257234726,
+      "grad_norm": 81.13125610351562,
+      "learning_rate": 4.416666666666667e-07,
+      "loss": 0.3586,
+      "num_input_tokens_seen": 716096,
+      "step": 53
+    },
+    {
+      "epoch": 0.6945337620578779,
+      "grad_norm": 24.126953125,
+      "learning_rate": 4.5000000000000003e-07,
+      "loss": 0.3023,
+      "num_input_tokens_seen": 728672,
+      "step": 54
+    },
+    {
+      "epoch": 0.707395498392283,
+      "grad_norm": 77.53913879394531,
+      "learning_rate": 4.583333333333333e-07,
+      "loss": 0.3547,
+      "num_input_tokens_seen": 742912,
+      "step": 55
+    },
+    {
+      "epoch": 0.7202572347266881,
+      "grad_norm": 95.46034240722656,
+      "learning_rate": 4.666666666666667e-07,
+      "loss": 0.3846,
+      "num_input_tokens_seen": 755808,
+      "step": 56
+    },
+    {
+      "epoch": 0.7331189710610932,
+      "grad_norm": 86.11170959472656,
+      "learning_rate": 4.7500000000000006e-07,
+      "loss": 0.3743,
+      "num_input_tokens_seen": 769696,
+      "step": 57
+    },
+    {
+      "epoch": 0.7459807073954984,
+      "grad_norm": 31.460582733154297,
+      "learning_rate": 4.833333333333334e-07,
+      "loss": 0.3091,
+      "num_input_tokens_seen": 783680,
+      "step": 58
+    },
+    {
+      "epoch": 0.7588424437299035,
+      "grad_norm": 34.459659576416016,
+      "learning_rate": 4.916666666666667e-07,
+      "loss": 0.3094,
+      "num_input_tokens_seen": 796672,
+      "step": 59
+    },
+    {
+      "epoch": 0.7717041800643086,
+      "grad_norm": 51.55731201171875,
+      "learning_rate": 5.000000000000001e-07,
+      "loss": 0.3309,
+      "num_input_tokens_seen": 810304,
+      "step": 60
+    },
+    {
+      "epoch": 0.7845659163987139,
+      "grad_norm": 35.633277893066406,
+      "learning_rate": 5.083333333333334e-07,
+      "loss": 0.3276,
+      "num_input_tokens_seen": 824160,
+      "step": 61
+    },
+    {
+      "epoch": 0.797427652733119,
+      "grad_norm": 9.230757713317871,
+      "learning_rate": 5.166666666666667e-07,
+      "loss": 0.3084,
+      "num_input_tokens_seen": 837312,
+      "step": 62
+    },
+    {
+      "epoch": 0.8102893890675241,
+      "grad_norm": 35.229454040527344,
+      "learning_rate": 5.250000000000001e-07,
+      "loss": 0.3182,
+      "num_input_tokens_seen": 851168,
+      "step": 63
+    },
+    {
+      "epoch": 0.8231511254019293,
+      "grad_norm": 47.88737487792969,
+      "learning_rate": 5.333333333333335e-07,
+      "loss": 0.3469,
+      "num_input_tokens_seen": 864096,
+      "step": 64
+    },
+    {
+      "epoch": 0.8360128617363344,
+      "grad_norm": 38.481746673583984,
+      "learning_rate": 5.416666666666667e-07,
+      "loss": 0.3253,
+      "num_input_tokens_seen": 878048,
+      "step": 65
+    },
+    {
+      "epoch": 0.8488745980707395,
+      "grad_norm": 11.656455993652344,
+      "learning_rate": 5.5e-07,
+      "loss": 0.2746,
+      "num_input_tokens_seen": 891904,
+      "step": 66
+    },
+    {
+      "epoch": 0.8617363344051447,
+      "grad_norm": 18.758018493652344,
+      "learning_rate": 5.583333333333333e-07,
+      "loss": 0.2893,
+      "num_input_tokens_seen": 906016,
+      "step": 67
+    },
+    {
+      "epoch": 0.8745980707395499,
+      "grad_norm": 19.316091537475586,
+      "learning_rate": 5.666666666666667e-07,
+      "loss": 0.2827,
+      "num_input_tokens_seen": 918944,
+      "step": 68
+    },
+    {
+      "epoch": 0.887459807073955,
+      "grad_norm": 32.92877960205078,
+      "learning_rate": 5.750000000000001e-07,
+      "loss": 0.2978,
+      "num_input_tokens_seen": 932672,
+      "step": 69
+    },
+    {
+      "epoch": 0.9003215434083601,
+      "grad_norm": 14.833833694458008,
+      "learning_rate": 5.833333333333334e-07,
+      "loss": 0.2703,
+      "num_input_tokens_seen": 945248,
+      "step": 70
+    },
+    {
+      "epoch": 0.9131832797427653,
+      "grad_norm": 23.44466209411621,
+      "learning_rate": 5.916666666666667e-07,
+      "loss": 0.2968,
+      "num_input_tokens_seen": 958784,
+      "step": 71
+    },
+    {
+      "epoch": 0.9260450160771704,
+      "grad_norm": 56.151981353759766,
+      "learning_rate": 6.000000000000001e-07,
+      "loss": 0.3035,
+      "num_input_tokens_seen": 972416,
+      "step": 72
+    },
+    {
+      "epoch": 0.9389067524115756,
+      "grad_norm": 77.62692260742188,
+      "learning_rate": 6.083333333333334e-07,
+      "loss": 0.3211,
+      "num_input_tokens_seen": 985888,
+      "step": 73
+    },
+    {
+      "epoch": 0.9517684887459807,
+      "grad_norm": 65.45875549316406,
+      "learning_rate": 6.166666666666668e-07,
+      "loss": 0.2913,
+      "num_input_tokens_seen": 999296,
+      "step": 74
+    },
+    {
+      "epoch": 0.9646302250803859,
+      "grad_norm": 16.159313201904297,
+      "learning_rate": 6.25e-07,
+      "loss": 0.2817,
+      "num_input_tokens_seen": 1012640,
+      "step": 75
+    },
+    {
+      "epoch": 0.977491961414791,
+      "grad_norm": 33.846290588378906,
+      "learning_rate": 6.333333333333334e-07,
+      "loss": 0.2827,
+      "num_input_tokens_seen": 1026112,
+      "step": 76
+    },
+    {
+      "epoch": 0.9903536977491961,
+      "grad_norm": 9.542298316955566,
+      "learning_rate": 6.416666666666667e-07,
+      "loss": 0.229,
+      "num_input_tokens_seen": 1039424,
+      "step": 77
+    },
+    {
+      "epoch": 1.0032154340836013,
+      "grad_norm": 10.284690856933594,
+      "learning_rate": 6.5e-07,
+      "loss": 0.2503,
+      "num_input_tokens_seen": 1053408,
+      "step": 78
+    },
+    {
+      "epoch": 1.0160771704180065,
+      "grad_norm": 19.490089416503906,
+      "learning_rate": 6.583333333333333e-07,
+      "loss": 0.2453,
+      "num_input_tokens_seen": 1067328,
+      "step": 79
+    },
+    {
+      "epoch": 1.0289389067524115,
+      "grad_norm": 18.234272003173828,
+      "learning_rate": 6.666666666666667e-07,
+      "loss": 0.2167,
+      "num_input_tokens_seen": 1080288,
+      "step": 80
+    },
+    {
+      "epoch": 1.0418006430868167,
+      "grad_norm": 9.240696907043457,
+      "learning_rate": 6.750000000000001e-07,
+      "loss": 0.2361,
+      "num_input_tokens_seen": 1093952,
+      "step": 81
+    },
+    {
+      "epoch": 1.0546623794212218,
+      "grad_norm": 22.255823135375977,
+      "learning_rate": 6.833333333333334e-07,
+      "loss": 0.2248,
+      "num_input_tokens_seen": 1107904,
+      "step": 82
+    },
+    {
+      "epoch": 1.067524115755627,
+      "grad_norm": 26.029281616210938,
+      "learning_rate": 6.916666666666668e-07,
+      "loss": 0.2491,
+      "num_input_tokens_seen": 1121120,
+      "step": 83
+    },
+    {
+      "epoch": 1.0803858520900322,
+      "grad_norm": 34.11492156982422,
+      "learning_rate": 7.000000000000001e-07,
+      "loss": 0.2352,
+      "num_input_tokens_seen": 1135040,
+      "step": 84
+    },
+    {
+      "epoch": 1.0932475884244373,
+      "grad_norm": 65.68599700927734,
+      "learning_rate": 7.083333333333334e-07,
+      "loss": 0.2365,
+      "num_input_tokens_seen": 1148992,
+      "step": 85
+    },
+    {
+      "epoch": 1.1061093247588425,
+      "grad_norm": 72.81002807617188,
+      "learning_rate": 7.166666666666668e-07,
+      "loss": 0.217,
+      "num_input_tokens_seen": 1162592,
+      "step": 86
+    },
+    {
+      "epoch": 1.1189710610932475,
+      "grad_norm": 21.043798446655273,
+      "learning_rate": 7.25e-07,
+      "loss": 0.2258,
+      "num_input_tokens_seen": 1175104,
+      "step": 87
+    },
+    {
+      "epoch": 1.1318327974276527,
+      "grad_norm": 71.2937240600586,
+      "learning_rate": 7.333333333333334e-07,
+      "loss": 0.245,
+      "num_input_tokens_seen": 1189376,
+      "step": 88
+    },
+    {
+      "epoch": 1.144694533762058,
+      "grad_norm": 106.12057495117188,
+      "learning_rate": 7.416666666666668e-07,
+      "loss": 0.3132,
+      "num_input_tokens_seen": 1202560,
+      "step": 89
+    },
+    {
+      "epoch": 1.157556270096463,
+      "grad_norm": 84.74403381347656,
+      "learning_rate": 7.5e-07,
+      "loss": 0.284,
+      "num_input_tokens_seen": 1216832,
+      "step": 90
+    },
+    {
+      "epoch": 1.1704180064308682,
+      "grad_norm": 10.08238410949707,
+      "learning_rate": 7.583333333333334e-07,
+      "loss": 0.1933,
+      "num_input_tokens_seen": 1230528,
+      "step": 91
+    },
+    {
+      "epoch": 1.1832797427652733,
+      "grad_norm": 56.27880859375,
+      "learning_rate": 7.666666666666667e-07,
+      "loss": 0.2154,
+      "num_input_tokens_seen": 1244352,
+      "step": 92
+    },
+    {
+      "epoch": 1.1961414790996785,
+      "grad_norm": 39.58720779418945,
+      "learning_rate": 7.750000000000001e-07,
+      "loss": 0.2064,
+      "num_input_tokens_seen": 1257472,
+      "step": 93
+    },
+    {
+      "epoch": 1.2090032154340835,
+      "grad_norm": 12.087499618530273,
+      "learning_rate": 7.833333333333335e-07,
+      "loss": 0.2038,
+      "num_input_tokens_seen": 1271392,
+      "step": 94
+    },
+    {
+      "epoch": 1.2218649517684887,
+      "grad_norm": 29.51178550720215,
+      "learning_rate": 7.916666666666667e-07,
+      "loss": 0.2152,
+      "num_input_tokens_seen": 1286432,
+      "step": 95
+    },
+    {
+      "epoch": 1.234726688102894,
+      "grad_norm": 12.675917625427246,
+      "learning_rate": 8.000000000000001e-07,
+      "loss": 0.1961,
+      "num_input_tokens_seen": 1300096,
+      "step": 96
+    },
+    {
+      "epoch": 1.247588424437299,
+      "grad_norm": 11.436767578125,
+      "learning_rate": 8.083333333333334e-07,
+      "loss": 0.1772,
+      "num_input_tokens_seen": 1313568,
+      "step": 97
+    },
+    {
+      "epoch": 1.2604501607717042,
+      "grad_norm": 25.415918350219727,
+      "learning_rate": 8.166666666666668e-07,
+      "loss": 0.1846,
+      "num_input_tokens_seen": 1327328,
+      "step": 98
+    },
+    {
+      "epoch": 1.2733118971061093,
+      "grad_norm": 11.161438941955566,
+      "learning_rate": 8.250000000000001e-07,
+      "loss": 0.1823,
+      "num_input_tokens_seen": 1340960,
+      "step": 99
+    },
+    {
+      "epoch": 1.2861736334405145,
+      "grad_norm": 7.966615200042725,
+      "learning_rate": 8.333333333333333e-07,
+      "loss": 0.1794,
+      "num_input_tokens_seen": 1353440,
+      "step": 100
+    },
+    {
+      "epoch": 1.2990353697749195,
+      "grad_norm": 32.296085357666016,
+      "learning_rate": 8.416666666666667e-07,
+      "loss": 0.2106,
+      "num_input_tokens_seen": 1367680,
+      "step": 101
+    },
+    {
+      "epoch": 1.3118971061093248,
+      "grad_norm": 26.521038055419922,
+      "learning_rate": 8.500000000000001e-07,
+      "loss": 0.2123,
+      "num_input_tokens_seen": 1380864,
+      "step": 102
+    },
+    {
+      "epoch": 1.32475884244373,
+      "grad_norm": 63.48737716674805,
+      "learning_rate": 8.583333333333334e-07,
+      "loss": 0.2413,
+      "num_input_tokens_seen": 1393888,
+      "step": 103
+    },
+    {
+      "epoch": 1.337620578778135,
+      "grad_norm": 55.781776428222656,
+      "learning_rate": 8.666666666666668e-07,
+      "loss": 0.2334,
+      "num_input_tokens_seen": 1406912,
+      "step": 104
+    },
+    {
+      "epoch": 1.3504823151125402,
+      "grad_norm": 13.036691665649414,
+      "learning_rate": 8.75e-07,
+      "loss": 0.2069,
+      "num_input_tokens_seen": 1420000,
+      "step": 105
+    },
+    {
+      "epoch": 1.3633440514469453,
+      "grad_norm": 45.98344421386719,
+      "learning_rate": 8.833333333333334e-07,
+      "loss": 0.2262,
+      "num_input_tokens_seen": 1433440,
+      "step": 106
+    },
+    {
+      "epoch": 1.3762057877813505,
+      "grad_norm": 34.49677276611328,
+      "learning_rate": 8.916666666666668e-07,
+      "loss": 0.1718,
+      "num_input_tokens_seen": 1446560,
+      "step": 107
+    },
+    {
+      "epoch": 1.3890675241157555,
+      "grad_norm": 21.14679718017578,
+      "learning_rate": 9.000000000000001e-07,
+      "loss": 0.204,
+      "num_input_tokens_seen": 1460320,
+      "step": 108
+    },
+    {
+      "epoch": 1.4019292604501608,
+      "grad_norm": 26.53287124633789,
+      "learning_rate": 9.083333333333335e-07,
+      "loss": 0.1849,
+      "num_input_tokens_seen": 1474272,
+      "step": 109
+    },
+    {
+      "epoch": 1.414790996784566,
+      "grad_norm": 34.011962890625,
+      "learning_rate": 9.166666666666666e-07,
+      "loss": 0.2028,
+      "num_input_tokens_seen": 1487904,
+      "step": 110
+    },
+    {
+      "epoch": 1.427652733118971,
+      "grad_norm": 17.4981689453125,
+      "learning_rate": 9.25e-07,
+      "loss": 0.179,
+      "num_input_tokens_seen": 1501664,
+      "step": 111
+    },
+    {
+      "epoch": 1.4405144694533762,
+      "grad_norm": 20.183582305908203,
+      "learning_rate": 9.333333333333334e-07,
+      "loss": 0.1813,
+      "num_input_tokens_seen": 1515264,
+      "step": 112
+    },
+    {
+      "epoch": 1.4533762057877815,
+      "grad_norm": 26.655994415283203,
+      "learning_rate": 9.416666666666667e-07,
+      "loss": 0.1955,
+      "num_input_tokens_seen": 1528640,
+      "step": 113
+    },
+    {
+      "epoch": 1.4662379421221865,
+      "grad_norm": 25.745370864868164,
+      "learning_rate": 9.500000000000001e-07,
+      "loss": 0.1577,
+      "num_input_tokens_seen": 1541632,
+      "step": 114
+    },
+    {
+      "epoch": 1.4790996784565915,
+      "grad_norm": 22.924352645874023,
+      "learning_rate": 9.583333333333334e-07,
+      "loss": 0.1509,
+      "num_input_tokens_seen": 1554592,
+      "step": 115
+    },
+    {
+      "epoch": 1.4919614147909968,
+      "grad_norm": 26.520856857299805,
+      "learning_rate": 9.666666666666668e-07,
+      "loss": 0.2052,
+      "num_input_tokens_seen": 1567296,
+      "step": 116
+    },
+    {
+      "epoch": 1.504823151125402,
+      "grad_norm": 23.989120483398438,
+      "learning_rate": 9.750000000000002e-07,
+      "loss": 0.1576,
+      "num_input_tokens_seen": 1580800,
+      "step": 117
+    },
+    {
+      "epoch": 1.517684887459807,
+      "grad_norm": 11.067609786987305,
+      "learning_rate": 9.833333333333334e-07,
+      "loss": 0.1459,
+      "num_input_tokens_seen": 1593792,
+      "step": 118
+    },
+    {
+      "epoch": 1.5305466237942122,
+      "grad_norm": 41.14239501953125,
+      "learning_rate": 9.916666666666668e-07,
+      "loss": 0.2694,
+      "num_input_tokens_seen": 1607648,
+      "step": 119
+    },
+    {
+      "epoch": 1.5434083601286175,
+      "grad_norm": 17.72841453552246,
+      "learning_rate": 1.0000000000000002e-06,
+      "loss": 0.1891,
+      "num_input_tokens_seen": 1621184,
+      "step": 120
+    },
+    {
+      "epoch": 1.5562700964630225,
+      "grad_norm": 13.838713645935059,
+      "learning_rate": 1.0083333333333333e-06,
+      "loss": 0.1655,
+      "num_input_tokens_seen": 1634240,
+      "step": 121
+    },
+    {
+      "epoch": 1.5691318327974275,
+      "grad_norm": 17.73114013671875,
+      "learning_rate": 1.0166666666666667e-06,
+      "loss": 0.1534,
+      "num_input_tokens_seen": 1647712,
+      "step": 122
+    },
+    {
+      "epoch": 1.5819935691318328,
+      "grad_norm": 7.044174671173096,
+      "learning_rate": 1.025e-06,
+      "loss": 0.1373,
+      "num_input_tokens_seen": 1661344,
+      "step": 123
+    },
+    {
+      "epoch": 1.594855305466238,
+      "grad_norm": 13.696454048156738,
+      "learning_rate": 1.0333333333333333e-06,
+      "loss": 0.1528,
+      "num_input_tokens_seen": 1674880,
+      "step": 124
+    },
+    {
+      "epoch": 1.607717041800643,
+      "grad_norm": 11.191144943237305,
+      "learning_rate": 1.0416666666666667e-06,
+      "loss": 0.2017,
+      "num_input_tokens_seen": 1688992,
+      "step": 125
+    },
+    {
+      "epoch": 1.6205787781350482,
+      "grad_norm": 10.479023933410645,
+      "learning_rate": 1.0500000000000001e-06,
+      "loss": 0.1554,
+      "num_input_tokens_seen": 1702944,
+      "step": 126
+    },
+    {
+      "epoch": 1.6334405144694535,
+      "grad_norm": 12.889703750610352,
+      "learning_rate": 1.0583333333333335e-06,
+      "loss": 0.1332,
+      "num_input_tokens_seen": 1716768,
+      "step": 127
+    },
+    {
+      "epoch": 1.6463022508038585,
+      "grad_norm": 8.173032760620117,
+      "learning_rate": 1.066666666666667e-06,
+      "loss": 0.115,
+      "num_input_tokens_seen": 1730848,
+      "step": 128
+    },
+    {
+      "epoch": 1.6591639871382635,
+      "grad_norm": 14.694462776184082,
+      "learning_rate": 1.075e-06,
+      "loss": 0.119,
+      "num_input_tokens_seen": 1743520,
+      "step": 129
+    },
+    {
+      "epoch": 1.6720257234726688,
+      "grad_norm": 8.472253799438477,
+      "learning_rate": 1.0833333333333335e-06,
+      "loss": 0.1164,
+      "num_input_tokens_seen": 1757696,
+      "step": 130
+    },
+    {
+      "epoch": 1.684887459807074,
+      "grad_norm": 20.757648468017578,
+      "learning_rate": 1.0916666666666667e-06,
+      "loss": 0.1981,
+      "num_input_tokens_seen": 1771264,
+      "step": 131
+    },
+    {
+      "epoch": 1.697749196141479,
+      "grad_norm": 20.149930953979492,
+      "learning_rate": 1.1e-06,
+      "loss": 0.168,
+      "num_input_tokens_seen": 1784480,
+      "step": 132
+    },
+    {
+      "epoch": 1.7106109324758842,
+      "grad_norm": 9.992751121520996,
+      "learning_rate": 1.1083333333333335e-06,
+      "loss": 0.0741,
+      "num_input_tokens_seen": 1798176,
+      "step": 133
+    },
+    {
+      "epoch": 1.7234726688102895,
+      "grad_norm": 26.80909538269043,
+      "learning_rate": 1.1166666666666666e-06,
+      "loss": 0.1847,
+      "num_input_tokens_seen": 1811264,
+      "step": 134
+    },
+    {
+      "epoch": 1.7363344051446945,
+      "grad_norm": 16.494613647460938,
+      "learning_rate": 1.125e-06,
+      "loss": 0.108,
+      "num_input_tokens_seen": 1825024,
+      "step": 135
+    },
+    {
+      "epoch": 1.7491961414790995,
+      "grad_norm": 18.742122650146484,
+      "learning_rate": 1.1333333333333334e-06,
+      "loss": 0.1214,
+      "num_input_tokens_seen": 1838624,
+      "step": 136
+    },
+    {
+      "epoch": 1.762057877813505,
+      "grad_norm": 10.102185249328613,
+      "learning_rate": 1.1416666666666668e-06,
+      "loss": 0.1252,
+      "num_input_tokens_seen": 1852288,
+      "step": 137
+    },
+    {
+      "epoch": 1.77491961414791,
+      "grad_norm": 23.612730026245117,
+      "learning_rate": 1.1500000000000002e-06,
+      "loss": 0.144,
+      "num_input_tokens_seen": 1865376,
+      "step": 138
+    },
+    {
+      "epoch": 1.787781350482315,
+      "grad_norm": 11.11936092376709,
+      "learning_rate": 1.1583333333333334e-06,
+      "loss": 0.1269,
+      "num_input_tokens_seen": 1878784,
+      "step": 139
+    },
+    {
+      "epoch": 1.8006430868167203,
+      "grad_norm": 7.341479778289795,
+      "learning_rate": 1.1666666666666668e-06,
+      "loss": 0.1283,
+      "num_input_tokens_seen": 1892320,
+      "step": 140
+    },
+    {
+      "epoch": 1.8135048231511255,
+      "grad_norm": 16.097875595092773,
+      "learning_rate": 1.175e-06,
+      "loss": 0.0929,
+      "num_input_tokens_seen": 1905728,
+      "step": 141
+    },
+    {
+      "epoch": 1.8263665594855305,
+      "grad_norm": 18.07818603515625,
+      "learning_rate": 1.1833333333333334e-06,
+      "loss": 0.1349,
+      "num_input_tokens_seen": 1918752,
+      "step": 142
+    },
+    {
+      "epoch": 1.8392282958199357,
+      "grad_norm": 18.349048614501953,
+      "learning_rate": 1.1916666666666668e-06,
+      "loss": 0.1277,
+      "num_input_tokens_seen": 1931808,
+      "step": 143
+    },
+    {
+      "epoch": 1.852090032154341,
+      "grad_norm": 12.308938980102539,
+      "learning_rate": 1.2000000000000002e-06,
+      "loss": 0.1585,
+      "num_input_tokens_seen": 1945312,
+      "step": 144
+    },
+    {
+      "epoch": 1.864951768488746,
+      "grad_norm": 19.95519256591797,
+      "learning_rate": 1.2083333333333333e-06,
+      "loss": 0.1468,
+      "num_input_tokens_seen": 1958016,
+      "step": 145
+    },
+    {
+      "epoch": 1.877813504823151,
+      "grad_norm": 11.419037818908691,
+      "learning_rate": 1.2166666666666667e-06,
+      "loss": 0.1049,
+      "num_input_tokens_seen": 1971392,
+      "step": 146
+    },
+    {
+      "epoch": 1.8906752411575563,
+      "grad_norm": 9.492958068847656,
+      "learning_rate": 1.2250000000000001e-06,
+      "loss": 0.1297,
+      "num_input_tokens_seen": 1984992,
+      "step": 147
+    },
+    {
+      "epoch": 1.9035369774919615,
+      "grad_norm": 6.134005546569824,
+      "learning_rate": 1.2333333333333335e-06,
+      "loss": 0.1111,
+      "num_input_tokens_seen": 1998144,
+      "step": 148
+    },
+    {
+      "epoch": 1.9163987138263665,
+      "grad_norm": 10.250520706176758,
+      "learning_rate": 1.2416666666666667e-06,
+      "loss": 0.1202,
+      "num_input_tokens_seen": 2012032,
+      "step": 149
+    },
+    {
+      "epoch": 1.9292604501607717,
+      "grad_norm": 10.018802642822266,
+      "learning_rate": 1.25e-06,
+      "loss": 0.0829,
+      "num_input_tokens_seen": 2025600,
+      "step": 150
+    },
+    {
+      "epoch": 1.942122186495177,
+      "grad_norm": 17.175769805908203,
+      "learning_rate": 1.2583333333333333e-06,
+      "loss": 0.1119,
+      "num_input_tokens_seen": 2040096,
+      "step": 151
+    },
+    {
+      "epoch": 1.954983922829582,
+      "grad_norm": 18.159189224243164,
+      "learning_rate": 1.2666666666666669e-06,
+      "loss": 0.1144,
+      "num_input_tokens_seen": 2053504,
+      "step": 152
+    },
+    {
+      "epoch": 1.967845659163987,
+      "grad_norm": 14.536725044250488,
+      "learning_rate": 1.275e-06,
+      "loss": 0.117,
+      "num_input_tokens_seen": 2065856,
+      "step": 153
+    },
+    {
+      "epoch": 1.9807073954983923,
+      "grad_norm": 10.24523639678955,
+      "learning_rate": 1.2833333333333335e-06,
+      "loss": 0.0998,
+      "num_input_tokens_seen": 2078656,
+      "step": 154
+    },
+    {
+      "epoch": 1.9935691318327975,
+      "grad_norm": 20.030460357666016,
+      "learning_rate": 1.2916666666666669e-06,
+      "loss": 0.1384,
+      "num_input_tokens_seen": 2092672,
+      "step": 155
+    },
+    {
+      "epoch": 2.0064308681672025,
+      "grad_norm": 12.654150009155273,
+      "learning_rate": 1.3e-06,
+      "loss": 0.1157,
+      "num_input_tokens_seen": 2105728,
+      "step": 156
+    },
+    {
+      "epoch": 2.0192926045016075,
+      "grad_norm": 5.51693058013916,
+      "learning_rate": 1.3083333333333334e-06,
+      "loss": 0.0696,
+      "num_input_tokens_seen": 2118976,
+      "step": 157
+    },
+    {
+      "epoch": 2.032154340836013,
+      "grad_norm": 9.432037353515625,
+      "learning_rate": 1.3166666666666666e-06,
+      "loss": 0.0665,
+      "num_input_tokens_seen": 2132352,
+      "step": 158
+    },
+    {
+      "epoch": 2.045016077170418,
+      "grad_norm": 15.199766159057617,
+      "learning_rate": 1.3250000000000002e-06,
+      "loss": 0.0783,
+      "num_input_tokens_seen": 2146048,
+      "step": 159
+    },
+    {
+      "epoch": 2.057877813504823,
+      "grad_norm": 10.740960121154785,
+      "learning_rate": 1.3333333333333334e-06,
+      "loss": 0.0749,
+      "num_input_tokens_seen": 2158752,
+      "step": 160
+    },
+    {
+      "epoch": 2.0707395498392285,
+      "grad_norm": 7.335105895996094,
+      "learning_rate": 1.3416666666666666e-06,
+      "loss": 0.0731,
+      "num_input_tokens_seen": 2171872,
+      "step": 161
+    },
+    {
+      "epoch": 2.0836012861736335,
+      "grad_norm": 14.225622177124023,
+      "learning_rate": 1.3500000000000002e-06,
+      "loss": 0.0913,
+      "num_input_tokens_seen": 2184800,
+      "step": 162
+    },
+    {
+      "epoch": 2.0964630225080385,
+      "grad_norm": 12.737798690795898,
+      "learning_rate": 1.3583333333333334e-06,
+      "loss": 0.0521,
+      "num_input_tokens_seen": 2198176,
+      "step": 163
+    },
+    {
+      "epoch": 2.1093247588424435,
+      "grad_norm": 10.609269142150879,
+      "learning_rate": 1.3666666666666668e-06,
+      "loss": 0.068,
+      "num_input_tokens_seen": 2211392,
+      "step": 164
+    },
+    {
+      "epoch": 2.122186495176849,
+      "grad_norm": 8.30199146270752,
+      "learning_rate": 1.3750000000000002e-06,
+      "loss": 0.0686,
+      "num_input_tokens_seen": 2224480,
+      "step": 165
+    },
+    {
+      "epoch": 2.135048231511254,
+      "grad_norm": 6.069125175476074,
+      "learning_rate": 1.3833333333333336e-06,
+      "loss": 0.0545,
+      "num_input_tokens_seen": 2237920,
+      "step": 166
+    },
+    {
+      "epoch": 2.147909967845659,
+      "grad_norm": 4.639097213745117,
+      "learning_rate": 1.3916666666666668e-06,
+      "loss": 0.0347,
+      "num_input_tokens_seen": 2251008,
+      "step": 167
+    },
+    {
+      "epoch": 2.1607717041800645,
+      "grad_norm": 9.093743324279785,
+      "learning_rate": 1.4000000000000001e-06,
+      "loss": 0.0993,
+      "num_input_tokens_seen": 2264448,
+      "step": 168
+    },
+    {
+      "epoch": 2.1736334405144695,
+      "grad_norm": 8.8851900100708,
+      "learning_rate": 1.4083333333333335e-06,
+      "loss": 0.1059,
+      "num_input_tokens_seen": 2277568,
+      "step": 169
+    },
+    {
+      "epoch": 2.1864951768488745,
+      "grad_norm": 10.823246955871582,
+      "learning_rate": 1.4166666666666667e-06,
+      "loss": 0.089,
+      "num_input_tokens_seen": 2291168,
+      "step": 170
+    },
+    {
+      "epoch": 2.19935691318328,
+      "grad_norm": 5.454392910003662,
+      "learning_rate": 1.425e-06,
+      "loss": 0.0379,
+      "num_input_tokens_seen": 2304192,
+      "step": 171
+    },
+    {
+      "epoch": 2.212218649517685,
+      "grad_norm": 6.2786030769348145,
+      "learning_rate": 1.4333333333333335e-06,
+      "loss": 0.0626,
+      "num_input_tokens_seen": 2317920,
+      "step": 172
+    },
+    {
+      "epoch": 2.22508038585209,
+      "grad_norm": 8.764283180236816,
+      "learning_rate": 1.4416666666666667e-06,
+      "loss": 0.0957,
+      "num_input_tokens_seen": 2332416,
+      "step": 173
+    },
+    {
+      "epoch": 2.237942122186495,
+      "grad_norm": 10.533244132995605,
+      "learning_rate": 1.45e-06,
+      "loss": 0.0636,
+      "num_input_tokens_seen": 2346112,
+      "step": 174
+    },
+    {
+      "epoch": 2.2508038585209005,
+      "grad_norm": 7.398101329803467,
+      "learning_rate": 1.4583333333333335e-06,
+      "loss": 0.074,
+      "num_input_tokens_seen": 2359488,
+      "step": 175
+    },
+    {
+      "epoch": 2.2636655948553055,
+      "grad_norm": 6.4459686279296875,
+      "learning_rate": 1.4666666666666669e-06,
+      "loss": 0.0685,
+      "num_input_tokens_seen": 2372928,
+      "step": 176
+    },
+    {
+      "epoch": 2.2765273311897105,
+      "grad_norm": 5.918321132659912,
+      "learning_rate": 1.475e-06,
+      "loss": 0.0574,
+      "num_input_tokens_seen": 2386016,
+      "step": 177
+    },
+    {
+      "epoch": 2.289389067524116,
+      "grad_norm": 7.045689582824707,
+      "learning_rate": 1.4833333333333337e-06,
+      "loss": 0.0619,
+      "num_input_tokens_seen": 2399936,
+      "step": 178
+    },
+    {
+      "epoch": 2.302250803858521,
+      "grad_norm": 9.59676742553711,
+      "learning_rate": 1.4916666666666669e-06,
+      "loss": 0.0683,
+      "num_input_tokens_seen": 2412480,
+      "step": 179
+    },
+    {
+      "epoch": 2.315112540192926,
+      "grad_norm": 7.824158668518066,
+      "learning_rate": 1.5e-06,
+      "loss": 0.07,
+      "num_input_tokens_seen": 2425888,
+      "step": 180
+    },
+    {
+      "epoch": 2.327974276527331,
+      "grad_norm": 10.920543670654297,
+      "learning_rate": 1.5083333333333336e-06,
+      "loss": 0.1154,
+      "num_input_tokens_seen": 2439168,
+      "step": 181
+    },
+    {
+      "epoch": 2.3408360128617365,
+      "grad_norm": 7.014737606048584,
+      "learning_rate": 1.5166666666666668e-06,
+      "loss": 0.0923,
+      "num_input_tokens_seen": 2452928,
+      "step": 182
+    },
+    {
+      "epoch": 2.3536977491961415,
+      "grad_norm": 7.072991371154785,
+      "learning_rate": 1.525e-06,
+      "loss": 0.0777,
+      "num_input_tokens_seen": 2465760,
+      "step": 183
+    },
+    {
+      "epoch": 2.3665594855305465,
+      "grad_norm": 13.426029205322266,
+      "learning_rate": 1.5333333333333334e-06,
+      "loss": 0.0754,
+      "num_input_tokens_seen": 2479744,
+      "step": 184
+    },
+    {
+      "epoch": 2.379421221864952,
+      "grad_norm": 9.223447799682617,
+      "learning_rate": 1.5416666666666668e-06,
+      "loss": 0.0704,
+      "num_input_tokens_seen": 2493024,
+      "step": 185
+    },
+    {
+      "epoch": 2.392282958199357,
+      "grad_norm": 6.772041320800781,
+      "learning_rate": 1.5500000000000002e-06,
+      "loss": 0.0915,
+      "num_input_tokens_seen": 2506720,
+      "step": 186
+    },
+    {
+      "epoch": 2.405144694533762,
+      "grad_norm": 7.140571117401123,
+      "learning_rate": 1.5583333333333334e-06,
+      "loss": 0.087,
+      "num_input_tokens_seen": 2519776,
+      "step": 187
+    },
+    {
+      "epoch": 2.418006430868167,
+      "grad_norm": 9.75447940826416,
+      "learning_rate": 1.566666666666667e-06,
+      "loss": 0.0566,
+      "num_input_tokens_seen": 2533536,
+      "step": 188
+    },
+    {
+      "epoch": 2.4308681672025725,
+      "grad_norm": 12.131728172302246,
+      "learning_rate": 1.5750000000000002e-06,
+      "loss": 0.1037,
+      "num_input_tokens_seen": 2547520,
+      "step": 189
+    },
+    {
+      "epoch": 2.4437299035369775,
+      "grad_norm": 10.324275970458984,
+      "learning_rate": 1.5833333333333333e-06,
+      "loss": 0.1143,
+      "num_input_tokens_seen": 2561280,
+      "step": 190
+    },
+    {
+      "epoch": 2.4565916398713825,
+      "grad_norm": 15.137426376342773,
+      "learning_rate": 1.591666666666667e-06,
+      "loss": 0.0829,
+      "num_input_tokens_seen": 2575136,
+      "step": 191
+    },
+    {
+      "epoch": 2.469453376205788,
+      "grad_norm": 4.877906322479248,
+      "learning_rate": 1.6000000000000001e-06,
+      "loss": 0.0422,
+      "num_input_tokens_seen": 2588736,
+      "step": 192
+    },
+    {
+      "epoch": 2.482315112540193,
+      "grad_norm": 6.23721170425415,
+      "learning_rate": 1.6083333333333333e-06,
+      "loss": 0.0727,
+      "num_input_tokens_seen": 2602400,
+      "step": 193
+    },
+    {
+      "epoch": 2.495176848874598,
+      "grad_norm": 10.653412818908691,
+      "learning_rate": 1.6166666666666667e-06,
+      "loss": 0.0836,
+      "num_input_tokens_seen": 2615648,
+      "step": 194
+    },
+    {
+      "epoch": 2.508038585209003,
+      "grad_norm": 8.391611099243164,
+      "learning_rate": 1.6250000000000001e-06,
+      "loss": 0.0803,
+      "num_input_tokens_seen": 2629024,
+      "step": 195
+    },
+    {
+      "epoch": 2.5209003215434085,
+      "grad_norm": 6.826237201690674,
+      "learning_rate": 1.6333333333333335e-06,
+      "loss": 0.0654,
+      "num_input_tokens_seen": 2642784,
+      "step": 196
+    },
+    {
+      "epoch": 2.5337620578778135,
+      "grad_norm": 7.795334339141846,
+      "learning_rate": 1.6416666666666667e-06,
+      "loss": 0.0587,
+      "num_input_tokens_seen": 2656064,
+      "step": 197
+    },
+    {
+      "epoch": 2.5466237942122185,
+      "grad_norm": 9.136542320251465,
+      "learning_rate": 1.6500000000000003e-06,
+      "loss": 0.0848,
+      "num_input_tokens_seen": 2669472,
+      "step": 198
+    },
+    {
+      "epoch": 2.559485530546624,
+      "grad_norm": 4.476882457733154,
+      "learning_rate": 1.6583333333333335e-06,
+      "loss": 0.0525,
+      "num_input_tokens_seen": 2682944,
+      "step": 199
+    },
+    {
+      "epoch": 2.572347266881029,
+      "grad_norm": 8.298385620117188,
+      "learning_rate": 1.6666666666666667e-06,
+      "loss": 0.0677,
+      "num_input_tokens_seen": 2695968,
+      "step": 200
+    },
+    {
+      "epoch": 2.585209003215434,
+      "grad_norm": 9.084216117858887,
+      "learning_rate": 1.6750000000000003e-06,
+      "loss": 0.062,
+      "num_input_tokens_seen": 2709504,
+      "step": 201
+    },
+    {
+      "epoch": 2.598070739549839,
+      "grad_norm": 10.124899864196777,
+      "learning_rate": 1.6833333333333335e-06,
+      "loss": 0.0674,
+      "num_input_tokens_seen": 2723264,
+      "step": 202
+    },
+    {
+      "epoch": 2.6109324758842445,
+      "grad_norm": 5.721729755401611,
+      "learning_rate": 1.6916666666666666e-06,
+      "loss": 0.0533,
+      "num_input_tokens_seen": 2736768,
+      "step": 203
+    },
+    {
+      "epoch": 2.6237942122186495,
+      "grad_norm": 6.966607570648193,
+      "learning_rate": 1.7000000000000002e-06,
+      "loss": 0.0757,
+      "num_input_tokens_seen": 2750816,
+      "step": 204
+    },
+    {
+      "epoch": 2.6366559485530545,
+      "grad_norm": 12.730990409851074,
+      "learning_rate": 1.7083333333333334e-06,
+      "loss": 0.0777,
+      "num_input_tokens_seen": 2764576,
+      "step": 205
+    },
+    {
+      "epoch": 2.64951768488746,
+      "grad_norm": 13.19457721710205,
+      "learning_rate": 1.7166666666666668e-06,
+      "loss": 0.0921,
+      "num_input_tokens_seen": 2778080,
+      "step": 206
+    },
+    {
+      "epoch": 2.662379421221865,
+      "grad_norm": 6.583987712860107,
+      "learning_rate": 1.725e-06,
+      "loss": 0.0378,
+      "num_input_tokens_seen": 2790976,
+      "step": 207
+    },
+    {
+      "epoch": 2.67524115755627,
+      "grad_norm": 6.989199161529541,
+      "learning_rate": 1.7333333333333336e-06,
+      "loss": 0.0671,
+      "num_input_tokens_seen": 2805152,
+      "step": 208
+    },
+    {
+      "epoch": 2.688102893890675,
+      "grad_norm": 8.903035163879395,
+      "learning_rate": 1.7416666666666668e-06,
+      "loss": 0.0664,
+      "num_input_tokens_seen": 2818688,
+      "step": 209
+    },
+    {
+      "epoch": 2.7009646302250805,
+      "grad_norm": 10.817940711975098,
+      "learning_rate": 1.75e-06,
+      "loss": 0.072,
+      "num_input_tokens_seen": 2831872,
+      "step": 210
+    },
+    {
+      "epoch": 2.7138263665594855,
+      "grad_norm": 9.068681716918945,
+      "learning_rate": 1.7583333333333336e-06,
+      "loss": 0.0883,
+      "num_input_tokens_seen": 2845984,
+      "step": 211
+    },
+    {
+      "epoch": 2.7266881028938905,
+      "grad_norm": 4.464389801025391,
+      "learning_rate": 1.7666666666666668e-06,
+      "loss": 0.0414,
+      "num_input_tokens_seen": 2859232,
+      "step": 212
+    },
+    {
+      "epoch": 2.739549839228296,
+      "grad_norm": 4.999591827392578,
+      "learning_rate": 1.7750000000000002e-06,
+      "loss": 0.031,
+      "num_input_tokens_seen": 2872192,
+      "step": 213
+    },
+    {
+      "epoch": 2.752411575562701,
+      "grad_norm": 6.336106300354004,
+      "learning_rate": 1.7833333333333336e-06,
+      "loss": 0.0634,
+      "num_input_tokens_seen": 2885312,
+      "step": 214
+    },
+    {
+      "epoch": 2.765273311897106,
+      "grad_norm": 7.798391819000244,
+      "learning_rate": 1.7916666666666667e-06,
+      "loss": 0.0837,
+      "num_input_tokens_seen": 2899072,
+      "step": 215
+    },
+    {
+      "epoch": 2.778135048231511,
+      "grad_norm": 8.36534595489502,
+      "learning_rate": 1.8000000000000001e-06,
+      "loss": 0.0855,
+      "num_input_tokens_seen": 2912448,
+      "step": 216
+    },
+    {
+      "epoch": 2.7909967845659165,
+      "grad_norm": 9.043126106262207,
+      "learning_rate": 1.8083333333333335e-06,
+      "loss": 0.0945,
+      "num_input_tokens_seen": 2925120,
+      "step": 217
+    },
+    {
+      "epoch": 2.8038585209003215,
+      "grad_norm": 5.886596202850342,
+      "learning_rate": 1.816666666666667e-06,
+      "loss": 0.078,
+      "num_input_tokens_seen": 2938336,
+      "step": 218
+    },
+    {
+      "epoch": 2.816720257234727,
+      "grad_norm": 6.200281620025635,
+      "learning_rate": 1.825e-06,
+      "loss": 0.0573,
+      "num_input_tokens_seen": 2951264,
+      "step": 219
+    },
+    {
+      "epoch": 2.829581993569132,
+      "grad_norm": 10.755176544189453,
+      "learning_rate": 1.8333333333333333e-06,
+      "loss": 0.0806,
+      "num_input_tokens_seen": 2965120,
+      "step": 220
+    },
+    {
+      "epoch": 2.842443729903537,
+      "grad_norm": 7.736689567565918,
+      "learning_rate": 1.8416666666666669e-06,
+      "loss": 0.0961,
+      "num_input_tokens_seen": 2978368,
+      "step": 221
+    },
+    {
+      "epoch": 2.855305466237942,
+      "grad_norm": 8.25919246673584,
+      "learning_rate": 1.85e-06,
+      "loss": 0.0732,
+      "num_input_tokens_seen": 2992000,
+      "step": 222
+    },
+    {
+      "epoch": 2.868167202572347,
+      "grad_norm": 9.59769344329834,
+      "learning_rate": 1.8583333333333335e-06,
+      "loss": 0.0957,
+      "num_input_tokens_seen": 3006048,
+      "step": 223
+    },
+    {
+      "epoch": 2.8810289389067525,
+      "grad_norm": 7.2752485275268555,
+      "learning_rate": 1.8666666666666669e-06,
+      "loss": 0.0774,
+      "num_input_tokens_seen": 3019744,
+      "step": 224
+    },
+    {
+      "epoch": 2.8938906752411575,
+      "grad_norm": 5.989022731781006,
+      "learning_rate": 1.8750000000000003e-06,
+      "loss": 0.0691,
+      "num_input_tokens_seen": 3033568,
+      "step": 225
+    },
+    {
+      "epoch": 2.906752411575563,
+      "grad_norm": 6.692146301269531,
+      "learning_rate": 1.8833333333333334e-06,
+      "loss": 0.0529,
+      "num_input_tokens_seen": 3047712,
+      "step": 226
+    },
+    {
+      "epoch": 2.919614147909968,
+      "grad_norm": 7.606103897094727,
+      "learning_rate": 1.8916666666666668e-06,
+      "loss": 0.0811,
+      "num_input_tokens_seen": 3060800,
+      "step": 227
+    },
+    {
+      "epoch": 2.932475884244373,
+      "grad_norm": 8.168295860290527,
+      "learning_rate": 1.9000000000000002e-06,
+      "loss": 0.1211,
+      "num_input_tokens_seen": 3074720,
+      "step": 228
+    },
+    {
+      "epoch": 2.945337620578778,
+      "grad_norm": 5.379516124725342,
+      "learning_rate": 1.9083333333333334e-06,
+      "loss": 0.0489,
+      "num_input_tokens_seen": 3088448,
+      "step": 229
+    },
+    {
+      "epoch": 2.958199356913183,
+      "grad_norm": 11.40697956085205,
+      "learning_rate": 1.916666666666667e-06,
+      "loss": 0.0947,
+      "num_input_tokens_seen": 3103008,
+      "step": 230
+    },
+    {
+      "epoch": 2.9710610932475885,
+      "grad_norm": 6.254599571228027,
+      "learning_rate": 1.925e-06,
+      "loss": 0.0561,
+      "num_input_tokens_seen": 3116320,
+      "step": 231
+    },
+    {
+      "epoch": 2.9839228295819935,
+      "grad_norm": 6.7563300132751465,
+      "learning_rate": 1.9333333333333336e-06,
+      "loss": 0.0629,
+      "num_input_tokens_seen": 3130592,
+      "step": 232
+    },
+    {
+      "epoch": 2.996784565916399,
+      "grad_norm": 3.745004177093506,
+      "learning_rate": 1.9416666666666666e-06,
+      "loss": 0.0579,
+      "num_input_tokens_seen": 3144000,
+      "step": 233
+    },
+    {
+      "epoch": 3.009646302250804,
+      "grad_norm": 3.5915324687957764,
+      "learning_rate": 1.9500000000000004e-06,
+      "loss": 0.0285,
+      "num_input_tokens_seen": 3157472,
+      "step": 234
+    },
+    {
+      "epoch": 3.022508038585209,
+      "grad_norm": 2.564159631729126,
+      "learning_rate": 1.9583333333333334e-06,
+      "loss": 0.0256,
+      "num_input_tokens_seen": 3170656,
+      "step": 235
+    },
+    {
+      "epoch": 3.035369774919614,
+      "grad_norm": 5.488891124725342,
+      "learning_rate": 1.9666666666666668e-06,
+      "loss": 0.0247,
+      "num_input_tokens_seen": 3184320,
+      "step": 236
+    },
+    {
+      "epoch": 3.0482315112540195,
+      "grad_norm": 4.964622497558594,
+      "learning_rate": 1.975e-06,
+      "loss": 0.0325,
+      "num_input_tokens_seen": 3197216,
+      "step": 237
+    },
+    {
+      "epoch": 3.0610932475884245,
+      "grad_norm": 5.399533748626709,
+      "learning_rate": 1.9833333333333335e-06,
+      "loss": 0.0172,
+      "num_input_tokens_seen": 3211008,
+      "step": 238
+    },
+    {
+      "epoch": 3.0739549839228295,
+      "grad_norm": 5.841439247131348,
+      "learning_rate": 1.991666666666667e-06,
+      "loss": 0.05,
+      "num_input_tokens_seen": 3225152,
+      "step": 239
+    },
+    {
+      "epoch": 3.0868167202572345,
+      "grad_norm": 3.352426052093506,
+      "learning_rate": 2.0000000000000003e-06,
+      "loss": 0.0134,
+      "num_input_tokens_seen": 3238752,
+      "step": 240
+    },
+    {
+      "epoch": 3.09967845659164,
+      "grad_norm": 8.727036476135254,
+      "learning_rate": 2.0083333333333337e-06,
+      "loss": 0.0434,
+      "num_input_tokens_seen": 3252224,
+      "step": 241
+    },
+    {
+      "epoch": 3.112540192926045,
+      "grad_norm": 6.6712727546691895,
+      "learning_rate": 2.0166666666666667e-06,
+      "loss": 0.0186,
+      "num_input_tokens_seen": 3264832,
+      "step": 242
+    },
+    {
+      "epoch": 3.12540192926045,
+      "grad_norm": 5.2496490478515625,
+      "learning_rate": 2.025e-06,
+      "loss": 0.0341,
+      "num_input_tokens_seen": 3278048,
+      "step": 243
+    },
+    {
+      "epoch": 3.1382636655948555,
+      "grad_norm": 6.601498126983643,
+      "learning_rate": 2.0333333333333335e-06,
+      "loss": 0.0386,
+      "num_input_tokens_seen": 3291680,
+      "step": 244
+    },
+    {
+      "epoch": 3.1511254019292605,
+      "grad_norm": 5.705418109893799,
+      "learning_rate": 2.041666666666667e-06,
+      "loss": 0.0389,
+      "num_input_tokens_seen": 3305440,
+      "step": 245
+    },
+    {
+      "epoch": 3.1639871382636655,
+      "grad_norm": 4.315848350524902,
+      "learning_rate": 2.05e-06,
+      "loss": 0.0227,
+      "num_input_tokens_seen": 3319360,
+      "step": 246
+    },
+    {
+      "epoch": 3.176848874598071,
+      "grad_norm": 3.5968470573425293,
+      "learning_rate": 2.0583333333333337e-06,
+      "loss": 0.0317,
+      "num_input_tokens_seen": 3333088,
+      "step": 247
+    },
+    {
+      "epoch": 3.189710610932476,
+      "grad_norm": 3.23504638671875,
+      "learning_rate": 2.0666666666666666e-06,
+      "loss": 0.0335,
+      "num_input_tokens_seen": 3345856,
+      "step": 248
+    },
+    {
+      "epoch": 3.202572347266881,
+      "grad_norm": 3.6574056148529053,
+      "learning_rate": 2.075e-06,
+      "loss": 0.0257,
+      "num_input_tokens_seen": 3358880,
+      "step": 249
+    },
+    {
+      "epoch": 3.215434083601286,
+      "grad_norm": 2.903745651245117,
+      "learning_rate": 2.0833333333333334e-06,
+      "loss": 0.0244,
+      "num_input_tokens_seen": 3373312,
+      "step": 250
+    },
+    {
+      "epoch": 3.2282958199356915,
+      "grad_norm": 5.933866024017334,
+      "learning_rate": 2.091666666666667e-06,
+      "loss": 0.0285,
+      "num_input_tokens_seen": 3386336,
+      "step": 251
+    },
+    {
+      "epoch": 3.2411575562700965,
+      "grad_norm": 2.2281062602996826,
+      "learning_rate": 2.1000000000000002e-06,
+      "loss": 0.0093,
+      "num_input_tokens_seen": 3399904,
+      "step": 252
+    },
+    {
+      "epoch": 3.2540192926045015,
+      "grad_norm": 5.6479573249816895,
+      "learning_rate": 2.1083333333333336e-06,
+      "loss": 0.0415,
+      "num_input_tokens_seen": 3412896,
+      "step": 253
+    },
+    {
+      "epoch": 3.266881028938907,
+      "grad_norm": 3.5148093700408936,
+      "learning_rate": 2.116666666666667e-06,
+      "loss": 0.0239,
+      "num_input_tokens_seen": 3426336,
+      "step": 254
+    },
+    {
+      "epoch": 3.279742765273312,
+      "grad_norm": 8.129992485046387,
+      "learning_rate": 2.125e-06,
+      "loss": 0.0412,
+      "num_input_tokens_seen": 3439680,
+      "step": 255
+    },
+    {
+      "epoch": 3.292604501607717,
+      "grad_norm": 10.018027305603027,
+      "learning_rate": 2.133333333333334e-06,
+      "loss": 0.0503,
+      "num_input_tokens_seen": 3453760,
+      "step": 256
+    },
+    {
+      "epoch": 3.305466237942122,
+      "grad_norm": 1.7141941785812378,
+      "learning_rate": 2.1416666666666668e-06,
+      "loss": 0.0046,
+      "num_input_tokens_seen": 3466496,
+      "step": 257
+    },
+    {
+      "epoch": 3.3183279742765275,
+      "grad_norm": 4.564700126647949,
+      "learning_rate": 2.15e-06,
+      "loss": 0.041,
+      "num_input_tokens_seen": 3480352,
+      "step": 258
+    },
+    {
+      "epoch": 3.3311897106109325,
+      "grad_norm": 5.49994421005249,
+      "learning_rate": 2.1583333333333336e-06,
+      "loss": 0.0257,
+      "num_input_tokens_seen": 3494112,
+      "step": 259
+    },
+    {
+      "epoch": 3.3440514469453375,
+      "grad_norm": 4.59380578994751,
+      "learning_rate": 2.166666666666667e-06,
+      "loss": 0.0168,
+      "num_input_tokens_seen": 3507520,
+      "step": 260
+    },
+    {
+      "epoch": 3.356913183279743,
+      "grad_norm": 5.633969306945801,
+      "learning_rate": 2.1750000000000004e-06,
+      "loss": 0.0439,
+      "num_input_tokens_seen": 3520352,
+      "step": 261
+    },
+    {
+      "epoch": 3.369774919614148,
+      "grad_norm": 7.95892333984375,
+      "learning_rate": 2.1833333333333333e-06,
+      "loss": 0.0204,
+      "num_input_tokens_seen": 3533984,
+      "step": 262
+    },
+    {
+      "epoch": 3.382636655948553,
+      "grad_norm": 4.728748798370361,
+      "learning_rate": 2.191666666666667e-06,
+      "loss": 0.0284,
+      "num_input_tokens_seen": 3547072,
+      "step": 263
+    },
+    {
+      "epoch": 3.395498392282958,
+      "grad_norm": 6.725067615509033,
+      "learning_rate": 2.2e-06,
+      "loss": 0.0684,
+      "num_input_tokens_seen": 3559872,
+      "step": 264
+    },
+    {
+      "epoch": 3.4083601286173635,
+      "grad_norm": 10.331790924072266,
+      "learning_rate": 2.2083333333333335e-06,
+      "loss": 0.0479,
+      "num_input_tokens_seen": 3572832,
+      "step": 265
+    },
+    {
+      "epoch": 3.4212218649517685,
+      "grad_norm": 5.8449296951293945,
+      "learning_rate": 2.216666666666667e-06,
+      "loss": 0.0434,
+      "num_input_tokens_seen": 3585792,
+      "step": 266
+    },
+    {
+      "epoch": 3.4340836012861735,
+      "grad_norm": 4.554612636566162,
+      "learning_rate": 2.2250000000000003e-06,
+      "loss": 0.0213,
+      "num_input_tokens_seen": 3599392,
+      "step": 267
+    },
+    {
+      "epoch": 3.446945337620579,
+      "grad_norm": 7.6923828125,
+      "learning_rate": 2.2333333333333333e-06,
+      "loss": 0.0415,
+      "num_input_tokens_seen": 3613088,
+      "step": 268
+    },
+    {
+      "epoch": 3.459807073954984,
+      "grad_norm": 5.714204788208008,
+      "learning_rate": 2.2416666666666667e-06,
+      "loss": 0.0404,
+      "num_input_tokens_seen": 3626304,
+      "step": 269
+    },
+    {
+      "epoch": 3.472668810289389,
+      "grad_norm": 5.449560165405273,
+      "learning_rate": 2.25e-06,
+      "loss": 0.0566,
+      "num_input_tokens_seen": 3639968,
+      "step": 270
+    },
+    {
+      "epoch": 3.485530546623794,
+      "grad_norm": 4.743899345397949,
+      "learning_rate": 2.2583333333333335e-06,
+      "loss": 0.0509,
+      "num_input_tokens_seen": 3653472,
+      "step": 271
+    },
+    {
+      "epoch": 3.4983922829581995,
+      "grad_norm": 4.767292022705078,
+      "learning_rate": 2.266666666666667e-06,
+      "loss": 0.0385,
+      "num_input_tokens_seen": 3667104,
+      "step": 272
+    },
+    {
+      "epoch": 3.5112540192926045,
+      "grad_norm": 2.342560291290283,
+      "learning_rate": 2.2750000000000002e-06,
+      "loss": 0.0225,
+      "num_input_tokens_seen": 3680480,
+      "step": 273
+    },
+    {
+      "epoch": 3.5241157556270095,
+      "grad_norm": 3.9172496795654297,
+      "learning_rate": 2.2833333333333336e-06,
+      "loss": 0.0255,
+      "num_input_tokens_seen": 3693568,
+      "step": 274
+    },
+    {
+      "epoch": 3.536977491961415,
+      "grad_norm": 6.793405532836914,
+      "learning_rate": 2.2916666666666666e-06,
+      "loss": 0.0531,
+      "num_input_tokens_seen": 3706496,
+      "step": 275
+    },
+    {
+      "epoch": 3.54983922829582,
+      "grad_norm": 2.142134189605713,
+      "learning_rate": 2.3000000000000004e-06,
+      "loss": 0.0095,
+      "num_input_tokens_seen": 3720352,
+      "step": 276
+    },
+    {
+      "epoch": 3.562700964630225,
+      "grad_norm": 6.340362071990967,
+      "learning_rate": 2.3083333333333334e-06,
+      "loss": 0.0229,
+      "num_input_tokens_seen": 3734176,
+      "step": 277
+    },
+    {
+      "epoch": 3.57556270096463,
+      "grad_norm": 5.204097747802734,
+      "learning_rate": 2.316666666666667e-06,
+      "loss": 0.038,
+      "num_input_tokens_seen": 3748544,
+      "step": 278
+    },
+    {
+      "epoch": 3.5884244372990355,
+      "grad_norm": 6.532016754150391,
+      "learning_rate": 2.325e-06,
+      "loss": 0.0316,
+      "num_input_tokens_seen": 3761984,
+      "step": 279
+    },
+    {
+      "epoch": 3.6012861736334405,
+      "grad_norm": 10.49596881866455,
+      "learning_rate": 2.3333333333333336e-06,
+      "loss": 0.0861,
+      "num_input_tokens_seen": 3774304,
+      "step": 280
+    },
+    {
+      "epoch": 3.6141479099678455,
+      "grad_norm": 5.68178129196167,
+      "learning_rate": 2.341666666666667e-06,
+      "loss": 0.0566,
+      "num_input_tokens_seen": 3788416,
+      "step": 281
+    },
+    {
+      "epoch": 3.627009646302251,
+      "grad_norm": 6.790673732757568,
+      "learning_rate": 2.35e-06,
+      "loss": 0.0804,
+      "num_input_tokens_seen": 3802112,
+      "step": 282
+    },
+    {
+      "epoch": 3.639871382636656,
+      "grad_norm": 7.294980049133301,
+      "learning_rate": 2.3583333333333338e-06,
+      "loss": 0.046,
+      "num_input_tokens_seen": 3815968,
+      "step": 283
+    },
+    {
+      "epoch": 3.652733118971061,
+      "grad_norm": 10.37634563446045,
+      "learning_rate": 2.3666666666666667e-06,
+      "loss": 0.0693,
+      "num_input_tokens_seen": 3829248,
+      "step": 284
+    },
+    {
+      "epoch": 3.665594855305466,
+      "grad_norm": 4.692931652069092,
+      "learning_rate": 2.375e-06,
+      "loss": 0.0342,
+      "num_input_tokens_seen": 3843072,
+      "step": 285
+    },
+    {
+      "epoch": 3.6784565916398715,
+      "grad_norm": 3.5910987854003906,
+      "learning_rate": 2.3833333333333335e-06,
+      "loss": 0.0479,
+      "num_input_tokens_seen": 3856288,
+      "step": 286
+    },
+    {
+      "epoch": 3.6913183279742765,
+      "grad_norm": 5.858832359313965,
+      "learning_rate": 2.391666666666667e-06,
+      "loss": 0.0388,
+      "num_input_tokens_seen": 3870240,
+      "step": 287
+    },
+    {
+      "epoch": 3.7041800643086815,
+      "grad_norm": 4.469853401184082,
+      "learning_rate": 2.4000000000000003e-06,
+      "loss": 0.0274,
+      "num_input_tokens_seen": 3884096,
+      "step": 288
+    },
+    {
+      "epoch": 3.717041800643087,
+      "grad_norm": 4.967989921569824,
+      "learning_rate": 2.4083333333333337e-06,
+      "loss": 0.0259,
+      "num_input_tokens_seen": 3898368,
+      "step": 289
+    },
+    {
+      "epoch": 3.729903536977492,
+      "grad_norm": 6.155540943145752,
+      "learning_rate": 2.4166666666666667e-06,
+      "loss": 0.0367,
+      "num_input_tokens_seen": 3911200,
+      "step": 290
+    },
+    {
+      "epoch": 3.742765273311897,
+      "grad_norm": 6.5608367919921875,
+      "learning_rate": 2.425e-06,
+      "loss": 0.0661,
+      "num_input_tokens_seen": 3924320,
+      "step": 291
+    },
+    {
+      "epoch": 3.755627009646302,
+      "grad_norm": 6.416184902191162,
+      "learning_rate": 2.4333333333333335e-06,
+      "loss": 0.0466,
+      "num_input_tokens_seen": 3937632,
+      "step": 292
+    },
+    {
+      "epoch": 3.7684887459807075,
+      "grad_norm": 5.672483444213867,
+      "learning_rate": 2.441666666666667e-06,
+      "loss": 0.0286,
+      "num_input_tokens_seen": 3951040,
+      "step": 293
+    },
+    {
+      "epoch": 3.7813504823151125,
+      "grad_norm": 9.215237617492676,
+      "learning_rate": 2.4500000000000003e-06,
+      "loss": 0.0586,
+      "num_input_tokens_seen": 3963936,
+      "step": 294
+    },
+    {
+      "epoch": 3.7942122186495175,
+      "grad_norm": 5.630943298339844,
+      "learning_rate": 2.4583333333333332e-06,
+      "loss": 0.0329,
+      "num_input_tokens_seen": 3976832,
+      "step": 295
+    },
+    {
+      "epoch": 3.807073954983923,
+      "grad_norm": 8.257805824279785,
+      "learning_rate": 2.466666666666667e-06,
+      "loss": 0.0582,
+      "num_input_tokens_seen": 3990592,
+      "step": 296
+    },
+    {
+      "epoch": 3.819935691318328,
+      "grad_norm": 3.351003408432007,
+      "learning_rate": 2.475e-06,
+      "loss": 0.0312,
+      "num_input_tokens_seen": 4004288,
+      "step": 297
+    },
+    {
+      "epoch": 3.832797427652733,
+      "grad_norm": 3.2983040809631348,
+      "learning_rate": 2.4833333333333334e-06,
+      "loss": 0.0329,
+      "num_input_tokens_seen": 4017856,
+      "step": 298
+    },
+    {
+      "epoch": 3.845659163987138,
+      "grad_norm": 4.082300662994385,
+      "learning_rate": 2.491666666666667e-06,
+      "loss": 0.0206,
+      "num_input_tokens_seen": 4031168,
+      "step": 299
+    },
+    {
+      "epoch": 3.8585209003215435,
+      "grad_norm": 3.222336530685425,
+      "learning_rate": 2.5e-06,
+      "loss": 0.0426,
+      "num_input_tokens_seen": 4044416,
+      "step": 300
+    },
+    {
+      "epoch": 3.8713826366559485,
+      "grad_norm": 3.699768304824829,
+      "learning_rate": 2.5083333333333336e-06,
+      "loss": 0.0179,
+      "num_input_tokens_seen": 4057824,
+      "step": 301
+    },
+    {
+      "epoch": 3.884244372990354,
+      "grad_norm": 2.819528341293335,
+      "learning_rate": 2.5166666666666666e-06,
+      "loss": 0.0289,
+      "num_input_tokens_seen": 4071936,
+      "step": 302
+    },
+    {
+      "epoch": 3.897106109324759,
+      "grad_norm": 4.0458478927612305,
+      "learning_rate": 2.5250000000000004e-06,
+      "loss": 0.0303,
+      "num_input_tokens_seen": 4085536,
+      "step": 303
+    },
+    {
+      "epoch": 3.909967845659164,
+      "grad_norm": 11.214617729187012,
+      "learning_rate": 2.5333333333333338e-06,
+      "loss": 0.046,
+      "num_input_tokens_seen": 4099552,
+      "step": 304
+    },
+    {
+      "epoch": 3.922829581993569,
+      "grad_norm": 6.758171081542969,
+      "learning_rate": 2.5416666666666668e-06,
+      "loss": 0.0523,
+      "num_input_tokens_seen": 4112736,
+      "step": 305
+    },
+    {
+      "epoch": 3.935691318327974,
+      "grad_norm": 5.622665882110596,
+      "learning_rate": 2.55e-06,
+      "loss": 0.0329,
+      "num_input_tokens_seen": 4126080,
+      "step": 306
+    },
+    {
+      "epoch": 3.9485530546623795,
+      "grad_norm": 1.4951086044311523,
+      "learning_rate": 2.558333333333334e-06,
+      "loss": 0.0072,
+      "num_input_tokens_seen": 4138912,
+      "step": 307
+    },
+    {
+      "epoch": 3.9614147909967845,
+      "grad_norm": 5.639418601989746,
+      "learning_rate": 2.566666666666667e-06,
+      "loss": 0.0415,
+      "num_input_tokens_seen": 4152768,
+      "step": 308
+    },
+    {
+      "epoch": 3.97427652733119,
+      "grad_norm": 7.496403694152832,
+      "learning_rate": 2.5750000000000003e-06,
+      "loss": 0.0233,
+      "num_input_tokens_seen": 4166528,
+      "step": 309
+    },
+    {
+      "epoch": 3.987138263665595,
+      "grad_norm": 8.74985408782959,
+      "learning_rate": 2.5833333333333337e-06,
+      "loss": 0.0423,
+      "num_input_tokens_seen": 4180832,
+      "step": 310
+    },
+    {
+      "epoch": 4.0,
+      "grad_norm": 6.229910850524902,
+      "learning_rate": 2.5916666666666667e-06,
+      "loss": 0.0295,
+      "num_input_tokens_seen": 4194048,
+      "step": 311
+    },
+    {
+      "epoch": 4.012861736334405,
+      "grad_norm": 3.404331922531128,
+      "learning_rate": 2.6e-06,
+      "loss": 0.0327,
+      "num_input_tokens_seen": 4207040,
+      "step": 312
+    },
+    {
+      "epoch": 4.02572347266881,
+      "grad_norm": 4.214801788330078,
+      "learning_rate": 2.608333333333333e-06,
+      "loss": 0.0301,
+      "num_input_tokens_seen": 4220000,
+      "step": 313
+    },
+    {
+      "epoch": 4.038585209003215,
+      "grad_norm": 6.241875171661377,
+      "learning_rate": 2.616666666666667e-06,
+      "loss": 0.0301,
+      "num_input_tokens_seen": 4233344,
+      "step": 314
+    },
+    {
+      "epoch": 4.051446945337621,
+      "grad_norm": 3.2021894454956055,
+      "learning_rate": 2.6250000000000003e-06,
+      "loss": 0.0281,
+      "num_input_tokens_seen": 4246144,
+      "step": 315
+    },
+    {
+      "epoch": 4.064308681672026,
+      "grad_norm": 3.5197951793670654,
+      "learning_rate": 2.6333333333333332e-06,
+      "loss": 0.0136,
+      "num_input_tokens_seen": 4260992,
+      "step": 316
+    },
+    {
+      "epoch": 4.077170418006431,
+      "grad_norm": 4.2946319580078125,
+      "learning_rate": 2.6416666666666666e-06,
+      "loss": 0.0219,
+      "num_input_tokens_seen": 4274112,
+      "step": 317
+    },
+    {
+      "epoch": 4.090032154340836,
+      "grad_norm": 1.3583834171295166,
+      "learning_rate": 2.6500000000000005e-06,
+      "loss": 0.0044,
+      "num_input_tokens_seen": 4287904,
+      "step": 318
+    },
+    {
+      "epoch": 4.102893890675241,
+      "grad_norm": 5.484881401062012,
+      "learning_rate": 2.6583333333333334e-06,
+      "loss": 0.0335,
+      "num_input_tokens_seen": 4301568,
+      "step": 319
+    },
+    {
+      "epoch": 4.115755627009646,
+      "grad_norm": 3.5873327255249023,
+      "learning_rate": 2.666666666666667e-06,
+      "loss": 0.0053,
+      "num_input_tokens_seen": 4314848,
+      "step": 320
+    },
+    {
+      "epoch": 4.128617363344051,
+      "grad_norm": 2.954930067062378,
+      "learning_rate": 2.6750000000000002e-06,
+      "loss": 0.0196,
+      "num_input_tokens_seen": 4327456,
+      "step": 321
+    },
+    {
+      "epoch": 4.141479099678457,
+      "grad_norm": 2.628854274749756,
+      "learning_rate": 2.683333333333333e-06,
+      "loss": 0.0309,
+      "num_input_tokens_seen": 4341024,
+      "step": 322
+    },
+    {
+      "epoch": 4.154340836012862,
+      "grad_norm": 6.691638469696045,
+      "learning_rate": 2.691666666666667e-06,
+      "loss": 0.0382,
+      "num_input_tokens_seen": 4353824,
+      "step": 323
+    },
+    {
+      "epoch": 4.167202572347267,
+      "grad_norm": 6.866171360015869,
+      "learning_rate": 2.7000000000000004e-06,
+      "loss": 0.046,
+      "num_input_tokens_seen": 4367616,
+      "step": 324
+    },
+    {
+      "epoch": 4.180064308681672,
+      "grad_norm": 4.016347408294678,
+      "learning_rate": 2.7083333333333334e-06,
+      "loss": 0.0133,
+      "num_input_tokens_seen": 4380448,
+      "step": 325
+    },
+    {
+      "epoch": 4.192926045016077,
+      "grad_norm": 4.948652744293213,
+      "learning_rate": 2.7166666666666668e-06,
+      "loss": 0.0265,
+      "num_input_tokens_seen": 4393920,
+      "step": 326
+    },
+    {
+      "epoch": 4.205787781350482,
+      "grad_norm": 2.7615582942962646,
+      "learning_rate": 2.7250000000000006e-06,
+      "loss": 0.0084,
+      "num_input_tokens_seen": 4407264,
+      "step": 327
+    },
+    {
+      "epoch": 4.218649517684887,
+      "grad_norm": 3.0052027702331543,
+      "learning_rate": 2.7333333333333336e-06,
+      "loss": 0.0382,
+      "num_input_tokens_seen": 4420352,
+      "step": 328
+    },
+    {
+      "epoch": 4.231511254019293,
+      "grad_norm": 3.0679566860198975,
+      "learning_rate": 2.741666666666667e-06,
+      "loss": 0.0101,
+      "num_input_tokens_seen": 4434688,
+      "step": 329
+    },
+    {
+      "epoch": 4.244372990353698,
+      "grad_norm": 4.612461566925049,
+      "learning_rate": 2.7500000000000004e-06,
+      "loss": 0.0174,
+      "num_input_tokens_seen": 4448160,
+      "step": 330
+    },
+    {
+      "epoch": 4.257234726688103,
+      "grad_norm": 4.299385070800781,
+      "learning_rate": 2.7583333333333333e-06,
+      "loss": 0.023,
+      "num_input_tokens_seen": 4461184,
+      "step": 331
+    },
+    {
+      "epoch": 4.270096463022508,
+      "grad_norm": 1.5578008890151978,
+      "learning_rate": 2.766666666666667e-06,
+      "loss": 0.0162,
+      "num_input_tokens_seen": 4475168,
+      "step": 332
+    },
+    {
+      "epoch": 4.282958199356913,
+      "grad_norm": 3.8595075607299805,
+      "learning_rate": 2.7750000000000005e-06,
+      "loss": 0.0261,
+      "num_input_tokens_seen": 4487872,
+      "step": 333
+    },
+    {
+      "epoch": 4.295819935691318,
+      "grad_norm": 8.00913143157959,
+      "learning_rate": 2.7833333333333335e-06,
+      "loss": 0.0266,
+      "num_input_tokens_seen": 4501216,
+      "step": 334
+    },
+    {
+      "epoch": 4.308681672025724,
+      "grad_norm": 2.2975409030914307,
+      "learning_rate": 2.791666666666667e-06,
+      "loss": 0.0194,
+      "num_input_tokens_seen": 4514336,
+      "step": 335
+    },
+    {
+      "epoch": 4.321543408360129,
+      "grad_norm": 1.9800870418548584,
+      "learning_rate": 2.8000000000000003e-06,
+      "loss": 0.0058,
+      "num_input_tokens_seen": 4528288,
+      "step": 336
+    },
+    {
+      "epoch": 4.334405144694534,
+      "grad_norm": 6.778807163238525,
+      "learning_rate": 2.8083333333333333e-06,
+      "loss": 0.0065,
+      "num_input_tokens_seen": 4541728,
+      "step": 337
+    },
+    {
+      "epoch": 4.347266881028939,
+      "grad_norm": 7.885656833648682,
+      "learning_rate": 2.816666666666667e-06,
+      "loss": 0.0202,
+      "num_input_tokens_seen": 4555552,
+      "step": 338
+    },
+    {
+      "epoch": 4.360128617363344,
+      "grad_norm": 9.400197982788086,
+      "learning_rate": 2.825e-06,
+      "loss": 0.0135,
+      "num_input_tokens_seen": 4568768,
+      "step": 339
+    },
+    {
+      "epoch": 4.372990353697749,
+      "grad_norm": 5.712286472320557,
+      "learning_rate": 2.8333333333333335e-06,
+      "loss": 0.01,
+      "num_input_tokens_seen": 4582400,
+      "step": 340
+    },
+    {
+      "epoch": 4.385852090032154,
+      "grad_norm": 4.751468181610107,
+      "learning_rate": 2.841666666666667e-06,
+      "loss": 0.0051,
+      "num_input_tokens_seen": 4595360,
+      "step": 341
+    },
+    {
+      "epoch": 4.39871382636656,
+      "grad_norm": 2.9044384956359863,
+      "learning_rate": 2.85e-06,
+      "loss": 0.0293,
+      "num_input_tokens_seen": 4609632,
+      "step": 342
+    },
+    {
+      "epoch": 4.411575562700965,
+      "grad_norm": 5.8322224617004395,
+      "learning_rate": 2.8583333333333336e-06,
+      "loss": 0.046,
+      "num_input_tokens_seen": 4623232,
+      "step": 343
+    },
+    {
+      "epoch": 4.42443729903537,
+      "grad_norm": 0.7529265284538269,
+      "learning_rate": 2.866666666666667e-06,
+      "loss": 0.0024,
+      "num_input_tokens_seen": 4636864,
+      "step": 344
+    },
+    {
+      "epoch": 4.437299035369775,
+      "grad_norm": 4.490261554718018,
+      "learning_rate": 2.875e-06,
+      "loss": 0.0211,
+      "num_input_tokens_seen": 4649600,
+      "step": 345
+    },
+    {
+      "epoch": 4.45016077170418,
+      "grad_norm": 9.859882354736328,
+      "learning_rate": 2.8833333333333334e-06,
+      "loss": 0.0229,
+      "num_input_tokens_seen": 4663072,
+      "step": 346
+    },
+    {
+      "epoch": 4.463022508038585,
+      "grad_norm": 3.8769121170043945,
+      "learning_rate": 2.8916666666666672e-06,
+      "loss": 0.0103,
+      "num_input_tokens_seen": 4676960,
+      "step": 347
+    },
+    {
+      "epoch": 4.47588424437299,
+      "grad_norm": 6.608347415924072,
+      "learning_rate": 2.9e-06,
+      "loss": 0.0262,
+      "num_input_tokens_seen": 4690080,
+      "step": 348
+    },
+    {
+      "epoch": 4.488745980707396,
+      "grad_norm": 3.058494806289673,
+      "learning_rate": 2.9083333333333336e-06,
+      "loss": 0.0295,
+      "num_input_tokens_seen": 4704064,
+      "step": 349
+    },
+    {
+      "epoch": 4.501607717041801,
+      "grad_norm": 4.893054485321045,
+      "learning_rate": 2.916666666666667e-06,
+      "loss": 0.0149,
+      "num_input_tokens_seen": 4717536,
+      "step": 350
+    },
+    {
+      "epoch": 4.514469453376206,
+      "grad_norm": 5.85437536239624,
+      "learning_rate": 2.925e-06,
+      "loss": 0.0337,
+      "num_input_tokens_seen": 4731040,
+      "step": 351
+    },
+    {
+      "epoch": 4.527331189710611,
+      "grad_norm": 5.814053058624268,
+      "learning_rate": 2.9333333333333338e-06,
+      "loss": 0.0318,
+      "num_input_tokens_seen": 4744864,
+      "step": 352
+    },
+    {
+      "epoch": 4.540192926045016,
+      "grad_norm": 6.330677509307861,
+      "learning_rate": 2.941666666666667e-06,
+      "loss": 0.0213,
+      "num_input_tokens_seen": 4758240,
+      "step": 353
+    },
+    {
+      "epoch": 4.553054662379421,
+      "grad_norm": 0.9806466698646545,
+      "learning_rate": 2.95e-06,
+      "loss": 0.0048,
+      "num_input_tokens_seen": 4770368,
+      "step": 354
+    },
+    {
+      "epoch": 4.565916398713826,
+      "grad_norm": 3.046319007873535,
+      "learning_rate": 2.9583333333333335e-06,
+      "loss": 0.0326,
+      "num_input_tokens_seen": 4783328,
+      "step": 355
+    },
+    {
+      "epoch": 4.578778135048232,
+      "grad_norm": 5.004611015319824,
+      "learning_rate": 2.9666666666666673e-06,
+      "loss": 0.013,
+      "num_input_tokens_seen": 4797088,
+      "step": 356
+    },
+    {
+      "epoch": 4.591639871382637,
+      "grad_norm": 5.062610626220703,
+      "learning_rate": 2.9750000000000003e-06,
+      "loss": 0.0293,
+      "num_input_tokens_seen": 4810464,
+      "step": 357
+    },
+    {
+      "epoch": 4.604501607717042,
+      "grad_norm": 9.05587100982666,
+      "learning_rate": 2.9833333333333337e-06,
+      "loss": 0.0411,
+      "num_input_tokens_seen": 4823744,
+      "step": 358
+    },
+    {
+      "epoch": 4.617363344051447,
+      "grad_norm": 7.2998175621032715,
+      "learning_rate": 2.991666666666667e-06,
+      "loss": 0.0389,
+      "num_input_tokens_seen": 4837824,
+      "step": 359
+    },
+    {
+      "epoch": 4.630225080385852,
+      "grad_norm": 6.108879566192627,
+      "learning_rate": 3e-06,
+      "loss": 0.0395,
+      "num_input_tokens_seen": 4852096,
+      "step": 360
+    },
+    {
+      "epoch": 4.643086816720257,
+      "grad_norm": 4.677525043487549,
+      "learning_rate": 3.0083333333333335e-06,
+      "loss": 0.0065,
+      "num_input_tokens_seen": 4865600,
+      "step": 361
+    },
+    {
+      "epoch": 4.655948553054662,
+      "grad_norm": 9.179205894470215,
+      "learning_rate": 3.0166666666666673e-06,
+      "loss": 0.0294,
+      "num_input_tokens_seen": 4878592,
+      "step": 362
+    },
+    {
+      "epoch": 4.668810289389068,
+      "grad_norm": 3.478533983230591,
+      "learning_rate": 3.0250000000000003e-06,
+      "loss": 0.0192,
+      "num_input_tokens_seen": 4892064,
+      "step": 363
+    },
+    {
+      "epoch": 4.681672025723473,
+      "grad_norm": 6.015811443328857,
+      "learning_rate": 3.0333333333333337e-06,
+      "loss": 0.0179,
+      "num_input_tokens_seen": 4906304,
+      "step": 364
+    },
+    {
+      "epoch": 4.694533762057878,
+      "grad_norm": 3.1433732509613037,
+      "learning_rate": 3.0416666666666666e-06,
+      "loss": 0.0131,
+      "num_input_tokens_seen": 4920480,
+      "step": 365
+    },
+    {
+      "epoch": 4.707395498392283,
+      "grad_norm": 3.894927501678467,
+      "learning_rate": 3.05e-06,
+      "loss": 0.0216,
+      "num_input_tokens_seen": 4933376,
+      "step": 366
+    },
+    {
+      "epoch": 4.720257234726688,
+      "grad_norm": 3.4421629905700684,
+      "learning_rate": 3.058333333333334e-06,
+      "loss": 0.0171,
+      "num_input_tokens_seen": 4947872,
+      "step": 367
+    },
+    {
+      "epoch": 4.733118971061093,
+      "grad_norm": 2.935985803604126,
+      "learning_rate": 3.066666666666667e-06,
+      "loss": 0.0129,
+      "num_input_tokens_seen": 4961280,
+      "step": 368
+    },
+    {
+      "epoch": 4.745980707395498,
+      "grad_norm": 7.003020763397217,
+      "learning_rate": 3.075e-06,
+      "loss": 0.0268,
+      "num_input_tokens_seen": 4975072,
+      "step": 369
+    },
+    {
+      "epoch": 4.758842443729904,
+      "grad_norm": 5.833902835845947,
+      "learning_rate": 3.0833333333333336e-06,
+      "loss": 0.0313,
+      "num_input_tokens_seen": 4988928,
+      "step": 370
+    },
+    {
+      "epoch": 4.771704180064309,
+      "grad_norm": 4.902740001678467,
+      "learning_rate": 3.0916666666666666e-06,
+      "loss": 0.0197,
+      "num_input_tokens_seen": 5002656,
+      "step": 371
+    },
+    {
+      "epoch": 4.784565916398714,
+      "grad_norm": 1.6855580806732178,
+      "learning_rate": 3.1000000000000004e-06,
+      "loss": 0.0051,
+      "num_input_tokens_seen": 5016928,
+      "step": 372
+    },
+    {
+      "epoch": 4.797427652733119,
+      "grad_norm": 3.116506814956665,
+      "learning_rate": 3.1083333333333338e-06,
+      "loss": 0.0107,
+      "num_input_tokens_seen": 5030432,
+      "step": 373
+    },
+    {
+      "epoch": 4.810289389067524,
+      "grad_norm": 2.606302261352539,
+      "learning_rate": 3.1166666666666668e-06,
+      "loss": 0.0299,
+      "num_input_tokens_seen": 5044128,
+      "step": 374
+    },
+    {
+      "epoch": 4.823151125401929,
+      "grad_norm": 5.198400497436523,
+      "learning_rate": 3.125e-06,
+      "loss": 0.0549,
+      "num_input_tokens_seen": 5058592,
+      "step": 375
+    },
+    {
+      "epoch": 4.836012861736334,
+      "grad_norm": 3.7756950855255127,
+      "learning_rate": 3.133333333333334e-06,
+      "loss": 0.0163,
+      "num_input_tokens_seen": 5071680,
+      "step": 376
+    },
+    {
+      "epoch": 4.84887459807074,
+      "grad_norm": 3.266502618789673,
+      "learning_rate": 3.141666666666667e-06,
+      "loss": 0.0133,
+      "num_input_tokens_seen": 5085760,
+      "step": 377
+    },
+    {
+      "epoch": 4.861736334405145,
+      "grad_norm": 3.7333478927612305,
+      "learning_rate": 3.1500000000000003e-06,
+      "loss": 0.0338,
+      "num_input_tokens_seen": 5099168,
+      "step": 378
+    },
+    {
+      "epoch": 4.87459807073955,
+      "grad_norm": 6.106029033660889,
+      "learning_rate": 3.1583333333333337e-06,
+      "loss": 0.0219,
+      "num_input_tokens_seen": 5112512,
+      "step": 379
+    },
+    {
+      "epoch": 4.887459807073955,
+      "grad_norm": 3.018552541732788,
+      "learning_rate": 3.1666666666666667e-06,
+      "loss": 0.0113,
+      "num_input_tokens_seen": 5125696,
+      "step": 380
+    },
+    {
+      "epoch": 4.90032154340836,
+      "grad_norm": 5.034224987030029,
+      "learning_rate": 3.175e-06,
+      "loss": 0.0297,
+      "num_input_tokens_seen": 5138816,
+      "step": 381
+    },
+    {
+      "epoch": 4.913183279742765,
+      "grad_norm": 6.260331630706787,
+      "learning_rate": 3.183333333333334e-06,
+      "loss": 0.0417,
+      "num_input_tokens_seen": 5152320,
+      "step": 382
+    },
+    {
+      "epoch": 4.92604501607717,
+      "grad_norm": 4.149762153625488,
+      "learning_rate": 3.191666666666667e-06,
+      "loss": 0.027,
+      "num_input_tokens_seen": 5165696,
+      "step": 383
+    },
+    {
+      "epoch": 4.938906752411576,
+      "grad_norm": 2.9361073970794678,
+      "learning_rate": 3.2000000000000003e-06,
+      "loss": 0.0271,
+      "num_input_tokens_seen": 5178368,
+      "step": 384
+    },
+    {
+      "epoch": 4.951768488745981,
+      "grad_norm": 5.155781269073486,
+      "learning_rate": 3.2083333333333337e-06,
+      "loss": 0.0207,
+      "num_input_tokens_seen": 5192736,
+      "step": 385
+    },
+    {
+      "epoch": 4.951768488745981,
+      "num_input_tokens_seen": 5192736,
+      "step": 385,
+      "total_flos": 2.3382655808988774e+17,
+      "train_loss": 0.7082552919200585,
+      "train_runtime": 5092.4527,
+      "train_samples_per_second": 19.519,
+      "train_steps_per_second": 0.076
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 385,
+  "num_input_tokens_seen": 5192736,
+  "num_train_epochs": 5,
+  "save_steps": 1000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.3382655808988774e+17,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e3de8a8e741121a5f4fde13b7396c2d104a757f5aaccd4dd320366c51b2f0ea
+size 6584