Model save

README.md

@@ -0,0 +1,66 @@
+---
+license: apache-2.0
+base_model: mistralai/Mistral-7B-v0.1
+tags:
+- trl
+- sft
+- generated_from_trainer
+datasets:
+- generator
+model-index:
+- name: zephyr-7b-sft-full
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# zephyr-7b-sft-full
+
+This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on the generator dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.9427
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 2e-05
+- train_batch_size: 16
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- total_train_batch_size: 128
+- total_eval_batch_size: 64
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss |
+|:-------------:|:-----:|:----:|:---------------:|
+| 0.9409        | 1.0   | 1084 | 0.9427          |
+
+
+### Framework versions
+
+- Transformers 4.42.0
+- Pytorch 2.4.0
+- Datasets 3.1.0
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 453935093514240.0,
+    "train_loss": 0.982847072101607,
+    "train_runtime": 5570.7798,
+    "train_samples": 207864,
+    "train_samples_per_second": 24.895,
+    "train_steps_per_second": 0.195
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.42.0"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7ae298b2cb2584b30f9139ae9482e60924927c75a9e23788bf3c4aba74ebc6d
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5adaeef7f323c71a6534120a1f5cb0bf33074c290d53cd38011630be0ce50c58
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0dc7cd81559d1a9fc41297d642f262cff193bb06e4c97847b12cc8b3b312a040
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

runs/Dec10_06-43-16_n124-104-101/events.out.tfevents.1733814335.n124-104-101.359094.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b0ad6c241be707aeec74043b2a5937dcda7bce8e6bd8ffbc8f34058640913c1f
-size 51240
+oid sha256:1b38f9bbe12812639e3ba26edd711f650ff10efe0c39ab34481a8100b5399557
+size 51865

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 453935093514240.0,
+    "train_loss": 0.982847072101607,
+    "train_runtime": 5570.7798,
+    "train_samples": 207864,
+    "train_samples_per_second": 24.895,
+    "train_steps_per_second": 0.195
+}

trainer_state.json

@@ -0,0 +1,1569 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 1084,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0009225092250922509,
+      "grad_norm": 9.870301733957042,
+      "learning_rate": 1.8348623853211012e-07,
+      "loss": 1.1607,
+      "step": 1
+    },
+    {
+      "epoch": 0.004612546125461255,
+      "grad_norm": 7.6358775502653735,
+      "learning_rate": 9.174311926605506e-07,
+      "loss": 1.1303,
+      "step": 5
+    },
+    {
+      "epoch": 0.00922509225092251,
+      "grad_norm": 3.9641069843195957,
+      "learning_rate": 1.8348623853211011e-06,
+      "loss": 1.0569,
+      "step": 10
+    },
+    {
+      "epoch": 0.013837638376383764,
+      "grad_norm": 2.5499940814970787,
+      "learning_rate": 2.7522935779816517e-06,
+      "loss": 1.0238,
+      "step": 15
+    },
+    {
+      "epoch": 0.01845018450184502,
+      "grad_norm": 2.083746857128064,
+      "learning_rate": 3.6697247706422022e-06,
+      "loss": 0.9931,
+      "step": 20
+    },
+    {
+      "epoch": 0.023062730627306273,
+      "grad_norm": 1.8847575524462474,
+      "learning_rate": 4.587155963302753e-06,
+      "loss": 0.982,
+      "step": 25
+    },
+    {
+      "epoch": 0.027675276752767528,
+      "grad_norm": 2.2303171581585435,
+      "learning_rate": 5.504587155963303e-06,
+      "loss": 0.9797,
+      "step": 30
+    },
+    {
+      "epoch": 0.03228782287822878,
+      "grad_norm": 2.790049451506377,
+      "learning_rate": 6.422018348623854e-06,
+      "loss": 0.9813,
+      "step": 35
+    },
+    {
+      "epoch": 0.03690036900369004,
+      "grad_norm": 1.9929053655008255,
+      "learning_rate": 7.3394495412844045e-06,
+      "loss": 0.9807,
+      "step": 40
+    },
+    {
+      "epoch": 0.04151291512915129,
+      "grad_norm": 2.7447280409420105,
+      "learning_rate": 8.256880733944956e-06,
+      "loss": 0.9867,
+      "step": 45
+    },
+    {
+      "epoch": 0.046125461254612546,
+      "grad_norm": 2.15994514361092,
+      "learning_rate": 9.174311926605506e-06,
+      "loss": 0.9966,
+      "step": 50
+    },
+    {
+      "epoch": 0.0507380073800738,
+      "grad_norm": 2.136357303154315,
+      "learning_rate": 1.0091743119266055e-05,
+      "loss": 0.9906,
+      "step": 55
+    },
+    {
+      "epoch": 0.055350553505535055,
+      "grad_norm": 2.274317609050521,
+      "learning_rate": 1.1009174311926607e-05,
+      "loss": 0.9921,
+      "step": 60
+    },
+    {
+      "epoch": 0.05996309963099631,
+      "grad_norm": 2.515953872720923,
+      "learning_rate": 1.1926605504587156e-05,
+      "loss": 1.0039,
+      "step": 65
+    },
+    {
+      "epoch": 0.06457564575645756,
+      "grad_norm": 2.6426205867787163,
+      "learning_rate": 1.2844036697247708e-05,
+      "loss": 0.9863,
+      "step": 70
+    },
+    {
+      "epoch": 0.06918819188191883,
+      "grad_norm": 2.209854625891484,
+      "learning_rate": 1.3761467889908258e-05,
+      "loss": 1.0025,
+      "step": 75
+    },
+    {
+      "epoch": 0.07380073800738007,
+      "grad_norm": 2.453800577899367,
+      "learning_rate": 1.4678899082568809e-05,
+      "loss": 1.0165,
+      "step": 80
+    },
+    {
+      "epoch": 0.07841328413284133,
+      "grad_norm": 2.284868701972977,
+      "learning_rate": 1.559633027522936e-05,
+      "loss": 1.0035,
+      "step": 85
+    },
+    {
+      "epoch": 0.08302583025830258,
+      "grad_norm": 2.6259331068874747,
+      "learning_rate": 1.6513761467889912e-05,
+      "loss": 0.9977,
+      "step": 90
+    },
+    {
+      "epoch": 0.08763837638376384,
+      "grad_norm": 2.2079127084296077,
+      "learning_rate": 1.743119266055046e-05,
+      "loss": 0.9878,
+      "step": 95
+    },
+    {
+      "epoch": 0.09225092250922509,
+      "grad_norm": 2.1954368041258663,
+      "learning_rate": 1.834862385321101e-05,
+      "loss": 1.0356,
+      "step": 100
+    },
+    {
+      "epoch": 0.09686346863468635,
+      "grad_norm": 2.531705339867925,
+      "learning_rate": 1.9266055045871563e-05,
+      "loss": 1.0098,
+      "step": 105
+    },
+    {
+      "epoch": 0.1014760147601476,
+      "grad_norm": 2.2792018031821497,
+      "learning_rate": 1.9999948088910656e-05,
+      "loss": 1.0133,
+      "step": 110
+    },
+    {
+      "epoch": 0.10608856088560886,
+      "grad_norm": 1.9782499003079985,
+      "learning_rate": 1.9998131257372878e-05,
+      "loss": 1.0238,
+      "step": 115
+    },
+    {
+      "epoch": 0.11070110701107011,
+      "grad_norm": 2.084336683738492,
+      "learning_rate": 1.999371941029485e-05,
+      "loss": 1.0146,
+      "step": 120
+    },
+    {
+      "epoch": 0.11531365313653137,
+      "grad_norm": 2.2430916805352776,
+      "learning_rate": 1.9986713692771732e-05,
+      "loss": 1.0375,
+      "step": 125
+    },
+    {
+      "epoch": 0.11992619926199262,
+      "grad_norm": 1.89148142251587,
+      "learning_rate": 1.9977115923137912e-05,
+      "loss": 1.0298,
+      "step": 130
+    },
+    {
+      "epoch": 0.12453874538745388,
+      "grad_norm": 1.9346322925017247,
+      "learning_rate": 1.9964928592495046e-05,
+      "loss": 1.0289,
+      "step": 135
+    },
+    {
+      "epoch": 0.12915129151291513,
+      "grad_norm": 2.063507608941714,
+      "learning_rate": 1.9950154864065497e-05,
+      "loss": 1.0283,
+      "step": 140
+    },
+    {
+      "epoch": 0.13376383763837638,
+      "grad_norm": 1.936455430343577,
+      "learning_rate": 1.993279857237133e-05,
+      "loss": 1.0321,
+      "step": 145
+    },
+    {
+      "epoch": 0.13837638376383765,
+      "grad_norm": 2.3172685570168743,
+      "learning_rate": 1.9912864222239045e-05,
+      "loss": 1.041,
+      "step": 150
+    },
+    {
+      "epoch": 0.1429889298892989,
+      "grad_norm": 2.2006105196246653,
+      "learning_rate": 1.9890356987630362e-05,
+      "loss": 1.0211,
+      "step": 155
+    },
+    {
+      "epoch": 0.14760147601476015,
+      "grad_norm": 1.945068405470377,
+      "learning_rate": 1.986528271029931e-05,
+      "loss": 1.0233,
+      "step": 160
+    },
+    {
+      "epoch": 0.1522140221402214,
+      "grad_norm": 1.8885138361710503,
+      "learning_rate": 1.9837647898276008e-05,
+      "loss": 1.0225,
+      "step": 165
+    },
+    {
+      "epoch": 0.15682656826568267,
+      "grad_norm": 2.10426243574591,
+      "learning_rate": 1.9807459724177497e-05,
+      "loss": 1.024,
+      "step": 170
+    },
+    {
+      "epoch": 0.16143911439114392,
+      "grad_norm": 1.95239943278475,
+      "learning_rate": 1.977472602334609e-05,
+      "loss": 1.0309,
+      "step": 175
+    },
+    {
+      "epoch": 0.16605166051660517,
+      "grad_norm": 2.0742390742666448,
+      "learning_rate": 1.973945529181572e-05,
+      "loss": 1.0351,
+      "step": 180
+    },
+    {
+      "epoch": 0.1706642066420664,
+      "grad_norm": 1.7368468057342068,
+      "learning_rate": 1.9701656684106764e-05,
+      "loss": 1.0405,
+      "step": 185
+    },
+    {
+      "epoch": 0.1752767527675277,
+      "grad_norm": 1.7716671273796425,
+      "learning_rate": 1.9661340010850025e-05,
+      "loss": 1.0287,
+      "step": 190
+    },
+    {
+      "epoch": 0.17988929889298894,
+      "grad_norm": 1.8588938948041192,
+      "learning_rate": 1.9618515736240353e-05,
+      "loss": 1.0498,
+      "step": 195
+    },
+    {
+      "epoch": 0.18450184501845018,
+      "grad_norm": 1.8452719555224244,
+      "learning_rate": 1.9573194975320672e-05,
+      "loss": 1.0369,
+      "step": 200
+    },
+    {
+      "epoch": 0.18911439114391143,
+      "grad_norm": 1.7827950944695834,
+      "learning_rate": 1.952538949109708e-05,
+      "loss": 1.0443,
+      "step": 205
+    },
+    {
+      "epoch": 0.1937269372693727,
+      "grad_norm": 1.9346364878018567,
+      "learning_rate": 1.9475111691485737e-05,
+      "loss": 1.0222,
+      "step": 210
+    },
+    {
+      "epoch": 0.19833948339483395,
+      "grad_norm": 1.6980455791090399,
+      "learning_rate": 1.9422374626092414e-05,
+      "loss": 1.0309,
+      "step": 215
+    },
+    {
+      "epoch": 0.2029520295202952,
+      "grad_norm": 1.5942581114601342,
+      "learning_rate": 1.936719198282545e-05,
+      "loss": 1.0296,
+      "step": 220
+    },
+    {
+      "epoch": 0.20756457564575645,
+      "grad_norm": 1.5677285054105923,
+      "learning_rate": 1.930957808434307e-05,
+      "loss": 1.0265,
+      "step": 225
+    },
+    {
+      "epoch": 0.21217712177121772,
+      "grad_norm": 1.6285717213585489,
+      "learning_rate": 1.9249547884335917e-05,
+      "loss": 1.0302,
+      "step": 230
+    },
+    {
+      "epoch": 0.21678966789667897,
+      "grad_norm": 1.7138451484563946,
+      "learning_rate": 1.9187116963645845e-05,
+      "loss": 1.0358,
+      "step": 235
+    },
+    {
+      "epoch": 0.22140221402214022,
+      "grad_norm": 1.849793007166441,
+      "learning_rate": 1.912230152622189e-05,
+      "loss": 1.0333,
+      "step": 240
+    },
+    {
+      "epoch": 0.22601476014760147,
+      "grad_norm": 1.8226869674776514,
+      "learning_rate": 1.9055118394914545e-05,
+      "loss": 1.0371,
+      "step": 245
+    },
+    {
+      "epoch": 0.23062730627306274,
+      "grad_norm": 1.6253243432899807,
+      "learning_rate": 1.898558500710939e-05,
+      "loss": 1.0336,
+      "step": 250
+    },
+    {
+      "epoch": 0.235239852398524,
+      "grad_norm": 1.6717598717297293,
+      "learning_rate": 1.891371941020121e-05,
+      "loss": 1.0541,
+      "step": 255
+    },
+    {
+      "epoch": 0.23985239852398524,
+      "grad_norm": 1.7951757710233756,
+      "learning_rate": 1.88395402569098e-05,
+      "loss": 1.017,
+      "step": 260
+    },
+    {
+      "epoch": 0.2444649446494465,
+      "grad_norm": 1.6096108295610199,
+      "learning_rate": 1.8763066800438638e-05,
+      "loss": 1.0269,
+      "step": 265
+    },
+    {
+      "epoch": 0.24907749077490776,
+      "grad_norm": 1.6416277343638672,
+      "learning_rate": 1.868431888947773e-05,
+      "loss": 1.037,
+      "step": 270
+    },
+    {
+      "epoch": 0.253690036900369,
+      "grad_norm": 1.4965013567584224,
+      "learning_rate": 1.860331696305188e-05,
+      "loss": 1.0267,
+      "step": 275
+    },
+    {
+      "epoch": 0.25830258302583026,
+      "grad_norm": 1.6330332258477454,
+      "learning_rate": 1.852008204521572e-05,
+      "loss": 1.0197,
+      "step": 280
+    },
+    {
+      "epoch": 0.2629151291512915,
+      "grad_norm": 1.5766862160608865,
+      "learning_rate": 1.8434635739596945e-05,
+      "loss": 1.0366,
+      "step": 285
+    },
+    {
+      "epoch": 0.26752767527675275,
+      "grad_norm": 1.6089055305536961,
+      "learning_rate": 1.834700022378907e-05,
+      "loss": 1.0533,
+      "step": 290
+    },
+    {
+      "epoch": 0.272140221402214,
+      "grad_norm": 1.6075644263468138,
+      "learning_rate": 1.825719824359524e-05,
+      "loss": 1.0392,
+      "step": 295
+    },
+    {
+      "epoch": 0.2767527675276753,
+      "grad_norm": 1.5507274436430425,
+      "learning_rate": 1.816525310712456e-05,
+      "loss": 1.0097,
+      "step": 300
+    },
+    {
+      "epoch": 0.28136531365313655,
+      "grad_norm": 1.5145472266368731,
+      "learning_rate": 1.8071188678742457e-05,
+      "loss": 1.0177,
+      "step": 305
+    },
+    {
+      "epoch": 0.2859778597785978,
+      "grad_norm": 1.9642560759333394,
+      "learning_rate": 1.7975029372876706e-05,
+      "loss": 1.0215,
+      "step": 310
+    },
+    {
+      "epoch": 0.29059040590405905,
+      "grad_norm": 1.5974962927976561,
+      "learning_rate": 1.787680014768065e-05,
+      "loss": 1.045,
+      "step": 315
+    },
+    {
+      "epoch": 0.2952029520295203,
+      "grad_norm": 1.6288780409656336,
+      "learning_rate": 1.777652649855531e-05,
+      "loss": 1.0112,
+      "step": 320
+    },
+    {
+      "epoch": 0.29981549815498154,
+      "grad_norm": 1.619084159391039,
+      "learning_rate": 1.7674234451532065e-05,
+      "loss": 1.0312,
+      "step": 325
+    },
+    {
+      "epoch": 0.3044280442804428,
+      "grad_norm": 1.5979104495922047,
+      "learning_rate": 1.7569950556517566e-05,
+      "loss": 1.0525,
+      "step": 330
+    },
+    {
+      "epoch": 0.30904059040590404,
+      "grad_norm": 1.4997831353273716,
+      "learning_rate": 1.7463701880402738e-05,
+      "loss": 1.0215,
+      "step": 335
+    },
+    {
+      "epoch": 0.31365313653136534,
+      "grad_norm": 1.5694872243699436,
+      "learning_rate": 1.7355516000037555e-05,
+      "loss": 1.0297,
+      "step": 340
+    },
+    {
+      "epoch": 0.3182656826568266,
+      "grad_norm": 1.6008099336404966,
+      "learning_rate": 1.7245420995073453e-05,
+      "loss": 1.0419,
+      "step": 345
+    },
+    {
+      "epoch": 0.32287822878228783,
+      "grad_norm": 1.6059180263249522,
+      "learning_rate": 1.7133445440675268e-05,
+      "loss": 1.0284,
+      "step": 350
+    },
+    {
+      "epoch": 0.3274907749077491,
+      "grad_norm": 1.6358881844781394,
+      "learning_rate": 1.7019618400104572e-05,
+      "loss": 1.0274,
+      "step": 355
+    },
+    {
+      "epoch": 0.33210332103321033,
+      "grad_norm": 1.5641614120804217,
+      "learning_rate": 1.6903969417176244e-05,
+      "loss": 1.0193,
+      "step": 360
+    },
+    {
+      "epoch": 0.3367158671586716,
+      "grad_norm": 1.4895862864423213,
+      "learning_rate": 1.6786528508590436e-05,
+      "loss": 1.0341,
+      "step": 365
+    },
+    {
+      "epoch": 0.3413284132841328,
+      "grad_norm": 1.564753891661441,
+      "learning_rate": 1.666732615614169e-05,
+      "loss": 1.0205,
+      "step": 370
+    },
+    {
+      "epoch": 0.3459409594095941,
+      "grad_norm": 1.4487617302488085,
+      "learning_rate": 1.6546393298807405e-05,
+      "loss": 1.018,
+      "step": 375
+    },
+    {
+      "epoch": 0.3505535055350554,
+      "grad_norm": 1.6533634720160688,
+      "learning_rate": 1.6423761324717636e-05,
+      "loss": 1.024,
+      "step": 380
+    },
+    {
+      "epoch": 0.3551660516605166,
+      "grad_norm": 1.493760972306007,
+      "learning_rate": 1.6299462063008272e-05,
+      "loss": 1.0166,
+      "step": 385
+    },
+    {
+      "epoch": 0.35977859778597787,
+      "grad_norm": 1.5564935885270512,
+      "learning_rate": 1.61735277755598e-05,
+      "loss": 1.0238,
+      "step": 390
+    },
+    {
+      "epoch": 0.3643911439114391,
+      "grad_norm": 1.8719539860792949,
+      "learning_rate": 1.6045991148623752e-05,
+      "loss": 1.0174,
+      "step": 395
+    },
+    {
+      "epoch": 0.36900369003690037,
+      "grad_norm": 1.509933150391116,
+      "learning_rate": 1.5916885284338937e-05,
+      "loss": 1.0047,
+      "step": 400
+    },
+    {
+      "epoch": 0.3736162361623616,
+      "grad_norm": 1.485803634928529,
+      "learning_rate": 1.5786243692139826e-05,
+      "loss": 1.0272,
+      "step": 405
+    },
+    {
+      "epoch": 0.37822878228782286,
+      "grad_norm": 1.5273070832476536,
+      "learning_rate": 1.5654100280059155e-05,
+      "loss": 1.0043,
+      "step": 410
+    },
+    {
+      "epoch": 0.3828413284132841,
+      "grad_norm": 1.469641191253245,
+      "learning_rate": 1.5520489345927095e-05,
+      "loss": 1.0033,
+      "step": 415
+    },
+    {
+      "epoch": 0.3874538745387454,
+      "grad_norm": 1.5621423725950956,
+      "learning_rate": 1.538544556846925e-05,
+      "loss": 1.0251,
+      "step": 420
+    },
+    {
+      "epoch": 0.39206642066420666,
+      "grad_norm": 1.4854792537280463,
+      "learning_rate": 1.5249003998305787e-05,
+      "loss": 1.0139,
+      "step": 425
+    },
+    {
+      "epoch": 0.3966789667896679,
+      "grad_norm": 1.540644154349171,
+      "learning_rate": 1.5111200048854055e-05,
+      "loss": 1.0204,
+      "step": 430
+    },
+    {
+      "epoch": 0.40129151291512916,
+      "grad_norm": 1.495564386831857,
+      "learning_rate": 1.4972069487137024e-05,
+      "loss": 1.0098,
+      "step": 435
+    },
+    {
+      "epoch": 0.4059040590405904,
+      "grad_norm": 1.6944058480092998,
+      "learning_rate": 1.4831648424499953e-05,
+      "loss": 1.0166,
+      "step": 440
+    },
+    {
+      "epoch": 0.41051660516605165,
+      "grad_norm": 1.4154740566727173,
+      "learning_rate": 1.4689973307237687e-05,
+      "loss": 1.0281,
+      "step": 445
+    },
+    {
+      "epoch": 0.4151291512915129,
+      "grad_norm": 1.456867007792944,
+      "learning_rate": 1.4547080907135024e-05,
+      "loss": 1.012,
+      "step": 450
+    },
+    {
+      "epoch": 0.41974169741697415,
+      "grad_norm": 1.5118739107040873,
+      "learning_rate": 1.4403008311922593e-05,
+      "loss": 1.0107,
+      "step": 455
+    },
+    {
+      "epoch": 0.42435424354243545,
+      "grad_norm": 1.4201069814359026,
+      "learning_rate": 1.4257792915650728e-05,
+      "loss": 1.0131,
+      "step": 460
+    },
+    {
+      "epoch": 0.4289667896678967,
+      "grad_norm": 1.3797821151089888,
+      "learning_rate": 1.4111472408983843e-05,
+      "loss": 1.0071,
+      "step": 465
+    },
+    {
+      "epoch": 0.43357933579335795,
+      "grad_norm": 1.547555594082307,
+      "learning_rate": 1.3964084769417823e-05,
+      "loss": 1.0294,
+      "step": 470
+    },
+    {
+      "epoch": 0.4381918819188192,
+      "grad_norm": 1.8253389041213868,
+      "learning_rate": 1.3815668251422953e-05,
+      "loss": 0.995,
+      "step": 475
+    },
+    {
+      "epoch": 0.44280442804428044,
+      "grad_norm": 1.4687976124888267,
+      "learning_rate": 1.3666261376514978e-05,
+      "loss": 0.9922,
+      "step": 480
+    },
+    {
+      "epoch": 0.4474169741697417,
+      "grad_norm": 1.4846214050363804,
+      "learning_rate": 1.3515902923256832e-05,
+      "loss": 1.0077,
+      "step": 485
+    },
+    {
+      "epoch": 0.45202952029520294,
+      "grad_norm": 1.5550431555961994,
+      "learning_rate": 1.3364631917193671e-05,
+      "loss": 1.0013,
+      "step": 490
+    },
+    {
+      "epoch": 0.4566420664206642,
+      "grad_norm": 1.541649391181409,
+      "learning_rate": 1.321248762072377e-05,
+      "loss": 1.0027,
+      "step": 495
+    },
+    {
+      "epoch": 0.4612546125461255,
+      "grad_norm": 1.4625591942436968,
+      "learning_rate": 1.3059509522907998e-05,
+      "loss": 0.9971,
+      "step": 500
+    },
+    {
+      "epoch": 0.46586715867158673,
+      "grad_norm": 1.4303485864613443,
+      "learning_rate": 1.2905737329220394e-05,
+      "loss": 0.9963,
+      "step": 505
+    },
+    {
+      "epoch": 0.470479704797048,
+      "grad_norm": 1.490615007478747,
+      "learning_rate": 1.2751210951242636e-05,
+      "loss": 1.0066,
+      "step": 510
+    },
+    {
+      "epoch": 0.47509225092250923,
+      "grad_norm": 1.453650333930737,
+      "learning_rate": 1.2595970496304975e-05,
+      "loss": 0.9939,
+      "step": 515
+    },
+    {
+      "epoch": 0.4797047970479705,
+      "grad_norm": 1.4965869855232417,
+      "learning_rate": 1.2440056257076376e-05,
+      "loss": 1.0079,
+      "step": 520
+    },
+    {
+      "epoch": 0.4843173431734317,
+      "grad_norm": 1.4489426823286569,
+      "learning_rate": 1.2283508701106559e-05,
+      "loss": 0.9781,
+      "step": 525
+    },
+    {
+      "epoch": 0.488929889298893,
+      "grad_norm": 1.4353569698033004,
+      "learning_rate": 1.2126368460322637e-05,
+      "loss": 1.0102,
+      "step": 530
+    },
+    {
+      "epoch": 0.4935424354243542,
+      "grad_norm": 1.3263470851244286,
+      "learning_rate": 1.1968676320483103e-05,
+      "loss": 0.9684,
+      "step": 535
+    },
+    {
+      "epoch": 0.4981549815498155,
+      "grad_norm": 1.4471529804335683,
+      "learning_rate": 1.1810473210591882e-05,
+      "loss": 0.9858,
+      "step": 540
+    },
+    {
+      "epoch": 0.5027675276752768,
+      "grad_norm": 1.6634718686097767,
+      "learning_rate": 1.1651800192275197e-05,
+      "loss": 0.9798,
+      "step": 545
+    },
+    {
+      "epoch": 0.507380073800738,
+      "grad_norm": 1.4077516065143243,
+      "learning_rate": 1.1492698449124042e-05,
+      "loss": 0.9904,
+      "step": 550
+    },
+    {
+      "epoch": 0.5119926199261993,
+      "grad_norm": 1.3842391940758678,
+      "learning_rate": 1.1333209276004959e-05,
+      "loss": 0.9971,
+      "step": 555
+    },
+    {
+      "epoch": 0.5166051660516605,
+      "grad_norm": 1.3552490385806142,
+      "learning_rate": 1.1173374068341962e-05,
+      "loss": 1.0064,
+      "step": 560
+    },
+    {
+      "epoch": 0.5212177121771218,
+      "grad_norm": 1.555289246108155,
+      "learning_rate": 1.1013234311372353e-05,
+      "loss": 0.9874,
+      "step": 565
+    },
+    {
+      "epoch": 0.525830258302583,
+      "grad_norm": 1.5905481722662995,
+      "learning_rate": 1.0852831569379217e-05,
+      "loss": 0.9915,
+      "step": 570
+    },
+    {
+      "epoch": 0.5304428044280443,
+      "grad_norm": 1.503857570008716,
+      "learning_rate": 1.0692207474903421e-05,
+      "loss": 0.9916,
+      "step": 575
+    },
+    {
+      "epoch": 0.5350553505535055,
+      "grad_norm": 1.4616116802333865,
+      "learning_rate": 1.0531403717937888e-05,
+      "loss": 0.9892,
+      "step": 580
+    },
+    {
+      "epoch": 0.5396678966789668,
+      "grad_norm": 1.4502706666625567,
+      "learning_rate": 1.037046203510694e-05,
+      "loss": 0.977,
+      "step": 585
+    },
+    {
+      "epoch": 0.544280442804428,
+      "grad_norm": 1.3734581487116164,
+      "learning_rate": 1.0209424198833571e-05,
+      "loss": 0.9862,
+      "step": 590
+    },
+    {
+      "epoch": 0.5488929889298892,
+      "grad_norm": 1.4945854406869938,
+      "learning_rate": 1.0048332006497406e-05,
+      "loss": 0.9936,
+      "step": 595
+    },
+    {
+      "epoch": 0.5535055350553506,
+      "grad_norm": 1.3930881740821102,
+      "learning_rate": 9.887227269586184e-06,
+      "loss": 0.9807,
+      "step": 600
+    },
+    {
+      "epoch": 0.5581180811808119,
+      "grad_norm": 1.478713291975006,
+      "learning_rate": 9.7261518028436e-06,
+      "loss": 0.9903,
+      "step": 605
+    },
+    {
+      "epoch": 0.5627306273062731,
+      "grad_norm": 1.420734873408918,
+      "learning_rate": 9.565147413416266e-06,
+      "loss": 0.9788,
+      "step": 610
+    },
+    {
+      "epoch": 0.5673431734317343,
+      "grad_norm": 1.4705269930380465,
+      "learning_rate": 9.404255890002677e-06,
+      "loss": 0.9967,
+      "step": 615
+    },
+    {
+      "epoch": 0.5719557195571956,
+      "grad_norm": 1.3329074523022386,
+      "learning_rate": 9.243518992006944e-06,
+      "loss": 0.9739,
+      "step": 620
+    },
+    {
+      "epoch": 0.5765682656826568,
+      "grad_norm": 1.3411458856414704,
+      "learning_rate": 9.082978438700138e-06,
+      "loss": 0.9682,
+      "step": 625
+    },
+    {
+      "epoch": 0.5811808118081181,
+      "grad_norm": 2.132001925450684,
+      "learning_rate": 8.922675898392072e-06,
+      "loss": 0.9666,
+      "step": 630
+    },
+    {
+      "epoch": 0.5857933579335793,
+      "grad_norm": 2.43102419510478,
+      "learning_rate": 8.762652977616258e-06,
+      "loss": 0.9874,
+      "step": 635
+    },
+    {
+      "epoch": 0.5904059040590406,
+      "grad_norm": 1.3738990911698028,
+      "learning_rate": 8.602951210330942e-06,
+      "loss": 0.967,
+      "step": 640
+    },
+    {
+      "epoch": 0.5950184501845018,
+      "grad_norm": 1.4174540748579312,
+      "learning_rate": 8.443612047138965e-06,
+      "loss": 0.982,
+      "step": 645
+    },
+    {
+      "epoch": 0.5996309963099631,
+      "grad_norm": 1.4357166892787867,
+      "learning_rate": 8.284676844529258e-06,
+      "loss": 0.9447,
+      "step": 650
+    },
+    {
+      "epoch": 0.6042435424354243,
+      "grad_norm": 1.3135487842803675,
+      "learning_rate": 8.126186854142752e-06,
+      "loss": 0.9521,
+      "step": 655
+    },
+    {
+      "epoch": 0.6088560885608856,
+      "grad_norm": 1.376281811405976,
+      "learning_rate": 7.968183212065537e-06,
+      "loss": 0.9639,
+      "step": 660
+    },
+    {
+      "epoch": 0.6134686346863468,
+      "grad_norm": 1.3531901458569717,
+      "learning_rate": 7.81070692815195e-06,
+      "loss": 0.9612,
+      "step": 665
+    },
+    {
+      "epoch": 0.6180811808118081,
+      "grad_norm": 1.4317488801453897,
+      "learning_rate": 7.6537988753805e-06,
+      "loss": 0.9559,
+      "step": 670
+    },
+    {
+      "epoch": 0.6226937269372693,
+      "grad_norm": 1.3488154766734353,
+      "learning_rate": 7.497499779245268e-06,
+      "loss": 0.9643,
+      "step": 675
+    },
+    {
+      "epoch": 0.6273062730627307,
+      "grad_norm": 1.3767572836176645,
+      "learning_rate": 7.3418502071856004e-06,
+      "loss": 0.9727,
+      "step": 680
+    },
+    {
+      "epoch": 0.6319188191881919,
+      "grad_norm": 1.3550732483616188,
+      "learning_rate": 7.186890558056836e-06,
+      "loss": 0.9524,
+      "step": 685
+    },
+    {
+      "epoch": 0.6365313653136532,
+      "grad_norm": 1.4366768084351662,
+      "learning_rate": 7.0326610516447825e-06,
+      "loss": 0.952,
+      "step": 690
+    },
+    {
+      "epoch": 0.6411439114391144,
+      "grad_norm": 1.3824939005375763,
+      "learning_rate": 6.879201718226658e-06,
+      "loss": 0.9432,
+      "step": 695
+    },
+    {
+      "epoch": 0.6457564575645757,
+      "grad_norm": 1.2522950876693184,
+      "learning_rate": 6.7265523881812335e-06,
+      "loss": 0.9566,
+      "step": 700
+    },
+    {
+      "epoch": 0.6503690036900369,
+      "grad_norm": 1.2822158525144027,
+      "learning_rate": 6.574752681650864e-06,
+      "loss": 0.9427,
+      "step": 705
+    },
+    {
+      "epoch": 0.6549815498154982,
+      "grad_norm": 1.281546005641374,
+      "learning_rate": 6.423841998258069e-06,
+      "loss": 0.9491,
+      "step": 710
+    },
+    {
+      "epoch": 0.6595940959409594,
+      "grad_norm": 1.3593173158577243,
+      "learning_rate": 6.273859506879365e-06,
+      "loss": 0.9681,
+      "step": 715
+    },
+    {
+      "epoch": 0.6642066420664207,
+      "grad_norm": 1.3175244538690682,
+      "learning_rate": 6.124844135478971e-06,
+      "loss": 0.9646,
+      "step": 720
+    },
+    {
+      "epoch": 0.6688191881918819,
+      "grad_norm": 1.2493446902612035,
+      "learning_rate": 5.976834561005069e-06,
+      "loss": 0.9596,
+      "step": 725
+    },
+    {
+      "epoch": 0.6734317343173432,
+      "grad_norm": 1.3442075840613932,
+      "learning_rate": 5.829869199351188e-06,
+      "loss": 0.9491,
+      "step": 730
+    },
+    {
+      "epoch": 0.6780442804428044,
+      "grad_norm": 1.3675030498037168,
+      "learning_rate": 5.68398619538536e-06,
+      "loss": 0.9427,
+      "step": 735
+    },
+    {
+      "epoch": 0.6826568265682657,
+      "grad_norm": 1.3920256857486872,
+      "learning_rate": 5.53922341304961e-06,
+      "loss": 0.9377,
+      "step": 740
+    },
+    {
+      "epoch": 0.6872693726937269,
+      "grad_norm": 1.4200210554910793,
+      "learning_rate": 5.39561842553239e-06,
+      "loss": 0.9765,
+      "step": 745
+    },
+    {
+      "epoch": 0.6918819188191881,
+      "grad_norm": 1.3292116459560566,
+      "learning_rate": 5.2532085055164205e-06,
+      "loss": 0.9501,
+      "step": 750
+    },
+    {
+      "epoch": 0.6964944649446494,
+      "grad_norm": 1.3169663796276798,
+      "learning_rate": 5.112030615504601e-06,
+      "loss": 0.9528,
+      "step": 755
+    },
+    {
+      "epoch": 0.7011070110701108,
+      "grad_norm": 1.2655423286923129,
+      "learning_rate": 4.972121398226371e-06,
+      "loss": 0.9387,
+      "step": 760
+    },
+    {
+      "epoch": 0.705719557195572,
+      "grad_norm": 1.3173634362259354,
+      "learning_rate": 4.833517167127077e-06,
+      "loss": 0.953,
+      "step": 765
+    },
+    {
+      "epoch": 0.7103321033210332,
+      "grad_norm": 1.2762603380948692,
+      "learning_rate": 4.6962538969428416e-06,
+      "loss": 0.9517,
+      "step": 770
+    },
+    {
+      "epoch": 0.7149446494464945,
+      "grad_norm": 1.2561610075530374,
+      "learning_rate": 4.560367214363295e-06,
+      "loss": 0.9459,
+      "step": 775
+    },
+    {
+      "epoch": 0.7195571955719557,
+      "grad_norm": 1.3911464801805598,
+      "learning_rate": 4.425892388784681e-06,
+      "loss": 0.9489,
+      "step": 780
+    },
+    {
+      "epoch": 0.724169741697417,
+      "grad_norm": 1.3779998222369714,
+      "learning_rate": 4.292864323155684e-06,
+      "loss": 0.9352,
+      "step": 785
+    },
+    {
+      "epoch": 0.7287822878228782,
+      "grad_norm": 1.3549667557199516,
+      "learning_rate": 4.161317544918345e-06,
+      "loss": 0.9385,
+      "step": 790
+    },
+    {
+      "epoch": 0.7333948339483395,
+      "grad_norm": 1.291259067589743,
+      "learning_rate": 4.031286197046493e-06,
+      "loss": 0.9427,
+      "step": 795
+    },
+    {
+      "epoch": 0.7380073800738007,
+      "grad_norm": 1.3100001731926842,
+      "learning_rate": 3.902804029183907e-06,
+      "loss": 0.9448,
+      "step": 800
+    },
+    {
+      "epoch": 0.742619926199262,
+      "grad_norm": 1.3365665745846784,
+      "learning_rate": 3.775904388884618e-06,
+      "loss": 0.9424,
+      "step": 805
+    },
+    {
+      "epoch": 0.7472324723247232,
+      "grad_norm": 1.3043985795677464,
+      "learning_rate": 3.650620212957524e-06,
+      "loss": 0.9411,
+      "step": 810
+    },
+    {
+      "epoch": 0.7518450184501845,
+      "grad_norm": 1.4271272496927294,
+      "learning_rate": 3.5269840189176616e-06,
+      "loss": 0.9283,
+      "step": 815
+    },
+    {
+      "epoch": 0.7564575645756457,
+      "grad_norm": 1.2952198347151347,
+      "learning_rate": 3.405027896546277e-06,
+      "loss": 0.9264,
+      "step": 820
+    },
+    {
+      "epoch": 0.761070110701107,
+      "grad_norm": 1.2957386450676602,
+      "learning_rate": 3.2847834995619067e-06,
+      "loss": 0.9453,
+      "step": 825
+    },
+    {
+      "epoch": 0.7656826568265682,
+      "grad_norm": 1.2793394772991085,
+      "learning_rate": 3.1662820374046776e-06,
+      "loss": 0.9355,
+      "step": 830
+    },
+    {
+      "epoch": 0.7702952029520295,
+      "grad_norm": 1.3073488709068672,
+      "learning_rate": 3.0495542671358745e-06,
+      "loss": 0.9295,
+      "step": 835
+    },
+    {
+      "epoch": 0.7749077490774908,
+      "grad_norm": 1.3060370644236081,
+      "learning_rate": 2.934630485454948e-06,
+      "loss": 0.9355,
+      "step": 840
+    },
+    {
+      "epoch": 0.7795202952029521,
+      "grad_norm": 1.3092461599305434,
+      "learning_rate": 2.8215405208360237e-06,
+      "loss": 0.9398,
+      "step": 845
+    },
+    {
+      "epoch": 0.7841328413284133,
+      "grad_norm": 1.3054882444271876,
+      "learning_rate": 2.7103137257858867e-06,
+      "loss": 0.9361,
+      "step": 850
+    },
+    {
+      "epoch": 0.7887453874538746,
+      "grad_norm": 1.3462042645343073,
+      "learning_rate": 2.600978969225558e-06,
+      "loss": 0.9405,
+      "step": 855
+    },
+    {
+      "epoch": 0.7933579335793358,
+      "grad_norm": 1.2852304461419937,
+      "learning_rate": 2.493564628997369e-06,
+      "loss": 0.9478,
+      "step": 860
+    },
+    {
+      "epoch": 0.7979704797047971,
+      "grad_norm": 1.3100445849640032,
+      "learning_rate": 2.3880985844994674e-06,
+      "loss": 0.9288,
+      "step": 865
+    },
+    {
+      "epoch": 0.8025830258302583,
+      "grad_norm": 1.304963117164403,
+      "learning_rate": 2.284608209449746e-06,
+      "loss": 0.9361,
+      "step": 870
+    },
+    {
+      "epoch": 0.8071955719557196,
+      "grad_norm": 1.328410545932146,
+      "learning_rate": 2.183120364780975e-06,
+      "loss": 0.9391,
+      "step": 875
+    },
+    {
+      "epoch": 0.8118081180811808,
+      "grad_norm": 1.2800033094593806,
+      "learning_rate": 2.083661391669043e-06,
+      "loss": 0.9256,
+      "step": 880
+    },
+    {
+      "epoch": 0.816420664206642,
+      "grad_norm": 1.319974772827423,
+      "learning_rate": 1.986257104696121e-06,
+      "loss": 0.9366,
+      "step": 885
+    },
+    {
+      "epoch": 0.8210332103321033,
+      "grad_norm": 1.4312306306233893,
+      "learning_rate": 1.8909327851504633e-06,
+      "loss": 0.9531,
+      "step": 890
+    },
+    {
+      "epoch": 0.8256457564575646,
+      "grad_norm": 1.3675933057252032,
+      "learning_rate": 1.7977131744646724e-06,
+      "loss": 0.9332,
+      "step": 895
+    },
+    {
+      "epoch": 0.8302583025830258,
+      "grad_norm": 1.3222296346463416,
+      "learning_rate": 1.7066224677940313e-06,
+      "loss": 0.9438,
+      "step": 900
+    },
+    {
+      "epoch": 0.834870848708487,
+      "grad_norm": 1.2650792366896617,
+      "learning_rate": 1.6176843077366755e-06,
+      "loss": 0.9137,
+      "step": 905
+    },
+    {
+      "epoch": 0.8394833948339483,
+      "grad_norm": 1.29693082614585,
+      "learning_rate": 1.5309217781971419e-06,
+      "loss": 0.9139,
+      "step": 910
+    },
+    {
+      "epoch": 0.8440959409594095,
+      "grad_norm": 1.3025564128700116,
+      "learning_rate": 1.446357398394934e-06,
+      "loss": 0.9231,
+      "step": 915
+    },
+    {
+      "epoch": 0.8487084870848709,
+      "grad_norm": 1.316701035916371,
+      "learning_rate": 1.3640131170196758e-06,
+      "loss": 0.9164,
+      "step": 920
+    },
+    {
+      "epoch": 0.8533210332103321,
+      "grad_norm": 1.2870882614562777,
+      "learning_rate": 1.2839103065343084e-06,
+      "loss": 0.9282,
+      "step": 925
+    },
+    {
+      "epoch": 0.8579335793357934,
+      "grad_norm": 1.317763154741275,
+      "learning_rate": 1.2060697576278812e-06,
+      "loss": 0.9337,
+      "step": 930
+    },
+    {
+      "epoch": 0.8625461254612546,
+      "grad_norm": 1.2660145443644573,
+      "learning_rate": 1.1305116738193211e-06,
+      "loss": 0.9347,
+      "step": 935
+    },
+    {
+      "epoch": 0.8671586715867159,
+      "grad_norm": 1.2871595461432421,
+      "learning_rate": 1.0572556662136036e-06,
+      "loss": 0.9303,
+      "step": 940
+    },
+    {
+      "epoch": 0.8717712177121771,
+      "grad_norm": 1.3507861246785304,
+      "learning_rate": 9.863207484116987e-07,
+      "loss": 0.9185,
+      "step": 945
+    },
+    {
+      "epoch": 0.8763837638376384,
+      "grad_norm": 1.308521612899775,
+      "learning_rate": 9.177253315755796e-07,
+      "loss": 0.9383,
+      "step": 950
+    },
+    {
+      "epoch": 0.8809963099630996,
+      "grad_norm": 1.2578224314813977,
+      "learning_rate": 8.514872196496182e-07,
+      "loss": 0.9347,
+      "step": 955
+    },
+    {
+      "epoch": 0.8856088560885609,
+      "grad_norm": 1.2195495659147317,
+      "learning_rate": 7.876236047395525e-07,
+      "loss": 0.9264,
+      "step": 960
+    },
+    {
+      "epoch": 0.8902214022140221,
+      "grad_norm": 1.2882456430497387,
+      "learning_rate": 7.26151062650291e-07,
+      "loss": 0.934,
+      "step": 965
+    },
+    {
+      "epoch": 0.8948339483394834,
+      "grad_norm": 1.2980016061518047,
+      "learning_rate": 6.670855485836525e-07,
+      "loss": 0.9283,
+      "step": 970
+    },
+    {
+      "epoch": 0.8994464944649446,
+      "grad_norm": 1.2554874300953622,
+      "learning_rate": 6.104423929971948e-07,
+      "loss": 0.9326,
+      "step": 975
+    },
+    {
+      "epoch": 0.9040590405904059,
+      "grad_norm": 1.2725343756638872,
+      "learning_rate": 5.562362976251901e-07,
+      "loss": 0.938,
+      "step": 980
+    },
+    {
+      "epoch": 0.9086715867158671,
+      "grad_norm": 1.2191555391690136,
+      "learning_rate": 5.044813316627994e-07,
+      "loss": 0.9273,
+      "step": 985
+    },
+    {
+      "epoch": 0.9132841328413284,
+      "grad_norm": 1.3000616540800916,
+      "learning_rate": 4.5519092811439627e-07,
+      "loss": 0.9251,
+      "step": 990
+    },
+    {
+      "epoch": 0.9178966789667896,
+      "grad_norm": 1.3096355630057495,
+      "learning_rate": 4.083778803070504e-07,
+      "loss": 0.9437,
+      "step": 995
+    },
+    {
+      "epoch": 0.922509225092251,
+      "grad_norm": 1.2507592130312455,
+      "learning_rate": 3.6405433856999684e-07,
+      "loss": 0.9568,
+      "step": 1000
+    },
+    {
+      "epoch": 0.9271217712177122,
+      "grad_norm": 1.3217593223715594,
+      "learning_rate": 3.2223180708102933e-07,
+      "loss": 0.9326,
+      "step": 1005
+    },
+    {
+      "epoch": 0.9317343173431735,
+      "grad_norm": 1.2786914534479203,
+      "learning_rate": 2.829211408805932e-07,
+      "loss": 0.918,
+      "step": 1010
+    },
+    {
+      "epoch": 0.9363468634686347,
+      "grad_norm": 1.2616013442947607,
+      "learning_rate": 2.461325430543482e-07,
+      "loss": 0.9318,
+      "step": 1015
+    },
+    {
+      "epoch": 0.940959409594096,
+      "grad_norm": 1.2293936869640694,
+      "learning_rate": 2.1187556208496885e-07,
+      "loss": 0.9213,
+      "step": 1020
+    },
+    {
+      "epoch": 0.9455719557195572,
+      "grad_norm": 1.2383337694708525,
+      "learning_rate": 1.8015908937382587e-07,
+      "loss": 0.9337,
+      "step": 1025
+    },
+    {
+      "epoch": 0.9501845018450185,
+      "grad_norm": 1.2844222178616886,
+      "learning_rate": 1.5099135693322776e-07,
+      "loss": 0.9339,
+      "step": 1030
+    },
+    {
+      "epoch": 0.9547970479704797,
+      "grad_norm": 1.2175607312454386,
+      "learning_rate": 1.2437993524979984e-07,
+      "loss": 0.9033,
+      "step": 1035
+    },
+    {
+      "epoch": 0.959409594095941,
+      "grad_norm": 1.3010316088129263,
+      "learning_rate": 1.0033173131956175e-07,
+      "loss": 0.9212,
+      "step": 1040
+    },
+    {
+      "epoch": 0.9640221402214022,
+      "grad_norm": 1.2327806598615183,
+      "learning_rate": 7.885298685522235e-08,
+      "loss": 0.9153,
+      "step": 1045
+    },
+    {
+      "epoch": 0.9686346863468634,
+      "grad_norm": 1.2746164722932198,
+      "learning_rate": 5.99492766661347e-08,
+      "loss": 0.9211,
+      "step": 1050
+    },
+    {
+      "epoch": 0.9732472324723247,
+      "grad_norm": 1.2783038098829709,
+      "learning_rate": 4.362550721136338e-08,
+      "loss": 0.9255,
+      "step": 1055
+    },
+    {
+      "epoch": 0.977859778597786,
+      "grad_norm": 1.2368705746608133,
+      "learning_rate": 2.988591532620322e-08,
+      "loss": 0.9157,
+      "step": 1060
+    },
+    {
+      "epoch": 0.9824723247232472,
+      "grad_norm": 1.249650501200988,
+      "learning_rate": 1.8734067122514464e-08,
+      "loss": 0.9224,
+      "step": 1065
+    },
+    {
+      "epoch": 0.9870848708487084,
+      "grad_norm": 1.5405800531112683,
+      "learning_rate": 1.0172857063137643e-08,
+      "loss": 0.925,
+      "step": 1070
+    },
+    {
+      "epoch": 0.9916974169741697,
+      "grad_norm": 1.2847817455085104,
+      "learning_rate": 4.204507210633368e-09,
+      "loss": 0.9462,
+      "step": 1075
+    },
+    {
+      "epoch": 0.996309963099631,
+      "grad_norm": 1.3071185274762656,
+      "learning_rate": 8.30566650548148e-10,
+      "loss": 0.9409,
+      "step": 1080
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.9426799416542053,
+      "eval_runtime": 141.1576,
+      "eval_samples_per_second": 108.744,
+      "eval_steps_per_second": 1.7,
+      "step": 1084
+    },
+    {
+      "epoch": 1.0,
+      "step": 1084,
+      "total_flos": 453935093514240.0,
+      "train_loss": 0.982847072101607,
+      "train_runtime": 5570.7798,
+      "train_samples_per_second": 24.895,
+      "train_steps_per_second": 0.195
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 1084,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 200,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 453935093514240.0,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}