Upload folder using huggingface_hub

command.txt

@@ -0,0 +1 @@
+main.py deepseek

config.json

@@ -0,0 +1,31 @@
+{
+  "_name_or_path": "deepseek-ai/deepseek-coder-1.3b-instruct",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 32013,
+  "eos_token_id": 32021,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 5504,
+  "max_position_embeddings": 16384,
+  "model_type": "llama",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 16,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": {
+    "factor": 4.0,
+    "type": "linear"
+  },
+  "rope_theta": 100000,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.38.2",
+  "use_cache": true,
+  "vocab_size": 32256
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 32013,
+  "eos_token_id": 32021,
+  "transformers_version": "4.38.2"
+}

model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc8b408211f3d6f12fa928573bae13aa44e32745654c884785497fd845f908d7
+size 4986380064

model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5700ee949e4ad2afc984948f6240588c65a0e30033d9643f20b5db0321dee689
+size 399532808

model.safetensors.index.json

@@ -0,0 +1,226 @@
+{
+  "metadata": {
+    "total_size": 5385887744
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00002-of-00002.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2305a178c90a806a5a0a24ecec62cadd0a11e730fcd4f791ac633f449b8c675
+size 2699039674

rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc5ad86f4f472e3fbe661a3e37dcb4cc0d3f931baf861c388a004f41ab765191
+size 14244

run_config.json

@@ -0,0 +1,604 @@
+{
+    "model": {
+        "codellama": {
+            "base_model_id": "codellama/CodeLlama-7b-hf",
+            "quantitize": "int8",
+            "dataset": "Arithmetic_Simple",
+            "data_collator": "DataCollatorForSeq2Seq",
+            "lora_config": {
+                "r": 16,
+                "lora_alpha": 16,
+                "target_modules": [
+                    "q_proj",
+                    "k_proj",
+                    "v_proj",
+                    "o_proj",
+                    "gate_proj",
+                    "up_proj",
+                    "down_proj"
+                ],
+                "lora_dropout": 0.05,
+                "bias": "none",
+                "task_type": "CAUSAL_LM"
+            },
+            "training_args": {
+                "output_dir": "codellama-output",
+                "warmup_steps": 100,
+                "per_device_train_batch_size": 1,
+                "per_device_eval_batch_size": 1,
+                "gradient_accumulation_steps": 4,
+                "max_steps": 10000,
+                "learning_rate": 0.0003,
+                "optim": "adamw_torch",
+                "logging_dir": "codellama-output-logs",
+                "logging_steps": 10,
+                "save_strategy": "steps",
+                "save_steps": 500,
+                "load_best_model_at_end": false,
+                "group_by_length": true,
+                "fp16": true,
+                "evaluation_strategy": "steps",
+                "eval_steps": 1000
+            },
+            "tokenizer": {
+                "tokenize_config": {
+                    "truncation": true,
+                    "max_length": 192,
+                    "padding": "max_length"
+                },
+                "prompt_template": "config/qa_template.txt"
+            }
+        },
+        "phi-2": {
+            "base_model_id": "microsoft/phi-2",
+            "quantitize": "fp16",
+            "dataset": "Arithmetic_Simple",
+            "data_collator": "DataCollatorForLanguageModeling",
+            "lora_config": {
+                "r": 32,
+                "lora_alpha": 64,
+                "target_modules": [
+                    "q_proj",
+                    "k_proj",
+                    "v_proj",
+                    "dense",
+                    "fc1",
+                    "fc2"
+                ],
+                "bias": "none",
+                "lora_dropout": 0.05,
+                "task_type": "CAUSAL_LM"
+            },
+            "training_args": {
+                "output_dir": "phi2-output",
+                "warmup_steps": 500,
+                "per_device_train_batch_size": 1,
+                "per_device_eval_batch_size": 1,
+                "gradient_accumulation_steps": 4,
+                "max_steps": 100000,
+                "learning_rate": 0.0003,
+                "optim": "paged_adamw_8bit",
+                "logging_dir": "phi2-output-logs",
+                "logging_steps": 100,
+                "save_strategy": "steps",
+                "save_steps": 500,
+                "evaluation_strategy": "steps",
+                "eval_steps": 500,
+                "fp16": true
+            },
+            "tokenizer": {
+                "tokenize_config": {
+                    "truncation": true,
+                    "max_length": 512,
+                    "padding": "max_length"
+                },
+                "prompt_template": "config/qa_template.txt"
+            }
+        },
+        "deepseek": {
+            "base_model_id": "deepseek-ai/deepseek-coder-1.3b-instruct",
+            "quantitize": "fp16",
+            "dataset": "ghcode_python",
+            "data_collator": "DataCollatorForLanguageModeling",
+            "lora_config": {
+                "r": 32,
+                "lora_alpha": 64,
+                "target_modules": [
+                    "q_proj",
+                    "k_proj",
+                    "v_proj",
+                    "o_proj",
+                    "gate_proj",
+                    "up_proj",
+                    "down_proj"
+                ],
+                "bias": "none",
+                "lora_dropout": 0.05,
+                "task_type": "CAUSAL_LM"
+            },
+            "lora_large_config": {
+                "r": 128,
+                "lora_alpha": 256,
+                "target_modules": [
+                    "q_proj",
+                    "k_proj",
+                    "v_proj",
+                    "o_proj",
+                    "gate_proj",
+                    "up_proj",
+                    "down_proj"
+                ],
+                "bias": "none",
+                "lora_dropout": 0.05,
+                "task_type": "CAUSAL_LM"
+            },
+            "p_tuning_config": {
+                "num_virtual_tokens": 16,
+                "num_transformer_submodules": 1,
+                "token_dim": 2048,
+                "encoder_hidden_size": 2048,
+                "task_type": "CAUSAL_LM"
+            },
+            "training_args": {
+                "output_dir": "runs/deepseek-ctrl-gh",
+                "warmup_steps": 0,
+                "per_device_train_batch_size": 8,
+                "per_device_eval_batch_size": 8,
+                "gradient_accumulation_steps": 4,
+                "max_steps": 15000,
+                "learning_rate": 2e-05,
+                "optim": "paged_adamw_8bit",
+                "logging_dir": "runs/deepseek-ctrl-gh/logs",
+                "logging_steps": 100,
+                "save_strategy": "steps",
+                "save_steps": 5000,
+                "evaluation_strategy": "steps",
+                "eval_steps": 5000,
+                "weight_decay": 0.01,
+                "fp16": true
+            },
+            "tokenizer": {
+                "tokenize_config": {
+                    "truncation": true,
+                    "max_length": 512,
+                    "padding": "max_length"
+                },
+                "prompt_template": "config/qa_template.txt"
+            }
+        }
+    },
+    "dataset": {
+        "simple_dataset": {
+            "type": "huggingface",
+            "dataset_purpose": "downstream",
+            "name": "b-mc2/sql-create-context",
+            "train_split": 0.9,
+            "max_train_size": 100,
+            "filling_field": [
+                "question",
+                "context",
+                "answer"
+            ]
+        },
+        "testdset": {
+            "type": "local",
+            "dataset_purpose": "downstream",
+            "train_file": "data/Test/TestDataset.json",
+            "val_file": "data/Test/TestDataset.json",
+            "test_file": "data/Test/TestDataset.json",
+            "filling_field": [
+                "prompted_question",
+                "answer"
+            ]
+        },
+        "mixture_codegen": {
+            "filling_field": [
+                "Question",
+                "Answer"
+            ],
+            "dataset_purpose": "downstream"
+        },
+        "MathQA_Python_loader": {
+            "type": "list-like",
+            "dataset_purpose": "downstream",
+            "train": "data/MathQA_Python_processed/mathqa_python_train_clean_final.json",
+            "val": "data/MathQA_Python_processed/mathqa_python_dev_clean_final.json",
+            "test": "data/MathQA_Python_processed/mathqa_python_test_clean_final.json",
+            "filling_field": [
+                "Question",
+                "Answer"
+            ]
+        },
+        "APPS_loader": {
+            "type": "list-like",
+            "dataset_purpose": "downstream",
+            "train": "data/APPS/apps_train.json",
+            "val": "data/APPS/apps_dev.json",
+            "test": "data/APPS/test/apps_test_75.json",
+            "filling_field": [
+                "Question",
+                "Answer"
+            ]
+        },
+        "MBPP_loader": {
+            "type": "list-like",
+            "dataset_purpose": "downstream",
+            "train": "data/MBPP/mbpp_train.json",
+            "val": "data/MBPP/mbpp_dev.json",
+            "test": "data/MBPP/mbpp_test.json",
+            "filling_field": [
+                "Question",
+                "Answer"
+            ]
+        },
+        "Arithmetic_Simple": {
+            "type": "list-like",
+            "dataset_purpose": "downstream",
+            "attributes": {
+                "subjects": [
+                    1,
+                    2,
+                    3,
+                    4,
+                    5,
+                    6,
+                    7,
+                    8,
+                    9
+                ],
+                "lessons": [
+                    "Max_Ops1_Bounds0_100",
+                    "Max_Ops1_Bounds0_1000",
+                    "Max_Ops2_Bounds0_100",
+                    "Max_Ops2_Bounds0_1000",
+                    "Max_Ops3_Bounds0_100",
+                    "Max_Ops3_Bounds0_1000",
+                    "Max_Ops4_Bounds0_100",
+                    "Max_Ops4_Bounds0_1000",
+                    "Max_Ops5_Bounds0_100",
+                    "Max_Ops5_Bounds0_1000"
+                ]
+            },
+            "train": "data/Arithmetic/Curriculum_Simple",
+            "val": "data/Arithmetic/Curriculum_Simple",
+            "test": "data/Arithmetic/Curriculum_Simple",
+            "filling_field": [
+                "Question",
+                "Answer"
+            ]
+        },
+        "Arithmetic_Hard": {
+            "type": "list-like",
+            "dataset_purpose": "downstream",
+            "attributes": {
+                "subjects": [
+                    1,
+                    2,
+                    3,
+                    4,
+                    5,
+                    6,
+                    7,
+                    8,
+                    9
+                ],
+                "lessons": [
+                    "Max_Ops1_Bounds-1000_1000",
+                    "Max_Ops1_Bounds-100_100",
+                    "Max_Ops1_Bounds0_100",
+                    "Max_Ops1_Bounds0_1000",
+                    "Max_Ops2_Bounds-1000_1000",
+                    "Max_Ops2_Bounds-100_100",
+                    "Max_Ops2_Bounds0_100",
+                    "Max_Ops2_Bounds0_1000",
+                    "Max_Ops3_Bounds-1000_1000",
+                    "Max_Ops3_Bounds-100_100",
+                    "Max_Ops3_Bounds0_100",
+                    "Max_Ops3_Bounds0_1000",
+                    "Max_Ops4_Bounds-1000_1000",
+                    "Max_Ops4_Bounds-100_100",
+                    "Max_Ops4_Bounds0_100",
+                    "Max_Ops4_Bounds0_1000",
+                    "Max_Ops5_Bounds-1000_1000",
+                    "Max_Ops5_Bounds-100_100",
+                    "Max_Ops5_Bounds0_100",
+                    "Max_Ops5_Bounds0_1000",
+                    "Max_Ops6_Bounds-1000_1000",
+                    "Max_Ops6_Bounds-100_100",
+                    "Max_Ops6_Bounds0_100",
+                    "Max_Ops6_Bounds0_1000",
+                    "Max_Ops7_Bounds-1000_1000",
+                    "Max_Ops7_Bounds-100_100",
+                    "Max_Ops7_Bounds0_100",
+                    "Max_Ops7_Bounds0_1000",
+                    "Max_Ops8_Bounds-1000_1000",
+                    "Max_Ops8_Bounds-100_100",
+                    "Max_Ops8_Bounds0_100",
+                    "Max_Ops8_Bounds0_1000",
+                    "Max_Ops9_Bounds-1000_1000",
+                    "Max_Ops9_Bounds-100_100",
+                    "Max_Ops9_Bounds0_100",
+                    "Max_Ops9_Bounds0_1000",
+                    "Max_Ops10_Bounds-1000_1000",
+                    "Max_Ops10_Bounds-100_100",
+                    "Max_Ops10_Bounds0_100",
+                    "Max_Ops10_Bounds0_1000"
+                ]
+            },
+            "train": "data/Arithmetic/Curriculum_Hard",
+            "val": "data/Arithmetic/Curriculum_Hard",
+            "test": "data/Arithmetic/Curriculum_Hard",
+            "filling_field": [
+                "Question",
+                "Answer"
+            ]
+        },
+        "Arithmetic_Hard_prompt_C11": {
+            "type": "list-like",
+            "dataset_purpose": "downstream",
+            "attributes": {
+                "subjects": [
+                    1,
+                    2,
+                    3,
+                    4,
+                    5,
+                    6,
+                    7,
+                    8,
+                    9
+                ],
+                "lessons": [
+                    "Max_Ops1_Bounds-1000_1000",
+                    "Max_Ops1_Bounds-100_100",
+                    "Max_Ops1_Bounds0_100",
+                    "Max_Ops1_Bounds0_1000",
+                    "Max_Ops2_Bounds-1000_1000",
+                    "Max_Ops2_Bounds-100_100",
+                    "Max_Ops2_Bounds0_100",
+                    "Max_Ops2_Bounds0_1000",
+                    "Max_Ops3_Bounds-1000_1000",
+                    "Max_Ops3_Bounds-100_100",
+                    "Max_Ops3_Bounds0_100",
+                    "Max_Ops3_Bounds0_1000",
+                    "Max_Ops4_Bounds-1000_1000",
+                    "Max_Ops4_Bounds-100_100",
+                    "Max_Ops4_Bounds0_100",
+                    "Max_Ops4_Bounds0_1000",
+                    "Max_Ops5_Bounds-1000_1000",
+                    "Max_Ops5_Bounds-100_100",
+                    "Max_Ops5_Bounds0_100",
+                    "Max_Ops5_Bounds0_1000",
+                    "Max_Ops6_Bounds-1000_1000",
+                    "Max_Ops6_Bounds-100_100",
+                    "Max_Ops6_Bounds0_100",
+                    "Max_Ops6_Bounds0_1000",
+                    "Max_Ops7_Bounds-1000_1000",
+                    "Max_Ops7_Bounds-100_100",
+                    "Max_Ops7_Bounds0_100",
+                    "Max_Ops7_Bounds0_1000",
+                    "Max_Ops8_Bounds-1000_1000",
+                    "Max_Ops8_Bounds-100_100",
+                    "Max_Ops8_Bounds0_100",
+                    "Max_Ops8_Bounds0_1000",
+                    "Max_Ops9_Bounds-1000_1000",
+                    "Max_Ops9_Bounds-100_100",
+                    "Max_Ops9_Bounds0_100",
+                    "Max_Ops9_Bounds0_1000",
+                    "Max_Ops10_Bounds-1000_1000",
+                    "Max_Ops10_Bounds-100_100",
+                    "Max_Ops10_Bounds0_100",
+                    "Max_Ops10_Bounds0_1000"
+                ]
+            },
+            "train": "data/Arithmetic/Curriculum_Hard",
+            "val": "data/Arithmetic/Curriculum_Hard",
+            "test": "data/Arithmetic/Curriculum_Hard",
+            "filling_field": [
+                "Question",
+                "Answer"
+            ]
+        },
+        "Arithmetic_Hard_prompt_C12": {
+            "type": "list-like",
+            "dataset_purpose": "downstream",
+            "attributes": {
+                "subjects": [
+                    1,
+                    2,
+                    3,
+                    4
+                ],
+                "lessons": [
+                    "Max_Ops1_Bounds-1000_1000",
+                    "Max_Ops1_Bounds-100_100",
+                    "Max_Ops1_Bounds0_100",
+                    "Max_Ops1_Bounds0_1000",
+                    "Max_Ops2_Bounds-1000_1000",
+                    "Max_Ops2_Bounds-100_100",
+                    "Max_Ops2_Bounds0_100",
+                    "Max_Ops2_Bounds0_1000",
+                    "Max_Ops3_Bounds-1000_1000",
+                    "Max_Ops3_Bounds-100_100",
+                    "Max_Ops3_Bounds0_100",
+                    "Max_Ops3_Bounds0_1000",
+                    "Max_Ops4_Bounds-1000_1000",
+                    "Max_Ops4_Bounds-100_100",
+                    "Max_Ops4_Bounds0_100",
+                    "Max_Ops4_Bounds0_1000",
+                    "Max_Ops5_Bounds-1000_1000",
+                    "Max_Ops5_Bounds-100_100",
+                    "Max_Ops5_Bounds0_100",
+                    "Max_Ops5_Bounds0_1000",
+                    "Max_Ops6_Bounds-1000_1000",
+                    "Max_Ops6_Bounds-100_100",
+                    "Max_Ops6_Bounds0_100",
+                    "Max_Ops6_Bounds0_1000",
+                    "Max_Ops7_Bounds-1000_1000",
+                    "Max_Ops7_Bounds-100_100",
+                    "Max_Ops7_Bounds0_100",
+                    "Max_Ops7_Bounds0_1000",
+                    "Max_Ops8_Bounds-1000_1000",
+                    "Max_Ops8_Bounds-100_100",
+                    "Max_Ops8_Bounds0_100",
+                    "Max_Ops8_Bounds0_1000",
+                    "Max_Ops9_Bounds-1000_1000",
+                    "Max_Ops9_Bounds-100_100",
+                    "Max_Ops9_Bounds0_100",
+                    "Max_Ops9_Bounds0_1000",
+                    "Max_Ops10_Bounds-1000_1000",
+                    "Max_Ops10_Bounds-100_100",
+                    "Max_Ops10_Bounds0_100",
+                    "Max_Ops10_Bounds0_1000"
+                ]
+            },
+            "train": "data/Arithmetic/Curriculum_Hard",
+            "val": "data/Arithmetic/Curriculum_Hard",
+            "test": "data/Arithmetic/Curriculum_Hard",
+            "filling_field": [
+                "Question",
+                "Answer"
+            ]
+        },
+        "Arithmetic_XHard": {
+            "type": "list-like",
+            "dataset_purpose": "downstream",
+            "attributes": {
+                "subjects": [
+                    1,
+                    2,
+                    3,
+                    4,
+                    5,
+                    6,
+                    7,
+                    8,
+                    9
+                ],
+                "lessons": [
+                    "Max_Ops10_Bounds0_10000.json",
+                    "Max_Ops10_Bounds0_1000.json",
+                    "Max_Ops10_Bounds-10000_10000.json",
+                    "Max_Ops10_Bounds-1000_1000.json",
+                    "Max_Ops11_Bounds0_10000.json",
+                    "Max_Ops11_Bounds0_1000.json",
+                    "Max_Ops11_Bounds-10000_10000.json",
+                    "Max_Ops11_Bounds-1000_1000.json",
+                    "Max_Ops12_Bounds0_10000.json",
+                    "Max_Ops12_Bounds0_1000.json",
+                    "Max_Ops12_Bounds-10000_10000.json",
+                    "Max_Ops12_Bounds-1000_1000.json",
+                    "Max_Ops13_Bounds0_10000.json",
+                    "Max_Ops13_Bounds0_1000.json",
+                    "Max_Ops13_Bounds-10000_10000.json",
+                    "Max_Ops13_Bounds-1000_1000.json",
+                    "Max_Ops14_Bounds0_10000.json",
+                    "Max_Ops14_Bounds0_1000.json",
+                    "Max_Ops14_Bounds-10000_10000.json",
+                    "Max_Ops14_Bounds-1000_1000.json",
+                    "Max_Ops15_Bounds0_10000.json",
+                    "Max_Ops15_Bounds0_1000.json",
+                    "Max_Ops15_Bounds-10000_10000.json",
+                    "Max_Ops15_Bounds-1000_1000.json",
+                    "Max_Ops16_Bounds0_10000.json",
+                    "Max_Ops16_Bounds0_1000.json",
+                    "Max_Ops16_Bounds-10000_10000.json",
+                    "Max_Ops16_Bounds-1000_1000.json",
+                    "Max_Ops17_Bounds0_10000.json",
+                    "Max_Ops17_Bounds0_1000.json",
+                    "Max_Ops17_Bounds-10000_10000.json",
+                    "Max_Ops17_Bounds-1000_1000.json",
+                    "Max_Ops18_Bounds0_10000.json",
+                    "Max_Ops18_Bounds0_1000.json",
+                    "Max_Ops18_Bounds-10000_10000.json",
+                    "Max_Ops18_Bounds-1000_1000.json",
+                    "Max_Ops19_Bounds0_10000.json",
+                    "Max_Ops19_Bounds0_1000.json",
+                    "Max_Ops19_Bounds-10000_10000.json",
+                    "Max_Ops19_Bounds-1000_1000.json",
+                    "Max_Ops1_Bounds0_10000.json",
+                    "Max_Ops1_Bounds0_1000.json",
+                    "Max_Ops1_Bounds-10000_10000.json",
+                    "Max_Ops1_Bounds-1000_1000.json",
+                    "Max_Ops20_Bounds0_10000.json",
+                    "Max_Ops20_Bounds0_1000.json",
+                    "Max_Ops20_Bounds-10000_10000.json",
+                    "Max_Ops20_Bounds-1000_1000.json",
+                    "Max_Ops2_Bounds0_10000.json",
+                    "Max_Ops2_Bounds0_1000.json",
+                    "Max_Ops2_Bounds-10000_10000.json",
+                    "Max_Ops2_Bounds-1000_1000.json",
+                    "Max_Ops3_Bounds0_10000.json",
+                    "Max_Ops3_Bounds0_1000.json",
+                    "Max_Ops3_Bounds-10000_10000.json",
+                    "Max_Ops3_Bounds-1000_1000.json",
+                    "Max_Ops4_Bounds0_10000.json",
+                    "Max_Ops4_Bounds0_1000.json",
+                    "Max_Ops4_Bounds-10000_10000.json",
+                    "Max_Ops4_Bounds-1000_1000.json",
+                    "Max_Ops5_Bounds0_10000.json",
+                    "Max_Ops5_Bounds0_1000.json",
+                    "Max_Ops5_Bounds-10000_10000.json",
+                    "Max_Ops5_Bounds-1000_1000.json",
+                    "Max_Ops6_Bounds0_10000.json",
+                    "Max_Ops6_Bounds0_1000.json",
+                    "Max_Ops6_Bounds-10000_10000.json",
+                    "Max_Ops6_Bounds-1000_1000.json",
+                    "Max_Ops7_Bounds0_10000.json",
+                    "Max_Ops7_Bounds0_1000.json",
+                    "Max_Ops7_Bounds-10000_10000.json",
+                    "Max_Ops7_Bounds-1000_1000.json",
+                    "Max_Ops8_Bounds0_10000.json",
+                    "Max_Ops8_Bounds0_1000.json",
+                    "Max_Ops8_Bounds-10000_10000.json",
+                    "Max_Ops8_Bounds-1000_1000.json",
+                    "Max_Ops9_Bounds0_10000.json",
+                    "Max_Ops9_Bounds0_1000.json",
+                    "Max_Ops9_Bounds-10000_10000.json",
+                    "Max_Ops9_Bounds-1000_1000.json"
+                ]
+            },
+            "train": "data/Arithmetic/Curriculum_XHard",
+            "val": "data/Arithmetic/Curriculum_XHard",
+            "test": "data/Arithmetic/Curriculum_XHard",
+            "filling_field": [
+                "Question",
+                "Answer"
+            ]
+        },
+        "GSM8K": {
+            "type": "local",
+            "dataset_purpose": "downstream",
+            "train_file": "data/GSM8K/GSM8K_train.json",
+            "val_file": "data/GSM8K/GSM8K_test.json",
+            "test_file": "data/GSM8K/GSM8K_dev.json",
+            "filling_field": [
+                "Body",
+                "Question",
+                "Answer"
+            ]
+        },
+        "APPS": {
+            "type": "local",
+            "dataset_purpose": "downstream",
+            "train_file": "data/APPS/apps_train.json",
+            "val_file": "data/APPS/apps_test.json",
+            "test_file": "data/APPS/apps_dev.json",
+            "filling_field": [
+                "Body",
+                "Question",
+                "Answer"
+            ]
+        },
+        "ghcode_python": {
+            "type": "huggingface",
+            "dataset_purpose": "pretrain",
+            "name": "slseanwu/ghcode_python_split_700k",
+            "max_eval_size": 1000,
+            "max_train_size": 160000,
+            "filling_field": [
+                "code"
+            ]
+        }
+    }
+}

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7dbf3c56b55bacde1773bb6b7271e30823177a28ab1b1a0ab88b4d8b5a78d8e8
+size 1064

trainer_state.json

@@ -0,0 +1,1095 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 5000,
+  "global_step": 15000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.02,
+      "grad_norm": 1.0555673837661743,
+      "learning_rate": 1.9866666666666667e-05,
+      "loss": 0.9803,
+      "step": 100
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 1.0609339475631714,
+      "learning_rate": 1.9733333333333336e-05,
+      "loss": 0.9477,
+      "step": 200
+    },
+    {
+      "epoch": 0.06,
+      "grad_norm": 0.8154662251472473,
+      "learning_rate": 1.9600000000000002e-05,
+      "loss": 0.9472,
+      "step": 300
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 0.8507415652275085,
+      "learning_rate": 1.9466666666666668e-05,
+      "loss": 0.9496,
+      "step": 400
+    },
+    {
+      "epoch": 0.1,
+      "grad_norm": 0.9045298099517822,
+      "learning_rate": 1.9333333333333333e-05,
+      "loss": 0.9416,
+      "step": 500
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 1.2995306253433228,
+      "learning_rate": 1.9200000000000003e-05,
+      "loss": 0.9501,
+      "step": 600
+    },
+    {
+      "epoch": 0.14,
+      "grad_norm": 0.8860793709754944,
+      "learning_rate": 1.9066666666666668e-05,
+      "loss": 0.929,
+      "step": 700
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 0.8791409134864807,
+      "learning_rate": 1.8933333333333334e-05,
+      "loss": 0.9323,
+      "step": 800
+    },
+    {
+      "epoch": 0.18,
+      "grad_norm": 0.868554949760437,
+      "learning_rate": 1.88e-05,
+      "loss": 0.9252,
+      "step": 900
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 0.7996535301208496,
+      "learning_rate": 1.866666666666667e-05,
+      "loss": 0.916,
+      "step": 1000
+    },
+    {
+      "epoch": 0.22,
+      "grad_norm": 0.9991319179534912,
+      "learning_rate": 1.8533333333333334e-05,
+      "loss": 0.9267,
+      "step": 1100
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 1.0026607513427734,
+      "learning_rate": 1.8400000000000003e-05,
+      "loss": 0.9281,
+      "step": 1200
+    },
+    {
+      "epoch": 0.26,
+      "grad_norm": 0.860996663570404,
+      "learning_rate": 1.826666666666667e-05,
+      "loss": 0.9272,
+      "step": 1300
+    },
+    {
+      "epoch": 0.28,
+      "grad_norm": 0.8613474369049072,
+      "learning_rate": 1.8133333333333335e-05,
+      "loss": 0.9277,
+      "step": 1400
+    },
+    {
+      "epoch": 0.3,
+      "grad_norm": 0.8491615653038025,
+      "learning_rate": 1.8e-05,
+      "loss": 0.9293,
+      "step": 1500
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 0.9264828562736511,
+      "learning_rate": 1.7866666666666666e-05,
+      "loss": 0.917,
+      "step": 1600
+    },
+    {
+      "epoch": 0.34,
+      "grad_norm": 0.8633733987808228,
+      "learning_rate": 1.7733333333333335e-05,
+      "loss": 0.9325,
+      "step": 1700
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 0.8906639218330383,
+      "learning_rate": 1.76e-05,
+      "loss": 0.9126,
+      "step": 1800
+    },
+    {
+      "epoch": 0.38,
+      "grad_norm": 0.885764479637146,
+      "learning_rate": 1.7466666666666667e-05,
+      "loss": 0.9065,
+      "step": 1900
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 0.8839893937110901,
+      "learning_rate": 1.7333333333333336e-05,
+      "loss": 0.9429,
+      "step": 2000
+    },
+    {
+      "epoch": 0.42,
+      "grad_norm": 0.913644552230835,
+      "learning_rate": 1.72e-05,
+      "loss": 0.9056,
+      "step": 2100
+    },
+    {
+      "epoch": 0.44,
+      "grad_norm": 0.8399360179901123,
+      "learning_rate": 1.706666666666667e-05,
+      "loss": 0.913,
+      "step": 2200
+    },
+    {
+      "epoch": 0.46,
+      "grad_norm": 0.8176397681236267,
+      "learning_rate": 1.6933333333333336e-05,
+      "loss": 0.9192,
+      "step": 2300
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 0.8771522045135498,
+      "learning_rate": 1.6800000000000002e-05,
+      "loss": 0.9222,
+      "step": 2400
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 0.7874905467033386,
+      "learning_rate": 1.6666666666666667e-05,
+      "loss": 0.9197,
+      "step": 2500
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 0.8051652312278748,
+      "learning_rate": 1.6533333333333333e-05,
+      "loss": 0.91,
+      "step": 2600
+    },
+    {
+      "epoch": 0.54,
+      "grad_norm": 0.80189049243927,
+      "learning_rate": 1.64e-05,
+      "loss": 0.9091,
+      "step": 2700
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 0.7678588032722473,
+      "learning_rate": 1.6266666666666668e-05,
+      "loss": 0.9111,
+      "step": 2800
+    },
+    {
+      "epoch": 0.58,
+      "grad_norm": 0.7835684418678284,
+      "learning_rate": 1.6133333333333334e-05,
+      "loss": 0.9127,
+      "step": 2900
+    },
+    {
+      "epoch": 0.6,
+      "grad_norm": 0.8735753297805786,
+      "learning_rate": 1.6000000000000003e-05,
+      "loss": 0.9038,
+      "step": 3000
+    },
+    {
+      "epoch": 0.62,
+      "grad_norm": 0.8635367155075073,
+      "learning_rate": 1.586666666666667e-05,
+      "loss": 0.9148,
+      "step": 3100
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 0.8477209210395813,
+      "learning_rate": 1.5733333333333334e-05,
+      "loss": 0.9123,
+      "step": 3200
+    },
+    {
+      "epoch": 0.66,
+      "grad_norm": 0.7765554785728455,
+      "learning_rate": 1.5600000000000003e-05,
+      "loss": 0.8898,
+      "step": 3300
+    },
+    {
+      "epoch": 0.68,
+      "grad_norm": 0.891146183013916,
+      "learning_rate": 1.546666666666667e-05,
+      "loss": 0.8992,
+      "step": 3400
+    },
+    {
+      "epoch": 0.7,
+      "grad_norm": 0.8695024251937866,
+      "learning_rate": 1.5333333333333334e-05,
+      "loss": 0.9114,
+      "step": 3500
+    },
+    {
+      "epoch": 0.72,
+      "grad_norm": 0.927379846572876,
+      "learning_rate": 1.5200000000000002e-05,
+      "loss": 0.89,
+      "step": 3600
+    },
+    {
+      "epoch": 0.74,
+      "grad_norm": 0.885179340839386,
+      "learning_rate": 1.5066666666666668e-05,
+      "loss": 0.9159,
+      "step": 3700
+    },
+    {
+      "epoch": 0.76,
+      "grad_norm": 0.9603422284126282,
+      "learning_rate": 1.4933333333333335e-05,
+      "loss": 0.9115,
+      "step": 3800
+    },
+    {
+      "epoch": 0.78,
+      "grad_norm": 0.8624181151390076,
+      "learning_rate": 1.48e-05,
+      "loss": 0.9025,
+      "step": 3900
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 0.9430868625640869,
+      "learning_rate": 1.4666666666666666e-05,
+      "loss": 0.8984,
+      "step": 4000
+    },
+    {
+      "epoch": 0.82,
+      "grad_norm": 0.7860404849052429,
+      "learning_rate": 1.4533333333333335e-05,
+      "loss": 0.8887,
+      "step": 4100
+    },
+    {
+      "epoch": 0.84,
+      "grad_norm": 0.793118417263031,
+      "learning_rate": 1.4400000000000001e-05,
+      "loss": 0.9097,
+      "step": 4200
+    },
+    {
+      "epoch": 0.86,
+      "grad_norm": 0.8741154074668884,
+      "learning_rate": 1.4266666666666668e-05,
+      "loss": 0.8919,
+      "step": 4300
+    },
+    {
+      "epoch": 0.88,
+      "grad_norm": 0.7793194651603699,
+      "learning_rate": 1.4133333333333334e-05,
+      "loss": 0.8927,
+      "step": 4400
+    },
+    {
+      "epoch": 0.9,
+      "grad_norm": 0.7923697233200073,
+      "learning_rate": 1.4e-05,
+      "loss": 0.9032,
+      "step": 4500
+    },
+    {
+      "epoch": 0.92,
+      "grad_norm": 0.8406550288200378,
+      "learning_rate": 1.3866666666666669e-05,
+      "loss": 0.8878,
+      "step": 4600
+    },
+    {
+      "epoch": 0.94,
+      "grad_norm": 1.0170931816101074,
+      "learning_rate": 1.3733333333333335e-05,
+      "loss": 0.9032,
+      "step": 4700
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 0.8855489492416382,
+      "learning_rate": 1.3600000000000002e-05,
+      "loss": 0.9026,
+      "step": 4800
+    },
+    {
+      "epoch": 0.98,
+      "grad_norm": 0.7720271348953247,
+      "learning_rate": 1.3466666666666668e-05,
+      "loss": 0.8978,
+      "step": 4900
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 0.841073751449585,
+      "learning_rate": 1.3333333333333333e-05,
+      "loss": 0.8958,
+      "step": 5000
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.88493812084198,
+      "eval_runtime": 62.115,
+      "eval_samples_per_second": 16.099,
+      "eval_steps_per_second": 2.012,
+      "step": 5000
+    },
+    {
+      "epoch": 1.02,
+      "grad_norm": 0.856351375579834,
+      "learning_rate": 1.3200000000000002e-05,
+      "loss": 0.7886,
+      "step": 5100
+    },
+    {
+      "epoch": 1.04,
+      "grad_norm": 0.92889404296875,
+      "learning_rate": 1.3066666666666668e-05,
+      "loss": 0.7957,
+      "step": 5200
+    },
+    {
+      "epoch": 1.06,
+      "grad_norm": 0.8512794375419617,
+      "learning_rate": 1.2933333333333334e-05,
+      "loss": 0.7887,
+      "step": 5300
+    },
+    {
+      "epoch": 1.08,
+      "grad_norm": 0.8943263292312622,
+      "learning_rate": 1.2800000000000001e-05,
+      "loss": 0.8119,
+      "step": 5400
+    },
+    {
+      "epoch": 1.1,
+      "grad_norm": 0.908469557762146,
+      "learning_rate": 1.2666666666666667e-05,
+      "loss": 0.7958,
+      "step": 5500
+    },
+    {
+      "epoch": 1.12,
+      "grad_norm": 0.790229856967926,
+      "learning_rate": 1.2533333333333336e-05,
+      "loss": 0.7958,
+      "step": 5600
+    },
+    {
+      "epoch": 1.14,
+      "grad_norm": 1.0110950469970703,
+      "learning_rate": 1.2400000000000002e-05,
+      "loss": 0.7907,
+      "step": 5700
+    },
+    {
+      "epoch": 1.16,
+      "grad_norm": 0.9110943078994751,
+      "learning_rate": 1.2266666666666667e-05,
+      "loss": 0.7939,
+      "step": 5800
+    },
+    {
+      "epoch": 1.18,
+      "grad_norm": 0.8515949249267578,
+      "learning_rate": 1.2133333333333335e-05,
+      "loss": 0.7849,
+      "step": 5900
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 0.8418060541152954,
+      "learning_rate": 1.2e-05,
+      "loss": 0.8036,
+      "step": 6000
+    },
+    {
+      "epoch": 1.22,
+      "grad_norm": 0.8303418159484863,
+      "learning_rate": 1.186666666666667e-05,
+      "loss": 0.8002,
+      "step": 6100
+    },
+    {
+      "epoch": 1.24,
+      "grad_norm": 0.8996633291244507,
+      "learning_rate": 1.1733333333333335e-05,
+      "loss": 0.8005,
+      "step": 6200
+    },
+    {
+      "epoch": 1.26,
+      "grad_norm": 0.8889008164405823,
+      "learning_rate": 1.16e-05,
+      "loss": 0.7956,
+      "step": 6300
+    },
+    {
+      "epoch": 1.28,
+      "grad_norm": 0.8718474507331848,
+      "learning_rate": 1.1466666666666668e-05,
+      "loss": 0.786,
+      "step": 6400
+    },
+    {
+      "epoch": 1.3,
+      "grad_norm": 0.931644856929779,
+      "learning_rate": 1.1333333333333334e-05,
+      "loss": 0.8017,
+      "step": 6500
+    },
+    {
+      "epoch": 1.32,
+      "grad_norm": 0.873427152633667,
+      "learning_rate": 1.1200000000000001e-05,
+      "loss": 0.7855,
+      "step": 6600
+    },
+    {
+      "epoch": 1.34,
+      "grad_norm": 0.8685030341148376,
+      "learning_rate": 1.1066666666666669e-05,
+      "loss": 0.802,
+      "step": 6700
+    },
+    {
+      "epoch": 1.36,
+      "grad_norm": 0.8744147419929504,
+      "learning_rate": 1.0933333333333334e-05,
+      "loss": 0.7936,
+      "step": 6800
+    },
+    {
+      "epoch": 1.38,
+      "grad_norm": 0.9263073205947876,
+      "learning_rate": 1.0800000000000002e-05,
+      "loss": 0.7864,
+      "step": 6900
+    },
+    {
+      "epoch": 1.4,
+      "grad_norm": 0.8322786688804626,
+      "learning_rate": 1.0666666666666667e-05,
+      "loss": 0.7886,
+      "step": 7000
+    },
+    {
+      "epoch": 1.42,
+      "grad_norm": 0.8734496235847473,
+      "learning_rate": 1.0533333333333333e-05,
+      "loss": 0.7889,
+      "step": 7100
+    },
+    {
+      "epoch": 1.44,
+      "grad_norm": 1.0558969974517822,
+      "learning_rate": 1.04e-05,
+      "loss": 0.8092,
+      "step": 7200
+    },
+    {
+      "epoch": 1.46,
+      "grad_norm": 0.8517795205116272,
+      "learning_rate": 1.0266666666666668e-05,
+      "loss": 0.8061,
+      "step": 7300
+    },
+    {
+      "epoch": 1.48,
+      "grad_norm": 0.8912076354026794,
+      "learning_rate": 1.0133333333333335e-05,
+      "loss": 0.8073,
+      "step": 7400
+    },
+    {
+      "epoch": 1.5,
+      "grad_norm": 0.8496484756469727,
+      "learning_rate": 1e-05,
+      "loss": 0.789,
+      "step": 7500
+    },
+    {
+      "epoch": 1.52,
+      "grad_norm": 0.8365699648857117,
+      "learning_rate": 9.866666666666668e-06,
+      "loss": 0.7813,
+      "step": 7600
+    },
+    {
+      "epoch": 1.54,
+      "grad_norm": 0.8612975478172302,
+      "learning_rate": 9.733333333333334e-06,
+      "loss": 0.7923,
+      "step": 7700
+    },
+    {
+      "epoch": 1.56,
+      "grad_norm": 0.825588047504425,
+      "learning_rate": 9.600000000000001e-06,
+      "loss": 0.8017,
+      "step": 7800
+    },
+    {
+      "epoch": 1.58,
+      "grad_norm": 0.9208545684814453,
+      "learning_rate": 9.466666666666667e-06,
+      "loss": 0.7975,
+      "step": 7900
+    },
+    {
+      "epoch": 1.6,
+      "grad_norm": 0.8390321731567383,
+      "learning_rate": 9.333333333333334e-06,
+      "loss": 0.8023,
+      "step": 8000
+    },
+    {
+      "epoch": 1.62,
+      "grad_norm": 0.9178739786148071,
+      "learning_rate": 9.200000000000002e-06,
+      "loss": 0.7965,
+      "step": 8100
+    },
+    {
+      "epoch": 1.64,
+      "grad_norm": 0.9349595904350281,
+      "learning_rate": 9.066666666666667e-06,
+      "loss": 0.7977,
+      "step": 8200
+    },
+    {
+      "epoch": 1.66,
+      "grad_norm": 0.8347993493080139,
+      "learning_rate": 8.933333333333333e-06,
+      "loss": 0.7898,
+      "step": 8300
+    },
+    {
+      "epoch": 1.68,
+      "grad_norm": 0.8217402696609497,
+      "learning_rate": 8.8e-06,
+      "loss": 0.7903,
+      "step": 8400
+    },
+    {
+      "epoch": 1.7,
+      "grad_norm": 0.8198782801628113,
+      "learning_rate": 8.666666666666668e-06,
+      "loss": 0.8032,
+      "step": 8500
+    },
+    {
+      "epoch": 1.72,
+      "grad_norm": 0.8704864382743835,
+      "learning_rate": 8.533333333333335e-06,
+      "loss": 0.7967,
+      "step": 8600
+    },
+    {
+      "epoch": 1.74,
+      "grad_norm": 0.892342746257782,
+      "learning_rate": 8.400000000000001e-06,
+      "loss": 0.7884,
+      "step": 8700
+    },
+    {
+      "epoch": 1.76,
+      "grad_norm": 0.9803333282470703,
+      "learning_rate": 8.266666666666667e-06,
+      "loss": 0.7859,
+      "step": 8800
+    },
+    {
+      "epoch": 1.78,
+      "grad_norm": 0.8991880416870117,
+      "learning_rate": 8.133333333333334e-06,
+      "loss": 0.7872,
+      "step": 8900
+    },
+    {
+      "epoch": 1.8,
+      "grad_norm": 1.0206116437911987,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 0.7967,
+      "step": 9000
+    },
+    {
+      "epoch": 1.82,
+      "grad_norm": 0.8060243725776672,
+      "learning_rate": 7.866666666666667e-06,
+      "loss": 0.7893,
+      "step": 9100
+    },
+    {
+      "epoch": 1.84,
+      "grad_norm": 0.8802576065063477,
+      "learning_rate": 7.733333333333334e-06,
+      "loss": 0.7985,
+      "step": 9200
+    },
+    {
+      "epoch": 1.86,
+      "grad_norm": 0.8549374341964722,
+      "learning_rate": 7.600000000000001e-06,
+      "loss": 0.7889,
+      "step": 9300
+    },
+    {
+      "epoch": 1.88,
+      "grad_norm": 0.9457976222038269,
+      "learning_rate": 7.4666666666666675e-06,
+      "loss": 0.8061,
+      "step": 9400
+    },
+    {
+      "epoch": 1.9,
+      "grad_norm": 0.8606815338134766,
+      "learning_rate": 7.333333333333333e-06,
+      "loss": 0.7866,
+      "step": 9500
+    },
+    {
+      "epoch": 1.92,
+      "grad_norm": 0.8577678799629211,
+      "learning_rate": 7.2000000000000005e-06,
+      "loss": 0.7951,
+      "step": 9600
+    },
+    {
+      "epoch": 1.94,
+      "grad_norm": 0.9072070717811584,
+      "learning_rate": 7.066666666666667e-06,
+      "loss": 0.7879,
+      "step": 9700
+    },
+    {
+      "epoch": 1.96,
+      "grad_norm": 0.8739129304885864,
+      "learning_rate": 6.9333333333333344e-06,
+      "loss": 0.7936,
+      "step": 9800
+    },
+    {
+      "epoch": 1.98,
+      "grad_norm": 0.8881368637084961,
+      "learning_rate": 6.800000000000001e-06,
+      "loss": 0.7825,
+      "step": 9900
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 0.9448174834251404,
+      "learning_rate": 6.666666666666667e-06,
+      "loss": 0.7837,
+      "step": 10000
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 0.8766704201698303,
+      "eval_runtime": 64.0882,
+      "eval_samples_per_second": 15.604,
+      "eval_steps_per_second": 1.95,
+      "step": 10000
+    },
+    {
+      "epoch": 2.02,
+      "grad_norm": 0.8892523050308228,
+      "learning_rate": 6.534666666666667e-06,
+      "loss": 0.7014,
+      "step": 10100
+    },
+    {
+      "epoch": 2.04,
+      "grad_norm": 0.9131534695625305,
+      "learning_rate": 6.4013333333333334e-06,
+      "loss": 0.6962,
+      "step": 10200
+    },
+    {
+      "epoch": 2.06,
+      "grad_norm": 0.9470012784004211,
+      "learning_rate": 6.268000000000001e-06,
+      "loss": 0.6874,
+      "step": 10300
+    },
+    {
+      "epoch": 2.08,
+      "grad_norm": 0.9785165190696716,
+      "learning_rate": 6.134666666666667e-06,
+      "loss": 0.7029,
+      "step": 10400
+    },
+    {
+      "epoch": 2.1,
+      "grad_norm": 1.0245338678359985,
+      "learning_rate": 6.001333333333334e-06,
+      "loss": 0.6922,
+      "step": 10500
+    },
+    {
+      "epoch": 2.12,
+      "grad_norm": 0.920975387096405,
+      "learning_rate": 5.868e-06,
+      "loss": 0.6973,
+      "step": 10600
+    },
+    {
+      "epoch": 2.14,
+      "grad_norm": 0.9391937851905823,
+      "learning_rate": 5.734666666666667e-06,
+      "loss": 0.6917,
+      "step": 10700
+    },
+    {
+      "epoch": 2.16,
+      "grad_norm": 0.9682599306106567,
+      "learning_rate": 5.601333333333334e-06,
+      "loss": 0.6965,
+      "step": 10800
+    },
+    {
+      "epoch": 2.18,
+      "grad_norm": 1.0669585466384888,
+      "learning_rate": 5.468e-06,
+      "loss": 0.698,
+      "step": 10900
+    },
+    {
+      "epoch": 2.2,
+      "grad_norm": 1.0977643728256226,
+      "learning_rate": 5.3346666666666665e-06,
+      "loss": 0.6961,
+      "step": 11000
+    },
+    {
+      "epoch": 2.22,
+      "grad_norm": 0.9608291387557983,
+      "learning_rate": 5.201333333333334e-06,
+      "loss": 0.6935,
+      "step": 11100
+    },
+    {
+      "epoch": 2.24,
+      "grad_norm": 0.9867018461227417,
+      "learning_rate": 5.069333333333334e-06,
+      "loss": 0.691,
+      "step": 11200
+    },
+    {
+      "epoch": 2.26,
+      "grad_norm": 0.9778249263763428,
+      "learning_rate": 4.936e-06,
+      "loss": 0.6973,
+      "step": 11300
+    },
+    {
+      "epoch": 2.28,
+      "grad_norm": 1.040911316871643,
+      "learning_rate": 4.802666666666667e-06,
+      "loss": 0.6958,
+      "step": 11400
+    },
+    {
+      "epoch": 2.3,
+      "grad_norm": 0.948652446269989,
+      "learning_rate": 4.669333333333334e-06,
+      "loss": 0.7075,
+      "step": 11500
+    },
+    {
+      "epoch": 2.32,
+      "grad_norm": 1.045653223991394,
+      "learning_rate": 4.536e-06,
+      "loss": 0.6966,
+      "step": 11600
+    },
+    {
+      "epoch": 2.34,
+      "grad_norm": 0.9575178623199463,
+      "learning_rate": 4.4040000000000005e-06,
+      "loss": 0.7036,
+      "step": 11700
+    },
+    {
+      "epoch": 2.36,
+      "grad_norm": 0.8968670964241028,
+      "learning_rate": 4.270666666666667e-06,
+      "loss": 0.7038,
+      "step": 11800
+    },
+    {
+      "epoch": 2.38,
+      "grad_norm": 0.9445046186447144,
+      "learning_rate": 4.137333333333334e-06,
+      "loss": 0.687,
+      "step": 11900
+    },
+    {
+      "epoch": 2.4,
+      "grad_norm": 1.0018490552902222,
+      "learning_rate": 4.004e-06,
+      "loss": 0.6978,
+      "step": 12000
+    },
+    {
+      "epoch": 2.42,
+      "grad_norm": 0.9981265664100647,
+      "learning_rate": 3.870666666666667e-06,
+      "loss": 0.6876,
+      "step": 12100
+    },
+    {
+      "epoch": 2.44,
+      "grad_norm": 0.9957240223884583,
+      "learning_rate": 3.737333333333333e-06,
+      "loss": 0.691,
+      "step": 12200
+    },
+    {
+      "epoch": 2.46,
+      "grad_norm": 1.0652461051940918,
+      "learning_rate": 3.604e-06,
+      "loss": 0.7026,
+      "step": 12300
+    },
+    {
+      "epoch": 2.48,
+      "grad_norm": 0.9067023992538452,
+      "learning_rate": 3.470666666666667e-06,
+      "loss": 0.6918,
+      "step": 12400
+    },
+    {
+      "epoch": 2.5,
+      "grad_norm": 0.997002899646759,
+      "learning_rate": 3.3373333333333336e-06,
+      "loss": 0.7013,
+      "step": 12500
+    },
+    {
+      "epoch": 2.52,
+      "grad_norm": 0.914594829082489,
+      "learning_rate": 3.2040000000000006e-06,
+      "loss": 0.7025,
+      "step": 12600
+    },
+    {
+      "epoch": 2.54,
+      "grad_norm": 1.197375774383545,
+      "learning_rate": 3.0706666666666667e-06,
+      "loss": 0.7053,
+      "step": 12700
+    },
+    {
+      "epoch": 2.56,
+      "grad_norm": 1.0545796155929565,
+      "learning_rate": 2.9373333333333336e-06,
+      "loss": 0.6888,
+      "step": 12800
+    },
+    {
+      "epoch": 2.58,
+      "grad_norm": 1.0479904413223267,
+      "learning_rate": 2.804e-06,
+      "loss": 0.6853,
+      "step": 12900
+    },
+    {
+      "epoch": 2.6,
+      "grad_norm": 0.9415286183357239,
+      "learning_rate": 2.670666666666667e-06,
+      "loss": 0.7053,
+      "step": 13000
+    },
+    {
+      "epoch": 2.62,
+      "grad_norm": 0.9765642881393433,
+      "learning_rate": 2.5373333333333332e-06,
+      "loss": 0.7093,
+      "step": 13100
+    },
+    {
+      "epoch": 2.64,
+      "grad_norm": 0.9646571278572083,
+      "learning_rate": 2.404e-06,
+      "loss": 0.7026,
+      "step": 13200
+    },
+    {
+      "epoch": 2.66,
+      "grad_norm": 0.9278367757797241,
+      "learning_rate": 2.2706666666666667e-06,
+      "loss": 0.6967,
+      "step": 13300
+    },
+    {
+      "epoch": 2.68,
+      "grad_norm": 0.9418883323669434,
+      "learning_rate": 2.1373333333333337e-06,
+      "loss": 0.699,
+      "step": 13400
+    },
+    {
+      "epoch": 2.7,
+      "grad_norm": 1.0113035440444946,
+      "learning_rate": 2.004e-06,
+      "loss": 0.6937,
+      "step": 13500
+    },
+    {
+      "epoch": 2.72,
+      "grad_norm": 1.0150426626205444,
+      "learning_rate": 1.8706666666666667e-06,
+      "loss": 0.6795,
+      "step": 13600
+    },
+    {
+      "epoch": 2.74,
+      "grad_norm": 1.177462100982666,
+      "learning_rate": 1.7373333333333333e-06,
+      "loss": 0.6822,
+      "step": 13700
+    },
+    {
+      "epoch": 2.76,
+      "grad_norm": 1.1305912733078003,
+      "learning_rate": 1.604e-06,
+      "loss": 0.7039,
+      "step": 13800
+    },
+    {
+      "epoch": 2.78,
+      "grad_norm": 0.9975621700286865,
+      "learning_rate": 1.470666666666667e-06,
+      "loss": 0.6864,
+      "step": 13900
+    },
+    {
+      "epoch": 2.8,
+      "grad_norm": 1.0691418647766113,
+      "learning_rate": 1.3373333333333335e-06,
+      "loss": 0.6945,
+      "step": 14000
+    },
+    {
+      "epoch": 2.82,
+      "grad_norm": 0.988650381565094,
+      "learning_rate": 1.204e-06,
+      "loss": 0.6891,
+      "step": 14100
+    },
+    {
+      "epoch": 2.84,
+      "grad_norm": 1.0204881429672241,
+      "learning_rate": 1.0706666666666668e-06,
+      "loss": 0.6873,
+      "step": 14200
+    },
+    {
+      "epoch": 2.86,
+      "grad_norm": 1.0628246068954468,
+      "learning_rate": 9.373333333333334e-07,
+      "loss": 0.691,
+      "step": 14300
+    },
+    {
+      "epoch": 2.88,
+      "grad_norm": 1.058432698249817,
+      "learning_rate": 8.04e-07,
+      "loss": 0.6938,
+      "step": 14400
+    },
+    {
+      "epoch": 2.9,
+      "grad_norm": 1.0059112310409546,
+      "learning_rate": 6.706666666666667e-07,
+      "loss": 0.7113,
+      "step": 14500
+    },
+    {
+      "epoch": 2.92,
+      "grad_norm": 0.963862955570221,
+      "learning_rate": 5.373333333333334e-07,
+      "loss": 0.6831,
+      "step": 14600
+    },
+    {
+      "epoch": 2.94,
+      "grad_norm": 0.9393936395645142,
+      "learning_rate": 4.04e-07,
+      "loss": 0.7055,
+      "step": 14700
+    },
+    {
+      "epoch": 2.96,
+      "grad_norm": 1.1199434995651245,
+      "learning_rate": 2.706666666666667e-07,
+      "loss": 0.6944,
+      "step": 14800
+    },
+    {
+      "epoch": 2.98,
+      "grad_norm": 1.1279562711715698,
+      "learning_rate": 1.3733333333333335e-07,
+      "loss": 0.7033,
+      "step": 14900
+    },
+    {
+      "epoch": 3.0,
+      "grad_norm": 0.9235104918479919,
+      "learning_rate": 4e-09,
+      "loss": 0.6902,
+      "step": 15000
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.8971832394599915,
+      "eval_runtime": 53.8519,
+      "eval_samples_per_second": 18.569,
+      "eval_steps_per_second": 2.321,
+      "step": 15000
+    }
+  ],
+  "logging_steps": 100,
+  "max_steps": 15000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 5000,
+  "total_flos": 1.88804379967488e+18,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3303ac2034c16851b9953b2116fac54a0666ba5d4a68472db2d7090bc1db3a4
+size 4856

training_logs.txt

@@ -0,0 +1,323 @@
+[train dset len] 160000
+[valid dset len] 1000
+/usr0/home/liangzel/anaconda3/envs/air2/lib/python3.11/site-packages/accelerate/accelerator.py:432:
+FutureWarning: Passing the following arguments to `Accelerator` is deprecated
+and will be removed in version 1.0 of Accelerate: dict_keys(['dispatch_batches',
+'split_batches', 'even_batches', 'use_seedable_sampler']). Please pass an
+`accelerate.DataLoaderConfiguration` instead:
+dataloader_config = DataLoaderConfiguration(dispatch_batches=None,
+split_batches=False, even_batches=True, use_seedable_sampler=True)
+  warnings.warn(
+  {'loss': 0.9803, 'grad_norm': 1.0555673837661743, 'learning_rate':
+  1.9866666666666667e-05, 'epoch': 0.02}
+  {'loss': 0.9477, 'grad_norm': 1.0609339475631714, 'learning_rate':
+  1.9733333333333336e-05, 'epoch': 0.04}
+  {'loss': 0.9472, 'grad_norm': 0.8154662251472473, 'learning_rate':
+  1.9600000000000002e-05, 'epoch': 0.06}
+  {'loss': 0.9496, 'grad_norm': 0.8507415652275085, 'learning_rate':
+  1.9466666666666668e-05, 'epoch': 0.08}
+  {'loss': 0.9416, 'grad_norm': 0.9045298099517822, 'learning_rate':
+  1.9333333333333333e-05, 'epoch': 0.1}
+  {'loss': 0.9501, 'grad_norm': 1.2995306253433228, 'learning_rate':
+  1.9200000000000003e-05, 'epoch': 0.12}
+  {'loss': 0.929, 'grad_norm': 0.8860793709754944, 'learning_rate':
+  1.9066666666666668e-05, 'epoch': 0.14}
+  {'loss': 0.9323, 'grad_norm': 0.8791409134864807, 'learning_rate':
+  1.8933333333333334e-05, 'epoch': 0.16}
+  {'loss': 0.9252, 'grad_norm': 0.868554949760437, 'learning_rate': 1.88e-05,
+  'epoch': 0.18}
+  {'loss': 0.916, 'grad_norm': 0.7996535301208496, 'learning_rate':
+  1.866666666666667e-05, 'epoch': 0.2}
+  {'loss': 0.9267, 'grad_norm': 0.9991319179534912, 'learning_rate':
+  1.8533333333333334e-05, 'epoch': 0.22}
+  {'loss': 0.9281, 'grad_norm': 1.0026607513427734, 'learning_rate':
+  1.8400000000000003e-05, 'epoch': 0.24}
+  {'loss': 0.9272, 'grad_norm': 0.860996663570404, 'learning_rate':
+  1.826666666666667e-05, 'epoch': 0.26}
+  {'loss': 0.9277, 'grad_norm': 0.8613474369049072, 'learning_rate':
+  1.8133333333333335e-05, 'epoch': 0.28}
+  {'loss': 0.9293, 'grad_norm': 0.8491615653038025, 'learning_rate': 1.8e-05,
+  'epoch': 0.3}
+  {'loss': 0.917, 'grad_norm': 0.9264828562736511, 'learning_rate':
+  1.7866666666666666e-05, 'epoch': 0.32}
+  {'loss': 0.9325, 'grad_norm': 0.8633733987808228, 'learning_rate':
+  1.7733333333333335e-05, 'epoch': 0.34}
+  {'loss': 0.9126, 'grad_norm': 0.8906639218330383, 'learning_rate': 1.76e-05,
+  'epoch': 0.36}
+  {'loss': 0.9065, 'grad_norm': 0.885764479637146, 'learning_rate':
+  1.7466666666666667e-05, 'epoch': 0.38}
+  {'loss': 0.9429, 'grad_norm': 0.8839893937110901, 'learning_rate':
+  1.7333333333333336e-05, 'epoch': 0.4}
+  {'loss': 0.9056, 'grad_norm': 0.913644552230835, 'learning_rate': 1.72e-05,
+  'epoch': 0.42}
+  {'loss': 0.913, 'grad_norm': 0.8399360179901123, 'learning_rate':
+  1.706666666666667e-05, 'epoch': 0.44}
+  {'loss': 0.9192, 'grad_norm': 0.8176397681236267, 'learning_rate':
+  1.6933333333333336e-05, 'epoch': 0.46}
+  {'loss': 0.9222, 'grad_norm': 0.8771522045135498, 'learning_rate':
+  1.6800000000000002e-05, 'epoch': 0.48}
+  {'loss': 0.9197, 'grad_norm': 0.7874905467033386, 'learning_rate':
+  1.6666666666666667e-05, 'epoch': 0.5}
+  {'loss': 0.91, 'grad_norm': 0.8051652312278748, 'learning_rate':
+  1.6533333333333333e-05, 'epoch': 0.52}
+  {'loss': 0.9091, 'grad_norm': 0.80189049243927, 'learning_rate': 1.64e-05,
+  'epoch': 0.54}
+  {'loss': 0.9111, 'grad_norm': 0.7678588032722473, 'learning_rate':
+  1.6266666666666668e-05, 'epoch': 0.56}
+  {'loss': 0.9127, 'grad_norm': 0.7835684418678284, 'learning_rate':
+  1.6133333333333334e-05, 'epoch': 0.58}
+  {'loss': 0.9038, 'grad_norm': 0.8735753297805786, 'learning_rate':
+  1.6000000000000003e-05, 'epoch': 0.6}
+  {'loss': 0.9148, 'grad_norm': 0.8635367155075073, 'learning_rate':
+  1.586666666666667e-05, 'epoch': 0.62}
+  {'loss': 0.9123, 'grad_norm': 0.8477209210395813, 'learning_rate':
+  1.5733333333333334e-05, 'epoch': 0.64}
+  {'loss': 0.8898, 'grad_norm': 0.7765554785728455, 'learning_rate':
+  1.5600000000000003e-05, 'epoch': 0.66}
+  {'loss': 0.8992, 'grad_norm': 0.891146183013916, 'learning_rate':
+  1.546666666666667e-05, 'epoch': 0.68}
+  {'loss': 0.9114, 'grad_norm': 0.8695024251937866, 'learning_rate':
+  1.5333333333333334e-05, 'epoch': 0.7}
+  {'loss': 0.89, 'grad_norm': 0.927379846572876, 'learning_rate':
+  1.5200000000000002e-05, 'epoch': 0.72}
+  {'loss': 0.9159, 'grad_norm': 0.885179340839386, 'learning_rate':
+  1.5066666666666668e-05, 'epoch': 0.74}
+  {'loss': 0.9115, 'grad_norm': 0.9603422284126282, 'learning_rate':
+  1.4933333333333335e-05, 'epoch': 0.76}
+  {'loss': 0.9025, 'grad_norm': 0.8624181151390076, 'learning_rate': 1.48e-05,
+  'epoch': 0.78}
+  {'loss': 0.8984, 'grad_norm': 0.9430868625640869, 'learning_rate':
+  1.4666666666666666e-05, 'epoch': 0.8}
+  {'loss': 0.8887, 'grad_norm': 0.7860404849052429, 'learning_rate':
+  1.4533333333333335e-05, 'epoch': 0.82}
+  {'loss': 0.9097, 'grad_norm': 0.793118417263031, 'learning_rate':
+  1.4400000000000001e-05, 'epoch': 0.84}
+  {'loss': 0.8919, 'grad_norm': 0.8741154074668884, 'learning_rate':
+  1.4266666666666668e-05, 'epoch': 0.86}
+  {'loss': 0.8927, 'grad_norm': 0.7793194651603699, 'learning_rate':
+  1.4133333333333334e-05, 'epoch': 0.88}
+  {'loss': 0.9032, 'grad_norm': 0.7923697233200073, 'learning_rate': 1.4e-05,
+  'epoch': 0.9}
+  {'loss': 0.8878, 'grad_norm': 0.8406550288200378, 'learning_rate':
+  1.3866666666666669e-05, 'epoch': 0.92}
+  {'loss': 0.9032, 'grad_norm': 1.0170931816101074, 'learning_rate':
+  1.3733333333333335e-05, 'epoch': 0.94}
+  {'loss': 0.9026, 'grad_norm': 0.8855489492416382, 'learning_rate':
+  1.3600000000000002e-05, 'epoch': 0.96}
+  {'loss': 0.8978, 'grad_norm': 0.7720271348953247, 'learning_rate':
+  1.3466666666666668e-05, 'epoch': 0.98}
+  {'loss': 0.8958, 'grad_norm': 0.841073751449585, 'learning_rate':
+  1.3333333333333333e-05, 'epoch': 1.0}
+  {'eval_loss': 0.88493812084198, 'eval_runtime': 62.115,
+  'eval_samples_per_second': 16.099, 'eval_steps_per_second': 2.012, 'epoch':
+  1.0}
+  {'loss': 0.7886, 'grad_norm': 0.856351375579834, 'learning_rate':
+  1.3200000000000002e-05, 'epoch': 1.02}
+  {'loss': 0.7957, 'grad_norm': 0.92889404296875, 'learning_rate':
+  1.3066666666666668e-05, 'epoch': 1.04}
+  {'loss': 0.7887, 'grad_norm': 0.8512794375419617, 'learning_rate':
+  1.2933333333333334e-05, 'epoch': 1.06}
+  {'loss': 0.8119, 'grad_norm': 0.8943263292312622, 'learning_rate':
+  1.2800000000000001e-05, 'epoch': 1.08}
+  {'loss': 0.7958, 'grad_norm': 0.908469557762146, 'learning_rate':
+  1.2666666666666667e-05, 'epoch': 1.1}
+  {'loss': 0.7958, 'grad_norm': 0.790229856967926, 'learning_rate':
+  1.2533333333333336e-05, 'epoch': 1.12}
+  {'loss': 0.7907, 'grad_norm': 1.0110950469970703, 'learning_rate':
+  1.2400000000000002e-05, 'epoch': 1.14}
+  {'loss': 0.7939, 'grad_norm': 0.9110943078994751, 'learning_rate':
+  1.2266666666666667e-05, 'epoch': 1.16}
+  {'loss': 0.7849, 'grad_norm': 0.8515949249267578, 'learning_rate':
+  1.2133333333333335e-05, 'epoch': 1.18}
+  {'loss': 0.8036, 'grad_norm': 0.8418060541152954, 'learning_rate': 1.2e-05,
+  'epoch': 1.2}
+  {'loss': 0.8002, 'grad_norm': 0.8303418159484863, 'learning_rate':
+  1.186666666666667e-05, 'epoch': 1.22}
+  {'loss': 0.8005, 'grad_norm': 0.8996633291244507, 'learning_rate':
+  1.1733333333333335e-05, 'epoch': 1.24}
+  {'loss': 0.7956, 'grad_norm': 0.8889008164405823, 'learning_rate': 1.16e-05,
+  'epoch': 1.26}
+  {'loss': 0.786, 'grad_norm': 0.8718474507331848, 'learning_rate':
+  1.1466666666666668e-05, 'epoch': 1.28}
+  {'loss': 0.8017, 'grad_norm': 0.931644856929779, 'learning_rate':
+  1.1333333333333334e-05, 'epoch': 1.3}
+  {'loss': 0.7855, 'grad_norm': 0.873427152633667, 'learning_rate':
+  1.1200000000000001e-05, 'epoch': 1.32}
+  {'loss': 0.802, 'grad_norm': 0.8685030341148376, 'learning_rate':
+  1.1066666666666669e-05, 'epoch': 1.34}
+  {'loss': 0.7936, 'grad_norm': 0.8744147419929504, 'learning_rate':
+  1.0933333333333334e-05, 'epoch': 1.36}
+  {'loss': 0.7864, 'grad_norm': 0.9263073205947876, 'learning_rate':
+  1.0800000000000002e-05, 'epoch': 1.38}
+  {'loss': 0.7886, 'grad_norm': 0.8322786688804626, 'learning_rate':
+  1.0666666666666667e-05, 'epoch': 1.4}
+  {'loss': 0.7889, 'grad_norm': 0.8734496235847473, 'learning_rate':
+  1.0533333333333333e-05, 'epoch': 1.42}
+  {'loss': 0.8092, 'grad_norm': 1.0558969974517822, 'learning_rate': 1.04e-05,
+  'epoch': 1.44}
+  {'loss': 0.8061, 'grad_norm': 0.8517795205116272, 'learning_rate':
+  1.0266666666666668e-05, 'epoch': 1.46}
+  {'loss': 0.8073, 'grad_norm': 0.8912076354026794, 'learning_rate':
+  1.0133333333333335e-05, 'epoch': 1.48}
+  {'loss': 0.789, 'grad_norm': 0.8496484756469727, 'learning_rate': 1e-05,
+  'epoch': 1.5}
+  {'loss': 0.7813, 'grad_norm': 0.8365699648857117, 'learning_rate':
+  9.866666666666668e-06, 'epoch': 1.52}
+  {'loss': 0.7923, 'grad_norm': 0.8612975478172302, 'learning_rate':
+  9.733333333333334e-06, 'epoch': 1.54}
+  {'loss': 0.8017, 'grad_norm': 0.825588047504425, 'learning_rate':
+  9.600000000000001e-06, 'epoch': 1.56}
+  {'loss': 0.7975, 'grad_norm': 0.9208545684814453, 'learning_rate':
+  9.466666666666667e-06, 'epoch': 1.58}
+  {'loss': 0.8023, 'grad_norm': 0.8390321731567383, 'learning_rate':
+  9.333333333333334e-06, 'epoch': 1.6}
+  {'loss': 0.7965, 'grad_norm': 0.9178739786148071, 'learning_rate':
+  9.200000000000002e-06, 'epoch': 1.62}
+  {'loss': 0.7977, 'grad_norm': 0.9349595904350281, 'learning_rate':
+  9.066666666666667e-06, 'epoch': 1.64}
+  {'loss': 0.7898, 'grad_norm': 0.8347993493080139, 'learning_rate':
+  8.933333333333333e-06, 'epoch': 1.66}
+  {'loss': 0.7903, 'grad_norm': 0.8217402696609497, 'learning_rate': 8.8e-06,
+  'epoch': 1.68}
+  {'loss': 0.8032, 'grad_norm': 0.8198782801628113, 'learning_rate':
+  8.666666666666668e-06, 'epoch': 1.7}
+  {'loss': 0.7967, 'grad_norm': 0.8704864382743835, 'learning_rate':
+  8.533333333333335e-06, 'epoch': 1.72}
+  {'loss': 0.7884, 'grad_norm': 0.892342746257782, 'learning_rate':
+  8.400000000000001e-06, 'epoch': 1.74}
+  {'loss': 0.7859, 'grad_norm': 0.9803333282470703, 'learning_rate':
+  8.266666666666667e-06, 'epoch': 1.76}
+  {'loss': 0.7872, 'grad_norm': 0.8991880416870117, 'learning_rate':
+  8.133333333333334e-06, 'epoch': 1.78}
+  {'loss': 0.7967, 'grad_norm': 1.0206116437911987, 'learning_rate':
+  8.000000000000001e-06, 'epoch': 1.8}
+  {'loss': 0.7893, 'grad_norm': 0.8060243725776672, 'learning_rate':
+  7.866666666666667e-06, 'epoch': 1.82}
+  {'loss': 0.7985, 'grad_norm': 0.8802576065063477, 'learning_rate':
+  7.733333333333334e-06, 'epoch': 1.84}
+  {'loss': 0.7889, 'grad_norm': 0.8549374341964722, 'learning_rate':
+  7.600000000000001e-06, 'epoch': 1.86}
+  {'loss': 0.8061, 'grad_norm': 0.9457976222038269, 'learning_rate':
+  7.4666666666666675e-06, 'epoch': 1.88}
+  {'loss': 0.7866, 'grad_norm': 0.8606815338134766, 'learning_rate':
+  7.333333333333333e-06, 'epoch': 1.9}
+  {'loss': 0.7951, 'grad_norm': 0.8577678799629211, 'learning_rate':
+  7.2000000000000005e-06, 'epoch': 1.92}
+  {'loss': 0.7879, 'grad_norm': 0.9072070717811584, 'learning_rate':
+  7.066666666666667e-06, 'epoch': 1.94}
+  {'loss': 0.7936, 'grad_norm': 0.8739129304885864, 'learning_rate':
+  6.9333333333333344e-06, 'epoch': 1.96}
+  {'loss': 0.7825, 'grad_norm': 0.8881368637084961, 'learning_rate':
+  6.800000000000001e-06, 'epoch': 1.98}
+  {'loss': 0.7837, 'grad_norm': 0.9448174834251404, 'learning_rate':
+  6.666666666666667e-06, 'epoch': 2.0}
+  {'eval_loss': 0.8766704201698303, 'eval_runtime': 64.0882,
+  'eval_samples_per_second': 15.604, 'eval_steps_per_second': 1.95, 'epoch':
+  2.0}
+  {'loss': 0.7014, 'grad_norm': 0.8892523050308228, 'learning_rate':
+  6.534666666666667e-06, 'epoch': 2.02}
+  {'loss': 0.6962, 'grad_norm': 0.9131534695625305, 'learning_rate':
+  6.4013333333333334e-06, 'epoch': 2.04}
+  {'loss': 0.6874, 'grad_norm': 0.9470012784004211, 'learning_rate':
+  6.268000000000001e-06, 'epoch': 2.06}
+  {'loss': 0.7029, 'grad_norm': 0.9785165190696716, 'learning_rate':
+  6.134666666666667e-06, 'epoch': 2.08}
+  {'loss': 0.6922, 'grad_norm': 1.0245338678359985, 'learning_rate':
+  6.001333333333334e-06, 'epoch': 2.1}
+  {'loss': 0.6973, 'grad_norm': 0.920975387096405, 'learning_rate': 5.868e-06,
+  'epoch': 2.12}
+  {'loss': 0.6917, 'grad_norm': 0.9391937851905823, 'learning_rate':
+  5.734666666666667e-06, 'epoch': 2.14}
+  {'loss': 0.6965, 'grad_norm': 0.9682599306106567, 'learning_rate':
+  5.601333333333334e-06, 'epoch': 2.16}
+  {'loss': 0.698, 'grad_norm': 1.0669585466384888, 'learning_rate': 5.468e-06,
+  'epoch': 2.18}
+  {'loss': 0.6961, 'grad_norm': 1.0977643728256226, 'learning_rate':
+  5.3346666666666665e-06, 'epoch': 2.2}
+  {'loss': 0.6935, 'grad_norm': 0.9608291387557983, 'learning_rate':
+  5.201333333333334e-06, 'epoch': 2.22}
+  {'loss': 0.691, 'grad_norm': 0.9867018461227417, 'learning_rate':
+  5.069333333333334e-06, 'epoch': 2.24}
+  {'loss': 0.6973, 'grad_norm': 0.9778249263763428, 'learning_rate': 4.936e-06,
+  'epoch': 2.26}
+  {'loss': 0.6958, 'grad_norm': 1.040911316871643, 'learning_rate':
+  4.802666666666667e-06, 'epoch': 2.28}
+  {'loss': 0.7075, 'grad_norm': 0.948652446269989, 'learning_rate':
+  4.669333333333334e-06, 'epoch': 2.3}
+  {'loss': 0.6966, 'grad_norm': 1.045653223991394, 'learning_rate': 4.536e-06,
+  'epoch': 2.32}
+  {'loss': 0.7036, 'grad_norm': 0.9575178623199463, 'learning_rate':
+  4.4040000000000005e-06, 'epoch': 2.34}
+  {'loss': 0.7038, 'grad_norm': 0.8968670964241028, 'learning_rate':
+  4.270666666666667e-06, 'epoch': 2.36}
+  {'loss': 0.687, 'grad_norm': 0.9445046186447144, 'learning_rate':
+  4.137333333333334e-06, 'epoch': 2.38}
+  {'loss': 0.6978, 'grad_norm': 1.0018490552902222, 'learning_rate': 4.004e-06,
+  'epoch': 2.4}
+  {'loss': 0.6876, 'grad_norm': 0.9981265664100647, 'learning_rate':
+  3.870666666666667e-06, 'epoch': 2.42}
+  {'loss': 0.691, 'grad_norm': 0.9957240223884583, 'learning_rate':
+  3.737333333333333e-06, 'epoch': 2.44}
+  {'loss': 0.7026, 'grad_norm': 1.0652461051940918, 'learning_rate': 3.604e-06,
+  'epoch': 2.46}
+  {'loss': 0.6918, 'grad_norm': 0.9067023992538452, 'learning_rate':
+  3.470666666666667e-06, 'epoch': 2.48}
+  {'loss': 0.7013, 'grad_norm': 0.997002899646759, 'learning_rate':
+  3.3373333333333336e-06, 'epoch': 2.5}
+  {'loss': 0.7025, 'grad_norm': 0.914594829082489, 'learning_rate':
+  3.2040000000000006e-06, 'epoch': 2.52}
+  {'loss': 0.7053, 'grad_norm': 1.197375774383545, 'learning_rate':
+  3.0706666666666667e-06, 'epoch': 2.54}
+  {'loss': 0.6888, 'grad_norm': 1.0545796155929565, 'learning_rate':
+  2.9373333333333336e-06, 'epoch': 2.56}
+  {'loss': 0.6853, 'grad_norm': 1.0479904413223267, 'learning_rate': 2.804e-06,
+  'epoch': 2.58}
+  {'loss': 0.7053, 'grad_norm': 0.9415286183357239, 'learning_rate':
+  2.670666666666667e-06, 'epoch': 2.6}
+  {'loss': 0.7093, 'grad_norm': 0.9765642881393433, 'learning_rate':
+  2.5373333333333332e-06, 'epoch': 2.62}
+  {'loss': 0.7026, 'grad_norm': 0.9646571278572083, 'learning_rate': 2.404e-06,
+  'epoch': 2.64}
+  {'loss': 0.6967, 'grad_norm': 0.9278367757797241, 'learning_rate':
+  2.2706666666666667e-06, 'epoch': 2.66}
+  {'loss': 0.699, 'grad_norm': 0.9418883323669434, 'learning_rate':
+  2.1373333333333337e-06, 'epoch': 2.68}
+  {'loss': 0.6937, 'grad_norm': 1.0113035440444946, 'learning_rate': 2.004e-06,
+  'epoch': 2.7}
+  {'loss': 0.6795, 'grad_norm': 1.0150426626205444, 'learning_rate':
+  1.8706666666666667e-06, 'epoch': 2.72}
+  {'loss': 0.6822, 'grad_norm': 1.177462100982666, 'learning_rate':
+  1.7373333333333333e-06, 'epoch': 2.74}
+  {'loss': 0.7039, 'grad_norm': 1.1305912733078003, 'learning_rate': 1.604e-06,
+  'epoch': 2.76}
+  {'loss': 0.6864, 'grad_norm': 0.9975621700286865, 'learning_rate':
+  1.470666666666667e-06, 'epoch': 2.78}
+  {'loss': 0.6945, 'grad_norm': 1.0691418647766113, 'learning_rate':
+  1.3373333333333335e-06, 'epoch': 2.8}
+  {'loss': 0.6891, 'grad_norm': 0.988650381565094, 'learning_rate': 1.204e-06,
+  'epoch': 2.82}
+  {'loss': 0.6873, 'grad_norm': 1.0204881429672241, 'learning_rate':
+  1.0706666666666668e-06, 'epoch': 2.84}
+  {'loss': 0.691, 'grad_norm': 1.0628246068954468, 'learning_rate':
+  9.373333333333334e-07, 'epoch': 2.86}
+  {'loss': 0.6938, 'grad_norm': 1.058432698249817, 'learning_rate': 8.04e-07,
+  'epoch': 2.88}
+  {'loss': 0.7113, 'grad_norm': 1.0059112310409546, 'learning_rate':
+  6.706666666666667e-07, 'epoch': 2.9}
+  {'loss': 0.6831, 'grad_norm': 0.963862955570221, 'learning_rate':
+  5.373333333333334e-07, 'epoch': 2.92}
+  {'loss': 0.7055, 'grad_norm': 0.9393936395645142, 'learning_rate': 4.04e-07,
+  'epoch': 2.94}
+  {'loss': 0.6944, 'grad_norm': 1.1199434995651245, 'learning_rate':
+  2.706666666666667e-07, 'epoch': 2.96}
+  {'loss': 0.7033, 'grad_norm': 1.1279562711715698, 'learning_rate':
+  1.3733333333333335e-07, 'epoch': 2.98}
+  {'loss': 0.6902, 'grad_norm': 0.9235104918479919, 'learning_rate': 4e-09,
+  'epoch': 3.0}
+  {'eval_loss': 0.8971832394599915, 'eval_runtime': 53.8519,
+  'eval_samples_per_second': 18.569, 'eval_steps_per_second': 2.321, 'epoch':
+  3.0}
+  {'train_runtime': 69618.6499, 'train_samples_per_second': 6.895,
+  'train_steps_per_second': 0.215, 'train_loss': 0.8021132176717123, 'epoch':
+  3.0}
+