Training in progress, step 5000, checkpoint

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+checkpoint-5000/tokenizer.json filter=lfs diff=lfs merge=lfs -text

checkpoint-5000/config.json

@@ -0,0 +1,40 @@
+{
+  "_name_or_path": "meta-llama/Llama-3.2-1B-Instruct",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 16,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.46.3",
+  "use_cache": true,
+  "vocab_size": 128256
+}

checkpoint-5000/generation_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.46.3"
+}

checkpoint-5000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf316d3d5e9ee3006804bd0a6c861e4460c97ce00980caa9762d304a122f7f9f
+size 4943274328

checkpoint-5000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2efc3228b3e84cd3b92c771fda9001c205dd2045dbe2a49a5721fa284e08863
+size 3724602

checkpoint-5000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23fc1d8ed676ab7decdc316d7a08124779270b79f1f3238b0cb8ca1ecef259b8
+size 14244

checkpoint-5000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2be67f6aac8e482bb2022409709d8774ffb125292c0c9cf025c0ae747f3a6d57
+size 1064

checkpoint-5000/special_tokens_map.json

@@ -0,0 +1,17 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|eot_id|>"
+}

checkpoint-5000/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

checkpoint-5000/tokenizer_config.json

@@ -0,0 +1,2064 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "chat_template": "{{- bos_token }}\n{%- if custom_tools is defined %}\n    {%- set tools = custom_tools %}\n{%- endif %}\n{%- if not tools_in_user_message is defined %}\n    {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n    {%- if strftime_now is defined %}\n        {%- set date_string = strftime_now(\"%d %b %Y\") %}\n    {%- else %}\n        {%- set date_string = \"26 Jul 2024\" %}\n    {%- endif %}\n{%- endif %}\n{%- if not tools is defined %}\n    {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n    {%- set system_message = messages[0]['content']|trim %}\n    {%- set messages = messages[1:] %}\n{%- else %}\n    {%- set system_message = \"\" %}\n{%- endif %}\n\n{#- System message #}\n{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n{%- if tools is not none %}\n    {{- \"Environment: ipython\\n\" }}\n{%- endif %}\n{{- \"Cutting Knowledge Date: December 2023\\n\" }}\n{{- \"Today Date: \" + date_string + \"\\n\\n\" }}\n{%- if tools is not none and not tools_in_user_message %}\n    {{- \"You have access to the following functions. To call a function, please respond with JSON for a function call.\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n{%- endif %}\n{{- system_message }}\n{{- \"<|eot_id|>\" }}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n    {#- Extract the first user message so we can plug it in here #}\n    {%- if messages | length != 0 %}\n        {%- set first_user_message = messages[0]['content']|trim %}\n        {%- set messages = messages[1:] %}\n    {%- else %}\n        {{- raise_exception(\"Cannot put tools in the first user message when there's no first user message!\") }}\n{%- endif %}\n    {{- '<|start_header_id|>user<|end_header_id|>\\n\\n' -}}\n    {{- \"Given the following functions, please respond with a JSON for a function call \" }}\n    {{- \"with its proper arguments that best answers the given prompt.\\n\\n\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n    {{- first_user_message + \"<|eot_id|>\"}}\n{%- endif %}\n\n{%- for message in messages %}\n    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' }}\n    {%- elif 'tool_calls' in message %}\n        {%- if not message.tool_calls|length == 1 %}\n            {{- raise_exception(\"This model only supports single tool-calls at once!\") }}\n        {%- endif %}\n        {%- set tool_call = message.tool_calls[0].function %}\n        {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n        {{- '{\"name\": \"' + tool_call.name + '\", ' }}\n        {{- '\"parameters\": ' }}\n        {{- tool_call.arguments | tojson }}\n        {{- \"}\" }}\n        {{- \"<|eot_id|>\" }}\n    {%- elif message.role == \"tool\" or message.role == \"ipython\" %}\n        {{- \"<|start_header_id|>ipython<|end_header_id|>\\n\\n\" }}\n        {%- if message.content is mapping or message.content is iterable %}\n            {{- message.content | tojson }}\n        {%- else %}\n            {{- message.content }}\n        {%- endif %}\n        {{- \"<|eot_id|>\" }}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|eot_id|>",
+  "padding_side": "left",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

checkpoint-5000/trainer_state.json

@@ -0,0 +1,3533 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.7648037322422133,
+  "eval_steps": 500,
+  "global_step": 5000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0015296074644844267,
+      "grad_norm": 39.202430725097656,
+      "learning_rate": 4.0000000000000003e-07,
+      "loss": 2.2544,
+      "step": 10
+    },
+    {
+      "epoch": 0.0030592149289688534,
+      "grad_norm": 6.937076568603516,
+      "learning_rate": 8.000000000000001e-07,
+      "loss": 1.9815,
+      "step": 20
+    },
+    {
+      "epoch": 0.00458882239345328,
+      "grad_norm": 12.800159454345703,
+      "learning_rate": 1.2000000000000002e-06,
+      "loss": 1.8989,
+      "step": 30
+    },
+    {
+      "epoch": 0.006118429857937707,
+      "grad_norm": 18.019540786743164,
+      "learning_rate": 1.6000000000000001e-06,
+      "loss": 1.894,
+      "step": 40
+    },
+    {
+      "epoch": 0.007648037322422133,
+      "grad_norm": 19.988229751586914,
+      "learning_rate": 2.0000000000000003e-06,
+      "loss": 1.8713,
+      "step": 50
+    },
+    {
+      "epoch": 0.00917764478690656,
+      "grad_norm": 21.80278968811035,
+      "learning_rate": 2.4000000000000003e-06,
+      "loss": 1.8662,
+      "step": 60
+    },
+    {
+      "epoch": 0.010707252251390987,
+      "grad_norm": 23.45534324645996,
+      "learning_rate": 2.8000000000000003e-06,
+      "loss": 1.8661,
+      "step": 70
+    },
+    {
+      "epoch": 0.012236859715875414,
+      "grad_norm": 23.260648727416992,
+      "learning_rate": 3.2000000000000003e-06,
+      "loss": 1.8222,
+      "step": 80
+    },
+    {
+      "epoch": 0.013766467180359841,
+      "grad_norm": 23.629636764526367,
+      "learning_rate": 3.6000000000000003e-06,
+      "loss": 1.8312,
+      "step": 90
+    },
+    {
+      "epoch": 0.015296074644844266,
+      "grad_norm": 25.211572647094727,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 1.8441,
+      "step": 100
+    },
+    {
+      "epoch": 0.016825682109328693,
+      "grad_norm": 23.6651668548584,
+      "learning_rate": 4.4e-06,
+      "loss": 1.8113,
+      "step": 110
+    },
+    {
+      "epoch": 0.01835528957381312,
+      "grad_norm": 24.198888778686523,
+      "learning_rate": 4.800000000000001e-06,
+      "loss": 1.8209,
+      "step": 120
+    },
+    {
+      "epoch": 0.019884897038297548,
+      "grad_norm": 21.0020694732666,
+      "learning_rate": 5.2e-06,
+      "loss": 1.8178,
+      "step": 130
+    },
+    {
+      "epoch": 0.021414504502781973,
+      "grad_norm": 22.20139503479004,
+      "learning_rate": 5.600000000000001e-06,
+      "loss": 1.8184,
+      "step": 140
+    },
+    {
+      "epoch": 0.0229441119672664,
+      "grad_norm": 20.499448776245117,
+      "learning_rate": 6e-06,
+      "loss": 1.7967,
+      "step": 150
+    },
+    {
+      "epoch": 0.024473719431750827,
+      "grad_norm": 27.421058654785156,
+      "learning_rate": 6.4000000000000006e-06,
+      "loss": 1.7887,
+      "step": 160
+    },
+    {
+      "epoch": 0.026003326896235253,
+      "grad_norm": 19.557756423950195,
+      "learning_rate": 6.800000000000001e-06,
+      "loss": 1.7954,
+      "step": 170
+    },
+    {
+      "epoch": 0.027532934360719682,
+      "grad_norm": 27.11305809020996,
+      "learning_rate": 7.2000000000000005e-06,
+      "loss": 1.7946,
+      "step": 180
+    },
+    {
+      "epoch": 0.029062541825204107,
+      "grad_norm": 19.00750160217285,
+      "learning_rate": 7.600000000000001e-06,
+      "loss": 1.7845,
+      "step": 190
+    },
+    {
+      "epoch": 0.030592149289688533,
+      "grad_norm": 18.32988929748535,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 1.7817,
+      "step": 200
+    },
+    {
+      "epoch": 0.03212175675417296,
+      "grad_norm": 22.121030807495117,
+      "learning_rate": 8.400000000000001e-06,
+      "loss": 1.8228,
+      "step": 210
+    },
+    {
+      "epoch": 0.03365136421865739,
+      "grad_norm": 30.522912979125977,
+      "learning_rate": 8.8e-06,
+      "loss": 1.8359,
+      "step": 220
+    },
+    {
+      "epoch": 0.03518097168314181,
+      "grad_norm": 26.51453399658203,
+      "learning_rate": 9.200000000000002e-06,
+      "loss": 1.7971,
+      "step": 230
+    },
+    {
+      "epoch": 0.03671057914762624,
+      "grad_norm": 28.695058822631836,
+      "learning_rate": 9.600000000000001e-06,
+      "loss": 1.8302,
+      "step": 240
+    },
+    {
+      "epoch": 0.03824018661211067,
+      "grad_norm": 27.09485626220703,
+      "learning_rate": 1e-05,
+      "loss": 1.829,
+      "step": 250
+    },
+    {
+      "epoch": 0.039769794076595096,
+      "grad_norm": 19.422021865844727,
+      "learning_rate": 1.04e-05,
+      "loss": 1.7814,
+      "step": 260
+    },
+    {
+      "epoch": 0.04129940154107952,
+      "grad_norm": 26.004735946655273,
+      "learning_rate": 1.0800000000000002e-05,
+      "loss": 1.8323,
+      "step": 270
+    },
+    {
+      "epoch": 0.042829009005563946,
+      "grad_norm": 21.66231346130371,
+      "learning_rate": 1.1200000000000001e-05,
+      "loss": 1.8503,
+      "step": 280
+    },
+    {
+      "epoch": 0.04435861647004837,
+      "grad_norm": 16.89419174194336,
+      "learning_rate": 1.16e-05,
+      "loss": 1.8079,
+      "step": 290
+    },
+    {
+      "epoch": 0.0458882239345328,
+      "grad_norm": 11.961407661437988,
+      "learning_rate": 1.2e-05,
+      "loss": 1.8598,
+      "step": 300
+    },
+    {
+      "epoch": 0.04741783139901723,
+      "grad_norm": 22.449304580688477,
+      "learning_rate": 1.2400000000000002e-05,
+      "loss": 1.8461,
+      "step": 310
+    },
+    {
+      "epoch": 0.048947438863501655,
+      "grad_norm": 17.80685806274414,
+      "learning_rate": 1.2800000000000001e-05,
+      "loss": 1.8374,
+      "step": 320
+    },
+    {
+      "epoch": 0.05047704632798608,
+      "grad_norm": 20.850351333618164,
+      "learning_rate": 1.3200000000000002e-05,
+      "loss": 1.8325,
+      "step": 330
+    },
+    {
+      "epoch": 0.052006653792470506,
+      "grad_norm": 18.351491928100586,
+      "learning_rate": 1.3600000000000002e-05,
+      "loss": 1.8575,
+      "step": 340
+    },
+    {
+      "epoch": 0.05353626125695493,
+      "grad_norm": 23.839378356933594,
+      "learning_rate": 1.4e-05,
+      "loss": 1.8521,
+      "step": 350
+    },
+    {
+      "epoch": 0.055065868721439364,
+      "grad_norm": 17.616167068481445,
+      "learning_rate": 1.4400000000000001e-05,
+      "loss": 1.8278,
+      "step": 360
+    },
+    {
+      "epoch": 0.05659547618592379,
+      "grad_norm": 25.248546600341797,
+      "learning_rate": 1.48e-05,
+      "loss": 1.8904,
+      "step": 370
+    },
+    {
+      "epoch": 0.058125083650408214,
+      "grad_norm": 23.11628532409668,
+      "learning_rate": 1.5200000000000002e-05,
+      "loss": 1.8552,
+      "step": 380
+    },
+    {
+      "epoch": 0.05965469111489264,
+      "grad_norm": 21.66451072692871,
+      "learning_rate": 1.5600000000000003e-05,
+      "loss": 1.8795,
+      "step": 390
+    },
+    {
+      "epoch": 0.061184298579377065,
+      "grad_norm": 22.456846237182617,
+      "learning_rate": 1.6000000000000003e-05,
+      "loss": 1.8819,
+      "step": 400
+    },
+    {
+      "epoch": 0.0627139060438615,
+      "grad_norm": 18.99188232421875,
+      "learning_rate": 1.64e-05,
+      "loss": 1.8874,
+      "step": 410
+    },
+    {
+      "epoch": 0.06424351350834592,
+      "grad_norm": 20.916900634765625,
+      "learning_rate": 1.6800000000000002e-05,
+      "loss": 1.8895,
+      "step": 420
+    },
+    {
+      "epoch": 0.06577312097283035,
+      "grad_norm": 25.91555404663086,
+      "learning_rate": 1.72e-05,
+      "loss": 1.8769,
+      "step": 430
+    },
+    {
+      "epoch": 0.06730272843731477,
+      "grad_norm": 20.888111114501953,
+      "learning_rate": 1.76e-05,
+      "loss": 1.8992,
+      "step": 440
+    },
+    {
+      "epoch": 0.0688323359017992,
+      "grad_norm": 18.96579933166504,
+      "learning_rate": 1.8e-05,
+      "loss": 1.9045,
+      "step": 450
+    },
+    {
+      "epoch": 0.07036194336628362,
+      "grad_norm": 27.60817527770996,
+      "learning_rate": 1.8400000000000003e-05,
+      "loss": 1.8765,
+      "step": 460
+    },
+    {
+      "epoch": 0.07189155083076805,
+      "grad_norm": 21.365825653076172,
+      "learning_rate": 1.88e-05,
+      "loss": 1.9315,
+      "step": 470
+    },
+    {
+      "epoch": 0.07342115829525248,
+      "grad_norm": 21.732152938842773,
+      "learning_rate": 1.9200000000000003e-05,
+      "loss": 1.9313,
+      "step": 480
+    },
+    {
+      "epoch": 0.0749507657597369,
+      "grad_norm": 21.766752243041992,
+      "learning_rate": 1.9600000000000002e-05,
+      "loss": 1.9151,
+      "step": 490
+    },
+    {
+      "epoch": 0.07648037322422134,
+      "grad_norm": 21.943374633789062,
+      "learning_rate": 2e-05,
+      "loss": 1.9768,
+      "step": 500
+    },
+    {
+      "epoch": 0.07800998068870577,
+      "grad_norm": 20.251033782958984,
+      "learning_rate": 1.9999756307053947e-05,
+      "loss": 1.9019,
+      "step": 510
+    },
+    {
+      "epoch": 0.07953958815319019,
+      "grad_norm": 26.903671264648438,
+      "learning_rate": 1.9999025240093045e-05,
+      "loss": 1.9229,
+      "step": 520
+    },
+    {
+      "epoch": 0.08106919561767462,
+      "grad_norm": 17.911582946777344,
+      "learning_rate": 1.9997806834748455e-05,
+      "loss": 1.9123,
+      "step": 530
+    },
+    {
+      "epoch": 0.08259880308215904,
+      "grad_norm": 17.10667610168457,
+      "learning_rate": 1.9996101150403543e-05,
+      "loss": 1.8725,
+      "step": 540
+    },
+    {
+      "epoch": 0.08412841054664347,
+      "grad_norm": 23.082595825195312,
+      "learning_rate": 1.999390827019096e-05,
+      "loss": 1.9008,
+      "step": 550
+    },
+    {
+      "epoch": 0.08565801801112789,
+      "grad_norm": 16.786951065063477,
+      "learning_rate": 1.9991228300988586e-05,
+      "loss": 1.9283,
+      "step": 560
+    },
+    {
+      "epoch": 0.08718762547561232,
+      "grad_norm": 22.904605865478516,
+      "learning_rate": 1.9988061373414342e-05,
+      "loss": 1.886,
+      "step": 570
+    },
+    {
+      "epoch": 0.08871723294009674,
+      "grad_norm": 18.19251823425293,
+      "learning_rate": 1.9984407641819812e-05,
+      "loss": 1.8846,
+      "step": 580
+    },
+    {
+      "epoch": 0.09024684040458117,
+      "grad_norm": 17.656436920166016,
+      "learning_rate": 1.9980267284282718e-05,
+      "loss": 1.8561,
+      "step": 590
+    },
+    {
+      "epoch": 0.0917764478690656,
+      "grad_norm": 20.029586791992188,
+      "learning_rate": 1.9975640502598243e-05,
+      "loss": 1.867,
+      "step": 600
+    },
+    {
+      "epoch": 0.09330605533355003,
+      "grad_norm": 15.413036346435547,
+      "learning_rate": 1.9970527522269204e-05,
+      "loss": 1.867,
+      "step": 610
+    },
+    {
+      "epoch": 0.09483566279803446,
+      "grad_norm": 17.78459930419922,
+      "learning_rate": 1.9964928592495046e-05,
+      "loss": 1.8522,
+      "step": 620
+    },
+    {
+      "epoch": 0.09636527026251888,
+      "grad_norm": 18.63687515258789,
+      "learning_rate": 1.9958843986159705e-05,
+      "loss": 1.8715,
+      "step": 630
+    },
+    {
+      "epoch": 0.09789487772700331,
+      "grad_norm": 18.381534576416016,
+      "learning_rate": 1.9952273999818312e-05,
+      "loss": 1.8483,
+      "step": 640
+    },
+    {
+      "epoch": 0.09942448519148774,
+      "grad_norm": 17.309951782226562,
+      "learning_rate": 1.9945218953682736e-05,
+      "loss": 1.8411,
+      "step": 650
+    },
+    {
+      "epoch": 0.10095409265597216,
+      "grad_norm": 16.78116798400879,
+      "learning_rate": 1.9937679191605964e-05,
+      "loss": 1.8417,
+      "step": 660
+    },
+    {
+      "epoch": 0.10248370012045659,
+      "grad_norm": 14.930190086364746,
+      "learning_rate": 1.992965508106537e-05,
+      "loss": 1.7848,
+      "step": 670
+    },
+    {
+      "epoch": 0.10401330758494101,
+      "grad_norm": 15.964579582214355,
+      "learning_rate": 1.9921147013144782e-05,
+      "loss": 1.8235,
+      "step": 680
+    },
+    {
+      "epoch": 0.10554291504942544,
+      "grad_norm": 19.765722274780273,
+      "learning_rate": 1.991215540251542e-05,
+      "loss": 1.8351,
+      "step": 690
+    },
+    {
+      "epoch": 0.10707252251390986,
+      "grad_norm": 22.259653091430664,
+      "learning_rate": 1.9902680687415704e-05,
+      "loss": 1.873,
+      "step": 700
+    },
+    {
+      "epoch": 0.10860212997839429,
+      "grad_norm": 17.007463455200195,
+      "learning_rate": 1.9892723329629885e-05,
+      "loss": 1.7792,
+      "step": 710
+    },
+    {
+      "epoch": 0.11013173744287873,
+      "grad_norm": 16.560100555419922,
+      "learning_rate": 1.988228381446553e-05,
+      "loss": 1.7872,
+      "step": 720
+    },
+    {
+      "epoch": 0.11166134490736315,
+      "grad_norm": 20.133487701416016,
+      "learning_rate": 1.987136265072988e-05,
+      "loss": 1.8089,
+      "step": 730
+    },
+    {
+      "epoch": 0.11319095237184758,
+      "grad_norm": 13.525500297546387,
+      "learning_rate": 1.985996037070505e-05,
+      "loss": 1.8169,
+      "step": 740
+    },
+    {
+      "epoch": 0.114720559836332,
+      "grad_norm": 17.305484771728516,
+      "learning_rate": 1.9848077530122083e-05,
+      "loss": 1.8002,
+      "step": 750
+    },
+    {
+      "epoch": 0.11625016730081643,
+      "grad_norm": 14.32529067993164,
+      "learning_rate": 1.983571470813386e-05,
+      "loss": 1.7969,
+      "step": 760
+    },
+    {
+      "epoch": 0.11777977476530085,
+      "grad_norm": 16.328943252563477,
+      "learning_rate": 1.982287250728689e-05,
+      "loss": 1.7679,
+      "step": 770
+    },
+    {
+      "epoch": 0.11930938222978528,
+      "grad_norm": 10.76165771484375,
+      "learning_rate": 1.9809551553491918e-05,
+      "loss": 1.8304,
+      "step": 780
+    },
+    {
+      "epoch": 0.1208389896942697,
+      "grad_norm": 14.789531707763672,
+      "learning_rate": 1.979575249599344e-05,
+      "loss": 1.8215,
+      "step": 790
+    },
+    {
+      "epoch": 0.12236859715875413,
+      "grad_norm": 16.026445388793945,
+      "learning_rate": 1.9781476007338058e-05,
+      "loss": 1.7581,
+      "step": 800
+    },
+    {
+      "epoch": 0.12389820462323856,
+      "grad_norm": 17.013036727905273,
+      "learning_rate": 1.9766722783341682e-05,
+      "loss": 1.7698,
+      "step": 810
+    },
+    {
+      "epoch": 0.125427812087723,
+      "grad_norm": 24.33189582824707,
+      "learning_rate": 1.9751493543055634e-05,
+      "loss": 1.7896,
+      "step": 820
+    },
+    {
+      "epoch": 0.1269574195522074,
+      "grad_norm": 18.09388542175293,
+      "learning_rate": 1.9735789028731603e-05,
+      "loss": 1.754,
+      "step": 830
+    },
+    {
+      "epoch": 0.12848702701669185,
+      "grad_norm": 12.945524215698242,
+      "learning_rate": 1.9719610005785466e-05,
+      "loss": 1.7693,
+      "step": 840
+    },
+    {
+      "epoch": 0.13001663448117626,
+      "grad_norm": 12.579715728759766,
+      "learning_rate": 1.9702957262759964e-05,
+      "loss": 1.7323,
+      "step": 850
+    },
+    {
+      "epoch": 0.1315462419456607,
+      "grad_norm": 13.929019927978516,
+      "learning_rate": 1.9685831611286312e-05,
+      "loss": 1.7814,
+      "step": 860
+    },
+    {
+      "epoch": 0.1330758494101451,
+      "grad_norm": 16.18221664428711,
+      "learning_rate": 1.9668233886044597e-05,
+      "loss": 1.7715,
+      "step": 870
+    },
+    {
+      "epoch": 0.13460545687462955,
+      "grad_norm": 17.0338191986084,
+      "learning_rate": 1.9650164944723116e-05,
+      "loss": 1.7783,
+      "step": 880
+    },
+    {
+      "epoch": 0.136135064339114,
+      "grad_norm": 12.997424125671387,
+      "learning_rate": 1.9631625667976584e-05,
+      "loss": 1.7303,
+      "step": 890
+    },
+    {
+      "epoch": 0.1376646718035984,
+      "grad_norm": 14.024730682373047,
+      "learning_rate": 1.961261695938319e-05,
+      "loss": 1.7659,
+      "step": 900
+    },
+    {
+      "epoch": 0.13919427926808284,
+      "grad_norm": 15.084492683410645,
+      "learning_rate": 1.9593139745400575e-05,
+      "loss": 1.7504,
+      "step": 910
+    },
+    {
+      "epoch": 0.14072388673256725,
+      "grad_norm": 13.797721862792969,
+      "learning_rate": 1.9573194975320672e-05,
+      "loss": 1.7514,
+      "step": 920
+    },
+    {
+      "epoch": 0.1422534941970517,
+      "grad_norm": 12.69803237915039,
+      "learning_rate": 1.9552783621223437e-05,
+      "loss": 1.7344,
+      "step": 930
+    },
+    {
+      "epoch": 0.1437831016615361,
+      "grad_norm": 11.39012622833252,
+      "learning_rate": 1.9531906677929472e-05,
+      "loss": 1.7235,
+      "step": 940
+    },
+    {
+      "epoch": 0.14531270912602054,
+      "grad_norm": 15.734513282775879,
+      "learning_rate": 1.9510565162951538e-05,
+      "loss": 1.7329,
+      "step": 950
+    },
+    {
+      "epoch": 0.14684231659050495,
+      "grad_norm": 11.408859252929688,
+      "learning_rate": 1.9488760116444966e-05,
+      "loss": 1.7163,
+      "step": 960
+    },
+    {
+      "epoch": 0.1483719240549894,
+      "grad_norm": 11.509827613830566,
+      "learning_rate": 1.9466492601156964e-05,
+      "loss": 1.7178,
+      "step": 970
+    },
+    {
+      "epoch": 0.1499015315194738,
+      "grad_norm": 6.658020496368408,
+      "learning_rate": 1.944376370237481e-05,
+      "loss": 1.7269,
+      "step": 980
+    },
+    {
+      "epoch": 0.15143113898395824,
+      "grad_norm": 15.084721565246582,
+      "learning_rate": 1.942057452787297e-05,
+      "loss": 1.7002,
+      "step": 990
+    },
+    {
+      "epoch": 0.15296074644844268,
+      "grad_norm": 12.061055183410645,
+      "learning_rate": 1.9396926207859085e-05,
+      "loss": 1.7466,
+      "step": 1000
+    },
+    {
+      "epoch": 0.1544903539129271,
+      "grad_norm": 12.866460800170898,
+      "learning_rate": 1.937281989491892e-05,
+      "loss": 1.7213,
+      "step": 1010
+    },
+    {
+      "epoch": 0.15601996137741153,
+      "grad_norm": 10.788905143737793,
+      "learning_rate": 1.9348256763960146e-05,
+      "loss": 1.7204,
+      "step": 1020
+    },
+    {
+      "epoch": 0.15754956884189594,
+      "grad_norm": 14.169967651367188,
+      "learning_rate": 1.9323238012155125e-05,
+      "loss": 1.6906,
+      "step": 1030
+    },
+    {
+      "epoch": 0.15907917630638038,
+      "grad_norm": 15.22304916381836,
+      "learning_rate": 1.9297764858882516e-05,
+      "loss": 1.7012,
+      "step": 1040
+    },
+    {
+      "epoch": 0.1606087837708648,
+      "grad_norm": 15.920389175415039,
+      "learning_rate": 1.9271838545667876e-05,
+      "loss": 1.7151,
+      "step": 1050
+    },
+    {
+      "epoch": 0.16213839123534923,
+      "grad_norm": 14.644919395446777,
+      "learning_rate": 1.9245460336123136e-05,
+      "loss": 1.7011,
+      "step": 1060
+    },
+    {
+      "epoch": 0.16366799869983364,
+      "grad_norm": 11.079008102416992,
+      "learning_rate": 1.9218631515885007e-05,
+      "loss": 1.6773,
+      "step": 1070
+    },
+    {
+      "epoch": 0.16519760616431808,
+      "grad_norm": 13.276355743408203,
+      "learning_rate": 1.9191353392552346e-05,
+      "loss": 1.6895,
+      "step": 1080
+    },
+    {
+      "epoch": 0.1667272136288025,
+      "grad_norm": 12.620210647583008,
+      "learning_rate": 1.9163627295622397e-05,
+      "loss": 1.7153,
+      "step": 1090
+    },
+    {
+      "epoch": 0.16825682109328693,
+      "grad_norm": 13.213116645812988,
+      "learning_rate": 1.913545457642601e-05,
+      "loss": 1.6805,
+      "step": 1100
+    },
+    {
+      "epoch": 0.16978642855777137,
+      "grad_norm": 13.620598793029785,
+      "learning_rate": 1.910683660806177e-05,
+      "loss": 1.6882,
+      "step": 1110
+    },
+    {
+      "epoch": 0.17131603602225579,
+      "grad_norm": 14.6329927444458,
+      "learning_rate": 1.907777478532909e-05,
+      "loss": 1.6843,
+      "step": 1120
+    },
+    {
+      "epoch": 0.17284564348674022,
+      "grad_norm": 12.677013397216797,
+      "learning_rate": 1.9048270524660197e-05,
+      "loss": 1.7041,
+      "step": 1130
+    },
+    {
+      "epoch": 0.17437525095122464,
+      "grad_norm": 12.261626243591309,
+      "learning_rate": 1.901832526405114e-05,
+      "loss": 1.6819,
+      "step": 1140
+    },
+    {
+      "epoch": 0.17590485841570908,
+      "grad_norm": 14.508549690246582,
+      "learning_rate": 1.8987940462991673e-05,
+      "loss": 1.6847,
+      "step": 1150
+    },
+    {
+      "epoch": 0.1774344658801935,
+      "grad_norm": 14.293961524963379,
+      "learning_rate": 1.895711760239413e-05,
+      "loss": 1.6928,
+      "step": 1160
+    },
+    {
+      "epoch": 0.17896407334467793,
+      "grad_norm": 13.377256393432617,
+      "learning_rate": 1.892585818452126e-05,
+      "loss": 1.6838,
+      "step": 1170
+    },
+    {
+      "epoch": 0.18049368080916234,
+      "grad_norm": 12.398565292358398,
+      "learning_rate": 1.889416373291298e-05,
+      "loss": 1.6692,
+      "step": 1180
+    },
+    {
+      "epoch": 0.18202328827364678,
+      "grad_norm": 11.622918128967285,
+      "learning_rate": 1.8862035792312148e-05,
+      "loss": 1.6429,
+      "step": 1190
+    },
+    {
+      "epoch": 0.1835528957381312,
+      "grad_norm": 9.335916519165039,
+      "learning_rate": 1.8829475928589272e-05,
+      "loss": 1.6535,
+      "step": 1200
+    },
+    {
+      "epoch": 0.18508250320261563,
+      "grad_norm": 30.566650390625,
+      "learning_rate": 1.879648572866617e-05,
+      "loss": 1.6625,
+      "step": 1210
+    },
+    {
+      "epoch": 0.18661211066710007,
+      "grad_norm": 10.248709678649902,
+      "learning_rate": 1.8763066800438638e-05,
+      "loss": 1.657,
+      "step": 1220
+    },
+    {
+      "epoch": 0.18814171813158448,
+      "grad_norm": 12.403678894042969,
+      "learning_rate": 1.8729220772698096e-05,
+      "loss": 1.6588,
+      "step": 1230
+    },
+    {
+      "epoch": 0.18967132559606892,
+      "grad_norm": 12.880125999450684,
+      "learning_rate": 1.869494929505219e-05,
+      "loss": 1.6782,
+      "step": 1240
+    },
+    {
+      "epoch": 0.19120093306055333,
+      "grad_norm": 11.847280502319336,
+      "learning_rate": 1.866025403784439e-05,
+      "loss": 1.6612,
+      "step": 1250
+    },
+    {
+      "epoch": 0.19273054052503777,
+      "grad_norm": 13.305404663085938,
+      "learning_rate": 1.8625136692072577e-05,
+      "loss": 1.6565,
+      "step": 1260
+    },
+    {
+      "epoch": 0.19426014798952218,
+      "grad_norm": 14.423601150512695,
+      "learning_rate": 1.8589598969306646e-05,
+      "loss": 1.677,
+      "step": 1270
+    },
+    {
+      "epoch": 0.19578975545400662,
+      "grad_norm": 15.052698135375977,
+      "learning_rate": 1.855364260160507e-05,
+      "loss": 1.6467,
+      "step": 1280
+    },
+    {
+      "epoch": 0.19731936291849103,
+      "grad_norm": 11.820367813110352,
+      "learning_rate": 1.851726934143048e-05,
+      "loss": 1.6384,
+      "step": 1290
+    },
+    {
+      "epoch": 0.19884897038297547,
+      "grad_norm": 15.453312873840332,
+      "learning_rate": 1.848048096156426e-05,
+      "loss": 1.637,
+      "step": 1300
+    },
+    {
+      "epoch": 0.20037857784745988,
+      "grad_norm": 10.9462308883667,
+      "learning_rate": 1.8443279255020153e-05,
+      "loss": 1.6397,
+      "step": 1310
+    },
+    {
+      "epoch": 0.20190818531194432,
+      "grad_norm": 12.902162551879883,
+      "learning_rate": 1.8405666034956842e-05,
+      "loss": 1.6289,
+      "step": 1320
+    },
+    {
+      "epoch": 0.20343779277642876,
+      "grad_norm": 9.770879745483398,
+      "learning_rate": 1.836764313458962e-05,
+      "loss": 1.6349,
+      "step": 1330
+    },
+    {
+      "epoch": 0.20496740024091317,
+      "grad_norm": 11.91503620147705,
+      "learning_rate": 1.8329212407100996e-05,
+      "loss": 1.6101,
+      "step": 1340
+    },
+    {
+      "epoch": 0.2064970077053976,
+      "grad_norm": 9.018235206604004,
+      "learning_rate": 1.8290375725550417e-05,
+      "loss": 1.6194,
+      "step": 1350
+    },
+    {
+      "epoch": 0.20802661516988202,
+      "grad_norm": 13.213310241699219,
+      "learning_rate": 1.8251134982782952e-05,
+      "loss": 1.6173,
+      "step": 1360
+    },
+    {
+      "epoch": 0.20955622263436646,
+      "grad_norm": 13.184313774108887,
+      "learning_rate": 1.821149209133704e-05,
+      "loss": 1.644,
+      "step": 1370
+    },
+    {
+      "epoch": 0.21108583009885087,
+      "grad_norm": 11.71191692352295,
+      "learning_rate": 1.8171448983351284e-05,
+      "loss": 1.6355,
+      "step": 1380
+    },
+    {
+      "epoch": 0.2126154375633353,
+      "grad_norm": 11.976449966430664,
+      "learning_rate": 1.8131007610470278e-05,
+      "loss": 1.6196,
+      "step": 1390
+    },
+    {
+      "epoch": 0.21414504502781972,
+      "grad_norm": 12.10886287689209,
+      "learning_rate": 1.8090169943749477e-05,
+      "loss": 1.637,
+      "step": 1400
+    },
+    {
+      "epoch": 0.21567465249230416,
+      "grad_norm": 9.17182731628418,
+      "learning_rate": 1.804893797355914e-05,
+      "loss": 1.5883,
+      "step": 1410
+    },
+    {
+      "epoch": 0.21720425995678858,
+      "grad_norm": 15.71194076538086,
+      "learning_rate": 1.8007313709487334e-05,
+      "loss": 1.6215,
+      "step": 1420
+    },
+    {
+      "epoch": 0.21873386742127301,
+      "grad_norm": 10.610137939453125,
+      "learning_rate": 1.7965299180241963e-05,
+      "loss": 1.6228,
+      "step": 1430
+    },
+    {
+      "epoch": 0.22026347488575745,
+      "grad_norm": 12.10354232788086,
+      "learning_rate": 1.792289643355191e-05,
+      "loss": 1.6052,
+      "step": 1440
+    },
+    {
+      "epoch": 0.22179308235024187,
+      "grad_norm": 10.069908142089844,
+      "learning_rate": 1.788010753606722e-05,
+      "loss": 1.5964,
+      "step": 1450
+    },
+    {
+      "epoch": 0.2233226898147263,
+      "grad_norm": 12.154913902282715,
+      "learning_rate": 1.78369345732584e-05,
+      "loss": 1.6269,
+      "step": 1460
+    },
+    {
+      "epoch": 0.22485229727921072,
+      "grad_norm": 12.322149276733398,
+      "learning_rate": 1.7793379649314743e-05,
+      "loss": 1.5855,
+      "step": 1470
+    },
+    {
+      "epoch": 0.22638190474369516,
+      "grad_norm": 10.875051498413086,
+      "learning_rate": 1.7749444887041797e-05,
+      "loss": 1.6005,
+      "step": 1480
+    },
+    {
+      "epoch": 0.22791151220817957,
+      "grad_norm": 11.17204761505127,
+      "learning_rate": 1.7705132427757895e-05,
+      "loss": 1.5849,
+      "step": 1490
+    },
+    {
+      "epoch": 0.229441119672664,
+      "grad_norm": 10.732039451599121,
+      "learning_rate": 1.766044443118978e-05,
+      "loss": 1.5825,
+      "step": 1500
+    },
+    {
+      "epoch": 0.23097072713714842,
+      "grad_norm": 13.450652122497559,
+      "learning_rate": 1.761538307536737e-05,
+      "loss": 1.6146,
+      "step": 1510
+    },
+    {
+      "epoch": 0.23250033460163286,
+      "grad_norm": 11.979947090148926,
+      "learning_rate": 1.7569950556517566e-05,
+      "loss": 1.5928,
+      "step": 1520
+    },
+    {
+      "epoch": 0.23402994206611727,
+      "grad_norm": 11.305917739868164,
+      "learning_rate": 1.7524149088957244e-05,
+      "loss": 1.5866,
+      "step": 1530
+    },
+    {
+      "epoch": 0.2355595495306017,
+      "grad_norm": 10.360641479492188,
+      "learning_rate": 1.747798090498532e-05,
+      "loss": 1.6028,
+      "step": 1540
+    },
+    {
+      "epoch": 0.23708915699508615,
+      "grad_norm": 8.656867027282715,
+      "learning_rate": 1.7431448254773943e-05,
+      "loss": 1.6014,
+      "step": 1550
+    },
+    {
+      "epoch": 0.23861876445957056,
+      "grad_norm": 11.687288284301758,
+      "learning_rate": 1.7384553406258842e-05,
+      "loss": 1.5948,
+      "step": 1560
+    },
+    {
+      "epoch": 0.240148371924055,
+      "grad_norm": 10.943402290344238,
+      "learning_rate": 1.7337298645028764e-05,
+      "loss": 1.5923,
+      "step": 1570
+    },
+    {
+      "epoch": 0.2416779793885394,
+      "grad_norm": 10.023202896118164,
+      "learning_rate": 1.7289686274214116e-05,
+      "loss": 1.5719,
+      "step": 1580
+    },
+    {
+      "epoch": 0.24320758685302385,
+      "grad_norm": 9.700725555419922,
+      "learning_rate": 1.7241718614374678e-05,
+      "loss": 1.5872,
+      "step": 1590
+    },
+    {
+      "epoch": 0.24473719431750826,
+      "grad_norm": 11.386409759521484,
+      "learning_rate": 1.7193398003386514e-05,
+      "loss": 1.5832,
+      "step": 1600
+    },
+    {
+      "epoch": 0.2462668017819927,
+      "grad_norm": 9.78838062286377,
+      "learning_rate": 1.7144726796328034e-05,
+      "loss": 1.5904,
+      "step": 1610
+    },
+    {
+      "epoch": 0.2477964092464771,
+      "grad_norm": 12.67321491241455,
+      "learning_rate": 1.709570736536521e-05,
+      "loss": 1.5779,
+      "step": 1620
+    },
+    {
+      "epoch": 0.24932601671096155,
+      "grad_norm": 10.230249404907227,
+      "learning_rate": 1.7046342099635948e-05,
+      "loss": 1.5931,
+      "step": 1630
+    },
+    {
+      "epoch": 0.250855624175446,
+      "grad_norm": 9.709312438964844,
+      "learning_rate": 1.6996633405133656e-05,
+      "loss": 1.5728,
+      "step": 1640
+    },
+    {
+      "epoch": 0.25238523163993043,
+      "grad_norm": 10.340200424194336,
+      "learning_rate": 1.6946583704589973e-05,
+      "loss": 1.561,
+      "step": 1650
+    },
+    {
+      "epoch": 0.2539148391044148,
+      "grad_norm": 9.730533599853516,
+      "learning_rate": 1.68961954373567e-05,
+      "loss": 1.5487,
+      "step": 1660
+    },
+    {
+      "epoch": 0.25544444656889925,
+      "grad_norm": 8.787372589111328,
+      "learning_rate": 1.684547105928689e-05,
+      "loss": 1.5704,
+      "step": 1670
+    },
+    {
+      "epoch": 0.2569740540333837,
+      "grad_norm": 9.325871467590332,
+      "learning_rate": 1.6794413042615168e-05,
+      "loss": 1.5352,
+      "step": 1680
+    },
+    {
+      "epoch": 0.25850366149786813,
+      "grad_norm": 10.094749450683594,
+      "learning_rate": 1.6743023875837233e-05,
+      "loss": 1.5718,
+      "step": 1690
+    },
+    {
+      "epoch": 0.2600332689623525,
+      "grad_norm": 9.554350852966309,
+      "learning_rate": 1.6691306063588583e-05,
+      "loss": 1.5563,
+      "step": 1700
+    },
+    {
+      "epoch": 0.26156287642683695,
+      "grad_norm": 10.526246070861816,
+      "learning_rate": 1.6639262126522417e-05,
+      "loss": 1.5399,
+      "step": 1710
+    },
+    {
+      "epoch": 0.2630924838913214,
+      "grad_norm": 8.69581413269043,
+      "learning_rate": 1.6586894601186804e-05,
+      "loss": 1.5456,
+      "step": 1720
+    },
+    {
+      "epoch": 0.26462209135580583,
+      "grad_norm": 9.803443908691406,
+      "learning_rate": 1.6534206039901057e-05,
+      "loss": 1.54,
+      "step": 1730
+    },
+    {
+      "epoch": 0.2661516988202902,
+      "grad_norm": 9.926239013671875,
+      "learning_rate": 1.6481199010631312e-05,
+      "loss": 1.5468,
+      "step": 1740
+    },
+    {
+      "epoch": 0.26768130628477466,
+      "grad_norm": 11.072277069091797,
+      "learning_rate": 1.6427876096865394e-05,
+      "loss": 1.5495,
+      "step": 1750
+    },
+    {
+      "epoch": 0.2692109137492591,
+      "grad_norm": 10.02304744720459,
+      "learning_rate": 1.63742398974869e-05,
+      "loss": 1.567,
+      "step": 1760
+    },
+    {
+      "epoch": 0.27074052121374353,
+      "grad_norm": 9.076205253601074,
+      "learning_rate": 1.632029302664851e-05,
+      "loss": 1.553,
+      "step": 1770
+    },
+    {
+      "epoch": 0.272270128678228,
+      "grad_norm": 7.942831039428711,
+      "learning_rate": 1.6266038113644605e-05,
+      "loss": 1.5408,
+      "step": 1780
+    },
+    {
+      "epoch": 0.27379973614271236,
+      "grad_norm": 10.165443420410156,
+      "learning_rate": 1.6211477802783105e-05,
+      "loss": 1.5317,
+      "step": 1790
+    },
+    {
+      "epoch": 0.2753293436071968,
+      "grad_norm": 10.491219520568848,
+      "learning_rate": 1.6156614753256583e-05,
+      "loss": 1.5528,
+      "step": 1800
+    },
+    {
+      "epoch": 0.27685895107168124,
+      "grad_norm": 4.210089206695557,
+      "learning_rate": 1.610145163901268e-05,
+      "loss": 1.5295,
+      "step": 1810
+    },
+    {
+      "epoch": 0.2783885585361657,
+      "grad_norm": 12.827298164367676,
+      "learning_rate": 1.6045991148623752e-05,
+      "loss": 1.5366,
+      "step": 1820
+    },
+    {
+      "epoch": 0.27991816600065006,
+      "grad_norm": 11.551325798034668,
+      "learning_rate": 1.599023598515586e-05,
+      "loss": 1.5226,
+      "step": 1830
+    },
+    {
+      "epoch": 0.2814477734651345,
+      "grad_norm": 11.47888469696045,
+      "learning_rate": 1.5934188866037017e-05,
+      "loss": 1.5285,
+      "step": 1840
+    },
+    {
+      "epoch": 0.28297738092961894,
+      "grad_norm": 9.28005599975586,
+      "learning_rate": 1.5877852522924733e-05,
+      "loss": 1.504,
+      "step": 1850
+    },
+    {
+      "epoch": 0.2845069883941034,
+      "grad_norm": 10.573029518127441,
+      "learning_rate": 1.5821229701572897e-05,
+      "loss": 1.5212,
+      "step": 1860
+    },
+    {
+      "epoch": 0.2860365958585878,
+      "grad_norm": 11.156023025512695,
+      "learning_rate": 1.5764323161697933e-05,
+      "loss": 1.5257,
+      "step": 1870
+    },
+    {
+      "epoch": 0.2875662033230722,
+      "grad_norm": 9.835415840148926,
+      "learning_rate": 1.570713567684432e-05,
+      "loss": 1.5119,
+      "step": 1880
+    },
+    {
+      "epoch": 0.28909581078755664,
+      "grad_norm": 8.080103874206543,
+      "learning_rate": 1.564967003424938e-05,
+      "loss": 1.5048,
+      "step": 1890
+    },
+    {
+      "epoch": 0.2906254182520411,
+      "grad_norm": 10.452638626098633,
+      "learning_rate": 1.5591929034707468e-05,
+      "loss": 1.5197,
+      "step": 1900
+    },
+    {
+      "epoch": 0.2921550257165255,
+      "grad_norm": 10.441474914550781,
+      "learning_rate": 1.553391549243344e-05,
+      "loss": 1.5317,
+      "step": 1910
+    },
+    {
+      "epoch": 0.2936846331810099,
+      "grad_norm": 9.752622604370117,
+      "learning_rate": 1.5475632234925505e-05,
+      "loss": 1.5069,
+      "step": 1920
+    },
+    {
+      "epoch": 0.29521424064549434,
+      "grad_norm": 8.496747016906738,
+      "learning_rate": 1.54170821028274e-05,
+      "loss": 1.5285,
+      "step": 1930
+    },
+    {
+      "epoch": 0.2967438481099788,
+      "grad_norm": 9.566315650939941,
+      "learning_rate": 1.5358267949789968e-05,
+      "loss": 1.5162,
+      "step": 1940
+    },
+    {
+      "epoch": 0.2982734555744632,
+      "grad_norm": 6.093721389770508,
+      "learning_rate": 1.529919264233205e-05,
+      "loss": 1.5008,
+      "step": 1950
+    },
+    {
+      "epoch": 0.2998030630389476,
+      "grad_norm": 7.963770866394043,
+      "learning_rate": 1.5239859059700794e-05,
+      "loss": 1.4732,
+      "step": 1960
+    },
+    {
+      "epoch": 0.30133267050343204,
+      "grad_norm": 8.183985710144043,
+      "learning_rate": 1.5180270093731305e-05,
+      "loss": 1.5126,
+      "step": 1970
+    },
+    {
+      "epoch": 0.3028622779679165,
+      "grad_norm": 10.149001121520996,
+      "learning_rate": 1.5120428648705716e-05,
+      "loss": 1.5145,
+      "step": 1980
+    },
+    {
+      "epoch": 0.3043918854324009,
+      "grad_norm": 8.544008255004883,
+      "learning_rate": 1.5060337641211637e-05,
+      "loss": 1.5061,
+      "step": 1990
+    },
+    {
+      "epoch": 0.30592149289688536,
+      "grad_norm": 9.953203201293945,
+      "learning_rate": 1.5000000000000002e-05,
+      "loss": 1.5209,
+      "step": 2000
+    },
+    {
+      "epoch": 0.30745110036136974,
+      "grad_norm": 10.6850004196167,
+      "learning_rate": 1.493941866584231e-05,
+      "loss": 1.5163,
+      "step": 2010
+    },
+    {
+      "epoch": 0.3089807078258542,
+      "grad_norm": 8.49984359741211,
+      "learning_rate": 1.4878596591387329e-05,
+      "loss": 1.4868,
+      "step": 2020
+    },
+    {
+      "epoch": 0.3105103152903386,
+      "grad_norm": 8.950973510742188,
+      "learning_rate": 1.4817536741017153e-05,
+      "loss": 1.481,
+      "step": 2030
+    },
+    {
+      "epoch": 0.31203992275482306,
+      "grad_norm": 10.185256958007812,
+      "learning_rate": 1.4756242090702756e-05,
+      "loss": 1.49,
+      "step": 2040
+    },
+    {
+      "epoch": 0.31356953021930745,
+      "grad_norm": 8.98540210723877,
+      "learning_rate": 1.469471562785891e-05,
+      "loss": 1.4905,
+      "step": 2050
+    },
+    {
+      "epoch": 0.3150991376837919,
+      "grad_norm": 7.6039299964904785,
+      "learning_rate": 1.463296035119862e-05,
+      "loss": 1.4794,
+      "step": 2060
+    },
+    {
+      "epoch": 0.3166287451482763,
+      "grad_norm": 8.91348934173584,
+      "learning_rate": 1.4570979270586944e-05,
+      "loss": 1.4697,
+      "step": 2070
+    },
+    {
+      "epoch": 0.31815835261276076,
+      "grad_norm": 8.245038986206055,
+      "learning_rate": 1.4508775406894308e-05,
+      "loss": 1.4827,
+      "step": 2080
+    },
+    {
+      "epoch": 0.3196879600772452,
+      "grad_norm": 9.471598625183105,
+      "learning_rate": 1.4446351791849276e-05,
+      "loss": 1.4912,
+      "step": 2090
+    },
+    {
+      "epoch": 0.3212175675417296,
+      "grad_norm": 5.9019975662231445,
+      "learning_rate": 1.4383711467890776e-05,
+      "loss": 1.4708,
+      "step": 2100
+    },
+    {
+      "epoch": 0.322747175006214,
+      "grad_norm": 7.020793437957764,
+      "learning_rate": 1.4320857488019826e-05,
+      "loss": 1.4702,
+      "step": 2110
+    },
+    {
+      "epoch": 0.32427678247069847,
+      "grad_norm": 9.424378395080566,
+      "learning_rate": 1.4257792915650728e-05,
+      "loss": 1.4805,
+      "step": 2120
+    },
+    {
+      "epoch": 0.3258063899351829,
+      "grad_norm": 7.374673366546631,
+      "learning_rate": 1.4194520824461773e-05,
+      "loss": 1.4682,
+      "step": 2130
+    },
+    {
+      "epoch": 0.3273359973996673,
+      "grad_norm": 10.07297134399414,
+      "learning_rate": 1.413104429824542e-05,
+      "loss": 1.4817,
+      "step": 2140
+    },
+    {
+      "epoch": 0.32886560486415173,
+      "grad_norm": 8.525253295898438,
+      "learning_rate": 1.4067366430758004e-05,
+      "loss": 1.4686,
+      "step": 2150
+    },
+    {
+      "epoch": 0.33039521232863617,
+      "grad_norm": 9.093647956848145,
+      "learning_rate": 1.4003490325568953e-05,
+      "loss": 1.4779,
+      "step": 2160
+    },
+    {
+      "epoch": 0.3319248197931206,
+      "grad_norm": 7.984882831573486,
+      "learning_rate": 1.3939419095909513e-05,
+      "loss": 1.4756,
+      "step": 2170
+    },
+    {
+      "epoch": 0.333454427257605,
+      "grad_norm": 7.52358341217041,
+      "learning_rate": 1.3875155864521031e-05,
+      "loss": 1.4531,
+      "step": 2180
+    },
+    {
+      "epoch": 0.33498403472208943,
+      "grad_norm": 9.23783016204834,
+      "learning_rate": 1.3810703763502744e-05,
+      "loss": 1.4685,
+      "step": 2190
+    },
+    {
+      "epoch": 0.33651364218657387,
+      "grad_norm": 10.533625602722168,
+      "learning_rate": 1.3746065934159123e-05,
+      "loss": 1.4532,
+      "step": 2200
+    },
+    {
+      "epoch": 0.3380432496510583,
+      "grad_norm": 10.983015060424805,
+      "learning_rate": 1.3681245526846782e-05,
+      "loss": 1.4631,
+      "step": 2210
+    },
+    {
+      "epoch": 0.33957285711554275,
+      "grad_norm": 7.50883150100708,
+      "learning_rate": 1.3616245700820922e-05,
+      "loss": 1.4627,
+      "step": 2220
+    },
+    {
+      "epoch": 0.34110246458002713,
+      "grad_norm": 6.438501834869385,
+      "learning_rate": 1.3551069624081372e-05,
+      "loss": 1.4517,
+      "step": 2230
+    },
+    {
+      "epoch": 0.34263207204451157,
+      "grad_norm": 8.066999435424805,
+      "learning_rate": 1.3485720473218153e-05,
+      "loss": 1.4548,
+      "step": 2240
+    },
+    {
+      "epoch": 0.344161679508996,
+      "grad_norm": 8.656118392944336,
+      "learning_rate": 1.342020143325669e-05,
+      "loss": 1.4445,
+      "step": 2250
+    },
+    {
+      "epoch": 0.34569128697348045,
+      "grad_norm": 7.250131607055664,
+      "learning_rate": 1.3354515697502552e-05,
+      "loss": 1.4321,
+      "step": 2260
+    },
+    {
+      "epoch": 0.34722089443796483,
+      "grad_norm": 8.546892166137695,
+      "learning_rate": 1.3288666467385834e-05,
+      "loss": 1.4464,
+      "step": 2270
+    },
+    {
+      "epoch": 0.3487505019024493,
+      "grad_norm": 8.729716300964355,
+      "learning_rate": 1.3222656952305113e-05,
+      "loss": 1.4541,
+      "step": 2280
+    },
+    {
+      "epoch": 0.3502801093669337,
+      "grad_norm": 8.721868515014648,
+      "learning_rate": 1.3156490369471026e-05,
+      "loss": 1.4475,
+      "step": 2290
+    },
+    {
+      "epoch": 0.35180971683141815,
+      "grad_norm": 8.831208229064941,
+      "learning_rate": 1.3090169943749475e-05,
+      "loss": 1.4302,
+      "step": 2300
+    },
+    {
+      "epoch": 0.3533393242959026,
+      "grad_norm": 8.883235931396484,
+      "learning_rate": 1.3023698907504447e-05,
+      "loss": 1.4432,
+      "step": 2310
+    },
+    {
+      "epoch": 0.354868931760387,
+      "grad_norm": 8.794004440307617,
+      "learning_rate": 1.2957080500440469e-05,
+      "loss": 1.4444,
+      "step": 2320
+    },
+    {
+      "epoch": 0.3563985392248714,
+      "grad_norm": 4.8828935623168945,
+      "learning_rate": 1.2890317969444716e-05,
+      "loss": 1.4384,
+      "step": 2330
+    },
+    {
+      "epoch": 0.35792814668935585,
+      "grad_norm": 15.948801040649414,
+      "learning_rate": 1.2823414568428767e-05,
+      "loss": 1.4551,
+      "step": 2340
+    },
+    {
+      "epoch": 0.3594577541538403,
+      "grad_norm": 7.99323844909668,
+      "learning_rate": 1.2756373558169992e-05,
+      "loss": 1.4202,
+      "step": 2350
+    },
+    {
+      "epoch": 0.3609873616183247,
+      "grad_norm": 5.047421455383301,
+      "learning_rate": 1.2689198206152657e-05,
+      "loss": 1.4278,
+      "step": 2360
+    },
+    {
+      "epoch": 0.3625169690828091,
+      "grad_norm": 5.64467191696167,
+      "learning_rate": 1.2621891786408648e-05,
+      "loss": 1.4138,
+      "step": 2370
+    },
+    {
+      "epoch": 0.36404657654729355,
+      "grad_norm": 8.789236068725586,
+      "learning_rate": 1.2554457579357906e-05,
+      "loss": 1.4153,
+      "step": 2380
+    },
+    {
+      "epoch": 0.365576184011778,
+      "grad_norm": 8.083710670471191,
+      "learning_rate": 1.2486898871648552e-05,
+      "loss": 1.4093,
+      "step": 2390
+    },
+    {
+      "epoch": 0.3671057914762624,
+      "grad_norm": 5.929847240447998,
+      "learning_rate": 1.2419218955996677e-05,
+      "loss": 1.4153,
+      "step": 2400
+    },
+    {
+      "epoch": 0.3686353989407468,
+      "grad_norm": 7.852176666259766,
+      "learning_rate": 1.23514211310259e-05,
+      "loss": 1.4145,
+      "step": 2410
+    },
+    {
+      "epoch": 0.37016500640523126,
+      "grad_norm": 9.10372257232666,
+      "learning_rate": 1.2283508701106559e-05,
+      "loss": 1.414,
+      "step": 2420
+    },
+    {
+      "epoch": 0.3716946138697157,
+      "grad_norm": 9.65334415435791,
+      "learning_rate": 1.2215484976194675e-05,
+      "loss": 1.4363,
+      "step": 2430
+    },
+    {
+      "epoch": 0.37322422133420013,
+      "grad_norm": 7.612096786499023,
+      "learning_rate": 1.2147353271670634e-05,
+      "loss": 1.4188,
+      "step": 2440
+    },
+    {
+      "epoch": 0.3747538287986845,
+      "grad_norm": 5.171387672424316,
+      "learning_rate": 1.2079116908177592e-05,
+      "loss": 1.4108,
+      "step": 2450
+    },
+    {
+      "epoch": 0.37628343626316896,
+      "grad_norm": 6.974627494812012,
+      "learning_rate": 1.2010779211459649e-05,
+      "loss": 1.4164,
+      "step": 2460
+    },
+    {
+      "epoch": 0.3778130437276534,
+      "grad_norm": 7.995100021362305,
+      "learning_rate": 1.194234351219972e-05,
+      "loss": 1.4241,
+      "step": 2470
+    },
+    {
+      "epoch": 0.37934265119213784,
+      "grad_norm": 9.874503135681152,
+      "learning_rate": 1.187381314585725e-05,
+      "loss": 1.3957,
+      "step": 2480
+    },
+    {
+      "epoch": 0.3808722586566222,
+      "grad_norm": 9.190123558044434,
+      "learning_rate": 1.1805191452505602e-05,
+      "loss": 1.4234,
+      "step": 2490
+    },
+    {
+      "epoch": 0.38240186612110666,
+      "grad_norm": 9.317654609680176,
+      "learning_rate": 1.1736481776669307e-05,
+      "loss": 1.4251,
+      "step": 2500
+    },
+    {
+      "epoch": 0.3839314735855911,
+      "grad_norm": 8.3729829788208,
+      "learning_rate": 1.1667687467161025e-05,
+      "loss": 1.4024,
+      "step": 2510
+    },
+    {
+      "epoch": 0.38546108105007554,
+      "grad_norm": 8.835628509521484,
+      "learning_rate": 1.159881187691835e-05,
+      "loss": 1.4098,
+      "step": 2520
+    },
+    {
+      "epoch": 0.38699068851456,
+      "grad_norm": 8.480125427246094,
+      "learning_rate": 1.1529858362840383e-05,
+      "loss": 1.4,
+      "step": 2530
+    },
+    {
+      "epoch": 0.38852029597904436,
+      "grad_norm": 8.7410306930542,
+      "learning_rate": 1.1460830285624119e-05,
+      "loss": 1.4084,
+      "step": 2540
+    },
+    {
+      "epoch": 0.3900499034435288,
+      "grad_norm": 5.247309684753418,
+      "learning_rate": 1.1391731009600655e-05,
+      "loss": 1.3918,
+      "step": 2550
+    },
+    {
+      "epoch": 0.39157951090801324,
+      "grad_norm": 6.82070779800415,
+      "learning_rate": 1.1322563902571227e-05,
+      "loss": 1.3829,
+      "step": 2560
+    },
+    {
+      "epoch": 0.3931091183724977,
+      "grad_norm": 7.315955638885498,
+      "learning_rate": 1.1253332335643043e-05,
+      "loss": 1.3879,
+      "step": 2570
+    },
+    {
+      "epoch": 0.39463872583698206,
+      "grad_norm": 8.81851863861084,
+      "learning_rate": 1.1184039683065014e-05,
+      "loss": 1.3809,
+      "step": 2580
+    },
+    {
+      "epoch": 0.3961683333014665,
+      "grad_norm": 7.224653720855713,
+      "learning_rate": 1.1114689322063255e-05,
+      "loss": 1.3958,
+      "step": 2590
+    },
+    {
+      "epoch": 0.39769794076595094,
+      "grad_norm": 6.905256748199463,
+      "learning_rate": 1.1045284632676535e-05,
+      "loss": 1.3854,
+      "step": 2600
+    },
+    {
+      "epoch": 0.3992275482304354,
+      "grad_norm": 7.848160266876221,
+      "learning_rate": 1.0975828997591496e-05,
+      "loss": 1.3876,
+      "step": 2610
+    },
+    {
+      "epoch": 0.40075715569491976,
+      "grad_norm": 6.962865352630615,
+      "learning_rate": 1.0906325801977804e-05,
+      "loss": 1.3909,
+      "step": 2620
+    },
+    {
+      "epoch": 0.4022867631594042,
+      "grad_norm": 7.401321887969971,
+      "learning_rate": 1.083677843332316e-05,
+      "loss": 1.3825,
+      "step": 2630
+    },
+    {
+      "epoch": 0.40381637062388864,
+      "grad_norm": 5.543237209320068,
+      "learning_rate": 1.0767190281268187e-05,
+      "loss": 1.3773,
+      "step": 2640
+    },
+    {
+      "epoch": 0.4053459780883731,
+      "grad_norm": 7.603894233703613,
+      "learning_rate": 1.0697564737441254e-05,
+      "loss": 1.3961,
+      "step": 2650
+    },
+    {
+      "epoch": 0.4068755855528575,
+      "grad_norm": 7.263538837432861,
+      "learning_rate": 1.0627905195293135e-05,
+      "loss": 1.3696,
+      "step": 2660
+    },
+    {
+      "epoch": 0.4084051930173419,
+      "grad_norm": 9.08191967010498,
+      "learning_rate": 1.055821504993164e-05,
+      "loss": 1.374,
+      "step": 2670
+    },
+    {
+      "epoch": 0.40993480048182634,
+      "grad_norm": 6.848121166229248,
+      "learning_rate": 1.0488497697956134e-05,
+      "loss": 1.3765,
+      "step": 2680
+    },
+    {
+      "epoch": 0.4114644079463108,
+      "grad_norm": 6.978294849395752,
+      "learning_rate": 1.0418756537291996e-05,
+      "loss": 1.375,
+      "step": 2690
+    },
+    {
+      "epoch": 0.4129940154107952,
+      "grad_norm": 7.876370429992676,
+      "learning_rate": 1.0348994967025012e-05,
+      "loss": 1.3681,
+      "step": 2700
+    },
+    {
+      "epoch": 0.4145236228752796,
+      "grad_norm": 5.700248718261719,
+      "learning_rate": 1.0279216387235691e-05,
+      "loss": 1.3733,
+      "step": 2710
+    },
+    {
+      "epoch": 0.41605323033976405,
+      "grad_norm": 5.646441459655762,
+      "learning_rate": 1.0209424198833571e-05,
+      "loss": 1.3487,
+      "step": 2720
+    },
+    {
+      "epoch": 0.4175828378042485,
+      "grad_norm": 6.236967086791992,
+      "learning_rate": 1.0139621803391454e-05,
+      "loss": 1.3621,
+      "step": 2730
+    },
+    {
+      "epoch": 0.4191124452687329,
+      "grad_norm": 6.606427192687988,
+      "learning_rate": 1.0069812602979617e-05,
+      "loss": 1.3551,
+      "step": 2740
+    },
+    {
+      "epoch": 0.42064205273321736,
+      "grad_norm": 5.275976181030273,
+      "learning_rate": 1e-05,
+      "loss": 1.3554,
+      "step": 2750
+    },
+    {
+      "epoch": 0.42217166019770175,
+      "grad_norm": 7.677177429199219,
+      "learning_rate": 9.930187397020385e-06,
+      "loss": 1.3601,
+      "step": 2760
+    },
+    {
+      "epoch": 0.4237012676621862,
+      "grad_norm": 7.980209827423096,
+      "learning_rate": 9.860378196608549e-06,
+      "loss": 1.3671,
+      "step": 2770
+    },
+    {
+      "epoch": 0.4252308751266706,
+      "grad_norm": 4.75565767288208,
+      "learning_rate": 9.790575801166432e-06,
+      "loss": 1.3557,
+      "step": 2780
+    },
+    {
+      "epoch": 0.42676048259115507,
+      "grad_norm": 5.737847328186035,
+      "learning_rate": 9.720783612764314e-06,
+      "loss": 1.3475,
+      "step": 2790
+    },
+    {
+      "epoch": 0.42829009005563945,
+      "grad_norm": 7.28814172744751,
+      "learning_rate": 9.651005032974994e-06,
+      "loss": 1.3516,
+      "step": 2800
+    },
+    {
+      "epoch": 0.4298196975201239,
+      "grad_norm": 6.400086879730225,
+      "learning_rate": 9.581243462708007e-06,
+      "loss": 1.3552,
+      "step": 2810
+    },
+    {
+      "epoch": 0.43134930498460833,
+      "grad_norm": 14.871561050415039,
+      "learning_rate": 9.511502302043867e-06,
+      "loss": 1.338,
+      "step": 2820
+    },
+    {
+      "epoch": 0.43287891244909277,
+      "grad_norm": 7.7634100914001465,
+      "learning_rate": 9.441784950068362e-06,
+      "loss": 1.346,
+      "step": 2830
+    },
+    {
+      "epoch": 0.43440851991357715,
+      "grad_norm": 7.211244106292725,
+      "learning_rate": 9.372094804706867e-06,
+      "loss": 1.3526,
+      "step": 2840
+    },
+    {
+      "epoch": 0.4359381273780616,
+      "grad_norm": 6.430041313171387,
+      "learning_rate": 9.302435262558748e-06,
+      "loss": 1.3343,
+      "step": 2850
+    },
+    {
+      "epoch": 0.43746773484254603,
+      "grad_norm": 5.903066635131836,
+      "learning_rate": 9.232809718731815e-06,
+      "loss": 1.3353,
+      "step": 2860
+    },
+    {
+      "epoch": 0.43899734230703047,
+      "grad_norm": 7.717010021209717,
+      "learning_rate": 9.163221566676847e-06,
+      "loss": 1.3438,
+      "step": 2870
+    },
+    {
+      "epoch": 0.4405269497715149,
+      "grad_norm": 7.673276901245117,
+      "learning_rate": 9.093674198022201e-06,
+      "loss": 1.3394,
+      "step": 2880
+    },
+    {
+      "epoch": 0.4420565572359993,
+      "grad_norm": 6.96506929397583,
+      "learning_rate": 9.024171002408507e-06,
+      "loss": 1.35,
+      "step": 2890
+    },
+    {
+      "epoch": 0.44358616470048373,
+      "grad_norm": 5.854609966278076,
+      "learning_rate": 8.954715367323468e-06,
+      "loss": 1.3376,
+      "step": 2900
+    },
+    {
+      "epoch": 0.44511577216496817,
+      "grad_norm": 6.3792572021484375,
+      "learning_rate": 8.885310677936746e-06,
+      "loss": 1.3359,
+      "step": 2910
+    },
+    {
+      "epoch": 0.4466453796294526,
+      "grad_norm": 6.389842510223389,
+      "learning_rate": 8.815960316934991e-06,
+      "loss": 1.3299,
+      "step": 2920
+    },
+    {
+      "epoch": 0.448174987093937,
+      "grad_norm": 6.603227138519287,
+      "learning_rate": 8.746667664356957e-06,
+      "loss": 1.323,
+      "step": 2930
+    },
+    {
+      "epoch": 0.44970459455842143,
+      "grad_norm": 6.4203338623046875,
+      "learning_rate": 8.677436097428775e-06,
+      "loss": 1.3459,
+      "step": 2940
+    },
+    {
+      "epoch": 0.4512342020229059,
+      "grad_norm": 5.568302154541016,
+      "learning_rate": 8.60826899039935e-06,
+      "loss": 1.3434,
+      "step": 2950
+    },
+    {
+      "epoch": 0.4527638094873903,
+      "grad_norm": 6.737658977508545,
+      "learning_rate": 8.539169714375885e-06,
+      "loss": 1.3146,
+      "step": 2960
+    },
+    {
+      "epoch": 0.45429341695187475,
+      "grad_norm": 5.4618940353393555,
+      "learning_rate": 8.47014163715962e-06,
+      "loss": 1.3205,
+      "step": 2970
+    },
+    {
+      "epoch": 0.45582302441635913,
+      "grad_norm": 6.360799312591553,
+      "learning_rate": 8.401188123081653e-06,
+      "loss": 1.3198,
+      "step": 2980
+    },
+    {
+      "epoch": 0.4573526318808436,
+      "grad_norm": 6.304644584655762,
+      "learning_rate": 8.332312532838978e-06,
+      "loss": 1.3261,
+      "step": 2990
+    },
+    {
+      "epoch": 0.458882239345328,
+      "grad_norm": 4.984965801239014,
+      "learning_rate": 8.263518223330698e-06,
+      "loss": 1.3181,
+      "step": 3000
+    },
+    {
+      "epoch": 0.46041184680981245,
+      "grad_norm": 6.168178081512451,
+      "learning_rate": 8.194808547494401e-06,
+      "loss": 1.3052,
+      "step": 3010
+    },
+    {
+      "epoch": 0.46194145427429684,
+      "grad_norm": 6.572290420532227,
+      "learning_rate": 8.126186854142752e-06,
+      "loss": 1.3082,
+      "step": 3020
+    },
+    {
+      "epoch": 0.4634710617387813,
+      "grad_norm": 5.937058448791504,
+      "learning_rate": 8.057656487800283e-06,
+      "loss": 1.327,
+      "step": 3030
+    },
+    {
+      "epoch": 0.4650006692032657,
+      "grad_norm": 5.716440677642822,
+      "learning_rate": 7.989220788540356e-06,
+      "loss": 1.3301,
+      "step": 3040
+    },
+    {
+      "epoch": 0.46653027666775015,
+      "grad_norm": 4.461696624755859,
+      "learning_rate": 7.92088309182241e-06,
+      "loss": 1.3234,
+      "step": 3050
+    },
+    {
+      "epoch": 0.46805988413223454,
+      "grad_norm": 6.483889579772949,
+      "learning_rate": 7.852646728329368e-06,
+      "loss": 1.3183,
+      "step": 3060
+    },
+    {
+      "epoch": 0.469589491596719,
+      "grad_norm": 5.030406475067139,
+      "learning_rate": 7.784515023805328e-06,
+      "loss": 1.3208,
+      "step": 3070
+    },
+    {
+      "epoch": 0.4711190990612034,
+      "grad_norm": 6.356899738311768,
+      "learning_rate": 7.716491298893443e-06,
+      "loss": 1.3277,
+      "step": 3080
+    },
+    {
+      "epoch": 0.47264870652568786,
+      "grad_norm": 5.9267578125,
+      "learning_rate": 7.6485788689741e-06,
+      "loss": 1.3257,
+      "step": 3090
+    },
+    {
+      "epoch": 0.4741783139901723,
+      "grad_norm": 5.1862473487854,
+      "learning_rate": 7.580781044003324e-06,
+      "loss": 1.2984,
+      "step": 3100
+    },
+    {
+      "epoch": 0.4757079214546567,
+      "grad_norm": 6.191176891326904,
+      "learning_rate": 7.513101128351454e-06,
+      "loss": 1.3031,
+      "step": 3110
+    },
+    {
+      "epoch": 0.4772375289191411,
+      "grad_norm": 5.009718894958496,
+      "learning_rate": 7.445542420642097e-06,
+      "loss": 1.3008,
+      "step": 3120
+    },
+    {
+      "epoch": 0.47876713638362556,
+      "grad_norm": 5.5182342529296875,
+      "learning_rate": 7.378108213591355e-06,
+      "loss": 1.307,
+      "step": 3130
+    },
+    {
+      "epoch": 0.48029674384811,
+      "grad_norm": 6.071241855621338,
+      "learning_rate": 7.310801793847344e-06,
+      "loss": 1.3108,
+      "step": 3140
+    },
+    {
+      "epoch": 0.4818263513125944,
+      "grad_norm": 5.727301597595215,
+      "learning_rate": 7.243626441830009e-06,
+      "loss": 1.2882,
+      "step": 3150
+    },
+    {
+      "epoch": 0.4833559587770788,
+      "grad_norm": 7.817323207855225,
+      "learning_rate": 7.176585431571235e-06,
+      "loss": 1.3124,
+      "step": 3160
+    },
+    {
+      "epoch": 0.48488556624156326,
+      "grad_norm": 4.9556884765625,
+      "learning_rate": 7.109682030555283e-06,
+      "loss": 1.3036,
+      "step": 3170
+    },
+    {
+      "epoch": 0.4864151737060477,
+      "grad_norm": 5.458242416381836,
+      "learning_rate": 7.042919499559538e-06,
+      "loss": 1.2807,
+      "step": 3180
+    },
+    {
+      "epoch": 0.48794478117053214,
+      "grad_norm": 4.993044853210449,
+      "learning_rate": 6.976301092495556e-06,
+      "loss": 1.2984,
+      "step": 3190
+    },
+    {
+      "epoch": 0.4894743886350165,
+      "grad_norm": 6.809391021728516,
+      "learning_rate": 6.909830056250527e-06,
+      "loss": 1.2957,
+      "step": 3200
+    },
+    {
+      "epoch": 0.49100399609950096,
+      "grad_norm": 4.602332592010498,
+      "learning_rate": 6.843509630528977e-06,
+      "loss": 1.2941,
+      "step": 3210
+    },
+    {
+      "epoch": 0.4925336035639854,
+      "grad_norm": 4.978896617889404,
+      "learning_rate": 6.777343047694891e-06,
+      "loss": 1.2996,
+      "step": 3220
+    },
+    {
+      "epoch": 0.49406321102846984,
+      "grad_norm": 4.655756950378418,
+      "learning_rate": 6.711333532614168e-06,
+      "loss": 1.3008,
+      "step": 3230
+    },
+    {
+      "epoch": 0.4955928184929542,
+      "grad_norm": 5.080605506896973,
+      "learning_rate": 6.645484302497452e-06,
+      "loss": 1.2914,
+      "step": 3240
+    },
+    {
+      "epoch": 0.49712242595743866,
+      "grad_norm": 5.634034633636475,
+      "learning_rate": 6.579798566743314e-06,
+      "loss": 1.2875,
+      "step": 3250
+    },
+    {
+      "epoch": 0.4986520334219231,
+      "grad_norm": 4.864494323730469,
+      "learning_rate": 6.5142795267818505e-06,
+      "loss": 1.2804,
+      "step": 3260
+    },
+    {
+      "epoch": 0.5001816408864075,
+      "grad_norm": 4.328612327575684,
+      "learning_rate": 6.448930375918632e-06,
+      "loss": 1.2837,
+      "step": 3270
+    },
+    {
+      "epoch": 0.501711248350892,
+      "grad_norm": 5.613663196563721,
+      "learning_rate": 6.383754299179079e-06,
+      "loss": 1.2845,
+      "step": 3280
+    },
+    {
+      "epoch": 0.5032408558153764,
+      "grad_norm": 4.356953144073486,
+      "learning_rate": 6.318754473153221e-06,
+      "loss": 1.2863,
+      "step": 3290
+    },
+    {
+      "epoch": 0.5047704632798609,
+      "grad_norm": 6.725609302520752,
+      "learning_rate": 6.25393406584088e-06,
+      "loss": 1.2821,
+      "step": 3300
+    },
+    {
+      "epoch": 0.5063000707443452,
+      "grad_norm": 6.448828220367432,
+      "learning_rate": 6.18929623649726e-06,
+      "loss": 1.2866,
+      "step": 3310
+    },
+    {
+      "epoch": 0.5078296782088296,
+      "grad_norm": 5.340694904327393,
+      "learning_rate": 6.124844135478971e-06,
+      "loss": 1.2896,
+      "step": 3320
+    },
+    {
+      "epoch": 0.5093592856733141,
+      "grad_norm": 4.572837829589844,
+      "learning_rate": 6.06058090409049e-06,
+      "loss": 1.2693,
+      "step": 3330
+    },
+    {
+      "epoch": 0.5108888931377985,
+      "grad_norm": 5.755439281463623,
+      "learning_rate": 5.996509674431053e-06,
+      "loss": 1.2622,
+      "step": 3340
+    },
+    {
+      "epoch": 0.5124185006022829,
+      "grad_norm": 3.928386926651001,
+      "learning_rate": 5.932633569242e-06,
+      "loss": 1.2801,
+      "step": 3350
+    },
+    {
+      "epoch": 0.5139481080667674,
+      "grad_norm": 4.075456142425537,
+      "learning_rate": 5.868955701754584e-06,
+      "loss": 1.3225,
+      "step": 3360
+    },
+    {
+      "epoch": 0.5154777155312518,
+      "grad_norm": 5.155795097351074,
+      "learning_rate": 5.8054791755382286e-06,
+      "loss": 1.2803,
+      "step": 3370
+    },
+    {
+      "epoch": 0.5170073229957363,
+      "grad_norm": 3.7002131938934326,
+      "learning_rate": 5.742207084349274e-06,
+      "loss": 1.2582,
+      "step": 3380
+    },
+    {
+      "epoch": 0.5185369304602206,
+      "grad_norm": 4.773938179016113,
+      "learning_rate": 5.679142511980176e-06,
+      "loss": 1.2735,
+      "step": 3390
+    },
+    {
+      "epoch": 0.520066537924705,
+      "grad_norm": 4.110844612121582,
+      "learning_rate": 5.616288532109225e-06,
+      "loss": 1.2655,
+      "step": 3400
+    },
+    {
+      "epoch": 0.5215961453891895,
+      "grad_norm": 5.912238121032715,
+      "learning_rate": 5.553648208150728e-06,
+      "loss": 1.2853,
+      "step": 3410
+    },
+    {
+      "epoch": 0.5231257528536739,
+      "grad_norm": 5.354110240936279,
+      "learning_rate": 5.491224593105695e-06,
+      "loss": 1.268,
+      "step": 3420
+    },
+    {
+      "epoch": 0.5246553603181584,
+      "grad_norm": 3.9772727489471436,
+      "learning_rate": 5.429020729413062e-06,
+      "loss": 1.2721,
+      "step": 3430
+    },
+    {
+      "epoch": 0.5261849677826428,
+      "grad_norm": 5.658088207244873,
+      "learning_rate": 5.367039648801386e-06,
+      "loss": 1.2643,
+      "step": 3440
+    },
+    {
+      "epoch": 0.5277145752471272,
+      "grad_norm": 5.285284042358398,
+      "learning_rate": 5.305284372141095e-06,
+      "loss": 1.2616,
+      "step": 3450
+    },
+    {
+      "epoch": 0.5292441827116117,
+      "grad_norm": 3.6706132888793945,
+      "learning_rate": 5.243757909297247e-06,
+      "loss": 1.2753,
+      "step": 3460
+    },
+    {
+      "epoch": 0.530773790176096,
+      "grad_norm": 4.3954176902771,
+      "learning_rate": 5.1824632589828465e-06,
+      "loss": 1.2637,
+      "step": 3470
+    },
+    {
+      "epoch": 0.5323033976405804,
+      "grad_norm": 3.5135397911071777,
+      "learning_rate": 5.121403408612672e-06,
+      "loss": 1.2662,
+      "step": 3480
+    },
+    {
+      "epoch": 0.5338330051050649,
+      "grad_norm": 4.423898220062256,
+      "learning_rate": 5.060581334157693e-06,
+      "loss": 1.2543,
+      "step": 3490
+    },
+    {
+      "epoch": 0.5353626125695493,
+      "grad_norm": 4.656537055969238,
+      "learning_rate": 5.000000000000003e-06,
+      "loss": 1.2666,
+      "step": 3500
+    },
+    {
+      "epoch": 0.5368922200340338,
+      "grad_norm": 3.9426023960113525,
+      "learning_rate": 4.939662358788364e-06,
+      "loss": 1.255,
+      "step": 3510
+    },
+    {
+      "epoch": 0.5384218274985182,
+      "grad_norm": 5.163910865783691,
+      "learning_rate": 4.879571351294287e-06,
+      "loss": 1.2504,
+      "step": 3520
+    },
+    {
+      "epoch": 0.5399514349630026,
+      "grad_norm": 3.8274786472320557,
+      "learning_rate": 4.8197299062687e-06,
+      "loss": 1.2566,
+      "step": 3530
+    },
+    {
+      "epoch": 0.5414810424274871,
+      "grad_norm": 4.835800647735596,
+      "learning_rate": 4.76014094029921e-06,
+      "loss": 1.2632,
+      "step": 3540
+    },
+    {
+      "epoch": 0.5430106498919715,
+      "grad_norm": 4.24269962310791,
+      "learning_rate": 4.700807357667953e-06,
+      "loss": 1.2501,
+      "step": 3550
+    },
+    {
+      "epoch": 0.544540257356456,
+      "grad_norm": 3.6899170875549316,
+      "learning_rate": 4.641732050210032e-06,
+      "loss": 1.26,
+      "step": 3560
+    },
+    {
+      "epoch": 0.5460698648209403,
+      "grad_norm": 3.939570665359497,
+      "learning_rate": 4.582917897172603e-06,
+      "loss": 1.2516,
+      "step": 3570
+    },
+    {
+      "epoch": 0.5475994722854247,
+      "grad_norm": 4.007904529571533,
+      "learning_rate": 4.524367765074499e-06,
+      "loss": 1.2509,
+      "step": 3580
+    },
+    {
+      "epoch": 0.5491290797499092,
+      "grad_norm": 3.8148136138916016,
+      "learning_rate": 4.46608450756656e-06,
+      "loss": 1.252,
+      "step": 3590
+    },
+    {
+      "epoch": 0.5506586872143936,
+      "grad_norm": 4.753785133361816,
+      "learning_rate": 4.408070965292534e-06,
+      "loss": 1.2515,
+      "step": 3600
+    },
+    {
+      "epoch": 0.552188294678878,
+      "grad_norm": 7.303796291351318,
+      "learning_rate": 4.350329965750622e-06,
+      "loss": 1.2536,
+      "step": 3610
+    },
+    {
+      "epoch": 0.5537179021433625,
+      "grad_norm": 4.17638635635376,
+      "learning_rate": 4.292864323155684e-06,
+      "loss": 1.2564,
+      "step": 3620
+    },
+    {
+      "epoch": 0.5552475096078469,
+      "grad_norm": 4.36394739151001,
+      "learning_rate": 4.235676838302069e-06,
+      "loss": 1.2454,
+      "step": 3630
+    },
+    {
+      "epoch": 0.5567771170723314,
+      "grad_norm": 4.1072187423706055,
+      "learning_rate": 4.178770298427107e-06,
+      "loss": 1.2449,
+      "step": 3640
+    },
+    {
+      "epoch": 0.5583067245368157,
+      "grad_norm": 4.247386455535889,
+      "learning_rate": 4.12214747707527e-06,
+      "loss": 1.2557,
+      "step": 3650
+    },
+    {
+      "epoch": 0.5598363320013001,
+      "grad_norm": 4.688133239746094,
+      "learning_rate": 4.065811133962987e-06,
+      "loss": 1.2468,
+      "step": 3660
+    },
+    {
+      "epoch": 0.5613659394657846,
+      "grad_norm": 3.2627274990081787,
+      "learning_rate": 4.009764014844143e-06,
+      "loss": 1.2499,
+      "step": 3670
+    },
+    {
+      "epoch": 0.562895546930269,
+      "grad_norm": 3.4146535396575928,
+      "learning_rate": 3.954008851376252e-06,
+      "loss": 1.2512,
+      "step": 3680
+    },
+    {
+      "epoch": 0.5644251543947535,
+      "grad_norm": 3.7398815155029297,
+      "learning_rate": 3.898548360987325e-06,
+      "loss": 1.2456,
+      "step": 3690
+    },
+    {
+      "epoch": 0.5659547618592379,
+      "grad_norm": 3.9804248809814453,
+      "learning_rate": 3.8433852467434175e-06,
+      "loss": 1.2586,
+      "step": 3700
+    },
+    {
+      "epoch": 0.5674843693237223,
+      "grad_norm": 4.1223602294921875,
+      "learning_rate": 3.7885221972168974e-06,
+      "loss": 1.2523,
+      "step": 3710
+    },
+    {
+      "epoch": 0.5690139767882068,
+      "grad_norm": 3.9871490001678467,
+      "learning_rate": 3.7339618863553983e-06,
+      "loss": 1.244,
+      "step": 3720
+    },
+    {
+      "epoch": 0.5705435842526911,
+      "grad_norm": 3.3211615085601807,
+      "learning_rate": 3.679706973351491e-06,
+      "loss": 1.2395,
+      "step": 3730
+    },
+    {
+      "epoch": 0.5720731917171756,
+      "grad_norm": 3.107677459716797,
+      "learning_rate": 3.625760102513103e-06,
+      "loss": 1.2336,
+      "step": 3740
+    },
+    {
+      "epoch": 0.57360279918166,
+      "grad_norm": 3.6052770614624023,
+      "learning_rate": 3.5721239031346067e-06,
+      "loss": 1.2402,
+      "step": 3750
+    },
+    {
+      "epoch": 0.5751324066461444,
+      "grad_norm": 3.1450812816619873,
+      "learning_rate": 3.5188009893686916e-06,
+      "loss": 1.2368,
+      "step": 3760
+    },
+    {
+      "epoch": 0.5766620141106289,
+      "grad_norm": 2.91536021232605,
+      "learning_rate": 3.4657939600989453e-06,
+      "loss": 1.2357,
+      "step": 3770
+    },
+    {
+      "epoch": 0.5781916215751133,
+      "grad_norm": 3.327528953552246,
+      "learning_rate": 3.4131053988131947e-06,
+      "loss": 1.2309,
+      "step": 3780
+    },
+    {
+      "epoch": 0.5797212290395977,
+      "grad_norm": 3.9608235359191895,
+      "learning_rate": 3.360737873477584e-06,
+      "loss": 1.2304,
+      "step": 3790
+    },
+    {
+      "epoch": 0.5812508365040822,
+      "grad_norm": 3.727250814437866,
+      "learning_rate": 3.308693936411421e-06,
+      "loss": 1.2401,
+      "step": 3800
+    },
+    {
+      "epoch": 0.5827804439685665,
+      "grad_norm": 3.9562222957611084,
+      "learning_rate": 3.2569761241627694e-06,
+      "loss": 1.2354,
+      "step": 3810
+    },
+    {
+      "epoch": 0.584310051433051,
+      "grad_norm": 3.277930736541748,
+      "learning_rate": 3.2055869573848374e-06,
+      "loss": 1.2414,
+      "step": 3820
+    },
+    {
+      "epoch": 0.5858396588975354,
+      "grad_norm": 7.952522277832031,
+      "learning_rate": 3.1545289407131128e-06,
+      "loss": 1.2468,
+      "step": 3830
+    },
+    {
+      "epoch": 0.5873692663620198,
+      "grad_norm": 3.296581506729126,
+      "learning_rate": 3.103804562643302e-06,
+      "loss": 1.238,
+      "step": 3840
+    },
+    {
+      "epoch": 0.5888988738265043,
+      "grad_norm": 3.134167432785034,
+      "learning_rate": 3.0534162954100264e-06,
+      "loss": 1.2284,
+      "step": 3850
+    },
+    {
+      "epoch": 0.5904284812909887,
+      "grad_norm": 3.1740708351135254,
+      "learning_rate": 3.003366594866345e-06,
+      "loss": 1.2455,
+      "step": 3860
+    },
+    {
+      "epoch": 0.5919580887554732,
+      "grad_norm": 3.03051495552063,
+      "learning_rate": 2.953657900364053e-06,
+      "loss": 1.2254,
+      "step": 3870
+    },
+    {
+      "epoch": 0.5934876962199576,
+      "grad_norm": 3.150672435760498,
+      "learning_rate": 2.9042926346347932e-06,
+      "loss": 1.2343,
+      "step": 3880
+    },
+    {
+      "epoch": 0.595017303684442,
+      "grad_norm": 3.540771961212158,
+      "learning_rate": 2.855273203671969e-06,
+      "loss": 1.2236,
+      "step": 3890
+    },
+    {
+      "epoch": 0.5965469111489264,
+      "grad_norm": 2.9037368297576904,
+      "learning_rate": 2.8066019966134907e-06,
+      "loss": 1.2253,
+      "step": 3900
+    },
+    {
+      "epoch": 0.5980765186134108,
+      "grad_norm": 4.331482410430908,
+      "learning_rate": 2.7582813856253276e-06,
+      "loss": 1.2364,
+      "step": 3910
+    },
+    {
+      "epoch": 0.5996061260778952,
+      "grad_norm": 3.0110764503479004,
+      "learning_rate": 2.7103137257858867e-06,
+      "loss": 1.2297,
+      "step": 3920
+    },
+    {
+      "epoch": 0.6011357335423797,
+      "grad_norm": 3.1451432704925537,
+      "learning_rate": 2.6627013549712355e-06,
+      "loss": 1.2375,
+      "step": 3930
+    },
+    {
+      "epoch": 0.6026653410068641,
+      "grad_norm": 2.8832802772521973,
+      "learning_rate": 2.615446593741161e-06,
+      "loss": 1.2285,
+      "step": 3940
+    },
+    {
+      "epoch": 0.6041949484713486,
+      "grad_norm": 3.073871374130249,
+      "learning_rate": 2.5685517452260566e-06,
+      "loss": 1.2339,
+      "step": 3950
+    },
+    {
+      "epoch": 0.605724555935833,
+      "grad_norm": 2.9287912845611572,
+      "learning_rate": 2.522019095014683e-06,
+      "loss": 1.215,
+      "step": 3960
+    },
+    {
+      "epoch": 0.6072541634003173,
+      "grad_norm": 2.7520062923431396,
+      "learning_rate": 2.4758509110427576e-06,
+      "loss": 1.2191,
+      "step": 3970
+    },
+    {
+      "epoch": 0.6087837708648018,
+      "grad_norm": 2.9838249683380127,
+      "learning_rate": 2.4300494434824373e-06,
+      "loss": 1.2257,
+      "step": 3980
+    },
+    {
+      "epoch": 0.6103133783292862,
+      "grad_norm": 2.9562954902648926,
+      "learning_rate": 2.3846169246326345e-06,
+      "loss": 1.214,
+      "step": 3990
+    },
+    {
+      "epoch": 0.6118429857937707,
+      "grad_norm": 2.8479607105255127,
+      "learning_rate": 2.339555568810221e-06,
+      "loss": 1.2229,
+      "step": 4000
+    },
+    {
+      "epoch": 0.6133725932582551,
+      "grad_norm": 2.977555990219116,
+      "learning_rate": 2.2948675722421086e-06,
+      "loss": 1.2283,
+      "step": 4010
+    },
+    {
+      "epoch": 0.6149022007227395,
+      "grad_norm": 3.5742952823638916,
+      "learning_rate": 2.2505551129582047e-06,
+      "loss": 1.2128,
+      "step": 4020
+    },
+    {
+      "epoch": 0.616431808187224,
+      "grad_norm": 2.897850513458252,
+      "learning_rate": 2.206620350685257e-06,
+      "loss": 1.2269,
+      "step": 4030
+    },
+    {
+      "epoch": 0.6179614156517084,
+      "grad_norm": 3.435910224914551,
+      "learning_rate": 2.163065426741603e-06,
+      "loss": 1.2249,
+      "step": 4040
+    },
+    {
+      "epoch": 0.6194910231161928,
+      "grad_norm": 3.5438225269317627,
+      "learning_rate": 2.119892463932781e-06,
+      "loss": 1.2275,
+      "step": 4050
+    },
+    {
+      "epoch": 0.6210206305806772,
+      "grad_norm": 3.0088398456573486,
+      "learning_rate": 2.0771035664480944e-06,
+      "loss": 1.222,
+      "step": 4060
+    },
+    {
+      "epoch": 0.6225502380451616,
+      "grad_norm": 2.5842950344085693,
+      "learning_rate": 2.0347008197580376e-06,
+      "loss": 1.2227,
+      "step": 4070
+    },
+    {
+      "epoch": 0.6240798455096461,
+      "grad_norm": 3.3006832599639893,
+      "learning_rate": 1.9926862905126663e-06,
+      "loss": 1.2216,
+      "step": 4080
+    },
+    {
+      "epoch": 0.6256094529741305,
+      "grad_norm": 2.9297842979431152,
+      "learning_rate": 1.95106202644086e-06,
+      "loss": 1.2227,
+      "step": 4090
+    },
+    {
+      "epoch": 0.6271390604386149,
+      "grad_norm": 2.8921964168548584,
+      "learning_rate": 1.9098300562505266e-06,
+      "loss": 1.2147,
+      "step": 4100
+    },
+    {
+      "epoch": 0.6286686679030994,
+      "grad_norm": 3.4011294841766357,
+      "learning_rate": 1.8689923895297247e-06,
+      "loss": 1.2056,
+      "step": 4110
+    },
+    {
+      "epoch": 0.6301982753675838,
+      "grad_norm": 5.375010967254639,
+      "learning_rate": 1.8285510166487154e-06,
+      "loss": 1.234,
+      "step": 4120
+    },
+    {
+      "epoch": 0.6317278828320683,
+      "grad_norm": 2.773867607116699,
+      "learning_rate": 1.7885079086629598e-06,
+      "loss": 1.2109,
+      "step": 4130
+    },
+    {
+      "epoch": 0.6332574902965526,
+      "grad_norm": 2.951381206512451,
+      "learning_rate": 1.7488650172170496e-06,
+      "loss": 1.2252,
+      "step": 4140
+    },
+    {
+      "epoch": 0.634787097761037,
+      "grad_norm": 3.2685885429382324,
+      "learning_rate": 1.709624274449584e-06,
+      "loss": 1.2227,
+      "step": 4150
+    },
+    {
+      "epoch": 0.6363167052255215,
+      "grad_norm": 3.015773057937622,
+      "learning_rate": 1.6707875928990059e-06,
+      "loss": 1.2276,
+      "step": 4160
+    },
+    {
+      "epoch": 0.6378463126900059,
+      "grad_norm": 3.0250165462493896,
+      "learning_rate": 1.6323568654103838e-06,
+      "loss": 1.2152,
+      "step": 4170
+    },
+    {
+      "epoch": 0.6393759201544904,
+      "grad_norm": 3.3289341926574707,
+      "learning_rate": 1.5943339650431578e-06,
+      "loss": 1.2208,
+      "step": 4180
+    },
+    {
+      "epoch": 0.6409055276189748,
+      "grad_norm": 2.703160524368286,
+      "learning_rate": 1.5567207449798517e-06,
+      "loss": 1.2139,
+      "step": 4190
+    },
+    {
+      "epoch": 0.6424351350834592,
+      "grad_norm": 3.5222249031066895,
+      "learning_rate": 1.5195190384357405e-06,
+      "loss": 1.2105,
+      "step": 4200
+    },
+    {
+      "epoch": 0.6439647425479437,
+      "grad_norm": 2.8859856128692627,
+      "learning_rate": 1.4827306585695234e-06,
+      "loss": 1.2039,
+      "step": 4210
+    },
+    {
+      "epoch": 0.645494350012428,
+      "grad_norm": 2.7604501247406006,
+      "learning_rate": 1.446357398394934e-06,
+      "loss": 1.2119,
+      "step": 4220
+    },
+    {
+      "epoch": 0.6470239574769124,
+      "grad_norm": 2.7769792079925537,
+      "learning_rate": 1.4104010306933558e-06,
+      "loss": 1.2122,
+      "step": 4230
+    },
+    {
+      "epoch": 0.6485535649413969,
+      "grad_norm": 2.9969465732574463,
+      "learning_rate": 1.3748633079274254e-06,
+      "loss": 1.2106,
+      "step": 4240
+    },
+    {
+      "epoch": 0.6500831724058813,
+      "grad_norm": 3.1720831394195557,
+      "learning_rate": 1.339745962155613e-06,
+      "loss": 1.2193,
+      "step": 4250
+    },
+    {
+      "epoch": 0.6516127798703658,
+      "grad_norm": 2.9160704612731934,
+      "learning_rate": 1.30505070494781e-06,
+      "loss": 1.2185,
+      "step": 4260
+    },
+    {
+      "epoch": 0.6531423873348502,
+      "grad_norm": 2.8743717670440674,
+      "learning_rate": 1.2707792273019049e-06,
+      "loss": 1.22,
+      "step": 4270
+    },
+    {
+      "epoch": 0.6546719947993346,
+      "grad_norm": 2.518815279006958,
+      "learning_rate": 1.2369331995613664e-06,
+      "loss": 1.1971,
+      "step": 4280
+    },
+    {
+      "epoch": 0.6562016022638191,
+      "grad_norm": 3.0868263244628906,
+      "learning_rate": 1.2035142713338366e-06,
+      "loss": 1.2196,
+      "step": 4290
+    },
+    {
+      "epoch": 0.6577312097283035,
+      "grad_norm": 2.997687816619873,
+      "learning_rate": 1.1705240714107301e-06,
+      "loss": 1.2153,
+      "step": 4300
+    },
+    {
+      "epoch": 0.659260817192788,
+      "grad_norm": 3.175381898880005,
+      "learning_rate": 1.1379642076878528e-06,
+      "loss": 1.2142,
+      "step": 4310
+    },
+    {
+      "epoch": 0.6607904246572723,
+      "grad_norm": 3.1321346759796143,
+      "learning_rate": 1.1058362670870248e-06,
+      "loss": 1.2176,
+      "step": 4320
+    },
+    {
+      "epoch": 0.6623200321217567,
+      "grad_norm": 3.027937889099121,
+      "learning_rate": 1.0741418154787443e-06,
+      "loss": 1.219,
+      "step": 4330
+    },
+    {
+      "epoch": 0.6638496395862412,
+      "grad_norm": 3.1587741374969482,
+      "learning_rate": 1.042882397605871e-06,
+      "loss": 1.211,
+      "step": 4340
+    },
+    {
+      "epoch": 0.6653792470507256,
+      "grad_norm": 3.0507407188415527,
+      "learning_rate": 1.012059537008332e-06,
+      "loss": 1.2139,
+      "step": 4350
+    },
+    {
+      "epoch": 0.66690885451521,
+      "grad_norm": 2.46720027923584,
+      "learning_rate": 9.816747359488632e-07,
+      "loss": 1.225,
+      "step": 4360
+    },
+    {
+      "epoch": 0.6684384619796945,
+      "grad_norm": 2.722921371459961,
+      "learning_rate": 9.517294753398066e-07,
+      "loss": 1.2207,
+      "step": 4370
+    },
+    {
+      "epoch": 0.6699680694441789,
+      "grad_norm": 2.5753180980682373,
+      "learning_rate": 9.222252146709143e-07,
+      "loss": 1.2149,
+      "step": 4380
+    },
+    {
+      "epoch": 0.6714976769086634,
+      "grad_norm": 2.9014554023742676,
+      "learning_rate": 8.931633919382299e-07,
+      "loss": 1.2203,
+      "step": 4390
+    },
+    {
+      "epoch": 0.6730272843731477,
+      "grad_norm": 2.6880147457122803,
+      "learning_rate": 8.645454235739903e-07,
+      "loss": 1.2106,
+      "step": 4400
+    },
+    {
+      "epoch": 0.6745568918376321,
+      "grad_norm": 2.7393648624420166,
+      "learning_rate": 8.363727043776037e-07,
+      "loss": 1.2177,
+      "step": 4410
+    },
+    {
+      "epoch": 0.6760864993021166,
+      "grad_norm": 2.766064405441284,
+      "learning_rate": 8.086466074476562e-07,
+      "loss": 1.2126,
+      "step": 4420
+    },
+    {
+      "epoch": 0.677616106766601,
+      "grad_norm": 2.6107726097106934,
+      "learning_rate": 7.81368484114996e-07,
+      "loss": 1.2087,
+      "step": 4430
+    },
+    {
+      "epoch": 0.6791457142310855,
+      "grad_norm": 2.6271679401397705,
+      "learning_rate": 7.545396638768698e-07,
+      "loss": 1.2066,
+      "step": 4440
+    },
+    {
+      "epoch": 0.6806753216955699,
+      "grad_norm": 3.3646392822265625,
+      "learning_rate": 7.281614543321269e-07,
+      "loss": 1.2123,
+      "step": 4450
+    },
+    {
+      "epoch": 0.6822049291600543,
+      "grad_norm": 2.6894729137420654,
+      "learning_rate": 7.022351411174866e-07,
+      "loss": 1.2101,
+      "step": 4460
+    },
+    {
+      "epoch": 0.6837345366245388,
+      "grad_norm": 3.0988988876342773,
+      "learning_rate": 6.767619878448783e-07,
+      "loss": 1.2079,
+      "step": 4470
+    },
+    {
+      "epoch": 0.6852641440890231,
+      "grad_norm": 2.667529821395874,
+      "learning_rate": 6.517432360398556e-07,
+      "loss": 1.2115,
+      "step": 4480
+    },
+    {
+      "epoch": 0.6867937515535076,
+      "grad_norm": 4.324584484100342,
+      "learning_rate": 6.271801050810856e-07,
+      "loss": 1.2049,
+      "step": 4490
+    },
+    {
+      "epoch": 0.688323359017992,
+      "grad_norm": 2.921243906021118,
+      "learning_rate": 6.030737921409169e-07,
+      "loss": 1.2145,
+      "step": 4500
+    },
+    {
+      "epoch": 0.6898529664824764,
+      "grad_norm": 2.8604378700256348,
+      "learning_rate": 5.794254721270331e-07,
+      "loss": 1.2017,
+      "step": 4510
+    },
+    {
+      "epoch": 0.6913825739469609,
+      "grad_norm": 2.7629590034484863,
+      "learning_rate": 5.562362976251901e-07,
+      "loss": 1.2048,
+      "step": 4520
+    },
+    {
+      "epoch": 0.6929121814114453,
+      "grad_norm": 3.2407941818237305,
+      "learning_rate": 5.335073988430373e-07,
+      "loss": 1.2305,
+      "step": 4530
+    },
+    {
+      "epoch": 0.6944417888759297,
+      "grad_norm": 11.863210678100586,
+      "learning_rate": 5.112398835550348e-07,
+      "loss": 1.2077,
+      "step": 4540
+    },
+    {
+      "epoch": 0.6959713963404142,
+      "grad_norm": 2.7767953872680664,
+      "learning_rate": 4.894348370484648e-07,
+      "loss": 1.2085,
+      "step": 4550
+    },
+    {
+      "epoch": 0.6975010038048985,
+      "grad_norm": 3.932037591934204,
+      "learning_rate": 4.6809332207053083e-07,
+      "loss": 1.1995,
+      "step": 4560
+    },
+    {
+      "epoch": 0.699030611269383,
+      "grad_norm": 2.76766037940979,
+      "learning_rate": 4.4721637877656377e-07,
+      "loss": 1.2048,
+      "step": 4570
+    },
+    {
+      "epoch": 0.7005602187338674,
+      "grad_norm": 2.9043328762054443,
+      "learning_rate": 4.268050246793276e-07,
+      "loss": 1.1983,
+      "step": 4580
+    },
+    {
+      "epoch": 0.7020898261983518,
+      "grad_norm": 2.993134021759033,
+      "learning_rate": 4.068602545994249e-07,
+      "loss": 1.2186,
+      "step": 4590
+    },
+    {
+      "epoch": 0.7036194336628363,
+      "grad_norm": 2.8081493377685547,
+      "learning_rate": 3.8738304061681107e-07,
+      "loss": 1.2079,
+      "step": 4600
+    },
+    {
+      "epoch": 0.7051490411273207,
+      "grad_norm": 2.9038686752319336,
+      "learning_rate": 3.68374332023419e-07,
+      "loss": 1.2116,
+      "step": 4610
+    },
+    {
+      "epoch": 0.7066786485918052,
+      "grad_norm": 2.708916425704956,
+      "learning_rate": 3.498350552768859e-07,
+      "loss": 1.2159,
+      "step": 4620
+    },
+    {
+      "epoch": 0.7082082560562896,
+      "grad_norm": 2.7816905975341797,
+      "learning_rate": 3.3176611395540625e-07,
+      "loss": 1.1976,
+      "step": 4630
+    },
+    {
+      "epoch": 0.709737863520774,
+      "grad_norm": 2.761017322540283,
+      "learning_rate": 3.1416838871368925e-07,
+      "loss": 1.2133,
+      "step": 4640
+    },
+    {
+      "epoch": 0.7112674709852584,
+      "grad_norm": 2.566500186920166,
+      "learning_rate": 2.970427372400353e-07,
+      "loss": 1.2096,
+      "step": 4650
+    },
+    {
+      "epoch": 0.7127970784497428,
+      "grad_norm": 2.5895071029663086,
+      "learning_rate": 2.8038999421453827e-07,
+      "loss": 1.1967,
+      "step": 4660
+    },
+    {
+      "epoch": 0.7143266859142272,
+      "grad_norm": 2.4453067779541016,
+      "learning_rate": 2.6421097126839714e-07,
+      "loss": 1.2091,
+      "step": 4670
+    },
+    {
+      "epoch": 0.7158562933787117,
+      "grad_norm": 3.3402082920074463,
+      "learning_rate": 2.4850645694436736e-07,
+      "loss": 1.2161,
+      "step": 4680
+    },
+    {
+      "epoch": 0.7173859008431961,
+      "grad_norm": 2.6519172191619873,
+      "learning_rate": 2.332772166583208e-07,
+      "loss": 1.208,
+      "step": 4690
+    },
+    {
+      "epoch": 0.7189155083076806,
+      "grad_norm": 2.572880983352661,
+      "learning_rate": 2.1852399266194312e-07,
+      "loss": 1.2123,
+      "step": 4700
+    },
+    {
+      "epoch": 0.720445115772165,
+      "grad_norm": 2.7262823581695557,
+      "learning_rate": 2.0424750400655947e-07,
+      "loss": 1.2068,
+      "step": 4710
+    },
+    {
+      "epoch": 0.7219747232366494,
+      "grad_norm": 3.2167742252349854,
+      "learning_rate": 1.9044844650808468e-07,
+      "loss": 1.2133,
+      "step": 4720
+    },
+    {
+      "epoch": 0.7235043307011338,
+      "grad_norm": 2.587249994277954,
+      "learning_rate": 1.7712749271311392e-07,
+      "loss": 1.2165,
+      "step": 4730
+    },
+    {
+      "epoch": 0.7250339381656182,
+      "grad_norm": 2.9766838550567627,
+      "learning_rate": 1.6428529186614195e-07,
+      "loss": 1.2017,
+      "step": 4740
+    },
+    {
+      "epoch": 0.7265635456301027,
+      "grad_norm": 2.9937245845794678,
+      "learning_rate": 1.519224698779198e-07,
+      "loss": 1.2068,
+      "step": 4750
+    },
+    {
+      "epoch": 0.7280931530945871,
+      "grad_norm": 2.7272565364837646,
+      "learning_rate": 1.400396292949513e-07,
+      "loss": 1.202,
+      "step": 4760
+    },
+    {
+      "epoch": 0.7296227605590715,
+      "grad_norm": 2.6886658668518066,
+      "learning_rate": 1.2863734927012094e-07,
+      "loss": 1.2144,
+      "step": 4770
+    },
+    {
+      "epoch": 0.731152368023556,
+      "grad_norm": 3.8151731491088867,
+      "learning_rate": 1.1771618553447217e-07,
+      "loss": 1.2095,
+      "step": 4780
+    },
+    {
+      "epoch": 0.7326819754880404,
+      "grad_norm": 2.7299585342407227,
+      "learning_rate": 1.0727667037011668e-07,
+      "loss": 1.2158,
+      "step": 4790
+    },
+    {
+      "epoch": 0.7342115829525248,
+      "grad_norm": 2.6803183555603027,
+      "learning_rate": 9.731931258429638e-08,
+      "loss": 1.2039,
+      "step": 4800
+    },
+    {
+      "epoch": 0.7357411904170092,
+      "grad_norm": 2.7974493503570557,
+      "learning_rate": 8.784459748458318e-08,
+      "loss": 1.2088,
+      "step": 4810
+    },
+    {
+      "epoch": 0.7372707978814936,
+      "grad_norm": 2.8328588008880615,
+      "learning_rate": 7.885298685522235e-08,
+      "loss": 1.204,
+      "step": 4820
+    },
+    {
+      "epoch": 0.7388004053459781,
+      "grad_norm": 3.2823333740234375,
+      "learning_rate": 7.034491893463059e-08,
+      "loss": 1.2204,
+      "step": 4830
+    },
+    {
+      "epoch": 0.7403300128104625,
+      "grad_norm": 2.736294746398926,
+      "learning_rate": 6.232080839403631e-08,
+      "loss": 1.2123,
+      "step": 4840
+    },
+    {
+      "epoch": 0.7418596202749469,
+      "grad_norm": 2.680877685546875,
+      "learning_rate": 5.4781046317267103e-08,
+      "loss": 1.2016,
+      "step": 4850
+    },
+    {
+      "epoch": 0.7433892277394314,
+      "grad_norm": 2.821756362915039,
+      "learning_rate": 4.772600018168816e-08,
+      "loss": 1.2056,
+      "step": 4860
+    },
+    {
+      "epoch": 0.7449188352039158,
+      "grad_norm": 2.4544124603271484,
+      "learning_rate": 4.115601384029666e-08,
+      "loss": 1.1988,
+      "step": 4870
+    },
+    {
+      "epoch": 0.7464484426684003,
+      "grad_norm": 2.5160040855407715,
+      "learning_rate": 3.50714075049563e-08,
+      "loss": 1.2098,
+      "step": 4880
+    },
+    {
+      "epoch": 0.7479780501328847,
+      "grad_norm": 3.101855754852295,
+      "learning_rate": 2.947247773079753e-08,
+      "loss": 1.2126,
+      "step": 4890
+    },
+    {
+      "epoch": 0.749507657597369,
+      "grad_norm": 2.718191385269165,
+      "learning_rate": 2.4359497401758026e-08,
+      "loss": 1.2102,
+      "step": 4900
+    },
+    {
+      "epoch": 0.7510372650618535,
+      "grad_norm": 2.9169981479644775,
+      "learning_rate": 1.973271571728441e-08,
+      "loss": 1.214,
+      "step": 4910
+    },
+    {
+      "epoch": 0.7525668725263379,
+      "grad_norm": 2.6038503646850586,
+      "learning_rate": 1.5592358180189782e-08,
+      "loss": 1.2092,
+      "step": 4920
+    },
+    {
+      "epoch": 0.7540964799908224,
+      "grad_norm": 2.4344565868377686,
+      "learning_rate": 1.1938626585660252e-08,
+      "loss": 1.2001,
+      "step": 4930
+    },
+    {
+      "epoch": 0.7556260874553068,
+      "grad_norm": 2.6881103515625,
+      "learning_rate": 8.771699011416169e-09,
+      "loss": 1.2274,
+      "step": 4940
+    },
+    {
+      "epoch": 0.7571556949197912,
+      "grad_norm": 2.711486339569092,
+      "learning_rate": 6.091729809042379e-09,
+      "loss": 1.2083,
+      "step": 4950
+    },
+    {
+      "epoch": 0.7586853023842757,
+      "grad_norm": 2.7651705741882324,
+      "learning_rate": 3.898849596456477e-09,
+      "loss": 1.2047,
+      "step": 4960
+    },
+    {
+      "epoch": 0.7602149098487601,
+      "grad_norm": 2.634777545928955,
+      "learning_rate": 2.193165251545004e-09,
+      "loss": 1.2038,
+      "step": 4970
+    },
+    {
+      "epoch": 0.7617445173132444,
+      "grad_norm": 2.7561585903167725,
+      "learning_rate": 9.74759906957612e-10,
+      "loss": 1.2029,
+      "step": 4980
+    },
+    {
+      "epoch": 0.7632741247777289,
+      "grad_norm": 2.701568841934204,
+      "learning_rate": 2.436929460525317e-10,
+      "loss": 1.2074,
+      "step": 4990
+    },
+    {
+      "epoch": 0.7648037322422133,
+      "grad_norm": 2.763172149658203,
+      "learning_rate": 0.0,
+      "loss": 1.2121,
+      "step": 5000
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 5000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 1000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 9.5664155394048e+17,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-5000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34016aeb1c60a2b2f37eab5658f24f3f89496572839a5af4ccf1303cef5cae5c
+size 5304